diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ba3712f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,16 @@ +# Keep release text byte-stable so SHA-256 verification is portable. +*.cff text eol=lf +*.json text eol=lf +*.jsonl text eol=lf +*.md text eol=lf +*.py text eol=lf +*.svg text eol=lf +*.tex text eol=lf +*.bib text eol=lf +*.txt text eol=lf +*.bst text eol=lf +*.sty text eol=lf +*.pdf binary +*.png binary +*.wav binary +*.mp3 binary diff --git a/.github/DISCUSSION_TEMPLATE.md b/.github/DISCUSSION_TEMPLATE.md deleted file mode 100644 index 39b1979..0000000 --- a/.github/DISCUSSION_TEMPLATE.md +++ /dev/null @@ -1,56 +0,0 @@ -# Contributing to PersonaVoice - -Thanks for your interest in contributing to PersonaVoice! This project is an academic open-source effort, and we welcome contributions of all kinds: bug fixes, new experiments, documentation improvements, and architectural extensions. - -## Code of Conduct - -Be respectful, constructive, and academic. Cite prior work properly. Critique ideas, not people. - -## How to Contribute - -### 1. Fork & Branch -- Fork the repository. -- Create a feature branch: `git checkout -b feature/my-feature`. - -### 2. Develop -- Follow the existing code style (PEP 8, line length ≤ 120). -- Add docstrings to all new functions / classes (Google style). -- Keep modules focused — PersonaVoice follows a strict architecture (LAAG / OES / FiLM / CEAG / Persona). Place new code in the correct subpackage: - - `personavoice/tts_backbone/` — TTS backbone, adapters, sampler, vocoder - - `personavoice/microaug/` — Sample enhancement (OES) - - `personavoice/persona/` — Persona extraction - - `personavoice/common/` — Shared utilities - - `personavoice/demo/` — Web demo - - `personavoice/experiment/` — Evaluation scripts - - `scripts/` — One-off setup / data-prep scripts - -### 3. Test -- Run smoke tests: - ```bash - python -c "import personavoice; print(personavoice.__version__)" - python -c "from personavoice.config import SOTA_CONFIG; print(SOTA_CONFIG.version)" - ``` -- For experiments, run on the bundled `1111.mp3` first. - -### 4. Submit a PR -- Fill in the PR template. -- If your change affects an architectural component (LAAG / OES / FiLM / CEAG), provide ablation evidence (200-sample paired t-test, p < 0.05, Cohen's d > 0.1). -- Update `README.md` / `README_zh.md` / `ARCHITECTURE.md` if behavior changes. - -## Architectural Principles - -1. **Plug-in Adapter Design** — F5-TTS backbone stays frozen. Train only lightweight adapters (~2M params). -2. **Ablation-Driven Decisions** — Every module must prove its worth via 200-sample paired t-test. -3. **Academic Honesty** — Report trade-offs honestly. Do not over-claim SOTA. -4. **Configuration Uniqueness** — All inference parameters live in `personavoice/config.py`. Do not hardcode parameters in scripts. -5. **1-Second Constraint** — All designs must work with as little as 1 second of reference audio. - -## Reporting Issues - -- **Bug**: use the Bug Report template. -- **Reproduction discrepancy**: use the Experiment Reproduction template. -- **Feature idea**: use the Feature Request template. - -## Citation - -If PersonaVoice helps your research, please cite it (see `CITATION.cff`). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index badd9cd..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: Bug Report -about: Report a reproducible issue with PersonaVoice -title: "[BUG] " -labels: bug, triage -assignees: '' ---- - -## Describe the bug -A clear and concise description of what the bug is. - -## To Reproduce -Steps to reproduce the behavior: -1. ... -2. ... -3. ... - -## Expected behavior -What you expected to happen. - -## Actual behavior -What actually happened. - -## Environment -- OS: [e.g. Windows 11 / Ubuntu 22.04 / macOS 14] -- Python version: [e.g. 3.10.11] -- PersonaVoice version: [e.g. 10.4.8] -- PyTorch version: [e.g. 2.1.0+cu118] -- GPU: [e.g. RTX 4070 8GB / CPU only] -- Did you run `scripts/setup_env.ps1` (or `.sh`)? [yes/no] -- Did you run `scripts/download_models.py`? [yes/no] - -## Logs -``` -Paste relevant error logs / stack traces here. -``` - -## Additional context -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/experiment_reproduction.md b/.github/ISSUE_TEMPLATE/experiment_reproduction.md deleted file mode 100644 index 7cbb76e..0000000 --- a/.github/ISSUE_TEMPLATE/experiment_reproduction.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: Experiment Reproduction -about: Discuss issues reproducing results from the paper / README -title: "[REPRO] " -labels: reproduction -assignees: '' ---- - -## Which experiment are you trying to reproduce? -- [ ] 200-sample LibriTTS evaluation (`personavoice/experiment/eval_200_samples.py`) -- [ ] External baseline comparison (`personavoice/experiment/baseline_external.py`) -- [ ] CFR analysis (`personavoice/experiment/cfr_analysis.py`) -- [ ] 1111.mp3 demo clone (`examples/clone_demo.py`) -- [ ] Web Demo (`personavoice/demo/api_server.py`) -- [ ] Other: ... - -## Your environment -- OS: -- Python: -- GPU: -- PersonaVoice version: -- Did you run `scripts/prepare_data.py`? [yes/no] - -## Expected vs. obtained metrics -| Metric | Expected (paper) | Obtained | -|--------|------------------|----------| -| SECS | | | -| WER | | | -| CFR | | | - -## Logs -``` -Paste relevant logs here. -``` - -## Additional context -Any other details that might help diagnose the discrepancy. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 6b6f9f1..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: Feature Request -about: Suggest a new feature or improvement for PersonaVoice -title: "[FEATURE] " -labels: enhancement -assignees: '' ---- - -## Is your feature request related to a problem? -A clear description of what the problem is. Ex. I'm always frustrated when [...] - -## Proposed solution -A clear description of what you want to happen. - -## Alternatives considered -A description of any alternative solutions or features you've considered. - -## Additional context -Add any other context, screenshots, or references here. - -## Relation to existing innovations -Does this relate to LAAG / OES / FiLM / CEAG / Silero VAD / Persona pipeline? -Or is it a brand-new direction? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index b2e090a..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,37 +0,0 @@ -## Description -Briefly describe what this PR changes and why. - -## Type of change -- [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] New feature (non-breaking change which adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) -- [ ] Documentation update -- [ ] Experiment / evaluation script -- [ ] Other - -## Related issue -Fixes # (issue number) - -## How has this been tested? -Describe the tests you ran to verify your changes. - -## Checklist -- [ ] My code follows the project's style guidelines -- [ ] I have performed a self-review of my own code -- [ ] I have commented my code, particularly in hard-to-understand areas -- [ ] I have updated the documentation accordingly -- [ ] My changes generate no new warnings -- [ ] I have added tests that prove my fix is effective or that my feature works -- [ ] New and existing unit tests pass locally with my changes - -## Architecture impact -Does this PR affect any of the core innovations? -- [ ] LAAG (Length-Aware Adaptive Generation) -- [ ] OES (Orthogonal Environment Sub-manifold) -- [ ] CEAG (Cross-Entropy Attention Guidance) -- [ ] FiLM Adapter -- [ ] Silero VAD / RMS preprocessing -- [ ] Persona extraction pipeline -- [ ] No architectural impact - -If yes, describe the change and provide ablation evidence. diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 219eec5..0000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Python Package - -on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install lint dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 black - - name: Flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 personavoice --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings - flake8 personavoice --count --exit-zero --max-complexity=20 --max-line-length=120 --statistics - - name: Black format check - run: | - black --check --line-length=100 personavoice || true - - import-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install minimal deps (CPU-only) - run: | - python -m pip install --upgrade pip - pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu - pip install -r requirements.txt - pip install -e . - - name: Smoke import - run: | - python -c "import personavoice; print('PersonaVoice', personavoice.__version__)" - python -c "from personavoice.config import SOTA_CONFIG; print('config OK:', SOTA_CONFIG.version)" - python -c "from personavoice.microaug import OrthogonalEnvironmentSubmanifold; print('OES OK')" - python -c "from personavoice.tts_backbone import VocosVocoder; print('Vocoder OK')" diff --git a/.gitignore b/.gitignore index a143195..115db2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,99 +1,58 @@ -# ============================================================ -# PersonaVoice .gitignore (v10.4.8) -# ============================================================ - -# ── Python ── __pycache__/ *.py[cod] -*$py.class -*.so -.Python *.egg-info/ .eggs/ build/ dist/ -.eggs/ -# ── Virtual Environments ── +# Local environments and tools .venv/ venv/ -env/ -ENV/ -ENV/ - -# ── IDE / Editor ── .idea/ .vscode/ -*.swp -*.swo -*~ -*.sublime-workspace -*.sublime-project - -# ── OS ── -.DS_Store -Thumbs.db -desktop.ini -ehthumbs.db - -# ── Large Data Files (NOT for GitHub — download via scripts/download_models.py) ── -data/LibriTTS/ -data/dev-clean.tar.gz -data/dpo_preference_pairs.pt -data/libritts_processed/ -data/*.pt -data/*.tar.gz -data/*.zip +.pytest_cache/ +test-results/ +.coverage +htmlcov/ -# ── Model Checkpoints (too large for GitHub — train locally) ── +# Licensed datasets and model artifacts are downloaded separately. +data/ checkpoints/ +pretrained_models/ +models/ *.pt *.pth *.ckpt *.bin *.safetensors -# ── Pretrained Models (download via scripts/download_models.py) ── -pretrained_models/ -models/ -models/** - -# ── Experiment Temp Files ── +# Generated run artifacts tmp/ tmp_*/ +.tmp_render2/ *.tmp *.log logs/ - -# ── Generated Audio Outputs (regenerable) ── outputs/ -outputs/** - -# ── Results: keep JSON + figures, ignore large audio samples ── -results/subjective_eval_samples/ -results/pdfm_audio/ results/*.wav results/*.mp3 results/*.flac +results/subjective_eval_samples/ -# ── Jupyter ── -.ipynb_checkpoints/ -*.ipynb - -# ── Local Demo Audio (regenerable from 1111.mp3) ── -# Keep 1111.mp3 / 1111.wav as bundled demo reference +# LaTeX build intermediates (keep source PDFs and publication figures) +*.aux +*.bbl +*.blg +*.fdb_latexmk +*.fls +*.out +*.synctex.gz -# ── Cache / HuggingFace cache (regenerable) ── +# Local caches and OS files .cache/ .huggingface/ *.cache - -# ── Coverage / Test Reports ── -.coverage -htmlcov/ -.pytest_cache/ -.tox/ - -# ── Lock files (keep requirements.txt as source of truth) ── -# poetry.lock -# Pipfile.lock +.DS_Store +._* +Thumbs.db +desktop.ini diff --git a/1111.mp3 b/1111.mp3 deleted file mode 100644 index 4ba3399..0000000 Binary files a/1111.mp3 and /dev/null differ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index 7527737..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,604 +0,0 @@ -# PersonaVoice: 架构设计文档 (v10.4.8 Trade-off Edition) - -> **版本说明**: 本文档描述 v10.4.8 满血架构设计与实现。实验数据见 [SOTA_VERIFICATION_REPORT.md](SOTA_VERIFICATION_REPORT.md)。 -> -> **v10.4.8 核心变更**: LAAG 长度自适应生成 + 动态 FiLM 激活 + F5-TTS 官方流程 + 移除 cudnn.deterministic + ECAPA 替代 WavLM (SIM-o) + 禁用 CEAG/Best-of-N (消融验证) + 移除 Duration Predictor/Reference Enhancer (消融验证)。 - -## 1. 项目概述 - -**PersonaVoice v10.4.8** 是一个面向**1 秒极限语音克隆**的 Plug-in Adapter 架构,支持**人格驱动的语音合成**与**零样本声学解缠**。给定少至 **1 秒**的参考音频、可选的聊天记录和结构化回忆,系统生成保留目标说话者音色、语言习惯和个性化情感表达模式的语音。 - -### 1.1 版本演进 - -| 版本 | 核心升级 | 关键指标 | -|------|---------|---------| -| v5.0 | 100% F5-TTS 权重加载 | SECS=0.6890 (20样本) | -| v7.0+ | IEAG / 动态长度 | WER=0.2405, SECS=0.4832 | -| v8.0 | 200 样本统计显著性 | p < 1e-49, Cohen's d=1.42 | -| v9.0 | OES + 消融实验 | SECS=0.4390, WER=0.1817 | -| v9.1 | 精简架构 (移除 4 个无效模块) | SECS=0.4832 (SOTA), WER=0.1817, CFR=18.0% | -| v10.0 | 三大地雷修复 + 统一配置 + CEAG 升级 | SECS=0.4832 (SOTA), WER=0.1817, CFR=18.0%, 1111.mp3 修复 | -| v10.1 | LAAG 长度自适应生成 | 长短文本性能均衡 | -| v10.3 | F5-TTS 官方流程集成, **移除 Duration Predictor** (F5 官方公式更准确) | SECS 稳定基线 | -| v10.4 | **移除 Reference Enhancer** (循环扩展伤害 SECS), 静态 CFG=2.5, steps=32 | RTF -45% | -| v10.4.5 | mel 拼接维度修复 (T, mel_dim) vs (mel_dim, T) | SECS +56.6% | -| v10.4.6 | 动态 FiLM 激活 (短 off / 长 on) | 长文本 +0.06 SECS | -| **v10.4.8** | **诚实 Trade-off 修订 + 禁用 CEAG/Best-of-N (消融验证)** | **200样本 SECS=0.4945, WER=0.1928, CFR=13.5%; 1111.mp3 长文本 SECS=0.5854, WER=0.0676** | - -### 1.2 v10.4.8 五大核心创新 (消融实验验证有效) - -| 创新 | 名称 | 解决的根本问题 | 核心公式 | -|------|------|---------------|----------| -| **A** | LAAG (长度自适应生成) | 1s 极限下长短文本性能不均 | 动态 Chunking + 动态 CFG + 动态 FiLM | -| **B** | 正交环境子流形 (OES) | 1s 样本环境音不匹配 | $Z_{audio} = Z_{timbre} + Z_{env}, \; Z_{timbre} \perp Z_{env}$ | -| **C** | 交叉熵注意力引导 (CEAG) | 1s 参考下注意力分散导致咬字崩溃 | $v_{final} = v_{cfg} - \lambda(t) \cdot \nabla_x H(A_{mel \to text})$ (v10.4.8 disabled) | -| **D** | 动态 FiLM 适配器 | persona/emotion 注入破坏预训练骨干 | $\gamma = \gamma_{net}(persona), \beta = \beta_{net}(emotion)$, 零初始化 + 短 off/长 on | -| **E** | Silero VAD + RMS 归一化 | 1s 样本前端切分粗糙 | 神经网络 VAD 替代 GMM-based WebRTC | - -**LAAG Demo**: 基于文本长度动态调整生成策略: -- 短文本 (单 chunk): FiLM off (稳定基线), CFG=2.5 -- 长文本 (多 chunk): FiLM on (防音色漂移, +0.06 SECS), 动态 CEAG λ (CEAG disabled 时为 0) - -**OES Demo**: 推理时调节环境权重 $\omega_2$: -- $\omega_2 = 0$: 纯净录音棚音质 (Studio Quality, **v10.0 默认 SOTA 配置**) -- $\omega_2 = 1$: 完美复现 1s 样本的麦克风质感 - -**CEAG Demo** (v10.4.8 disabled, 历史效果): 在 ODE 积分对齐关键期 ($t \in [0.1, 0.4]$),通过最小化**文本-音频交叉注意力熵**引导速度场: -- 历史 WER: 0.2405 (无引导) → 0.1817 (有 CEAG, **-24.6%**) -- v10.4.8 状态: 在 LAAG + F5 官方流程基线上增量收益可忽略, 已禁用以保持架构简洁 - -### 1.3 v10.4.8 架构决策 (实验验证) - -| 决策 | 原因 | 效果 | -|------|------|------| -| 移除 `cudnn.deterministic` | 强制不同卷积算法,改变数值精度 | 对齐 F5 基线行为 | -| 动态 FiLM (短 off / 长 on) | 短文本步数少易过校正;长文本需音色锚定 | 长文本 +0.06 SECS | -| ECAPA 替代 WavLM (SIM-o) | WavLM 离线不可用;均为 x-vector 提取器 | 功能等价 | -| F5-TTS 官方 `infer_batch_process` | 最权威的零样本克隆流程 | 稳定 SECS 基线 | -| **移除 BERT Duration Predictor** | F5-TTS 官方时长公式更准确 | 简化架构 (v10.3) | -| **移除 Reference Enhancer** | 循环扩展伤害 SECS, 与 F5 官方基线不一致 | 对齐基线 (v10.4) | -| 禁用 CEAG | v10.4.8 消融: 在 LAAG 基线上增量收益可忽略 | 简化, 保留代码 | -| 禁用 Best-of-N + CFG boost | 恶化短文本 WER (0.4047→0.6964), 流形崩塌是架构性的 | 回退到单次 CFG=2.5 | -| mel 拼接修复 | (T, mel_dim) vs (mel_dim, T) 维度错误 | SECS +56.6% | -| steps: 96→32 | RTF 优化 | -45% 推理时间 | - -### 1.4 已移除模块 (消融实验证实无效) - -| 模块 | 消融 p-value | Cohen's d | 移除原因 | -|------|-------------|-----------|---------| -| IBOP (信息瓶颈正交投影) | 0.85/0.51 | <0.05 | 无统计显著效果 | -| AM-ODE (声学动量连续动力学) | 零初始化 | 恒等映射 | 未训练, 无贡献 | -| TD-CFG (时间衰减 CFG) | 0.41/0.74 | <0.06 | 无统计显著效果 | -| SBM (子流形薛定谔桥) | 0.71/0.90 | <0.03 | 无统计显著效果 | -| GRPO (引导强化优化) | 仅测试时 | 非核心 | v10.0 移除, 非核心创新 | -| Duration Predictor (BERT 时长预测) | — | — | v10.3 移除, F5 官方时长公式更准确 | -| Reference Enhancer (参考音频增强) | — | — | v10.4 移除, 循环扩展伤害 SECS | -| Best-of-N (N-best 采样) | — | — | v10.4.8 移除, 流形崩塌无法通过采样解决 | - ---- - -## 2. 系统架构 - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ PersonaVoice v10.4.8 满血架构 │ -│ │ -│ 输入层 │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ -│ │ 超短音频 │ │ 聊天历史 │ │ 结构化回忆 │ │ -│ │ (1-5秒) │ │ (可选) │ │ (问卷) │ │ -│ └──────┬───────┘ └──────┬───────┘ └──────────┬───────────┘ │ -│ │ │ │ │ -│ ┌──────▼─────┐ │ │ │ -│ │★ Silero VAD│ │ │ │ -│ │ + RMS │ │ │ │ -│ │ 归一化 │ │ │ │ -│ └──────┬─────┘ │ │ │ -│ │ │ │ │ -│ ┌──────▼─────────────────▼──────────────────────▼─────────────┐ │ -│ │ 特征提取层 │ │ -│ │ ECAPA-TDNN (192-d) │ BERT (文本风格) │ Persona (64-d) │ │ -│ └──────┬─────────────────┬──────────────────────┬───────────┘ │ -│ │ │ │ │ -│ ┌──────▼──────────────────────────────────────────────────────┐ │ -│ │ ★ v10.0 核心 B: 正交环境子流形 (OES) │ │ -│ │ Z_audio = Z_timbre + Z_env, Z_timbre ⊥ Z_env │ │ -│ │ env_scale=0.1 (v10.0 渐进初始化, 修复 1111.mp3) │ │ -│ │ 推理时: Z_output = Z_timbre + ω2·Z_env (ω2=0: 纯净音色) │ │ -│ └──────┬──────────────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────▼──────────────────────────────────────────────┐ │ -│ │ ★ v10.1 核心 A: LAAG (长度自适应生成) │ │ -│ │ 动态 Chunking (max 135 chars) + 动态 CFG (2.5) │ │ -│ │ + 动态 FiLM (短 off / 长 on) + mel 拼接修复 │ │ -│ └──────┬──────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────▼──────────────────────────────────────────────┐ │ -│ │ ★ v10.4.6 核心 D: FiLM Adapter (动态激活) │ │ -│ │ γ = γ_net(persona), β = β_net(emotion) │ │ -│ │ h_modulated = γ · h + β, 零初始化安全 │ │ -│ │ 实现位置: f5_pretrained_backbone.py:PersonaEmotionFiLM│ │ -│ └──────┬──────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────▼──────────────────────────────────────────────┐ │ -│ │ F5-TTS DiT 骨干 (Flow Matching + AdaLN-Zero) │ │ -│ │ 22层, dim=1024, 前20层冻结, 最后2层解冻+FiLM Hook │ │ -│ │ 官方 infer_batch_process (时长公式内部计算) │ │ -│ │ → Mel → Vocos 声码器 → 24kHz 波形 │ │ -│ └──────┬──────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────▼──────────────────────────────────────────────┐ │ -│ │ ★ v10.0 核心 C: 交叉熵注意力引导 (CEAG) │ │ -│ │ v_final = v_cfg - λ(t) · ∇_x H(A_mel→text) │ │ -│ │ 带 text_mask/mel_mask (修复地雷1, 防 NaN) │ │ -│ │ v10.4.8: disabled (代码保留, use_ceag=False) │ │ -│ └──────┬──────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────▼──────────────────────────────────────────────┐ │ -│ │ 静态 CFG + Sway Sampling + 统一配置 (config.py) │ │ -│ │ CFG=2.5, sway_coef=-1.0, steps=32, best_of_n=1 │ │ -│ └──────────────────────────────────────────────────────┘ │ -│ │ -│ 已移除 (消融实验证实无效或被替代): │ -│ ✗ IBOP ✗ AM-ODE ✗ TD-CFG ✗ SBM ✗ GRPO │ -│ ✗ Duration Predictor (v10.3: F5 官方公式更准确) │ -│ ✗ Reference Enhancer (v10.4: 伤害 SECS) │ -│ ✗ Best-of-N (v10.4.8: 流形崩塌无法解决) │ -└─────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## 3. v10.4.8 核心创新详解 - -### 3.1 核心 A: LAAG (长度自适应生成) - -**问题**: 1s 极限参考音频下, 长文本生成存在音色漂移和对齐失败. 根本原因是信息论层面的不对称: 1s 参考 (94帧 mel) 信息量固定, 但目标文本长度变化导致信息瓶颈 — 短文本"信息过剩", 长文本"信息稀释". - -**解决方案** (LAAG 三大机制, 详见 `personavoice/tts_backbone/laag_generator.py`): - -1. **动态 Chunking**: 长文本切分为多个 chunk (上限 135 字符), 每 chunk 充分利用 1s 参考. 短文本 (≤10字符) 不切分. Cross-fade 拼接 (mel 域 + audio 域双重). - -2. **动态 CEAG λ**: $\lambda(L) = \lambda_{base} \times \text{clamp}(1 + \alpha \times (L - L_{ref}) / L_{ref}, \lambda_{min}, \lambda_{max})$ - - 短文本 (LL_ref): λ 提高, 强化对齐 - - v10.4.8: CEAG disabled, 此机制保留代码但实际 λ=0 - -3. **动态 CFG**: 短文本 CFG 增强, 长文本 CFG 保持. v10.4.5 实测 alpha=0.0 (固定 CFG=2.5) 在中文短/长文本上最均衡, 故当前为固定 CFG. - -4. **动态 FiLM 激活 (v10.4.6+)**: 短文本 (单 chunk) FiLM off, 长文本 (多 chunk) FiLM on. 实验数据驱动决策, 非盲目参数调整. - -**效果**: 长文本 SECS 从 0.27 提升到 0.50 (+89%), WER 从 0.51 降到 0.12 (-77%) - -### 3.2 核心 B: 正交环境子流形 (OES) - -**问题**: 1s 样本听起来不像, 是因为人耳对"空间混响 (Room Acoustics)"极度敏感. ECAPA 特征中同时包含音色和环境信息, 直接使用会导致生成音频带有 1s 样本的麦克风/房间质感. - -**数学重构**: 将 speaker embedding 空间分解为音色子空间与环境子空间的正交直和: - -$$Z_{audio} = Z_{timbre} + Z_{env}, \quad Z_{timbre} \perp Z_{env}$$ - -**实现**: 通过可学习的正交基底 $B \in \mathbb{R}^{D \times R}$ (R=32) 投影到环境子空间: - -$$Z_{env} = (Z_{audio} B) B^T \cdot \sigma_{env}, \quad Z_{timbre} = Z_{audio} - Z_{env}$$ - -**推理时调节**: -$$Z_{output} = Z_{timbre} + \omega_2 \cdot Z_{env}$$ - -**实现细节** (`personavoice/microaug/cross_manifold_refiner.py` → `OrthogonalEnvironmentSubmanifold`): -- `env_basis`: 可学习正交基底 (D=192, R=32), QR 正交化 -- `env_scale`: 环境缩放因子, **v10.0 配置 `env_scale=0.1` (渐进初始化, 修复 1111.mp3)** -- `decompose()`: 正交分解 Z_audio → (Z_timbre, Z_env) -- `forward(z_audio, env_weight)`: 推理时调节环境权重 - -**v10.0 关键修复 (1111.mp3 灾难)**: -- v9.x 问题: `env_scale=1.0` 初始化, 未训练的随机正交基直接全量投影, 破坏音色 → SECS=0.0363 -- v10.0 修复: `env_scale=0.1` 渐进初始化, 训练初期几乎恒等映射, 逐步学习环境子空间 → SECS=0.4832 - -**数值稳定性**: -- `torch.linalg.qr` 正交化基底, 确保 $B^T B = I$ -- `env_scale` 控制分解强度, 0.1 初始化保证训练稳定 - -**消融实验结果**: -- env_weight=0.0: SECS=0.4832 (**SOTA**) -- env_weight=1.0: SECS=0.4390 (下降 9.1%) -- 结论: env_weight=0.0 (纯净录音棚音色) 是最优配置 - -### 3.3 核心 C: 交叉熵注意力引导 (CEAG) — v10.4.8 disabled - -**问题**: 1s 参考音频过短, DiT 自注意力分散, 导致生成音频咬字崩溃, WER 升高. v9.x 的 IEAG 使用全自注意力熵, 未针对文本-音频对齐优化. - -**数学公式**: 在 ODE 积分对齐关键期 ($t \in [t_{start}, t_{end}]$), 通过最小化**文本-音频交叉注意力熵**引导速度场: - -$$v_{final} = v_{cfg} - \lambda(t) \cdot \nabla_x H(A_{mel \to text})$$ - -其中: -- $A_{mel \to text} \in \mathbb{R}^{T_{mel} \times T_{text}}$ 是 mel-to-text 交叉注意力子矩阵 -- $H(A_{mel \to text}) = -\sum_j p_{ij} \log p_{ij}$ 是每个 mel token 对 text 的注意力分布熵 -- $\lambda(t) = \lambda_{max} \cdot \mathbb{1}[t_{start} \leq t \leq t_{end}]$ 是时间步感知引导强度 -- $\nabla_x H(A)$ 是交叉注意力熵对输入 mel 的梯度 - -**v10.0 地雷1修复 (Padding Mask)**: -- 问题: F5-TTS 的自注意力是 `[text_tokens, mel_tokens]` 拼接后的全局注意力, Batch 推理时文本和音频都有 padding. 直接对截取矩阵求梯度, 会把极大梯度施加在 padding 区域 → NaN -- 修复: 在 `_compute_cross_attention_entropy` 中引入 `text_mask` 和 `mel_mask`, 在 `log_softmax` 之前将 mask 外的注意力权重设为 `-1e9`: - -```python -# 提取 mel-to-text 交叉注意力子矩阵 -A_mt = w[:, :, T_text:T_text+T_mel, :T_text] # (B, H, T_mel, T_text) - -# v10.0 地雷1修复: Padding Mask (防止梯度爆炸) -text_mask = self.text_mask[:w.shape[0]] # (B, 1, 1, T_text) -mel_mask = self.mel_mask[:w.shape[0]] # (B, 1, T_mel, 1) -A_mt_masked = A_mt.masked_fill(~text_mask, -1e9) - -# 计算熵 (仅在有效区域) -log_A_mt = F.log_softmax(A_mt_masked, dim=-1) -A_mt_norm = log_A_mt.exp() -entropy_per_mel = -(A_mt_norm * log_A_mt).sum(dim=-1) # (B, H, T_mel) - -# 应用 mel mask 并平均 -entropy_masked = entropy_per_mel * mel_mask.squeeze(-1) -``` - -**v10.4.8 参数** (从 `config.py` 读取, 由于 `use_ceag=False` 实际不生效): -- $\lambda_{max} = 0.20$ (v10.1: 从 0.25 降至 0.20, 配合动态 LAAG) -- $t_{start} = 0.1$, $t_{end} = 0.4$ (v10.1: 收窄激活窗口, 3x 速度提升) -- 激活层: $(-2, -1)$ (v10.1: 从 (-3,-2,-1) 减少到 2 层, 1.5x 速度提升) -- 梯度裁剪: `ceag_grad_max_norm=1.0` (逐层, 防止某层梯度淹没其他层) - -**实现细节** (`personavoice/tts_backbone/ceag_sampler.py` → `CEAGGuidance`): -- `guided_forward(x, fn, t)`: 在 ODE 积分步中注入交叉注意力熵梯度 -- 梯度计算: `torch.autograd.grad(H_cross_attn, x)` -- **Padding Mask**: `text_mask` 和 `mel_mask` 在 `log_softmax` 前应用 - -**消融实验结果** (历史, v10.0 时代): -- 无引导: WER=0.2405 -- 有 CEAG (λ=0.25): WER=0.1817 (**-24.6%**) -- 结论: CEAG 是 WER 改善的主要贡献者, 且 padding mask 防止了 Batch 推理的 NaN - -**v10.4.8 状态**: 在 LAAG + F5 官方流程基线上增量收益可忽略, 已禁用 (`use_ceag=False`). 代码路径完整保留以供复现和后续研究. - -### 3.4 核心 D: 动态 FiLM 适配器 (v10.4.6+) - -**问题**: persona/emotion 注入会破坏预训练骨干; FiLM 对短文本效果不一致 (CUDA 非确定性), 但对长文本有 +0.06 SECS 提升. - -**数学公式**: -$$h_{modulated} = \gamma \cdot h + \beta$$ - -其中: -- $\gamma = \gamma_{net}(persona\_emb)$ (persona 条件缩放) -- $\beta = \beta_{net}(emotion\_emb)$ (emotion 条件偏移) - -**零初始化安全**: -- `gamma_net` 和 `beta_net` 的最后一层零初始化 -- 训练初期: $\gamma=1, \beta=0$ (恒等映射), 不破坏预训练骨干 -- 训练后: persona/emotion 信息逐步注入 - -**动态激活 (v10.4.6+)**: -```python -if len(chunks) == 1: - persona_emb_effective = torch.zeros_like(persona_emb) # FiLM off - info["film_active"] = False -else: - persona_emb_effective = persona_emb # FiLM on - info["film_active"] = True -``` - -**原理**: 短文本单 chunk 生成稳定, FiLM 引入不确定性; 长文本多 chunk 需要 FiLM 防止音色漂移. - -**实现位置**: `personavoice/tts_backbone/f5_pretrained_backbone.py` → `PersonaEmotionFiLM` (FiLM 适配器直接集成在骨干文件中, 无单独 `film_adapter.py`). - -### 3.5 核心 E: Silero VAD + RMS 归一化 (v10.0 地雷3修复) - -**问题 (地雷3)**: v9.x 使用能量阈值法去除静音, 对 1s 极短音频的开头呼吸声、微弱辅音 (/s/, /f/) 切分生硬, 导致参考音频"吃字". - -**v10.0 方案**: 改用 Silero VAD (神经网络, <100MB 显存): - -**实现细节** (`personavoice/demo/audio_preprocess.py` → `preprocess_ref_audio`): -- Silero VAD: `snakers4/silero-vad` (PyTorch 实现) -- 阈值: 0.5 (可配置) -- RMS 归一化: `target_rms = 0.1`, `waveform *= (target_rms / rms).clamp(0.1, 10.0)` -- 尾部静音: 50ms (F5-TTS 官方要求) -- Fallback: Silero VAD 失败时回退到能量阈值法 - -**Pipeline**: -1. Silero VAD 精准语音端点检测 → 去除前后静音 -2. 重采样到 24kHz (F5-TTS 要求) -3. RMS 能量归一化 → 响度一致性 -4. 补 50ms 尾部静音 - ---- - -## 4. F5-TTS 骨干集成 - -### 4.1 骨干配置 - -| 项目 | 配置 | -|------|------| -| 模型 | F5-TTS v1 Base (SWivid/F5-TTS) | -| 权重加载 | 364/364 keys (100%) | -| DiT | 22 层, dim=1024, mel_dim=100 | -| 冻结策略 | 前 20 层冻结, 最后 2 层解冻 | -| FiLM Hook | 注册在 blocks [20, 21] | -| 可训练参数 | 31,876,709 (9.39%) | -| 总参数 | 339,586,761 | -| 声码器 | Vocos (24kHz, 100 mel bins) | - -### 4.2 适配器集成 - -**实现**: `personavoice/tts_backbone/f5_pretrained_backbone.py` → `F5TTSPretrainedBackbone` - -v10.4.8 在 `synthesize` 方法中的数据流: - -``` -1. speaker_emb (B, 192) → OES.decompose(env_weight=0.0) → z_timbre (B, 192) -2. mel_ref (B, T_ref, 100) → F5-TTS cond encoder → cond (B, T_ref, 1024) -3. text_tokens (B, T_text) → F5-TTS text encoder → text_emb (B, T_text, 512) -4. persona_emb (B, 64) → FiLM γ_net → γ (B, 1024) -5. emotion_emb (B, 64) → FiLM β_net → β (B, 1024) -6. v10.3+: F5-TTS 官方 infer_batch_process 自动计算时长 (无需 Duration Predictor) - - 官方公式: duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed) -7. sample_with_ceag(cond, text, mel_ref, steps=32, cfg=2.5, use_ceag=False) - → mel_gen (B, T_target, 100) -8. mel_gen → Vocos → waveform (B, T_audio) -``` - -### 4.3 FiLM Adapter (集成在骨干文件) - -**实现位置**: `personavoice/tts_backbone/f5_pretrained_backbone.py` → `PersonaEmotionFiLM` class - -**数学公式**: -$$h_{modulated} = \gamma \cdot h + \beta$$ - -其中: -- $\gamma = \gamma_{net}(persona\_emb)$ (persona 条件缩放) -- $\beta = \beta_{net}(emotion\_emb)$ (emotion 条件偏移) - -**零初始化安全**: -- `gamma_net` 和 `beta_net` 的最后一层零初始化 -- 训练初期: $\gamma=1, \beta=0$ (恒等映射), 不破坏预训练骨干 -- 训练后: persona/emotion 信息逐步注入 - -**Hook 机制**: 通过 `register_forward_hook` 挂载到 DiT blocks [20, 21], 在前向传播时自动调制隐层. - ---- - -## 5. 推理优化组件 - -### 5.1 静态 CFG (替代无效的 TD-CFG) - -**数学公式**: -$$v_{cfg} = v_{cond} + \text{CFG} \cdot (v_{cond} - v_{uncond})$$ - -**v10.4 配置**: CFG=2.5 (从 `config.py` 读取, v10.4 参数搜索验证最优) - -**为什么移除 TD-CFG**: 消融实验显示 TD-CFG 的动态 CFG 调度无统计显著效果 (p=0.41/0.74, Cohen's d<0.06), 静态 CFG 更简单且等效. - -### 5.2 Sway Sampling - -**数学公式**: 时间步采样采用 Sway Sampling: - -$$t_i = \text{sway\_sampling}(i, N, \text{coef}=-1.0)$$ - -**配置**: coef=-1.0 (F5-TTS 官方默认), 偏向早期时间步, 改善生成质量. - -### 5.3 v10.4.8 统一配置 (config.py) - -**实现**: `personavoice/config.py` → `SOTAConfig` (frozen dataclass) - -**设计原则**: 单一配置源, 消除模块间参数矛盾 (v9.x 的 api_server.py steps=64 vs README steps=96) - -```python -@dataclass(frozen=True) -class SOTAConfig: - # ODE integration (v10.4: 96→32, RTF -45%) - steps: int = 32 - sway_sampling_coef: float = -1.0 - # 静态 CFG (v10.4: 2.0→2.5, 参数搜索最优) - cfg_strength: float = 2.5 - # OES - env_weight: float = 0.0 - oes_env_scale_init: float = 0.1 # v10.0 critical fix - # CEAG (v10.4.8: disabled, 代码保留) - use_ceag: bool = False - ceag_lambda_max: float = 0.20 # v10.1: 从 0.25 调低配合 LAAG - ceag_t_start: float = 0.1 - ceag_t_end: float = 0.4 - ceag_layers: Tuple[int, ...] = (-2, -1) # v10.1: 从 (-3,-2,-1) 减层 - # LAAG (v10.1+ 核心创新) - use_laag: bool = True - laag_chunk_max_chars: int = 135 - laag_cfg_base: float = 2.5 - laag_cfg_alpha: float = 0.0 # 固定 CFG, 动态 α 关闭 - best_of_n: int = 1 # v10.4.8: 禁用 (流形崩塌无法解决) - # 前端预处理 - use_silero_vad: bool = True - use_rms_normalize: bool = True - silero_vad_threshold: float = 0.5 - rms_target: float = 0.1 - # 版本 - version: str = "10.4.8" -``` - ---- - -## 6. 模块文件结构 - -``` -personavoice/ -├── config.py # ★ v10.4.8 统一 SOTA 配置 (唯一配置源) -├── tts_backbone/ -│ ├── f5_pretrained_backbone.py # F5-TTS 骨干封装 + PersonaEmotionFiLM (v10.4.6 动态激活) -│ ├── ceag_sampler.py # ★ CEAG + 采样器 (v10.0 核心 C, 带 padding mask; v10.4.8 disabled) -│ ├── laag_generator.py # ★ LAAG (v10.1 核心 A, 动态 FiLM + mel 拼接修复) -│ └── vocoder.py # Vocos 神经声码器 -├── microaug/ -│ ├── __init__.py # 包入口 (仅暴露 OES) -│ └── cross_manifold_refiner.py # OES 正交环境子流形 (v10.0 核心 B, env_scale=0.1) -├── persona/ -│ ├── __init__.py # 人格管线入口 -│ └── extractor.py # BERT 聊天记录 → Big Five → persona_emb -├── common/ -│ └── local_models.py # 本地模型路径 (离线模式) -└── demo/ - ├── api_server.py # 前端演示 API (v10.4.8 统一配置 + 人格集成) - ├── audio_preprocess.py # ★ Silero VAD + RMS 归一化 (v10.0 核心 E) - └── index.html # 交互式前端 -``` - -**v10.4.8 已移除文件** (消融实验验证无效或被替代): -- `grpo_inference.py` (GRPO, v10.0 移除, 非核心) -- `grpo_curvature_optimizer.py` (TCO, 未集成) -- `schrodinger_bridge.py` (SBM, p=0.71/0.90) -- `flow_matching.py` 中的 Langevin 引导 (IMLG, 未验证) -- `cross_manifold_refiner.py` 中的 IBOP/AM-ODE/MINE -- `ieag_sampler.py` (被 `ceag_sampler.py` 替代) -- `duration_predictor.py` (v10.3 移除, F5 官方时长公式更准确) -- `reference_enhancer.py` (v10.4 移除, 循环扩展伤害 SECS) -- `film_adapter.py` (FiLM 直接集成在 `f5_pretrained_backbone.py`) - ---- - -## 7. 实验配置 - -### 7.1 v10.4.8 SOTA 配置 (统一 config.py) - -| 参数 | 值 | 说明 | -|------|-----|------| -| steps | 32 | ODE 积分步数 (v10.4: 从 96 降为 32, RTF -45%) | -| sway_sampling_coef | -1.0 | F5-TTS 官方默认 | -| cfg_strength | 2.5 | 静态 CFG (v10.4: 为 SECS 调优, 替代无效的 TD-CFG) | -| env_weight | 0.0 | OES: 录音棚纯净 (SECS SOTA) | -| oes_env_scale_init | 0.1 | OES 渐进初始化 (v10.0 修复 1111.mp3) | -| use_ceag | False | CEAG v10.4.8 禁用 (增量收益可忽略; 代码保留) | -| ceag_lambda_max | 0.20 | CEAG 引导强度 (v10.1: 从 0.25 调低配合 LAAG; 启用时使用) | -| ceag_t_start | 0.1 | CEAG 激活起始 (v10.1: 从 0.05 收窄) | -| ceag_t_end | 0.4 | CEAG 激活结束 (v10.1: 从 0.5 收窄, 3x 速度提升) | -| ceag_layers | (-2,-1) | CEAG 注意力提取层 (v10.1: 从 (-3,-2,-1) 减层, 1.5x 速度提升) | -| use_laag | True | LAAG 动态 Chunking + 动态 FiLM | -| laag_chunk_max_chars | 135 | LAAG chunk 上限 (F5-TTS 官方默认) | -| laag_cfg_base | 2.5 | LAAG 基准 CFG | -| laag_cfg_alpha | 0.0 | 动态 CFG α (关闭, v10.4.5 实测固定 CFG=2.5 最均衡) | -| best_of_n | 1 | Best-of-N 禁用 (v10.4.8: 流形崩塌无法通过采样解决) | -| use_silero_vad | True | Silero VAD 预处理 (v10.0) | -| use_rms_normalize | True | RMS 能量归一化 (v10.0) | -| rms_target | 0.1 | RMS 目标响度 | -| silero_vad_threshold | 0.5 | Silero VAD 触发阈值 | - -### 7.2 评估指标 - -| 指标 | 定义 | SOTA 归属 | -|------|------|-----------| -| ECAPA SECS | 生成音频与参考的 ECAPA 嵌入余弦相似度 | **PV v10.4.8 (0.4945)** | -| WER | Whisper-large-v3-turbo 转录的词错误率 | XTTS v2 (0.1296) | -| CFR | 灾难失败率 (WER>0.5) | XTTS v2 (~5%) | - -### 7.3 v10.4.8 实验结果 (200 样本, 诚实呈现) - -| 指标 | 均值 ± 标准差 | 95% CI | 中位数 | 样本数 | -|------|--------------|--------|--------|--------| -| **ECAPA SECS** | **0.4945 ± 0.1800** | [0.4694, 0.5196] | 0.5226 | 200 | -| **WER** | **0.1928 ± 0.3077** | [0.1499, 0.2358] | 0.0588 | 200 | -| **CFR (WER>0.5)** | **13.5%** | - | - | 200 | - -**按文本长度分组** (LAAG 动态机制验证): - -| 文本类型 | 样本数 | SECS | WER | CFR | 说明 | -|----------|--------|------|-----|-----|------| -| **短文本 (≤8 词)** | 53 | 0.4671 ± 0.21 | 0.4066 ± 0.42 | **35.8%** | FiLM off, 流形崩塌 | -| **长文本 (>8 词)** | 147 | **0.5044 ± 0.17** | **0.1158 ± 0.22** | **5.4%** | FiLM on, LAAG chunking | -| **极短文本 (≤4 词)** | 22 | 0.4282 ± 0.23 | 0.6629 ± 0.45 | **63.6%** | 流形崩塌重灾区 | - -**统计显著性** (vs F5-TTS 原版, 配对 t 检验): -- SECS: t=20.08, p < 1e-49, Cohen's d=1.42 (large) -- WER: t=-28.32, p < 1e-50, Cohen's d=-2.00 (large) - ---- - -## 8. 设计哲学 - -### 8.1 Plug-in Adapter 架构 - -**核心原则**: 冻结预训练骨干, 仅训练轻量适配器 - -| 组件 | 参数量 | 训练状态 | -|------|--------|---------| -| F5-TTS DiT (前 20 层) | ~307M | 冻结 | -| F5-TTS DiT (最后 2 层) | ~25M | 解冻 | -| FiLM Adapter (PersonaEmotionFiLM) | ~2M | 训练 | -| OES | ~6K | 训练 | -| CEAG | 0 | 推理时 (零训练) | -| **总可训练** | **~31.9M (9.39%)** | - | - -**优势**: -1. 8GB 显存即可训练 -2. 不破坏预训练骨干的泛化能力 -3. 适配器可插拔, 灵活组合 - -### 8.2 消融实验驱动决策 - -**原则**: 所有模块必须通过消融实验验证有效性, 无效模块立即移除 - -| 模块 | 验证方法 | 结果 | 决策 | -|------|---------|------|------| -| OES | env_weight=0.0 vs 1.0 | SECS 0.4832 vs 0.4390 | **保留** | -| CEAG | λ=0 vs λ=0.25 (历史) | WER 0.2405 vs 0.1817 | **保留代码, v10.4.8 disabled** | -| LAAG | 长文本 SECS 0.27→0.50 | +89% SECS | **保留** | -| 动态 FiLM | 长文本 +0.06 SECS | CUDA 非确定性已通过短 off/长 on 解决 | **保留** | -| IBOP | 200 样本配对 t 检验 | p=0.85/0.51 | **移除** | -| AM-ODE | 零初始化检查 | 恒等映射 | **移除** | -| TD-CFG | 200 样本配对 t 检验 | p=0.41/0.74 | **移除** | -| SBM | 200 样本配对 t 检验 | p=0.71/0.90 | **移除** | -| GRPO | 测试时分析 | 非核心 | **移除 (v10.0)** | -| Duration Predictor | F5 官方公式对比 | 官方公式更准确 | **移除 (v10.3)** | -| Reference Enhancer | SECS 对比 | 循环扩展伤害 SECS | **移除 (v10.4)** | -| Best-of-N | WER 0.4047→0.6964 | 恶化, 流形崩塌是架构性的 | **移除 (v10.4.8)** | - -### 8.3 学术诚实 - -**原则**: 公开证伪无效模块, 不保留"看起来创新但实际无效"的组件 - -v10.0 移除了 5 个曾经被认为是创新的模块 (IBOP, AM-ODE, TD-CFG, SBM, GRPO). v10.3 移除了 Duration Predictor (F5 官方公式更准确). v10.4 移除了 Reference Enhancer (伤害 SECS). v10.4.8 禁用了 CEAG 和 Best-of-N (消融验证无效). 这种诚实性本身就是学术贡献. - -### 8.4 v10.0 地雷修复的工程严谨性 - -v10.0 不仅保留有效创新, 更主动识别并修复了 3 个隐藏的致命地雷: -1. **CEAG Padding Mask**: 数学上证明 padding token 会导致梯度爆炸, 工程上用 mask 解决 -2. **OES env_scale=0.1**: 实验上证明 1.0 初始化破坏 1111.mp3, 改用 0.1 渐进初始化 -3. **Silero VAD**: 实验上证明 WebRTC VAD 损伤辅音, 改用神经网络 VAD - -v10.4.5+ 修复了 mel 拼接维度错误 ((T, mel_dim) vs (mel_dim, T)), 带来 SECS +56.6% 提升. - -这种"发现问题 → 分析根因 → 工程修复"的闭环, 体现了顶会级的工程严谨性. - ---- - -## 9. 未来工作 - -### 9.1 WER SOTA 优化 - -**目标**: WER < 0.1296 (超越 XTTS v2) - -**方案**: -1. 重新评估 CEAG 在更大消融样本上的效果, 或设计改进版本 -2. 主观评估 (MOS, 20+ 听众) -3. 多数据集验证 (VCTK + CommonVoice) -4. 短文本流形崩塌: 探索 CTC 对齐先验 / 短文本专用积分器 / 混合架构 fallback - -### 9.2 顶会投稿补充 - -**要冲顶会 (ICML/NeurIPS/ACL/Interspeech Main)**, 需补充: -1. **Human MOS 主观评测** (强制): 20+ 听众, Naturalness MOS + Similarity MOS -2. **CFR 实测对比** (建议): 对 XTTS v2 / CosyVoice 实测 CFR, 替代估算值 -3. **短文本 Future Work 章节** (建议): 详细阐述流形崩塌理论与未来解决方案 -4. **多数据集验证** (建议): VCTK + CommonVoice -5. **效率分析** (建议): Pareto 前沿: VRAM/Time/Data Efficiency -6. **定性展示** (建议): 波形图 + 梅尔频谱图对比 - ---- - -## 10. 参考文档 - -- [SOTA 验证报告](SOTA_VERIFICATION_REPORT.md) — 完整实验数据与统计分析 -- [README (中文)](README_zh.md) — 项目介绍与快速开始 -- [README (English)](README.md) — Project introduction and quick start diff --git a/CITATION.cff b/CITATION.cff index ff8cca6..99d3484 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,33 +1,33 @@ cff-version: 1.2.0 -title: "PersonaVoice: Plug-in Adapters for Persona-Driven 1-Second Voice Cloning" -message: "If you use PersonaVoice in your research, please cite it as below." +title: "PersonaVoice research preprints and reproducible extensions" +message: "If you use the PersonaVoice research artifacts, please cite the relevant Zenodo preprint." type: software authors: - - alias: PersonaVoice Team -repository-code: "https://github.com/personavoice/personavoice" -version: "10.4.8" -date-released: "2026-06-25" + - family-names: Luchang + given-names: Jiang + email: auroral.sunflower@gmail.com + orcid: "https://orcid.org/0009-0004-4579-4243" +repository-code: "https://github.com/FinalSunFlower/PersonaVoice" +version: "1.0.0" +date-released: "2026-08-25" license: MIT keywords: - - voice-cloning - - zero-shot-tts - - few-shot-voice-cloning - - flow-matching - - F5-TTS - - speech-synthesis - - persona-driven-tts - - emotional-tts - - plug-in-adapter - - 1-second-cloning - - speaker-embedding - - ECAPA-TDNN - - orthogonal-decomposition - - length-adaptive-generation - - FiLM-modulation + - provenance-constrained generation + - selective prediction + - citation attribution + - language style + - abstention + - voice safety + - text-to-speech preferred-citation: type: article - title: "PersonaVoice v10.4.8: Plug-in Adapters for Persona-Driven 1-Second Voice Cloning — A Trade-off Analysis of Timbre Fidelity and Text-Alignment Stability in Flow-Matching TTS" + title: "Provenance-Constrained Selective Language Style Planning from Sparse Evidence" authors: - - alias: PersonaVoice Team + - family-names: Luchang + given-names: Jiang + email: auroral.sunflower@gmail.com + orcid: "https://orcid.org/0009-0004-4579-4243" year: 2026 - notes: "Work in progress. arXiv preprint to appear." + month: 8 + day: 25 + notes: "Zenodo preprint, version 1.0.0." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index a5aa866..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,56 +0,0 @@ -# Contributing to PersonaVoice - -Thanks for your interest in contributing to PersonaVoice! This project is an academic open-source effort, and we welcome contributions of all kinds: bug fixes, new experiments, documentation improvements, and architectural extensions. - -## Code of Conduct - -Be respectful, constructive, and academic. Cite prior work properly. Critique ideas, not people. - -## How to Contribute - -### 1. Fork & Branch -- Fork the repository. -- Create a feature branch: `git checkout -b feature/my-feature`. - -### 2. Develop -- Follow the existing code style (PEP 8, line length ≤ 120). -- Add docstrings to all new functions / classes (Google style). -- Keep modules focused — PersonaVoice follows a strict architecture (LAAG / OES / FiLM / CEAG / Persona). Place new code in the correct subpackage: - - `personavoice/tts_backbone/` — TTS backbone, adapters, sampler, vocoder - - `personavoice/microaug/` — Sample enhancement (OES) - - `personavoice/persona/` — Persona extraction - - `personavoice/common/` — Shared utilities - - `personavoice/demo/` — Web demo - - `personavoice/experiment/` — Evaluation scripts - - `scripts/` — One-off setup / data-prep scripts - -### 3. Test -- Run smoke tests: - ```bash - python -c "import personavoice; print(personavoice.__version__)" - python -c "from personavoice.config import SOTA_CONFIG; print(SOTA_CONFIG.version)" - ``` -- For experiments, run on the bundled `1111.mp3` first. - -### 4. Submit a PR -- Fill in the PR template (`.github/PULL_REQUEST_TEMPLATE.md`). -- If your change affects an architectural component (LAAG / OES / FiLM / CEAG), provide ablation evidence (200-sample paired t-test, p < 0.05, Cohen's d > 0.1). -- Update `README.md` / `README_zh.md` / `ARCHITECTURE.md` if behavior changes. - -## Architectural Principles - -1. **Plug-in Adapter Design** — F5-TTS backbone stays frozen. Train only lightweight adapters (~2M params). -2. **Ablation-Driven Decisions** — Every module must prove its worth via 200-sample paired t-test. -3. **Academic Honesty** — Report trade-offs honestly. Do not over-claim SOTA. -4. **Configuration Uniqueness** — All inference parameters live in `personavoice/config.py`. Do not hardcode parameters in scripts. -5. **1-Second Constraint** — All designs must work with as little as 1 second of reference audio. - -## Reporting Issues - -- **Bug**: use the Bug Report template. -- **Reproduction discrepancy**: use the Experiment Reproduction template. -- **Feature idea**: use the Feature Request template. - -## Citation - -If PersonaVoice helps your research, please cite it (see `CITATION.cff`). diff --git a/README.md b/README.md index 39cad0e..ebc01e9 100644 --- a/README.md +++ b/README.md @@ -1,555 +1,34 @@ -# PersonaVoice: Plug-in Adapters for Persona-Driven 1-Second Voice Cloning +# PersonaVoice: preprint supplement snapshot -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) -[![PyTorch 2.0+](https://img.shields.io/badge/PyTorch-2.0+-ee4c2c.svg)](https://pytorch.org/) -[![Version 10.4.8](https://img.shields.io/badge/version-10.4.8--Trade--off-red.svg)](ARCHITECTURE.md) -[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) +This branch is the reproducibility companion for two independent Zenodo +preprints by Jiang Luchang (ORCID +[`0009-0004-4579-4243`](https://orcid.org/0009-0004-4579-4243)). It is deliberately +separate from the historical runtime repository: switching to +`paper-supplement-v1.0` exposes only public derived artifacts and reproduction +code. The paper PDFs and LaTeX sources are intentionally deposited separately +on Zenodo and are not duplicated in this GitHub support branch. -**English** | **[中文](README_zh.md)** +## Canonical papers -> **PersonaVoice v10.4.8** — Clone any voice from just **1 second** of audio with optional **persona / emotion injection** via plug-in adapters over a frozen F5-TTS backbone. A systematic trade-off analysis of timbre fidelity (SECS SOTA) vs. text-alignment stability. +1. **Provenance-Constrained Selective Language Style Planning from Sparse Evidence** - typed evidence, citation attribution, conflict preservation, selective abstention, and a measured calibration-transfer failure. The matching support snapshot is tagged `paper-language-v1.0`. +2. **PersonaVoice: One-Second Voice Cloning with Length-Adaptive Flow Matching** - a 200-item one-second reference evaluation, protocol-matched baselines, ablations, and short-text catastrophic-failure analysis. The matching support snapshot is tagged `paper-speech-v1.0`. ---- +## Reproducibility package -## Table of Contents -- [What is PersonaVoice?](#what-is-personavoice) -- [Key Results](#key-results) -- [Core Innovations](#core-innovations) -- [Architecture Overview](#architecture-overview) -- [Installation](#installation) -- [Data & Model Download](#data--model-download) -- [Quick Start](#quick-start) -- [Reproducing the Paper Results](#reproducing-the-paper-results) -- [Inference Parameters](#inference-parameters) -- [Project Structure](#project-structure) -- [Design Philosophy](#design-philosophy) -- [Related Work & Acknowledgements](#related-work--acknowledgements) -- [Citation](#citation) - ---- - -## What is PersonaVoice? - -PersonaVoice is a **persona-driven plug-in adapter architecture for 1-second voice cloning**. It addresses a fundamental challenge: - -> **Given only 1 second of a person's voice and optional text records (chat history / structured recalls), generate speech that preserves their timbre, linguistic habits, and individual emotional expression patterns.** - -The narrative of this project is **not** a "full SOTA" claim but a **trade-off analysis**: -- **Flow-Matching** achieves dominating timbre similarity (SECS) but suffers short-text manifold collapse. -- PersonaVoice's contribution is to **maximize that SECS lead** while using **LAAG + dynamic FiLM** to recover long-text alignment, leaving short-text WER as honest Future Work. - -### Why 1 Second? - -Most zero-shot TTS systems assume ≥3s of reference audio (XTTS v2, CosyVoice, VALL-E, VoiceBox). The 1-second regime exposes fundamental architectural limits: -- Information bottleneck (94 mel frames @ 24 kHz / hop 256) -- Manifold collapse under Flow-Matching for very short texts -- Reference audio impurity (breath, fricatives) dominates the cloning signal - -PersonaVoice is the first open-source system to systematically study this regime. - -### Interactive Web Demo - -Below is a screenshot of the PersonaVoice v10.4.8 interactive dashboard. Users upload a 1-second reference audio, optionally provide chat history or structured recollections, and obtain a cloned utterance together with a real-time Big-Five persona radar, mel-spectrogram visualization, and architecture condition inspection. - -

- PersonaVoice v10.4.8 Interactive Demo -

- ---- - -## Key Results - -**Setup**: 1-second reference audio, 200-sample LibriTTS dev-clean evaluation, ECAPA-TDNN SECS, Whisper-large-v3-turbo WER. - -| Metric | PersonaVoice v10.4.8 | XTTS v2 (1s ref) | CosyVoice (1s ref) | F5-TTS Official | SOTA | -|--------|----------------------|-------------------|---------------------|-----------------|------| -| **ECAPA SECS (overall)** | **0.4945** | 0.3349 | 0.3595 | 0.2508 | **PV ✓** (+47.6% vs XTTS v2) | -| **ECAPA SECS (long text)** | **0.5044** | 0.31 | 0.33 | 0.22 | **PV ✓** (+62.7% vs XTTS v2) | -| **WER (overall)** | 0.1928 | **0.1296** | 0.6130 | 0.8982 | XTTS v2 (AR) | -| **WER (long text)** | 0.1158 | **0.10** | 0.55 | 0.85 | XTTS v2 (acknowledged trade-off) | -| **WER (short text ≤8 words)** | 0.4066 | **0.16** | 0.68 | 0.95 | XTTS v2 (Future Work) | -| **CFR (overall, WER>0.5)** | 13.5% | ~5%* | ~30%* | ~45%* | XTTS v2 | -| **CFR (short text)** | 35.8% | ~10%* | ~50%* | ~70%* | XTTS v2 | -| **UTMOS (Objective Proxy)** | 4.0-4.5 | 3.8 | 4.0 | 3.5 | PV (Human MOS pending) | -| **RTF** | 0.55-0.66 | 0.85 | 0.72 | **0.50** | F5-TTS | -| **1111.mp3 SECS (long text)** | **0.5854** | - | - | - | PV ✓ | -| **1111.mp3 WER (long text)** | **0.0676** | - | - | - | PV ✓ | - -> `*` CFR for XTTS v2 / CosyVoice / F5-TTS are estimated from WER distribution; measured comparison is planned. Full evaluation details: [SOTA_VERIFICATION_REPORT.md](SOTA_VERIFICATION_REPORT.md). - ---- - -## Core Innovations - -PersonaVoice introduces **5 core innovations**, all validated by 200-sample paired t-tests (p < 0.05, Cohen's d > 0.1). - -### Innovation A: LAAG (Length-Aware Adaptive Generation) — v10.1+ - -Solves the fundamental flaw of uneven performance between short and long text under 1s extreme cloning: - -- **Dynamic Chunking**: Long text split into chunks (max 135 chars), each fully utilizes the 1s reference audio -- **Dynamic CFG base**: `cfg_base = 2.5` tuned for short/long-text balance -- **Dynamic FiLM Activation (v10.4.6+)**: Short text (single chunk) FiLM off (stable baseline); long text (multi-chunk) FiLM on (prevent timbre drift, +0.06 SECS) -- **mel splice fix (v10.4.5)**: Corrected `(T, mel_dim)` vs `(mel_dim, T)` dimension order across chunks (+56.6% SECS) -- **F5-TTS official flow integration (v10.3+)**: `infer_batch_process` with `ref_text` + `audio_ref` cond flow, replacing the custom Duration Predictor -- **v10.4.8 boundary hardening**: ref_text punctuation + space ensures clean ref/gen boundary; gen_text leading-space sacrifice prevents head-truncation; lead-silence trimming removes artifacts - -### Innovation B: Orthogonal Environment Sub-manifold (OES) - -Decomposes the speaker embedding into timbre and environment sub-spaces: - -$$Z_{audio} = Z_{timbre} + Z_{env}, \quad Z_{timbre} \perp Z_{env}$$ - -- `env_basis`: learnable orthogonal basis $B \in \mathbb{R}^{192 \times 32}$, QR-orthogonalized -- **v10.0 fix**: `env_scale = 0.1` gradual initialization (v9.x's 1.0 init caused the 1111.mp3 SECS=0.0363 disaster) -- Inference-time adjustment: `env_weight = 0` → studio-clean timbre (**v10.0 default SOTA config**), `env_weight = 1` → original mic texture - -### Innovation C: Cross-Entropy Attention Guidance (CEAG) — implemented, disabled in v10.4.7+ - -Upgraded from IEAG (full self-attention entropy) to CEAG (text-audio cross-attention entropy), directly targeting short-text alignment collapse: - -$$v_{final} = v_{cfg} - \lambda(t) \cdot \nabla_x H(A_{mel \to text})$$ - -- **v10.0 Landmine 1 fix**: Added `text_mask` and `mel_mask` before `log_softmax` to prevent gradient explosion from `` tokens (NaN defense) -- **v10.4.7+ status**: Code path preserved in `ceag_sampler.py` for reproducibility, but `use_ceag=False` in `SOTAConfig`. v10.4.7 ablations showed negligible incremental effect over the LAAG + F5-official-flow baseline; re-enabling is left to follow-up work. - -### Innovation D: Dynamic FiLM Adapter — v10.4.6+ - -Persona + emotion conditioned FiLM modulating DiT hidden layers: -- Zero-initialized `gamma_net` / `beta_net` (starts as identity mapping, safe for pretrained backbone) -- v10.4.6+ **dynamic activation**: FiLM off when `len(chunks) == 1` (short text, avoids over-correction); FiLM on for multi-chunk (long text, anchors timbre across chunks) - -### Innovation E: Silero VAD + RMS Normalization (v10.0 Landmine 3 fix) - -Replaces legacy WebRTC VAD (GMM-based, harsh on breath/fricatives) with **Silero VAD** (neural network, <100 MB VRAM), dramatically improving 1s sample reference purity for consonants like /s/, /f/. RMS normalization (`target_rms = 0.1`) ensures loudness consistency. - -### Removed Modules (Ablation-Verified Ineffective) - -| Module | Ablation p-value | Cohen's d | Status | -|--------|-----------------|-----------|--------| -| IBOP | 0.85 / 0.51 | <0.05 | **Removed** | -| AM-ODE | Zero-init | Identity mapping | **Removed** | -| TD-CFG | 0.41 / 0.74 | <0.06 | **Removed** | -| SBM | 0.71 / 0.90 | <0.03 | **Removed** | -| GRPO | Test-time only | Non-core | **Removed (v10.0)** | -| Duration Predictor | v10.3+ F5-TTS official formula is more accurate | — | **Removed (v10.3)** | -| Reference Enhancer | Loop-extension hurt SECS, deviates from F5-TTS baseline | — | **Removed (v10.4)** | -| Best-of-N | Manifold collapse is architectural; sampling can't fix | — | **Removed (v10.4.7)** | - ---- - -## Architecture Overview - -

- PersonaVoice Architecture -

- -The figure above illustrates the full inference pipeline of PersonaVoice v10.4.8. A 1-second reference audio is first pre-processed by **Silero VAD + RMS normalization**, then encoded by **ECAPA-TDNN** into a 192-d speaker embedding. Optional **chat history** and **structured recollections** are distilled by a BERT-based persona extractor into 64-d text-style and 64-d persona embeddings. The **OES** module decomposes the audio embedding into orthogonal timbre and environment components, while **LAAG** decides chunking strategy and FiLM activation based on text length. Finally, the frozen F5-TTS DiT blocks 0–19 plus two trainable blocks 20–21 generate the mel spectrogram, which is converted to 24 kHz waveform by the Vocos vocoder. - -> Full architecture details: [ARCHITECTURE.md](ARCHITECTURE.md) - ---- - -## Installation - -### Option A: One-Click Setup (Recommended) - -**Windows (PowerShell):** -```powershell -git clone https://github.com/personavoice/personavoice.git -cd personavoice -powershell -ExecutionPolicy Bypass -File scripts\setup_env.ps1 -``` - -**Linux / macOS:** -```bash -git clone https://github.com/personavoice/personavoice.git -cd personavoice -bash scripts/setup_env.sh -``` - -The setup script will: -1. Create a `.venv` virtual environment (inherits system torch/speechbrain/vocos) -2. Install PyTorch (CUDA 11.8 or CPU) -3. Install all PersonaVoice dependencies -4. Install `silero-vad`, `f5-tts`, `imageio-ffmpeg` -5. Install PersonaVoice in editable mode (`pip install -e .`) -6. Verify the installation -7. Download pretrained models (~3 GB, optional — pass `--skip-models` / `-SkipModels` to skip) - -### Option B: Manual Install - -```bash -# 1. Create virtual environment -python -m venv --system-site-packages .venv - -# 2. Activate it -# Windows: .\.venv\Scripts\activate -# Linux: source .venv/bin/activate - -# 3. Install PyTorch (CUDA 11.8) -pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu118 - -# 4. Install dependencies -pip install -r requirements.txt -pip install silero-vad f5-tts imageio-ffmpeg -pip install -e . -``` - -### Requirements - -- Python ≥ 3.10 -- PyTorch ≥ 2.0 (CUDA 11.8 recommended, CPU works but slow) -- GPU: 8 GB VRAM sufficient (tested on RTX 4070) -- ffmpeg (for Whisper ASR; auto-provided by `imageio-ffmpeg`) - ---- - -## Data & Model Download - -PersonaVoice does **not** bundle large data or model weights. Use the scripts below. - -### 1. Pretrained Models (~3 GB) - -All models are downloaded from HuggingFace. For users in mainland China, set the mirror: -```bash -export HF_ENDPOINT=https://hf-mirror.com # Linux/macOS -$env:HF_ENDPOINT="https://hf-mirror.com" # Windows PowerShell -``` - -Then run: -```bash -python scripts/download_models.py # All required models -python scripts/download_models.py --optional # Also optional models (Qwen2.5, MiniLM) -python scripts/download_models.py --check # Check cache status only -``` - -**Required models:** - -| Model | HuggingFace ID | Size | Purpose | -|-------|----------------|------|---------| -| F5-TTS | `SWivid/F5-TTS` | ~1.5 GB | TTS backbone (frozen) | -| ECAPA-TDNN | `speechbrain/spkrec-ecapa-voxceleb` | ~80 MB | Speaker encoder (SECS) | -| Vocos | `charactr/vocos-mel-24khz` | ~50 MB | Neural vocoder | -| BERT (Chinese) | `bert-base-chinese` | ~400 MB | Persona text encoder | -| Whisper | `openai/whisper-large-v3-turbo` | ~1.5 GB | ASR for WER evaluation & ref_text | - -**Optional models:** - -| Model | HuggingFace ID | Size | Purpose | -|-------|----------------|------|---------| -| Qwen2.5-0.5B | `Qwen/Qwen2.5-0.5B` | ~1 GB | LLM persona dialogue (experimental) | -| all-MiniLM-L6-v2 | `sentence-transformers/all-MiniLM-L6-v2` | ~90 MB | Text embedding baseline | - -### 2. Evaluation Dataset (LibriTTS dev-clean, ~1.2 GB) - -For reproducing the 200-sample evaluation: -```bash -python scripts/prepare_data.py -# Or skip download if LibriTTS already exists: -python scripts/prepare_data.py --skip-download -# Quick test with only 20 samples: -python scripts/prepare_data.py --n_samples 20 -``` - -This downloads LibriTTS dev-clean from [OpenSLR #60](https://openslr.org/60/), extracts mel spectrograms + ECAPA embeddings, and saves to `data/libritts_processed/libritts_devclean_processed.pt`. - -### 3. Bundled Demo Audio - -The repo includes a single 1-second reference audio `1111.mp3` (~35 KB) for quick testing — no download required. - ---- - -## Quick Start - -### Web Demo (Recommended First Run) - -```bash -python -m personavoice.demo.api_server -# Open http://localhost:8000 in browser -``` - -The web UI exposes: -- `/api/health` — backend health check -- `/api/architecture` — full v10.4.8 module graph (powers the frontend visualization) -- `/api/clone` — voice cloning with optional chat history (persona injection) -- `/api/persona/analyze` — Big Five trait radar from chat history -- `/api/persona/calibrate` — interactive persona recalibration -- `/api/demo/sample` — bundled demo sample - -### Programmatic Usage - -```python -from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone -from personavoice.config import SOTA_CONFIG - -# 1. Load backbone (F5-TTS + FiLM + OES adapters) -backbone, _ = load_pretrained_f5tts_backbone(device="cuda", use_film=True) - -# 2. All inference parameters auto-loaded from config.py — single source of truth -mel_gen, laag_info = backbone.laag_synthesize( - text_str="Hello world, this is a voice cloning test.", - mel_ref=mel_ref, # 1-second reference mel (T_ref, 100) - speaker_emb=speaker_emb, # ECAPA 192-d - persona_emb=persona_emb, # 64-d (zeros for no-persona) - emotion_emb=emotion_emb, # 4-d (zeros for no-emotion) - tokenizer=tokenizer, - ref_text=ref_text, # F5-TTS official ref_text concatenation - audio_ref=audio_ref, # F5-TTS official cond=audio flow -) - -# 3. Adjust OES environment weight (0.0 = studio quality, SOTA config) -backbone.set_env_weight(SOTA_CONFIG.env_weight) # 0.0 -``` - -### 1111.mp3 Quick Test - -```bash -python examples/clone_demo.py -# Outputs saved to outputs/1111_clone_test_*.wav -``` - ---- - -## Reproducing the Paper Results - -### 1. 200-Sample Statistical Evaluation - -```bash -# (requires LibriTTS prepared via scripts/prepare_data.py) -python -m personavoice.experiment.eval_200_samples -# Outputs: -# results/eval_200_samples.json — raw per-sample results -# results/eval_200_statistics.json — statistical significance -# results/eval_200_by_length.json — stratified by text length -``` - -### 2. External Baseline Comparison (CosyVoice / XTTS v2) - -```bash -# Install CosyVoice (https://github.com/FunAudioLLM/CosyVoice) and/or TTS (pip install TTS) -python -m personavoice.experiment.baseline_external --n_samples 200 --models cosyvoice xtts -# Output: results/baseline_external_200.json -``` - -### 3. CFR (Catastrophic Failure Rate) Analysis - -```bash -python -m personavoice.experiment.cfr_analysis -# Output: results/cfr_analysis.json -``` - -### 4. Visualization (Pareto / Waveform / Attention) +- [`supplement/language_provenance`](supplement/language_provenance) contains three frozen seed-level prediction files, ID-only evidence, failure audits, and deterministic figures. +- [`supplement/speech_shortref`](supplement/speech_shortref) contains the frozen 200-item metric rows, length-stratified analyses, trade-off/ablation ledgers, and a clearly marked pending-condition manifest. +- [`reproducibility/config.json`](reproducibility/config.json) records versions, seeds, metrics, data fingerprints, and the policy for excluded model weights. +- [`reproducibility/SHA256SUMS.txt`](reproducibility/SHA256SUMS.txt) is the release-wide SHA-256 manifest; each supplement also carries a package-level manifest. +- [`REPRODUCE.md`](REPRODUCE.md) and [`reproduce.py`](reproduce.py) provide the minimal CPU-only verification and derivation commands. ```bash -python -m personavoice.experiment.visualize -# Outputs: -# results/figures/figure4_pareto_frontier.png -# results/figures/figure5_waveform_spectrogram.png -# results/figures/figure6_attention_heatmap.png +python reproduce.py ``` ---- - -## Inference Parameters - -All parameters are centralized in [`personavoice/config.py`](personavoice/config.py) — the single source of truth. - -| Parameter | Value | Description | -|-----------|-------|-------------| -| steps | 32 | ODE integration steps (v10.4: reduced from 96 for speed, RTF -45%) | -| sway_coef | -1.0 | Sway Sampling (F5-TTS official default) | -| cfg_strength | 2.5 | Static CFG (v10.4: tuned for SECS, replaces TD-CFG) | -| env_weight | 0.0 | OES: studio quality (SECS SOTA) | -| oes_env_scale_init | 0.1 | OES gradual init (v10.0 fix for 1111.mp3) | -| use_ceag | False | CEAG disabled in v10.4.7+ (negligible incremental effect; code preserved) | -| ceag_lambda_max | 0.20 | CEAG guidance strength (used when enabled) | -| ceag_t_start | 0.1 | CEAG activation start | -| ceag_t_end | 0.4 | CEAG activation end | -| ceag_layers | (-2,-1) | CEAG attention extraction layers | -| use_laag | True | LAAG dynamic chunking + dynamic FiLM | -| laag_chunk_max_chars | 135 | LAAG chunk size cap | -| laag_cfg_base | 2.5 | LAAG base CFG | -| laag_cfg_alpha | 0.0 | Dynamic CFG α (disabled) | -| best_of_n | 1 | Best-of-N disabled (manifold collapse can't be fixed by sampling) | -| use_silero_vad | True | Silero VAD preprocessing (v10.0) | -| use_rms_normalize | True | RMS energy normalization (v10.0) | -| rms_target | 0.1 | RMS target level | -| silero_vad_threshold | 0.5 | Silero VAD trigger threshold | - ---- - -## Project Structure - -``` -PersonaVoice/ -├── ARCHITECTURE.md # Architecture design (v10.4.8) -├── SOTA_VERIFICATION_REPORT.md # SOTA verification report (honest trade-off edition) -├── README.md / README_zh.md # Documentation -├── CONTRIBUTING.md # Contribution guidelines -├── CITATION.cff # Citation metadata -├── LICENSE # MIT (note F5-TTS backbone is CC-BY-NC-4.0) -├── requirements.txt # Python dependencies -├── setup.py # pip-installable package -├── 1111.mp3 # Bundled 1s reference demo audio (~35KB) -├── scripts/ # Setup & data prep -│ ├── download_models.py # One-click model download -│ ├── prepare_data.py # LibriTTS download + preprocess -│ ├── setup_env.ps1 # Windows one-click env setup -│ └── setup_env.sh # Linux/macOS one-click env setup -├── .github/ # CI, issue/PR templates, discussion template -│ ├── workflows/python-package.yml -│ └── ISSUE_TEMPLATE/ # bug_report, feature_request, experiment_reproduction -├── personavoice/ -│ ├── config.py # ★ Unified SOTA config (single source of truth) -│ ├── microaug/ # Module: Ultra-short sample enhancement -│ │ └── cross_manifold_refiner.py # OES (v10.0 Core B, env_scale=0.1) -│ ├── tts_backbone/ # Module: F5-TTS DiT backbone -│ │ ├── f5_pretrained_backbone.py # Backbone + adapters (F5 official flow) -│ │ ├── ceag_sampler.py # ★ CEAG (v10.0 Core C, with padding mask; disabled) -│ │ ├── laag_generator.py # ★ LAAG (v10.1 Core A, dynamic FiLM + mel splice fix) -│ │ └── vocoder.py # Vocos vocoder -│ ├── persona/ # Module: Persona extraction pipeline -│ │ └── extractor.py # BERT chat → Big Five → persona_emb -│ ├── common/ # Shared utilities -│ │ └── local_models.py # Local model paths (offline mode) -│ ├── demo/ # Interactive demo -│ │ ├── api_server.py # FastAPI server (unified config + persona integration) -│ │ ├── audio_preprocess.py # ★ Silero VAD + RMS normalize (v10.0 Core E) -│ │ └── index.html # Web UI (architecture visualization + cloning + persona radar) -│ └── experiment/ # Evaluation scripts (top-conference experiments) -│ ├── eval_200_samples.py # 200-sample statistical evaluation (PV vs F5 baseline) -│ ├── baseline_external.py # External SOTA comparison (CosyVoice / XTTS v2) -│ ├── cfr_analysis.py # Catastrophic Failure Rate analysis -│ ├── comprehensive_evaluator.py # SECS + WER + UTMOS + RTF + SIM-o -│ ├── ecapa_evaluator.py # ECAPA-TDNN SECS evaluation -│ ├── wer_evaluator.py # Whisper-based WER evaluation -│ ├── visualize.py # Pareto frontier + waveform/spectrogram + attention heatmap -│ └── utils.py # Shared utilities (data loading, tokenizer, logger) -├── examples/ # Usage examples & quick-start demos -│ ├── clone_demo.py # 1111.mp3 clone verification (Quick Start) -│ └── api_demo.py # Frontend API end-to-end test -└── results/ # Experiment results (JSON + figures) - ├── eval_200_samples.json - ├── eval_200_statistics.json - ├── honest_metrics_v10.4.7.json - ├── ablation_200_samples.json - ├── ablation_200_statistics.json - ├── cfr_analysis.json - ├── baseline_external_200.json - └── figures/ - ├── figure4_pareto_frontier.png - ├── figure5_waveform_spectrogram.png - └── figure6_attention_heatmap.png -``` - ---- - -## Design Philosophy - -### Plug-in Adapter Architecture - -- Freeze F5-TTS pretrained backbone (first 20 layers) -- Train only lightweight adapters (FiLM + OES, ~2 M parameters) -- CEAG is an inference-time optimization, zero training cost -- 8 GB VRAM sufficient for training and inference -- Total trainable: ~31.9 M (9.39% of 339.6 M) - -### Ablation-Driven Decision Making - -All modules must pass 200-sample paired t-test (p < 0.05) and Cohen's d (> 0.1) verification. v10.0 removed 5 ineffective modules (IBOP, AM-ODE, TD-CFG, SBM, GRPO). v10.3+ removed Duration Predictor (F5 official formula is more accurate). v10.4.7+ disabled CEAG and Best-of-N after ablations showed no incremental benefit on the LAAG baseline. - -### Academic Honesty - -- UTMOS is reported as an **Objective Proxy**, not Human MOS -- Short-text WER (0.4066, CFR 35.8%) is openly reported as a Flow-Matching architectural limitation, left as Future Work -- Long-text WER (0.1158) is honestly reported as worse than XTTS v2 (0.10) — a trade-off, not a defeat -- Estimated CFR values for baselines are explicitly marked `*` -- Human MOS evaluation is pending (planned: 20 raters, Naturalness MOS + Similarity MOS) - ---- - -## Related Work & Acknowledgements - -PersonaVoice builds on and references the following prior work. We gratefully acknowledge the open-source community. - -### Backbone & Architecture - -- **F5-TTS** ([SWivid/F5-TTS](https://github.com/SWivid/F5-TTS), Cheng et al., 2024) — Flow-Matching with DiT backbone, CC-BY-NC-4.0. PersonaVoice freezes the first 20 layers and trains lightweight FiLM adapters. -- **Vocos** ([gemelo-ai/vocos](https://github.com/gemelo-ai/vocos), Siuzdak, 2023) — Neural vocoder, MIT. Used for mel → waveform. -- **Flow Matching** (Lipman et al., 2023) — Generative framework underlying F5-TTS. - -### Speaker & Text Encoders - -- **ECAPA-TDNN** ([SpeechBrain](https://github.com/speechbrain/speechbrain), Desplanques et al., 2020) — Speaker encoder for SECS evaluation, Apache-2.0. -- **BERT** ([bert-base-chinese](https://huggingface.co/bert-base-chinese), Devlin et al., 2019) — Persona text style encoder. -- **Whisper** ([openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo), Radford et al., 2023) — ASR for WER evaluation & ref_text transcription, MIT. - -### Voice Cloning Baselines - -- **CosyVoice** ([FunAudioLLM/CosyVoice](https://github.com/FunAudioLLM/CosyVoice), 2024) — Alibaba's zero-shot TTS, used as external baseline. -- **XTTS v2** ([coqui/XTTS-v2](https://huggingface.co/coqui/XTTS-v2), Casanova et al., 2024) — Coqui's 3s voice cloning baseline. -- **VALL-E** (Wang et al., 2023) — Early neural codec language model for TTS. -- **VoiceBox** (Le et al., 2024) — Flow-Matching TTS at scale. - -### Persona & Emotion Modeling - -- **Big Five Personality Traits** (McCrae & Costa, 1992) — Theoretical basis for persona extraction. -- **FiLM** (Perez et al., 2018) — Feature-wise Linear Modulation, the adapter mechanism we use. -- **LoRA** (Hu et al., 2022) — Inspired our plug-in adapter design philosophy. - -### Voice Activity Detection - -- **Silero VAD** ([snakers4/silero-vad](https://github.com/snakers4/silero-vad), 2021) — Neural VAD replacing WebRTC GMM, MIT. - -### Evaluation Metrics - -- **UTMOS** (Saeki et al., 2022) — Objective MOS prediction (reported as Objective Proxy). -- **SECS** — Speaker Encoder Cosine Similarity, standard in zero-shot TTS evaluation. -- **WER** — Word Error Rate via Whisper ASR. -- **CFR** — Catastrophic Failure Rate, proposed in this project for tail-risk analysis. - -> If we missed any work that should be cited, please open an issue — academic accuracy is a priority. - ---- - -## Citation - -If PersonaVoice helps your research, please cite: - -```bibtex -@article{personavoice2026, - title = {PersonaVoice v10.4.8: Plug-in Adapters for Persona-Driven 1-Second - Voice Cloning — A Trade-off Analysis of Timbre Fidelity - and Text-Alignment Stability in Flow-Matching TTS}, - author = {PersonaVoice Team}, - journal = {arXiv preprint}, - year = {2026}, - note = {Version 10.4.8. Code: https://github.com/personavoice/personavoice} -} -``` - -See also [`CITATION.cff`](CITATION.cff) for machine-readable metadata. - ---- - -## License - -- PersonaVoice codebase (adapters, integrations, novel modules OES/LAAG/CEAG): **MIT** -- F5-TTS pretrained backbone: **CC-BY-NC-4.0** (non-commercial) -- Vocos, ECAPA-TDNN, Whisper, Silero VAD: see their respective licenses - -Commercial use of F5-TTS pretrained backbone requires compliance with CC-BY-NC-4.0. See [LICENSE](LICENSE) for details. - ---- - -## Contact & Community - -- **Issues**: [GitHub Issues](https://github.com/personavoice/personavoice/issues) — use the appropriate template (bug / feature / reproduction) -- **Discussions**: [GitHub Discussions](https://github.com/personavoice/personavoice/discussions) — research questions, design debate, use cases -- **Contributing**: see [CONTRIBUTING.md](CONTRIBUTING.md) - ---- +The package does not distribute checkpoints, private/raw audio, API keys, or a +voice-cloning service. Objective SECS/WER and the controlled language proxy are +measurement endpoints, not evidence of psychological recovery, authorization to +impersonate a person, or a deployment safety guarantee. -**Keywords** (for academic search discoverability): -voice cloning, zero-shot TTS, few-shot voice cloning, 1-second voice cloning, flow matching, F5-TTS, speech synthesis, persona-driven TTS, emotional TTS, plug-in adapter, FiLM modulation, speaker embedding, ECAPA-TDNN, orthogonal decomposition, length-adaptive generation, manifold collapse, trade-off analysis, LibriTTS, Vocos vocoder, Silero VAD, Big Five personality, BERT persona, Whisper ASR, WER, SECS, UTMOS, CFR, catastrophic failure rate. +See [`ZENODO.md`](ZENODO.md) and [`CITATION.cff`](CITATION.cff) for deposition +metadata. The paper package is uploaded to Zenodo separately from this branch. diff --git a/README_zh.md b/README_zh.md deleted file mode 100644 index 24a997f..0000000 --- a/README_zh.md +++ /dev/null @@ -1,555 +0,0 @@ -# PersonaVoice: 插件化 Adapter 实现人格驱动的 1 秒极限声音克隆 - -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) -[![PyTorch 2.0+](https://img.shields.io/badge/PyTorch-2.0+-ee4c2c.svg)](https://pytorch.org/) -[![Version 10.4.8](https://img.shields.io/badge/version-10.4.8--Trade--off-red.svg)](ARCHITECTURE.md) -[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) - -**[English](README.md)** | **中文** - -> **PersonaVoice v10.4.8** — 仅需 **1 秒**音频即可克隆任意声音, 并可选注入人格/情绪。基于 F5-TTS Flow-Matching 冻结骨干 + 插件化 Adapter 架构, 对音色保真度 (SECS SOTA) 与文本对齐稳定性的 Trade-off 进行系统分析。 - ---- - -## 目录 -- [PersonaVoice 是什么?](#personavoice-是什么) -- [核心结果](#核心结果) -- [核心创新点](#核心创新点) -- [架构总览](#架构总览) -- [安装](#安装) -- [数据与模型下载](#数据与模型下载) -- [快速开始](#快速开始) -- [复现论文结果](#复现论文结果) -- [推理参数](#推理参数) -- [项目结构](#项目结构) -- [设计哲学](#设计哲学) -- [相关工作与致谢](#相关工作与致谢) -- [引用](#引用) - ---- - -## PersonaVoice 是什么? - -PersonaVoice 是一个**人格驱动的 1 秒极限声音克隆插件化 Adapter 架构**。它解决的核心问题: - -> **仅给 1 秒目标说话人的音频 + 可选文本记录 (聊天记录 / 结构化回忆), 生成保留其音色、语言习惯、个体情绪表达模式的语音。** - -本项目的叙事**不是**"全 SOTA"声明, 而是**Trade-off 分析**: -- **Flow-Matching** 在音色相似度 (SECS) 上具有统治性优势, 但在短文本下存在流形塌缩问题; -- PersonaVoice 的贡献是用 **LAAG + 动态 FiLM** 在保住 SECS 领先的同时恢复长文本对齐, 短文本 WER 则作为诚实的 Future Work。 - -### 为什么是 1 秒? - -主流 zero-shot TTS 系统通常假设 ≥3 秒参考音频 (XTTS v2、CosyVoice、VALL-E、VoiceBox)。1 秒场景暴露了架构的根本性限制: -- 信息瓶颈 (24 kHz / hop 256 下仅 94 mel 帧) -- Flow-Matching 在极短文本下的流形塌缩 -- 参考音频中的呼吸/摩擦音杂质主导克隆信号 - -PersonaVoice 是首个系统研究 1 秒极限克隆场景的开源项目。 - -### 交互式 Web 演示 - -下面是 PersonaVoice v10.4.8 交互式 Dashboard 的截图。用户上传 1 秒参考音频后,可選填入聊天记录或结构化回忆,系统即可生成克隆语音,并实时展示 Big Five 人格雷达图、Mel 频谱可视化以及架构条件注入状态。 - -

- PersonaVoice v10.4.8 交互式演示 -

- ---- - -## 核心结果 - -**实验设置**: 1 秒参考音频, 200 样本 LibriTTS dev-clean 评估, ECAPA-TDNN SECS, Whisper-large-v3-turbo WER。 - -| 指标 | PersonaVoice v10.4.8 | XTTS v2 (1s ref) | CosyVoice (1s ref) | F5-TTS 官方 | SOTA | -|------|----------------------|-------------------|---------------------|-------------|------| -| **ECAPA SECS (整体)** | **0.4945** | 0.3349 | 0.3595 | 0.2508 | **PV ✓** (+47.6% vs XTTS v2) | -| **ECAPA SECS (长文本)** | **0.5044** | 0.31 | 0.33 | 0.22 | **PV ✓** (+62.7% vs XTTS v2) | -| **WER (整体)** | 0.1928 | **0.1296** | 0.6130 | 0.8982 | XTTS v2 (AR 路线) | -| **WER (长文本)** | 0.1158 | **0.10** | 0.55 | 0.85 | XTTS v2 (诚实的 trade-off) | -| **WER (短文本 ≤8 词)** | 0.4066 | **0.16** | 0.68 | 0.95 | XTTS v2 (Future Work) | -| **CFR (整体, WER>0.5)** | 13.5% | ~5%* | ~30%* | ~45%* | XTTS v2 | -| **CFR (短文本)** | 35.8% | ~10%* | ~50%* | ~70%* | XTTS v2 | -| **UTMOS (客观代理)** | 4.0-4.5 | 3.8 | 4.0 | 3.5 | PV (人工 MOS 待做) | -| **RTF** | 0.55-0.66 | 0.85 | 0.72 | **0.50** | F5-TTS | -| **1111.mp3 SECS (长文本)** | **0.5854** | - | - | - | PV ✓ | -| **1111.mp3 WER (长文本)** | **0.0676** | - | - | - | PV ✓ | - -> `*` XTTS v2 / CosyVoice / F5-TTS 的 CFR 是从 WER 分布估算的, 实测对比计划中。完整评估细节: [SOTA_VERIFICATION_REPORT.md](SOTA_VERIFICATION_REPORT.md)。 - ---- - -## 核心创新点 - -PersonaVoice 引入了 **5 个核心创新**, 全部经 200 样本配对 t 检验 (p < 0.05, Cohen's d > 0.1) 验证。 - -### 创新 A: LAAG (Length-Aware Adaptive Generation, 长度自适应生成) — v10.1+ - -解决 1 秒极限克隆下短长文本性能不均的根本性缺陷: - -- **动态 Chunking**: 长文本按 135 字符上限切分, 每个 chunk 充分利用 1 秒参考音频 -- **动态 CFG base**: `cfg_base = 2.5`, 短长文本平衡 -- **动态 FiLM 激活 (v10.4.6+)**: 短文本 (单 chunk) 关 FiLM (稳定基线); 长文本 (多 chunk) 开 FiLM (防止跨 chunk 音色漂移, +0.06 SECS) -- **mel 拼接修复 (v10.4.5)**: 修正跨 chunk 的 `(T, mel_dim)` vs `(mel_dim, T)` 维度顺序错误 (+56.6% SECS) -- **F5-TTS 官方流程整合 (v10.3+)**: `infer_batch_process` + `ref_text` + `audio_ref` cond 流, 替代自定义 Duration Predictor -- **v10.4.8 边界硬化**: ref_text 加标点 + 空格确保 ref/gen 边界清晰; gen_text 起始空格牺牲防止头部截断; 起始静音裁剪去除伪影 - -### 创新 B: OES (Orthogonal Environment Sub-manifold, 正交环境子流形) - -将说话人嵌入分解为音色子空间与环境子空间: - -$$Z_{audio} = Z_{timbre} + Z_{env}, \quad Z_{timbre} \perp Z_{env}$$ - -- `env_basis`: 可学习正交基 $B \in \mathbb{R}^{192 \times 32}$, QR 正交化 -- **v10.0 修复**: `env_scale = 0.1` 渐进初始化 (v9.x 的 1.0 初始化导致 1111.mp3 SECS=0.0363 灾难) -- 推理时调整: `env_weight = 0` → 录音棚干净音色 (**v10.0 默认 SOTA 配置**), `env_weight = 1` → 原始麦克风质感 - -### 创新 C: CEAG (Cross-Entropy Attention Guidance, 交叉熵注意力引导) — 已实现, v10.4.7+ 禁用 - -从 IEAG (全自注意力熵) 升级为 CEAG (文本-音频交叉注意力熵), 直击短文本对齐塌缩: - -$$v_{final} = v_{cfg} - \lambda(t) \cdot \nabla_x H(A_{mel \to text})$$ - -- **v10.0 地雷 1 修复**: 在 `log_softmax` 前加 `text_mask` 和 `mel_mask`, 防止 `` token 导致梯度爆炸 (NaN 防御) -- **v10.4.7+ 状态**: 代码路径保留在 `ceag_sampler.py` 以便复现, 但 `SOTAConfig` 中 `use_ceag=False`。v10.4.7 消融实验显示在 LAAG + F5 官方流基线上增量效果可忽略, 重新启用留作后续工作。 - -### 创新 D: 动态 FiLM Adapter — v10.4.6+ - -persona + emotion 条件化的 FiLM 调制 DiT 隐藏层: -- `gamma_net` / `beta_net` 零初始化 (起始为恒等映射, 对预训练骨干安全) -- v10.4.6+ **动态激活**: `len(chunks) == 1` (短文本) 时关闭 FiLM (避免过校正); 多 chunk (长文本) 时开启 FiLM (锚定跨 chunk 音色) - -### 创新 E: Silero VAD + RMS 归一化 (v10.0 地雷 3 修复) - -用 **Silero VAD** (神经网络, <100 MB 显存) 替代老旧 WebRTC VAD (GMM 基, 对呼吸/摩擦音处理粗糙), 大幅提升 1 秒样本参考纯度, 尤其是像 /s/、/f/ 这样的辅音。RMS 归一化 (`target_rms = 0.1`) 确保响度一致。 - -### 已移除模块 (消融验证无效) - -| 模块 | 消融 p 值 | Cohen's d | 状态 | -|------|----------|-----------|------| -| IBOP | 0.85 / 0.51 | <0.05 | **已移除** | -| AM-ODE | 零初始化 | 恒等映射 | **已移除** | -| TD-CFG | 0.41 / 0.74 | <0.06 | **已移除** | -| SBM | 0.71 / 0.90 | <0.03 | **已移除** | -| GRPO | 仅测试时 | 非核心 | **已移除 (v10.0)** | -| Duration Predictor | v10.3+ F5 官方公式更准 | — | **已移除 (v10.3)** | -| Reference Enhancer | 循环延伸损害 SECS, 偏离 F5 基线 | — | **已移除 (v10.4)** | -| Best-of-N | 流形塌缩是架构问题, 采样无法修复 | — | **已移除 (v10.4.7)** | - ---- - -## 架构总览 - -

- PersonaVoice 架构图 -

- -上图展示了 PersonaVoice v10.4.8 的完整推理流程。1 秒参考音频首先经过 **Silero VAD + RMS 归一化** 预处理,再由 **ECAPA-TDNN** 编码为 192 维说话人嵌入;可选的**聊天记录**和**结构化回忆**通过 BERT-based 人格提取器蒸馏为 64 维文本风格与 64 维人格嵌入。**OES** 模块将音频嵌入分解为正交的音色与环境分量,**LAAG** 根据文本长度决定分块策略与 FiLM 开关;最终冻结的 F5-TTS DiT 0–19 层与两个可训练块 20–21 生成 Mel 谱,经 Vocos 声码器转换为 24 kHz 波形。 - -> 完整架构详情: [ARCHITECTURE.md](ARCHITECTURE.md) - ---- - -## 安装 - -### 方式 A: 一键搭建 (推荐) - -**Windows (PowerShell):** -```powershell -git clone https://github.com/personavoice/personavoice.git -cd personavoice -powershell -ExecutionPolicy Bypass -File scripts\setup_env.ps1 -``` - -**Linux / macOS:** -```bash -git clone https://github.com/personavoice/personavoice.git -cd personavoice -bash scripts/setup_env.sh -``` - -搭建脚本会: -1. 创建 `.venv` 虚拟环境 (继承系统 torch/speechbrain/vocos) -2. 安装 PyTorch (CUDA 11.8 或 CPU) -3. 安装所有 PersonaVoice 依赖 -4. 安装 `silero-vad`、`f5-tts`、`imageio-ffmpeg` -5. 以可编辑模式安装 PersonaVoice (`pip install -e .`) -6. 验证安装 -7. 下载预训练模型 (~3 GB, 可选 — 加 `--skip-models` / `-SkipModels` 跳过) - -### 方式 B: 手动安装 - -```bash -# 1. 创建虚拟环境 -python -m venv --system-site-packages .venv - -# 2. 激活 -# Windows: .\.venv\Scripts\activate -# Linux: source .venv/bin/activate - -# 3. 安装 PyTorch (CUDA 11.8) -pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu118 - -# 4. 安装依赖 -pip install -r requirements.txt -pip install silero-vad f5-tts imageio-ffmpeg -pip install -e . -``` - -### 环境要求 - -- Python ≥ 3.10 -- PyTorch ≥ 2.0 (推荐 CUDA 11.8, CPU 也可但较慢) -- GPU: 8 GB 显存足够 (在 RTX 4070 上测试) -- ffmpeg (Whisper ASR 需要; `imageio-ffmpeg` 已自动提供) - ---- - -## 数据与模型下载 - -PersonaVoice **不**附带大型数据或模型权重。请使用下面的脚本。 - -### 1. 预训练模型 (~3 GB) - -所有模型从 HuggingFace 下载。中国大陆用户可设置镜像: -```bash -export HF_ENDPOINT=https://hf-mirror.com # Linux/macOS -$env:HF_ENDPOINT="https://hf-mirror.com" # Windows PowerShell -``` - -然后运行: -```bash -python scripts/download_models.py # 所有必需模型 -python scripts/download_models.py --optional # 同时下载可选模型 (Qwen2.5, MiniLM) -python scripts/download_models.py --check # 仅检查缓存状态 -``` - -**必需模型:** - -| 模型 | HuggingFace ID | 大小 | 用途 | -|------|----------------|------|------| -| F5-TTS | `SWivid/F5-TTS` | ~1.5 GB | TTS 骨干 (冻结) | -| ECAPA-TDNN | `speechbrain/spkrec-ecapa-voxceleb` | ~80 MB | 说话人编码器 (SECS) | -| Vocos | `charactr/vocos-mel-24khz` | ~50 MB | 神经声码器 | -| BERT (中文) | `bert-base-chinese` | ~400 MB | 人格文本编码器 | -| Whisper | `openai/whisper-large-v3-turbo` | ~1.5 GB | WER 评估 ASR & ref_text 转写 | - -**可选模型:** - -| 模型 | HuggingFace ID | 大小 | 用途 | -|------|----------------|------|------| -| Qwen2.5-0.5B | `Qwen/Qwen2.5-0.5B` | ~1 GB | LLM 人格对话 (实验性) | -| all-MiniLM-L6-v2 | `sentence-transformers/all-MiniLM-L6-v2` | ~90 MB | 文本嵌入基线 | - -### 2. 评估数据集 (LibriTTS dev-clean, ~1.2 GB) - -复现 200 样本评估: -```bash -python scripts/prepare_data.py -# 如果 LibriTTS 已存在, 跳过下载: -python scripts/prepare_data.py --skip-download -# 快速测试, 仅处理 20 个样本: -python scripts/prepare_data.py --n_samples 20 -``` - -会从 [OpenSLR #60](https://openslr.org/60/) 下载 LibriTTS dev-clean, 提取 mel 频谱 + ECAPA 嵌入, 保存到 `data/libritts_processed/libritts_devclean_processed.pt`。 - -### 3. 内置演示音频 - -仓库内置 1 个 1 秒参考音频 `1111.mp3` (~35 KB), 用于快速测试 — 无需下载。 - ---- - -## 快速开始 - -### Web Demo (推荐首次运行) - -```bash -python -m personavoice.demo.api_server -# 浏览器打开 http://localhost:8000 -``` - -Web UI 提供接口: -- `/api/health` — 后端健康检查 -- `/api/architecture` — 完整 v10.4.8 模块图 (驱动前端可视化) -- `/api/clone` — 声音克隆 (可选注入聊天记录 persona) -- `/api/persona/analyze` — 从聊天记录分析 Big Five 雷达图 -- `/api/persona/calibrate` — 交互式 persona 重新校准 -- `/api/demo/sample` — 内置 demo 样本 - -### 编程式使用 - -```python -from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone -from personavoice.config import SOTA_CONFIG - -# 1. 加载骨干 (F5-TTS + FiLM + OES adapters) -backbone, _ = load_pretrained_f5tts_backbone(device="cuda", use_film=True) - -# 2. 所有推理参数自动从 config.py 加载 — 单一真相源 -mel_gen, laag_info = backbone.laag_synthesize( - text_str="你好, 这是一个声音克隆测试。", - mel_ref=mel_ref, # 1 秒参考 mel (T_ref, 100) - speaker_emb=speaker_emb, # ECAPA 192-d - persona_emb=persona_emb, # 64-d (无 persona 时全零) - emotion_emb=emotion_emb, # 4-d (无 emotion 时全零) - tokenizer=tokenizer, - ref_text=ref_text, # F5-TTS 官方 ref_text 拼接 - audio_ref=audio_ref, # F5-TTS 官方 cond=audio 流 -) - -# 3. 调整 OES 环境权重 (0.0 = 录音棚质量, SOTA 配置) -backbone.set_env_weight(SOTA_CONFIG.env_weight) # 0.0 -``` - -### 1111.mp3 快速测试 - -```bash -python examples/clone_demo.py -# 输出保存到 outputs/1111_clone_test_*.wav -``` - ---- - -## 复现论文结果 - -### 1. 200 样本统计评估 - -```bash -# (需要先通过 scripts/prepare_data.py 准备 LibriTTS) -python -m personavoice.experiment.eval_200_samples -# 输出: -# results/eval_200_samples.json — 原始每样本结果 -# results/eval_200_statistics.json — 统计显著性 -# results/eval_200_by_length.json — 按文本长度分层 -``` - -### 2. 外部基线对比 (CosyVoice / XTTS v2) - -```bash -# 安装 CosyVoice (https://github.com/FunAudioLLM/CosyVoice) 和/或 TTS (pip install TTS) -python -m personavoice.experiment.baseline_external --n_samples 200 --models cosyvoice xtts -# 输出: results/baseline_external_200.json -``` - -### 3. CFR (灾难性失败率) 分析 - -```bash -python -m personavoice.experiment.cfr_analysis -# 输出: results/cfr_analysis.json -``` - -### 4. 可视化 (Pareto / 波形 / 注意力) - -```bash -python -m personavoice.experiment.visualize -# 输出: -# results/figures/figure4_pareto_frontier.png -# results/figures/figure5_waveform_spectrogram.png -# results/figures/figure6_attention_heatmap.png -``` - ---- - -## 推理参数 - -所有参数集中在 [`personavoice/config.py`](personavoice/config.py) — 单一真相源。 - -| 参数 | 取值 | 说明 | -|------|------|------| -| steps | 32 | ODE 积分步数 (v10.4: 从 96 降低, RTF -45%) | -| sway_coef | -1.0 | Sway Sampling (F5-TTS 官方默认) | -| cfg_strength | 2.5 | 静态 CFG (v10.4: 调优 SECS, 替代 TD-CFG) | -| env_weight | 0.0 | OES: 录音棚质量 (SECS SOTA) | -| oes_env_scale_init | 0.1 | OES 渐进初始化 (v10.0 修复 1111.mp3) | -| use_ceag | False | CEAG 在 v10.4.7+ 禁用 (增量效果可忽略; 代码保留) | -| ceag_lambda_max | 0.20 | CEAG 引导强度 (启用时使用) | -| ceag_t_start | 0.1 | CEAG 激活起始 | -| ceag_t_end | 0.4 | CEAG 激活结束 | -| ceag_layers | (-2,-1) | CEAG 注意力提取层 | -| use_laag | True | LAAG 动态 Chunking + 动态 FiLM | -| laag_chunk_max_chars | 135 | LAAG chunk 大小上限 | -| laag_cfg_base | 2.5 | LAAG 基础 CFG | -| laag_cfg_alpha | 0.0 | 动态 CFG α (禁用) | -| best_of_n | 1 | Best-of-N 禁用 (流形塌缩无法靠采样修复) | -| use_silero_vad | True | Silero VAD 预处理 (v10.0) | -| use_rms_normalize | True | RMS 能量归一化 (v10.0) | -| rms_target | 0.1 | RMS 目标电平 | -| silero_vad_threshold | 0.5 | Silero VAD 触发阈值 | - ---- - -## 项目结构 - -``` -PersonaVoice/ -├── ARCHITECTURE.md # 架构设计 (v10.4.8) -├── SOTA_VERIFICATION_REPORT.md # SOTA 验证报告 (诚实 trade-off 版) -├── README.md / README_zh.md # 文档 -├── CONTRIBUTING.md # 贡献指南 -├── CITATION.cff # 引用元数据 -├── LICENSE # MIT (注意 F5-TTS 骨干为 CC-BY-NC-4.0) -├── requirements.txt # Python 依赖 -├── setup.py # pip 可安装包 -├── 1111.mp3 # 内置 1 秒参考 demo 音频 (~35KB) -├── scripts/ # 搭建与数据准备 -│ ├── download_models.py # 一键模型下载 -│ ├── prepare_data.py # LibriTTS 下载 + 预处理 -│ ├── setup_env.ps1 # Windows 一键环境搭建 -│ └── setup_env.sh # Linux/macOS 一键环境搭建 -├── .github/ # CI、issue/PR 模板、讨论模板 -│ ├── workflows/python-package.yml -│ └── ISSUE_TEMPLATE/ # bug_report、feature_request、experiment_reproduction -├── personavoice/ -│ ├── config.py # ★ 统一 SOTA 配置 (单一真相源) -│ ├── microaug/ # 模块: 超短样本增强 -│ │ └── cross_manifold_refiner.py # OES (v10.0 核心 B, env_scale=0.1) -│ ├── tts_backbone/ # 模块: F5-TTS DiT 骨干 -│ │ ├── f5_pretrained_backbone.py # 骨干 + adapters (F5 官方流) -│ │ ├── ceag_sampler.py # ★ CEAG (v10.0 核心 C, 带 padding mask; 已禁用) -│ │ ├── laag_generator.py # ★ LAAG (v10.1 核心 A, 动态 FiLM + mel 拼接修复) -│ │ └── vocoder.py # Vocos 声码器 -│ ├── persona/ # 模块: persona 提取管线 -│ │ └── extractor.py # BERT 聊天记录 → Big Five → persona_emb -│ ├── common/ # 共享工具 -│ │ └── local_models.py # 本地模型路径 (离线模式) -│ ├── demo/ # 交互式 demo -│ │ ├── api_server.py # FastAPI 服务器 (统一配置 + persona 集成) -│ │ ├── audio_preprocess.py # ★ Silero VAD + RMS 归一化 (v10.0 核心 E) -│ │ └── index.html # Web UI (架构可视化 + 克隆 + persona 雷达) -│ └── experiment/ # 评估脚本 (顶会实验) -│ ├── eval_200_samples.py # 200 样本统计评估 (PV vs F5 基线) -│ ├── baseline_external.py # 外部 SOTA 对比 (CosyVoice / XTTS v2) -│ ├── cfr_analysis.py # 灾难性失败率分析 -│ ├── comprehensive_evaluator.py # SECS + WER + UTMOS + RTF + SIM-o -│ ├── ecapa_evaluator.py # ECAPA-TDNN SECS 评估 -│ ├── wer_evaluator.py # Whisper WER 评估 -│ ├── visualize.py # Pareto 前沿 + 波形/频谱 + 注意力热图 -│ └── utils.py # 共享工具 (数据加载, tokenizer, 日志) -├── examples/ # 使用示例与快速测试 -│ ├── clone_demo.py # 1111.mp3 克隆验证 (快速开始) -│ └── api_demo.py # 前端 API 端到端测试 -└── results/ # 实验结果 (JSON + 图) - ├── eval_200_samples.json - ├── eval_200_statistics.json - ├── honest_metrics_v10.4.7.json - ├── ablation_200_samples.json - ├── ablation_200_statistics.json - ├── cfr_analysis.json - ├── baseline_external_200.json - └── figures/ - ├── figure4_pareto_frontier.png - ├── figure5_waveform_spectrogram.png - └── figure6_attention_heatmap.png -``` - ---- - -## 设计哲学 - -### 插件化 Adapter 架构 - -- 冻结 F5-TTS 预训练骨干 (前 20 层) -- 仅训练轻量级 adapter (FiLM + OES, ~2M 参数) -- CEAG 是推理时优化, 零训练成本 -- 8 GB 显存即可训练与推理 -- 总可训练参数: ~31.9M (占 339.6M 的 9.39%) - -### 消融驱动决策 - -所有模块必须通过 200 样本配对 t 检验 (p < 0.05) 和 Cohen's d (> 0.1) 验证。v10.0 移除了 5 个无效模块 (IBOP、AM-ODE、TD-CFG、SBM、GRPO)。v10.3+ 移除 Duration Predictor (F5 官方公式更准)。v10.4.7+ 在消融显示对 LAAG 基线无增量收益后, 禁用 CEAG 与 Best-of-N。 - -### 学术诚实 - -- UTMOS 作为**客观代理**报告, 非人工 MOS -- 短文本 WER (0.4066, CFR 35.8%) 公开报告为 Flow-Matching 架构限制, 留作 Future Work -- 长文本 WER (0.1158) 诚实报告为逊于 XTTS v2 (0.10) — 这是 trade-off, 不是完败 -- 基线的估算 CFR 值明确标注 `*` -- 人工 MOS 评估待做 (计划: 20 名评分者, Naturalness MOS + Similarity MOS) - ---- - -## 相关工作与致谢 - -PersonaVoice 构建在以下前期工作之上。我们衷心感谢开源社区。 - -### 骨干与架构 - -- **F5-TTS** ([SWivid/F5-TTS](https://github.com/SWivid/F5-TTS), Cheng et al., 2024) — Flow-Matching + DiT 骨干, CC-BY-NC-4.0。PersonaVoice 冻结前 20 层并训练轻量 FiLM adapter。 -- **Vocos** ([gemelo-ai/vocos](https://github.com/gemelo-ai/vocos), Siuzdak, 2023) — 神经声码器, MIT。用于 mel → 波形。 -- **Flow Matching** (Lipman et al., 2023) — F5-TTS 底层生成框架。 - -### 说话人与文本编码器 - -- **ECAPA-TDNN** ([SpeechBrain](https://github.com/speechbrain/speechbrain), Desplanques et al., 2020) — SECS 评估用说话人编码器, Apache-2.0。 -- **BERT** ([bert-base-chinese](https://huggingface.co/bert-base-chinese), Devlin et al., 2019) — persona 文本风格编码器。 -- **Whisper** ([openai/whisper-large-v3-turbo](https://huggingface.co/openai/whisper-large-v3-turbo), Radford et al., 2023) — WER 评估 ASR & ref_text 转写, MIT。 - -### 声音克隆基线 - -- **CosyVoice** ([FunAudioLLM/CosyVoice](https://github.com/FunAudioLLM/CosyVoice), 2024) — 阿里 zero-shot TTS, 外部基线。 -- **XTTS v2** ([coqui/XTTS-v2](https://huggingface.co/coqui/XTTS-v2), Casanova et al., 2024) — Coqui 3 秒声音克隆基线。 -- **VALL-E** (Wang et al., 2023) — 早期神经编解码 TTS 语言模型。 -- **VoiceBox** (Le et al., 2024) — 大规模 Flow-Matching TTS。 - -### Persona 与情绪建模 - -- **Big Five 人格特质** (McCrae & Costa, 1992) — persona 提取的理论基础。 -- **FiLM** (Perez et al., 2018) — Feature-wise Linear Modulation, 我们使用的 adapter 机制。 -- **LoRA** (Hu et al., 2022) — 启发了我们的插件化 adapter 设计哲学。 - -### 语音活动检测 - -- **Silero VAD** ([snakers4/silero-vad](https://github.com/snakers4/silero-vad), 2021) — 替代 WebRTC GMM 的神经 VAD, MIT。 - -### 评估指标 - -- **UTMOS** (Saeki et al., 2022) — 客观 MOS 预测 (报告为客观代理)。 -- **SECS** — Speaker Encoder Cosine Similarity, zero-shot TTS 评估标准。 -- **WER** — 基于 Whisper ASR 的词错率。 -- **CFR** — 灾难性失败率, 本项目提出的尾部风险分析指标。 - -> 如有遗漏应引的工作, 请开 issue — 学术准确性是优先事项。 - ---- - -## 引用 - -如果 PersonaVoice 对你的研究有帮助, 请引用: - -```bibtex -@article{personavoice2026, - title = {PersonaVoice v10.4.8: Plug-in Adapters for Persona-Driven 1-Second - Voice Cloning — A Trade-off Analysis of Timbre Fidelity - and Text-Alignment Stability in Flow-Matching TTS}, - author = {PersonaVoice Team}, - journal = {arXiv preprint}, - year = {2026}, - note = {Version 10.4.8. Code: https://github.com/personavoice/personavoice} -} -``` - -机器可读元数据见 [`CITATION.cff`](CITATION.cff)。 - ---- - -## 许可证 - -- PersonaVoice 代码库 (adapter、集成、新模块 OES/LAAG/CEAG): **MIT** -- F5-TTS 预训练骨干: **CC-BY-NC-4.0** (非商业) -- Vocos、ECAPA-TDNN、Whisper、Silero VAD: 见各自许可证 - -F5-TTS 预训练骨干的商业使用需遵守 CC-BY-NC-4.0。详见 [LICENSE](LICENSE)。 - ---- - -## 联系与社区 - -- **Issues**: [GitHub Issues](https://github.com/personavoice/personavoice/issues) — 使用对应模板 (bug / feature / 复现) -- **Discussions**: [GitHub Discussions](https://github.com/personavoice/personavoice/discussions) — 研究问题、设计讨论、用例 -- **贡献指南**: 见 [CONTRIBUTING.md](CONTRIBUTING.md) - ---- - -**关键词** (学术搜索可发现性): -声音克隆, 零样本 TTS, 少样本声音克隆, 1 秒声音克隆, flow matching, F5-TTS, 语音合成, persona 驱动 TTS, 情绪 TTS, 插件化 adapter, FiLM 调制, 说话人嵌入, ECAPA-TDNN, 正交分解, 长度自适应生成, 流形塌缩, trade-off 分析, LibriTTS, Vocos 声码器, Silero VAD, Big Five 人格, BERT persona, Whisper ASR, WER, SECS, UTMOS, CFR, 灾难性失败率。 diff --git a/REPRODUCE.md b/REPRODUCE.md new file mode 100644 index 0000000..229e0f1 --- /dev/null +++ b/REPRODUCE.md @@ -0,0 +1,49 @@ +# Reproduction and release contract + +This branch is the standalone supplement snapshot for the two Zenodo preprints. +The files in this branch correspond to **language preprint version +`paper-language-v1.0`** and **speech preprint version `paper-speech-v1.0`**. +It intentionally excludes the historical PersonaVoice runtime, model weights, +private/raw audio, API keys, deployment code, paper PDFs, LaTeX sources, and +LaTeX template dependencies. The paper package is deposited separately on +Zenodo. + +## Quick verification + +From the repository root, Python 3.10+ is sufficient for the offline checks: + +```bash +python reproduce.py +``` + +This verifies every release file against [`reproducibility/SHA256SUMS.txt`](reproducibility/SHA256SUMS.txt) and runs the four CPU-only tests. To regenerate the derived JSON summaries or publication figures: + +```bash +python reproduce.py --derive +python reproduce.py --figures +``` + +The figure command emits PDF and editable SVG architecture figures directly +inside the supplement packages; no paper source is required. + +## Frozen item-level predictions + +- Language: three seed snapshots in [`supplement/language_provenance/artifacts`](supplement/language_provenance/artifacts), with immutable episode IDs and predictions. +- Speech: the 200-item per-item metric snapshot [`eval_200_samples.json`](supplement/speech_shortref/artifacts/eval_200_samples.json), using seed 42 and one-second references. + +The package-level `SHA256SUMS.txt` files and the top-level manifest provide two +independent integrity checks. Configuration, dataset fingerprints, evaluator +definitions, and random seeds are recorded in [`reproducibility/config.json`](reproducibility/config.json). + +## Paper relationship + +The support package, frozen predictions, and SHA-256 manifests are available at: +. +The exact artifact snapshot corresponding to the language preprint is tagged +`paper-language-v1.0`; the speech snapshot is tagged `paper-speech-v1.0`. +The corresponding paper PDFs and LaTeX sources are distributed through the +separate Zenodo deposition. + +Large model weights remain upstream under their original licenses. The frozen +JSON snapshots are sufficient to audit the numerical claims without distributing +a voice-cloning service. diff --git a/SOTA_VERIFICATION_REPORT.md b/SOTA_VERIFICATION_REPORT.md deleted file mode 100644 index 217600e..0000000 --- a/SOTA_VERIFICATION_REPORT.md +++ /dev/null @@ -1,374 +0,0 @@ -# PersonaVoice v10.4.8 SOTA 验证报告 (诚实修订版) - -生成时间: 2026-06-25 -**版本: v10.4.8 Trade-off Edition** - -> **修订说明**: 本报告相对 v10.4.6 进行了关键修订,纠正了以下学术诚实性问题: -> 1. **删除误导性 SOTA 声明**: 长文本 WER (0.1259) 实际劣于 XTTS v2 (0.10),不应标注 SOTA -> 2. **恢复 CFR 指标**: 灾难失败率 (整体 13.5%, 短文本 35.8%) 重新纳入核心指标 -> 3. **澄清 UTMOS 性质**: 明确标注为 Objective Proxy, 非人类主观评测 -> 4. **重写故事主线**: 从"全面碾压"改为"Trade-off 分析", 承认短文本限制作为 Future Work - ---- - -## 1. 执行摘要 - -### 1.1 核心定位: Trade-off 分析 (非全面 SOTA) - -**论文故事主线**: 《探究连续时间语音生成中,音色保真度与文本对齐稳定性的根本博弈》 - -**核心论点**: -- 现有自回归模型 (XTTS v2) 在文本对齐上极稳定,但 1s 极限场景下音色容易模糊 -- 流匹配模型 (Flow Matching, 本项目基座) 能无损重建高频细节,在音色相似度 (SECS) 上具有统治力,但代价是极度缺乏局部时序约束,导致短文本极易产生流形崩塌 -- 本项目贡献: 提出 CEAG 和 LAAG, 在不牺牲音色 SOTA 的前提下, 极大弥补流匹配架构在长文本上的对齐缺陷; 短文本限制作为 Flow Matching 固有架构限制, 留作 Future Work - -### 1.2 诚实指标对比 (1s 极限场景) - -| 指标 | PV v10.4.8 | XTTS v2 (1s ref) | CosyVoice (1s ref) | F5-TTS 官方原版 | SOTA 归属 | -|------|------------|-------------------|---------------------|-----------------|-----------| -| **ECAPA SECS (整体)** | **0.4945** | 0.3349 | 0.3595 | 0.2508 | **PV v10.4.8 ✓ SOTA** | -| **ECAPA SECS (长文本)** | **0.5044** | 0.31 | 0.33 | 0.22 | **PV v10.4.8 ✓ SOTA** | -| **WER mean (整体)** | 0.1928 | **0.1296** | 0.6130 | 0.8982 | **XTTS v2 ✓ SOTA** | -| **WER (长文本)** | 0.1158 | **0.10** | 0.55 | 0.85 | **XTTS v2 ✓ SOTA** | -| **WER (短文本)** | 0.4066 | **0.16** | 0.68 | 0.95 | **XTTS v2 ✓ SOTA** | -| **CFR (整体, WER>0.5)** | 13.5% | ~5%* | ~30%* | ~45%* | **XTTS v2 ✓ SOTA** | -| **CFR (短文本)** | 35.8% | ~10%* | ~50%* | ~70%* | **XTTS v2 ✓ SOTA** | -| **UTMOS (Objective Proxy)** | 4.0-4.5 | 3.8 | 4.0 | 3.5 | PV v10.4.8 (待 Human MOS 验证) | -| **RTF** | 0.55-0.66 | 0.85 | 0.72 | **0.50** | F5-TTS ✓ SOTA | - -*注: XTTS v2 / CosyVoice / F5-TTS 的 CFR 为基于其 WER 分布的估算值, 严格对比需后续实测。 - -**诚实结论**: -- **SECS SOTA ✓**: PV v10.4.8 (0.4945) 大幅领先 XTTS v2 (0.3349, +47.6%) 和 CosyVoice (0.3595, +37.6%) — **这是本项目的核心贡献** -- **WER 劣于 XTTS v2 ✗**: PV v10.4.8 整体 WER (0.1928) > XTTS v2 (0.1296), 长文本 WER (0.1158) > XTTS v2 (0.10) — **承认 Trade-off** -- **短文本 WER 灾难 ✗**: 0.4066, CFR 35.8% — **Flow Matching 流形崩塌, Future Work** -- **UTMOS 待验证**: 4.0-4.5 为 Objective Proxy, **顶会要求 Human MOS, 计划招募 20 人主观评测** - -### 1.3 核心 Trade-off: SECS vs WER - -``` - ┌─────────────────────────────────────────┐ - │ SECS (音色保真度) │ - │ ↑ │ - │ │ +47.6% │ - │ │ │ - PV v10.4.8 │ ●───────────┤ │ - │ │ │ - XTTS v2 │ ●─────┐ │ - │ │ -32.8% │ - │ ↓ │ - │ WER (文本对齐稳定性) │ - └─────────────────────────────────────────┘ -``` - -**核心发现**: Flow Matching 架构在 1s 极限场景下呈现明显的 SECS-WER Trade-off, 自回归架构呈现相反 Trade-off。这不是缺陷, 而是两种架构范式的根本博弈。 - ---- - -## 2. CFR 指标深度分析 (Flow Matching 流形崩塌) - -### 2.1 CFR 定义与计算 - -**CFR (Catastrophic Failure Rate)**: WER > 0.5 的样本比例, 表示生成结果不可用的灾难性失败率。 - -| 文本类型 | 样本数 | CFR | 说明 | -|----------|--------|-----|------| -| **整体** | 200 | **13.5%** (27/200) | 主要由短文本贡献 | -| **长文本 (>8 词)** | 147 | **5.4%** (8/147) | 接近可用水平 | -| **短文本 (≤8 词)** | 53 | **35.8%** (19/53) | 灾难级别 | -| **极短文本 (≤4 词)** | 22 | **63.6%** (14/22) | 流形崩塌重灾区 | - -### 2.2 极短文本双峰分布 (Flow Matching 流形崩塌特征) - -对 22 个极短文本 (≤4 词) 样本的 WER 分布分析: - -| WER 区间 | 样本数 | 比例 | 说明 | -|----------|--------|------|------| -| WER = 0.0 (完美) | 5 | 22.7% | Flow Matching 成功收敛 | -| 0.0 < WER ≤ 0.5 | 4 | 18.2% | 轻微对齐偏移 | -| WER > 0.8 (灾难) | 13 | **59.1%** | **流形崩塌, 生成乱码/复读** | - -**关键发现**: 极短文本呈现明显的**双峰分布** — 22.7% 完美 vs 59.1% 灾难, 中间状态极少。这是 Flow Matching 在极短上下文下的流形崩塌特征, 而非渐进式退化。 - -### 2.3 灾难样本分析 (Flow Matching 流形崩塌实例) - -| 文本 | 词数 | WER | SECS | 失败模式 | -|------|------|-----|------|----------| -| `King Kaliko` | 2 | 1.000 | -0.031 | 完全乱码 | -| `Mainhall laughed.` | 2 | 1.000 | 0.541 | 复读 (音色保真但文本错) | -| `four hundred.` | 2 | 1.000 | 0.116 | 数字幻觉 | -| `seventeen seventeen.` | 2 | 1.000 | 0.822 | **高 SECS 但 WER=1** (Trade-off 极端案例) | -| `RICE BREAD.` | 2 | 1.000 | 0.141 | 完全乱码 | -| `[Illustration: ROLLS.]` | 2 | 1.000 | 0.569 | 复读 | - -**关键观察**: 即使在 WER=1.0 的灾难样本中, SECS 仍可达 0.822 (`seventeen seventeen.`), 证实 Flow Matching 优先保音色、牺牲对齐的 Trade-off 特性。 - -### 2.4 v10.4.7 短文本优化实验 (已废弃) - -**实验**: 尝试通过 Best-of-N=3 + 动态 CFG=3.5 解决短文本流形崩塌。 - -**结果** (10 个极短文本样本): -| 指标 | v10.4.6 (基线) | v10.4.7 (Best-of-N + CFG boost) | 变化 | -|------|----------------|----------------------------------|------| -| WER | 0.4047 | **0.6964** | **+72.2% (恶化)** | -| CFR | 35.8% | **70.0%** | **+95.5% (恶化)** | -| SECS | 0.4689 | 0.4019 | -14.3% | - -**结论**: Best-of-N + CFG boost 不仅未改善, 反而显著恶化短文本表现。**证实 Flow Matching 流形崩塌是根本性架构限制, 无法通过采样策略解决**。已回滚到 v10.4.6 稳定配置。 - -**理论解释**: 当所有 N 个样本都崩塌时, Best-of-N 无法选出好的; 过强 CFG (3.5) 扰动 ODE 积分轨迹, 加剧流形崩塌。 - ---- - -## 3. UTMOS 评测信度澄清 - -### 3.1 UTMOS 性质明确声明 - -**重要澄清**: 本报告中的 UTMOS 评分 (4.0-4.5) 是使用 **UTMOS 预测模型 (Objective Proxy)** 计算得出, **不是真实人类主观评测 (Human Subjective MOS)**。 - -| 评测类型 | 方法 | 本报告状态 | 顶会要求 | -|----------|------|------------|----------| -| **UTMOS (Objective Proxy)** | 开源 UTMOS 预测模型 | ✓ 已完成 | 仅作内部参考 | -| **Human MOS (Subjective)** | 15-20 人盲测 | **✗ 未完成** | **顶会强制要求** | -| **Human SMOS (Similarity MOS)** | 15-20 人相似度盲测 | **✗ 未完成** | **顶会强制要求** | - -### 3.2 UTMOS 作为 Objective Proxy 的局限性 - -1. **预测模型偏差**: UTMOS 训练于特定数据集, 对 Out-of-Distribution 样本预测可能不准 -2. **无法捕捉细微瑕疵**: 预测模型对人类敏感的微小 artifacts (如呼吸声、齿音) 不敏感 -3. **顶会不接受**: Interspeech/ICASSP/ACL 强制要求 Human MOS, 机器打分只能作为补充 - -### 3.3 Human MOS 主观评测计划 (待执行) - -**计划详情**: -- **评测规模**: 20 名听者 (15-25 岁, 正常听力) -- **评测内容**: - - Naturalness MOS (1-5 分): 自然度主观评分 - - Similarity MOS (1-5 分): 与参考音频的相似度主观评分 -- **盲测设计**: PV v10.4.8 vs XTTS v2 vs CosyVoice vs Ground Truth (随机顺序) -- **样本选择**: 30 个代表性样本 (10 短文本 + 10 中文本 + 10 长文本) -- **评测平台**: GitHub 开源社群招募 + 线下评测 -- **预计耗时**: 半天招募 + 1 天评测 - -**招募渠道**: -- GitHub Issue 发布盲测音频 -- 语音技术社群 (微信/QQ/Discord) -- 高校语音实验室合作 - ---- - -## 4. v10.4.8 核心创新 (LAAG + 动态 FiLM + mel 拼接修复) - -| 创新 | 名称 | 核心机制 | 实现位置 | 贡献 | -|------|------|----------|----------|------| -| **A** | 正交环境子流形 (OES) | $Z_{audio} = Z_{timbre} + Z_{env}, \; Z_{timbre} \perp Z_{env}$, env_scale=0.1 | `cross_manifold_refiner.py` | 音色纯净度 | -| **B** | 交叉熵注意力引导 (CEAG) | $v_{final} = v_{cfg} - \lambda(t) \cdot \nabla_x H(A_{mel \to text})$ | `ceag_sampler.py` | 长文本对齐 | -| **C** | **LAAG (长度自适应生成)** | 动态 Chunking + 动态 CEAG λ + 动态 CFG | `laag_generator.py` | 长文本 SECS +88.7% | -| **D** | **动态 FiLM 激活** | 短文本 FiLM off (稳定), 长文本 FiLM on (防漂移) | `laag_generator.py` | 长文本稳定性 | -| **E** | **mel 拼接修复** | 修正多 chunk mel 维度 (T, mel_dim) 拼接 | `laag_generator.py` | SECS +56.6% | -| **F** | Silero VAD + RMS 归一化 | 神经网络 VAD 替代 GMM-based WebRTC | `audio_preprocess.py` | 1s 样本纯度 | - ---- - -## 5. 200 样本统计显著性评估 - -### 5.1 评估配置 - -- 数据集: LibriTTS dev-clean, 200 个未见样本 (seed=42) -- 参考音频: 1 秒 (94 帧 @ 24kHz/hop=256) -- 评估链路: mel → Vocos → wav → ECAPA-TDNN (真实链路) -- WER 评估: Whisper-large-v3-turbo -- 声码器: Vocos (24kHz, 100 mel bins) - -### 5.2 v10.4.8 真实结果 (200 样本, 诚实呈现) - -| 指标 | 均值 ± 标准差 | 95% CI | 中位数 | 样本数 | -|------|--------------|--------|--------|--------| -| **ECAPA SECS** | **0.4945 ± 0.1800** | [0.4694, 0.5196] | 0.5226 | 200 | -| **WER** | **0.1928 ± 0.3077** | [0.1499, 0.2358] | 0.0588 | 200 | -| **CFR (WER>0.5)** | **13.5%** | - | - | 200 | - -### 5.3 按文本长度分组 (LAAG 动态机制验证) - -| 文本类型 | 样本数 | SECS | WER | CFR | 说明 | -|----------|--------|------|-----|-----|------| -| **短文本 (≤8 词)** | 53 | 0.4671 ± 0.21 | 0.4066 ± 0.42 | **35.8%** | FiLM off, 流形崩塌 | -| **长文本 (>8 词)** | 147 | **0.5044 ± 0.17** | **0.1158 ± 0.22** | **5.4%** | FiLM on, LAAG chunking | -| **极短文本 (≤4 词)** | 22 | 0.4282 ± 0.23 | 0.6629 ± 0.45 | **63.6%** | 流形崩塌重灾区 | - -**关键发现**: -- **长文本优势**: SECS (0.5044) > 短文本 (0.4671), LAAG 动态机制成功 -- **短文本劣势**: WER (0.4066) >> 长文本 (0.1158), Flow Matching 流形崩塌 -- **Trade-off 明确**: 长文本接近可用, 短文本需 Future Work 解决 - -### 5.4 v10.4.7 vs v10.0 对比 (bug 修复效果) - -| 指标 | v10.0 (bug) | v10.4.7 (修复) | 改进 | -|------|-------------|----------------|------| -| **整体 SECS** | 0.3158 | **0.4945** | **+56.6%** | -| **整体 WER** | 0.4836 | **0.1928** | **-60.1%** | -| **长文本 SECS** | 0.2663 | **0.5044** | **+89.4%** | -| **长文本 WER** | 0.5076 | **0.1158** | **-77.2%** | - ---- - -## 6. 1111.mp3 完美克隆验证 - -### 6.1 测试配置 - -- 参考音频: `d:\desktop\voice\1111.mp3` (1 秒) -- 测试文本: 4 个 (短/中/长) -- 评估指标: SECS, WER, UTMOS, RTF - -### 6.2 克隆结果 (v10.4.8 稳定配置) - -| 测试 | 文本类型 | SECS | WER | UTMOS (Proxy) | RTF | 达标 | -|------|----------|------|-----|-------|-----|------| -| 1 | 短 (12 字) | **0.4874** | 0.0625 | 4.5000 | 0.9449 | ✓ | -| 2 | 短 (英文) | 0.3423 | 0.0000 | 4.0000 | 2.3214 | 部分 (SECS) | -| 3 | 中 (17 字) | **0.5588** | 0.0625 | 4.5000 | 0.7317 | ✓ | -| 4 | 长 (120 字) | **0.6180** | 0.0676 | **4.5000** | 0.6196 | ✓ | - -**结论**: 1111.mp3 在中长文本上完美克隆 (SECS≥0.55, WER≤0.10, UTMOS=4.5), 短文本 SECS 略低但 WER 达标。 - ---- - -## 7. 与开源模型对比 (诚实版) - -### 7.1 SECS 对比 (音色相似度 — PV 核心优势) - -| 模型 | 整体 SECS | 长文本 SECS | 短文本 SECS | 架构 | -|------|-----------|-------------|-------------|------| -| **PV v10.4.8** | **0.4945** | **0.5044** | 0.4671 | Flow Matching | -| XTTS v2 | 0.3349 | 0.31 | 0.36 | Autoregressive | -| CosyVoice | 0.3595 | 0.33 | 0.39 | Flow Matching | -| F5-TTS 官方 | 0.2508 | 0.22 | 0.28 | Flow Matching | - -**PV v10.4.8 SECS 优势** (核心贡献): -- vs XTTS v2: +47.6% (整体), +62.7% (长文本) -- vs CosyVoice: +37.6% (整体), +52.8% (长文本) -- vs F5-TTS: +97.2% (整体), +129.3% (长文本) - -### 7.2 WER 对比 (可懂度 — PV 承认劣势) - -| 模型 | 整体 WER | 长文本 WER | 短文本 WER | 架构 | -|------|----------|------------|------------|------| -| **XTTS v2** | **0.1296** | **0.10** | **0.16** | Autoregressive | -| PV v10.4.8 | 0.1928 | 0.1158 | 0.4066 | Flow Matching | -| CosyVoice | 0.6130 | 0.55 | 0.68 | Flow Matching | -| F5-TTS 官方 | 0.8982 | 0.85 | 0.95 | Flow Matching | - -**诚实分析**: -- **PV 劣于 XTTS v2**: 整体 WER +48.6%, 长文本 WER +15.8%, 短文本 WER +154.1% -- **PV 优于其他 Flow Matching**: vs CosyVoice -68.5%, vs F5-TTS -78.5% (CEAG + LAAG 贡献) -- **Trade-off 根源**: Flow Matching 缺乏自回归的局部时序约束, 短文本对齐崩塌 - -### 7.3 CFR 对比 (灾难失败率 — 新增诚实指标) - -| 模型 | 整体 CFR | 短文本 CFR | 极短文本 CFR | 说明 | -|------|----------|------------|--------------|------| -| **XTTS v2** (估算) | ~5% | ~10% | ~15% | 自回归稳定 | -| **PV v10.4.8** | 13.5% | 35.8% | 63.6% | Flow Matching 流形崩塌 | -| CosyVoice (估算) | ~30% | ~50% | ~70% | Flow Matching | -| F5-TTS (估算) | ~45% | ~70% | ~90% | Flow Matching 未优化 | - -*注: 带"估算"的 CFR 为基于 WER 分布的推算, 需后续实测确认。 - ---- - -## 8. 架构创新深度分析 - -### 8.1 LAAG (Length-Aware Adaptive Generation) - -**问题**: 1s 极限参考音频下, 长文本生成存在音色漂移和对齐失败。 - -**解决方案**: -1. **动态 Chunking**: 长文本切分为多个 chunk, 每 chunk 充分利用 1s 参考 -2. **动态 CEAG λ**: $\lambda(t) = \lambda_{min} + (\lambda_{max} - \lambda_{min}) \cdot \min(1, \frac{T_{tokens}}{T_{ref}})$ -3. **动态 CFG**: 短文本增强引导, 长文本保持稳定 -4. **Cross-fade 拼接**: mel 域 + audio 域双重 cross-fade - -**效果**: 长文本 SECS 从 0.27 提升到 0.50 (+89%), WER 从 0.51 降到 0.12 (-77%) - -### 8.2 动态 FiLM 激活 - -**问题**: FiLM adapter 对短文本效果不一致 (CUDA 非确定性), 但对长文本有 +0.06 SECS 提升。 - -**解决方案**: -```python -if len(chunks) == 1: - persona_emb_effective = torch.zeros_like(persona_emb) # FiLM off -else: - persona_emb_effective = persona_emb # FiLM on -``` - -**原理**: 短文本单 chunk 生成稳定, FiLM 引入不确定性; 长文本多 chunk 需要 FiLM 防止音色漂移。 - -### 8.3 短文本流形崩塌理论分析 (Future Work) - -**现象**: 极短文本 (≤4 词) 呈现双峰分布 — 22.7% 完美 vs 59.1% 灾难。 - -**根因分析**: -1. **ODE 积分缺乏时序约束**: Flow Matching 通过 ODE 积分生成 mel, 短文本下积分轨迹缺乏局部时序锚点 -2. **流形崩塌 (Manifold Collapse)**: 极短文本的文本条件信息不足, ODE 积分易陷入错误流形 -3. **信息不对称**: 1s 参考 (丰富音色) vs 极短文本 (稀疏对齐) 导致模型优先保音色、牺牲对齐 - -**未来解决方案 (Future Work)**: -- **轻量级对齐先验**: 在输入端用 CTC 模型或强制对齐工具, 给每个 text token 预分配粗糙的 mel 帧长度分布 -- **短文本专用积分器**: 设计带显式时序约束的 ODE 积分器 -- **混合架构**: 短文本 fallback 到自回归, 长文本用 Flow Matching - ---- - -## 9. 推理速度优化 - -| 优化项 | v10.0 | v10.4.7 | 改进 | -|--------|-------|---------|------| -| ODE steps | 96 | **32** | -67% (3x 加速) | -| CFG strength | 3.0 | 2.5 | -17% | -| RTF (短文本) | ~1.2 | **0.66** | -45% | -| RTF (长文本) | ~1.5 | **0.55** | -63% | - ---- - -## 10. 结论 - -### 10.1 核心贡献 (诚实总结) - -| 贡献 | 状态 | 说明 | -|------|------|------| -| **SECS SOTA** | ✓ 达成 | 1s 极限场景下音色相似度全面领先 (+47.6% vs XTTS v2) | -| **长文本 LAAG** | ✓ 达成 | 长文本 SECS=0.5044, WER=0.1158, CFR=5.4% (接近可用) | -| **Flow Matching Trade-off 分析** | ✓ 达成 | 首次系统揭示 SECS-WER Trade-off 根源 | -| **短文本 WER** | ✗ 未解决 | 0.4066, CFR 35.8% — Flow Matching 固有限制, Future Work | -| **Human MOS** | ✗ 待完成 | 计划招募 20 人主观评测 (顶会强制要求) | - -### 10.2 论文故事主线 (Trade-off 视角) - -**标题**: 《探究连续时间语音生成中,音色保真度与文本对齐稳定性的根本博弈》 - -**核心论点**: -1. 自回归模型 (XTTS v2): 对齐稳定, 但 1s 极限场景音色模糊 -2. Flow Matching (本项目): 音色保真, 但短文本流形崩塌 -3. 本项目贡献: CEAG + LAAG 在不牺牲音色 SOTA 的前提下, 极大弥补长文本对齐缺陷 -4. 诚实承认: 短文本限制作为 Flow Matching 固有架构限制, 留作 Future Work - -### 10.3 顶会投稿前必做事项 - -1. **Human MOS 主观评测** (强制): 招募 20 人, Naturalness MOS + Similarity MOS -2. **CFR 实测对比** (建议): 对 XTTS v2 / CosyVoice 实测 CFR, 替代估算值 -3. **短文本 Future Work 章节** (建议): 详细阐述流形崩塌理论与未来解决方案 - ---- - -## 附录: 评估数据 - -- 评估结果: `results/eval_200_samples.json` -- 统计数据: `results/eval_200_statistics.json` -- 诚实指标 (CFR + 分层 WER): `results/honest_metrics_v10.4.7.json` -- CFR 分析脚本: `personavoice/experiment/cfr_analysis.py` -- 1111.mp3 克隆音频: `outputs/1111_clone_test_*.wav` -- 200 样本评估脚本: `personavoice/experiment/eval_200_samples.py` -- 综合评估器: `personavoice/experiment/comprehensive_evaluator.py` -- 外部基线对比: `personavoice/experiment/baseline_external.py` -- 顶会图生成: `personavoice/experiment/visualize.py` diff --git a/ZENODO.md b/ZENODO.md new file mode 100644 index 0000000..608421c --- /dev/null +++ b/ZENODO.md @@ -0,0 +1,46 @@ +# Zenodo release package + +This repository is released as version `1.0.0` (25 August 2026) by Jiang +Luchang. The release contains two canonical research objects. The primary +language preprint PDF is supplied separately to Zenodo. + +It reports a provenance-constrained selective language planner and measured +results on a public, speaker-disjoint controlled composition. It also includes +a separately labelled DailyDialog lexical-evidence audit; that auxiliary audit +is not pooled with the planner endpoint and does not use entailment or +authorization labels. The paper does not claim natural memorial speech, +psychological reconstruction, safe deployment, or an end-to-end acoustic +result. + +The second research object is the independent speech preprint PDF supplied +separately to Zenodo. It reports measured one-second +voice-cloning results from the repository's v10.4.x release, including +SECS/WER trade-offs, external baselines, ablations, and catastrophic-failure +analysis. It does not claim validated perceptual persona or emotion control. + +The protocol-only draft is not part of the canonical release. + +## Upload set + +Upload the two PDFs and their LaTeX source bundles from the paper workspace, +plus `CITATION.cff`, `LICENSE`, and the two `supplement/` packages from this +branch. Do not upload private speaker material, +raw consent records, model weights, API keys, or local environment files. + +## Suggested Zenodo metadata + +- **Titles:** Provenance-Constrained Selective Language Style Planning from Sparse Evidence; PersonaVoice: One-Second Voice Cloning with Length-Adaptive Flow Matching +- **Creator:** Jiang Luchang +- **ORCID:** 0009-0004-4579-4243 +- **Resource type:** Publication / Preprint +- **Publication date:** 2026-08-25 +- **License:** MIT for code; cite dataset licenses separately +- **Keywords:** provenance-constrained generation; selective prediction; citation attribution; language style; abstention; voice safety +- **Related identifiers:** project repository URL and the ORCID URL +- **Version:** 1.0.0 + +Zenodo assigns the DOI after deposition. Once assigned, add it to the final +record and the preferred citation; do not invent a DOI in source files. + +The paper PDFs are not duplicated in this GitHub support branch; record their +checksums in the Zenodo deposition metadata when the final PDFs are uploaded. diff --git a/ZENODO_METADATA.json b/ZENODO_METADATA.json new file mode 100644 index 0000000..6d666b7 --- /dev/null +++ b/ZENODO_METADATA.json @@ -0,0 +1,37 @@ +{ + "title": "PersonaVoice research preprints and reproducible extensions", + "upload_type": "publication", + "publication_type": "preprint", + "publication_date": "2026-08-25", + "version": "1.0.0", + "description": "Two independent preprints by Jiang Luchang: provenance-constrained selective language planning and measured one-second speech generation, with standalone public analysis supplements.", + "creators": [ + { + "name": "Jiang Luchang", + "orcid": "0009-0004-4579-4243", + "email": "auroral.sunflower@gmail.com" + } + ], + "keywords": [ + "provenance-constrained generation", + "selective prediction", + "citation attribution", + "language style", + "one-second voice cloning", + "speech synthesis", + "abstention" + ], + "license": "MIT", + "related_identifiers": [ + { + "identifier": "https://github.com/FinalSunFlower/PersonaVoice", + "relation": "isSupplementTo", + "resource_type": "software" + }, + { + "identifier": "https://orcid.org/0009-0004-4579-4243", + "relation": "isIdenticalTo", + "resource_type": "person" + } + ] +} diff --git a/examples/api_demo.py b/examples/api_demo.py deleted file mode 100644 index bc5a7e0..0000000 --- a/examples/api_demo.py +++ /dev/null @@ -1,115 +0,0 @@ -"""前端 API 全流程测试 (1111.mp3 + 人格联动).""" -import os -import requests, base64, json, time, sys -from pathlib import Path - -# examples/api_demo.py → 项目根目录 = 上两级 -PROJECT_ROOT = Path(__file__).resolve().parent.parent -API = os.environ.get("PERSONAVOICE_API", "http://localhost:8000") -AUDIO = str(PROJECT_ROOT / "1111.mp3") - -print("="*70) -print("前端 API 全流程测试 (1111.mp3 + 人格联动)") -print("="*70) - -# ── 测试 1: 健康检查 ── -print("\n[1/5] 健康检查") -r = requests.get(f"{API}/api/health", timeout=10) -data = r.json() -assert data["status"] == "ok", f"健康检查失败: {data}" -assert data["personavoice_loaded"], "PersonaVoice 未加载" -print(f" ✓ status={data['status']}, version={data['version']}") -print(f" ✓ 所有模块: {data['modules']}") - -# ── 测试 2: 架构状态 ── -print("\n[2/5] 架构状态查询") -r = requests.get(f"{API}/api/architecture", timeout=10) -arch = r.json() -print(f" ✓ 架构状态: {list(arch.keys())[:5]}...") - -# ── 测试 3: 纯声音克隆 (无人格) ── -print("\n[3/5] 纯声音克隆 (1111.mp3, 无人格)") -t0 = time.time() -with open(AUDIO, "rb") as f: - files = {"audio": ("1111.mp3", f, "audio/mpeg")} - data = {"text": "你好,这是前端 API 全流程测试。"} - r = requests.post(f"{API}/api/clone", files=files, data=data, timeout=120) -elapsed = time.time() - t0 -assert r.status_code == 200, f"克隆失败: {r.status_code} {r.text[:200]}" -result = r.json() -assert "audio_base64" in result, f"返回缺少 audio_base64 字段: {list(result.keys())}" -audio_b64 = result["audio_base64"] -print(f" ✓ 耗时: {elapsed:.2f}s") -print(f" ✓ audio_base64 长度: {len(audio_b64)}") -print(f" ✓ 返回字段: {list(result.keys())}") -if "speaker_embedding_norm" in result: - print(f" ✓ speaker_emb norm: {result['speaker_embedding_norm']:.4f}") -if "wave_duration" in result: - print(f" ✓ 音频时长: {result['wave_duration']:.2f}s") - -# 保存音频 -audio_data = base64.b64decode(audio_b64) -out_path = "outputs/frontend_test_3.wav" -Path("outputs").mkdir(exist_ok=True) -Path(out_path).write_bytes(audio_data) -print(f" ✓ 音频保存: {out_path} ({len(audio_data)} bytes)") - -# ── 测试 4: 声音克隆 + 人格联动 ── -print("\n[4/5] 声音克隆 + 人格联动 (1111.mp3 + 聊天记录)") -chat_history = json.dumps([ - {"role": "user", "content": "你好,我今天心情很好,想和你聊聊。"}, - {"role": "assistant", "content": "你好!很高兴听到你心情不错,我也很期待我们的对话。你今天有什么特别的事情想分享吗?"}, - {"role": "user", "content": "我刚完成了一个重要的项目,感觉非常有成就感。"}, - {"role": "assistant", "content": "太棒了!恭喜你完成项目。能感受到你的兴奋和自豪。这是什么类型的项目呢?"}, -], ensure_ascii=False) - -t0 = time.time() -with open(AUDIO, "rb") as f: - files = {"audio": ("1111.mp3", f, "audio/mpeg")} - data = { - "text": "刚完成了一个重要项目,感觉非常有成就感,想和你分享一下这份喜悦!", - "chat_history": chat_history, - } - r = requests.post(f"{API}/api/clone", files=files, data=data, timeout=180) -elapsed = time.time() - t0 -assert r.status_code == 200, f"人格克隆失败: {r.status_code} {r.text[:200]}" -result = r.json() -assert "audio_base64" in result, f"返回缺少 audio_base64 字段: {list(result.keys())}" -audio_b64 = result["audio_base64"] -print(f" ✓ 耗时: {elapsed:.2f}s") -print(f" ✓ audio_base64 长度: {len(audio_b64)}") -if "persona_representation_norm" in result: - print(f" ✓ persona_repr norm: {result['persona_representation_norm']:.4f} (应 > 0 表示人格注入)") - -# 保存音频 -audio_data = base64.b64decode(audio_b64) -out_path = "outputs/frontend_test_4_persona.wav" -Path(out_path).write_bytes(audio_data) -print(f" ✓ 音频保存: {out_path} ({len(audio_data)} bytes)") - -# ── 测试 5: 人格分析接口 ── -print("\n[5/5] 人格分析接口") -# api_server 期望 chat_history 是 List[str] -chat_str_list = [ - "你好,我今天心情很好,想和你聊聊。", - "我刚完成了一个重要的项目,感觉非常有成就感。", - "太棒了!恭喜你完成项目。能感受到你的兴奋和自豪。", -] -r = requests.post( - f"{API}/api/persona/analyze", - json={"chat_history": chat_str_list}, - timeout=60, -) -assert r.status_code == 200, f"人格分析失败: {r.status_code} {r.text[:200]}" -persona = r.json() -print(f" ✓ 人格分析返回: {list(persona.keys())}") -if "trait_radar" in persona: - print(f" ✓ Big Five traits: {persona['trait_radar']['labels']}") - print(f" ✓ Big Five values: {[f'{v:.3f}' for v in persona['trait_radar']['values']]}") -if "persona_representation" in persona: - print(f" ✓ persona_emb: shape={persona['persona_representation']['shape']}, " - f"norm={persona['persona_representation']['norm']:.4f}") - -print("\n" + "="*70) -print("✓ 前端全流程测试通过!") -print("="*70) diff --git a/examples/clone_demo.py b/examples/clone_demo.py deleted file mode 100644 index 52f247f..0000000 --- a/examples/clone_demo.py +++ /dev/null @@ -1,286 +0,0 @@ -"""验证 1111.mp3 完美克隆 - SECS SOTA 测试 (v10.4 满血架构, 无备用方案)""" -import os -# 设置离线模式, 避免连接 huggingface.co 超时 (模型已在本地缓存) -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" -os.environ["HF_DATASETS_OFFLINE"] = "1" -# v10.4: 禁用 wandb (f5_tts.trainer 依赖 wandb, 与 speechbrain lazy import 冲突) -os.environ["WANDB_DISABLED"] = "true" -os.environ["WANDB_MODE"] = "disabled" - -import sys -import time -import torch -import torchaudio -import numpy as np -from pathlib import Path - -# 添加项目路径 (兼容直接运行与 -m 运行) -# examples/clone_demo.py → 项目根目录 = 上两级 -PROJECT_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# v10.4 关键: 先导入 f5_tts (避免 speechbrain lazy import 与 wandb/pydantic 冲突) -from f5_tts.api import F5TTS -from f5_tts.model.utils import list_str_to_idx, get_tokenizer - -from personavoice.config import SOTA_CONFIG -from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone - - -def compute_secs(ref_audio_path, gen_audio_path, device="cpu"): - """计算 SECS (Speaker Encoder Cosine Similarity) - 满血架构, 无备用方案. - - 使用项目自带的 EcapaEvaluator (已处理 speechbrain lazy import 问题). - """ - import torchaudio - from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator - - evaluator = ECAPAEvaluator(device=device, vocoder_device=device) - if not evaluator.available: - raise RuntimeError("ECAPA evaluator 不可用") - - # 加载参考音频 (16kHz mono) - ref_wav, ref_sr = torchaudio.load(ref_audio_path) - if ref_sr != 16000: - ref_wav = torchaudio.functional.resample(ref_wav, ref_sr, 16000) - if ref_wav.shape[0] > 1: - ref_wav = ref_wav.mean(dim=0, keepdim=True) - - # 加载生成音频 (16kHz mono) - gen_wav, gen_sr = torchaudio.load(gen_audio_path) - if gen_sr != 16000: - gen_wav = torchaudio.functional.resample(gen_wav, gen_sr, 16000) - if gen_wav.shape[0] > 1: - gen_wav = gen_wav.mean(dim=0, keepdim=True) - - return evaluator.compute_secs(gen_wav, ref_wav, sr=16000) - - -def main(): - print("=" * 70) - print("PersonaVoice v10.4.8 - 1111 完美克隆验证 (F5-TTS 官方流程)") - print("=" * 70) - - device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"Device: {device}") - - # 1. 加载参考音频 1111.mp3 (35KB, 项目内置 demo 参考) - print("\n[1/3] 加载 1111.mp3...") - ref_audio_path = str(PROJECT_ROOT / "1111.mp3") - if not os.path.exists(ref_audio_path): - print(f" [ERROR] 文件不存在: {ref_audio_path}") - return - - t0 = time.time() - # v10.4: 直接加载音频 (与参数搜索一致, 不用 preprocess_ref_audio) - # preprocess_ref_audio 会添加 50ms 尾部静音, 导致 SECS 下降 - audio_ref, sr_ref = torchaudio.load(ref_audio_path) - if audio_ref.shape[0] > 1: - audio_ref = audio_ref.mean(dim=0, keepdim=True) - # 重采样到 24kHz (F5-TTS 要求) - if sr_ref != SOTA_CONFIG.sample_rate: - audio_ref = torchaudio.functional.resample(audio_ref, sr_ref, SOTA_CONFIG.sample_rate) - sr_ref = SOTA_CONFIG.sample_rate - print(f" 预处理耗时: {time.time()-t0:.2f}s") - print(f" audio_ref shape: {audio_ref.shape}, sr: {sr_ref}") - - # 提取 mel (用于签名兼容, 实际 v10.3 用 audio_ref) - mel_transform = torchaudio.transforms.MelSpectrogram( - sample_rate=sr_ref, - n_fft=1024, - hop_length=SOTA_CONFIG.hop_length, - win_length=1024, - n_mels=SOTA_CONFIG.n_mels, - ) - if audio_ref.ndim == 1: - audio_ref = audio_ref.unsqueeze(0) - mel_ref = mel_transform(audio_ref) # (1, n_mels, T) - mel_ref = torch.log(torch.clamp(mel_ref, min=1e-5)) - print(f" mel_ref shape: {mel_ref.shape}") - - # 2. 加载 backbone - print("\n[2/3] 加载 F5-TTS backbone...") - t0 = time.time() - backbone_result = load_pretrained_f5tts_backbone(device=device, use_film=True) - if isinstance(backbone_result, tuple): - backbone = backbone_result[0] - else: - backbone = backbone_result - print(f" Backbone 加载耗时: {time.time()-t0:.2f}s") - print(f" Backbone type: {type(backbone).__name__}") - - # 2.5 提取真实 speaker_emb (ECAPA 从 1111.mp3 提取, 满血架构) - print("\n[2.5/3] 提取真实 speaker_emb (ECAPA) + 参考音频增强 + 综合评估器...") - from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator - from personavoice.experiment.comprehensive_evaluator import ComprehensiveEvaluator - ecapa = ECAPAEvaluator(device=device, vocoder_device=device) - # 综合评估器 (SECS + WER + UTMOS + RTF + SIM-o) - evaluator = ComprehensiveEvaluator(device=device, vocoder_device=device) - # v10.4: 移除参考音频增强 (循环扩展伤害 SECS, 官方基线直接用原始 1s 音频) - # 直接用原始 1s 音频提取 ECAPA 嵌入 (与 F5-TTS 官方基线一致) - audio_ref_16k = torchaudio.functional.resample(audio_ref, sr_ref, 16000) - if audio_ref_16k.ndim == 1: - audio_ref_16k = audio_ref_16k.unsqueeze(0) - # v10.4: 直接从原始 1s 音频提取 ECAPA 嵌入 (无增强, 与官方基线一致) - speaker_emb_real = ecapa.extract_embedding(audio_ref_16k, sr=16000).to(device) - if speaker_emb_real.dim() == 1: - speaker_emb_real = speaker_emb_real.unsqueeze(0) - print(f" speaker_emb shape: {speaker_emb_real.shape}, norm: {speaker_emb_real.norm():.4f}") - - # 2.6 用 Whisper 转录 1111.mp3 得到 ref_text (F5-TTS 官方正确用法) - print("\n[2.6/3] Whisper 转录参考音频得到 ref_text...") - from transformers import pipeline as hf_pipeline - import numpy as np - # 直接传入 numpy 数组, 避免 ffmpeg 依赖 - asr_pipe = hf_pipeline( - "automatic-speech-recognition", - model="openai/whisper-large-v3-turbo", - torch_dtype=torch.float16 if "cuda" in device else torch.float32, - device=device, - ) - audio_ref_np = audio_ref_16k.squeeze(0).cpu().numpy().astype(np.float32) - asr_result = asr_pipe(audio_ref_np, generate_kwargs={"language": "chinese"}) - ref_text = asr_result["text"].strip() - print(f" ref_text: '{ref_text}'") - - # 3. 克隆合成 (v10.2: LAAG + F5-TTS 官方 ref_text 拼接) - print("\n[3/3] 执行克隆合成...") - test_texts = [ - "你好,这是一个完美的声音克隆测试。", - "Hello, this is a perfect voice cloning test.", - "今天天气真好,我们一起出去走走吧。", - # v10.1 LAAG 长文本测试 (触发 chunk 机制) - "人工智能是计算机科学的一个分支,它企图了解智能的实质,并生产出一种新的能以人类智能相似的方式做出反应的智能机器。该领域的研究包括机器人、语言识别、图像识别、自然语言处理和专家系统等。人工智能从诞生以来,理论和技术日益成熟,应用领域也不断扩大,可以设想,未来人工智能带来的科技产品,将会是人类智慧的容器。", - ] - - os.makedirs("outputs", exist_ok=True) - - # 准备 tokenizer - from f5_tts.model.utils import list_str_to_idx, get_tokenizer - # 自动查找 F5-TTS vocab.txt 位置 (兼容不同安装方式) - import f5_tts - f5_pkg_dir = Path(f5_tts.__file__).resolve().parent - vocab_candidates = [ - f5_pkg_dir / "infer" / "examples" / "vocab.txt", - f5_pkg_dir.parent / "f5_tts" / "infer" / "examples" / "vocab.txt", - PROJECT_ROOT / "f5_tts" / "infer" / "examples" / "vocab.txt", - ] - vocab_file = next((str(p) for p in vocab_candidates if p.exists()), None) - if vocab_file is None: - raise FileNotFoundError( - "无法找到 F5-TTS vocab.txt, 请确认 f5-tts 已安装。候选路径: " - + ", ".join(str(p) for p in vocab_candidates) - ) - vocab_char_map, vocab_size = get_tokenizer(vocab_file, tokenizer="custom") - - for i, text in enumerate(test_texts): - print(f"\n--- 测试 {i+1}/{len(test_texts)} ---") - print(f" 文本: {text}") - - t0 = time.time() - - # 准备输入 - mel_ref_b = mel_ref.to(device).float() - if mel_ref_b.ndim == 3: - mel_ref_b = mel_ref_b.permute(0, 2, 1) # (B, n_mels, T) -> (B, T, n_mels) - - # 用真实 ECAPA speaker_emb (满血架构, 无零向量降级) - speaker_emb = speaker_emb_real - # v10.4.6: persona_emb 由 LAAG 动态决定 (短文本=0, 长文本=speaker_emb) - # 见 laag_generator.py 的动态 FiLM 机制 - persona_emb = speaker_emb.clone() # 传递给 LAAG, 由 LAAG 决定是否使用 - emotion_emb = torch.zeros(1, 4, device=device) - - # v10.2.1 LAAG + F5-TTS 官方 cond=audio + ref_text 拼接 (满血架构) - print(f" [LAAG] 长度自适应生成 + cond=audio + ref_text 拼接") - # audio_ref: (B, T_samples) 24kHz 音频波形 (F5-TTS 官方 cond) - audio_ref_b = audio_ref.to(device).float() - if audio_ref_b.ndim == 1: - audio_ref_b = audio_ref_b.unsqueeze(0) - mel_gen, laag_info = backbone.laag_synthesize( - text_str=text, - mel_ref=mel_ref_b, - speaker_emb=speaker_emb, - persona_emb=persona_emb, - emotion_emb=emotion_emb, - tokenizer=type("T", (), {"encode": staticmethod(lambda t: list_str_to_idx([t], vocab_char_map=vocab_char_map).squeeze(0))})(), - ref_text=ref_text, - audio_ref=audio_ref_b, - ) - print(f" [LAAG] 策略: {laag_info.get('strategy')}, chunks: {laag_info.get('chunks')}, has_ref_text: {laag_info.get('has_ref_text')}") - - # v10.3: 优先用官方流程生成的 audio (避免重复 vocoder 转换) - gen_time = time.time() - t0 - if hasattr(backbone, '_last_audio_gen') and backbone._last_audio_gen is not None: - audio_gen_np = backbone._last_audio_gen.cpu().numpy().squeeze() - sr_gen = backbone._last_sr_gen - print(f" [v10.3] 使用官方流程生成的 audio: shape={audio_gen_np.shape}, sr={sr_gen}") - else: - # 备选: 用 vocoder 转换 mel (理论上不会走到这里) - if isinstance(mel_gen, tuple): - mel_gen = mel_gen[0] - mel_gen_np = mel_gen.cpu().numpy() - vocoder = backbone._f5tts.vocoder - # v10.4.6 修复: mel_gen 是 (B, 100, T), vocoder.decode 期望 (B, n_mels, T) - # 之前错误地 permute(0,2,1) 变成 (B, T, 100), 导致音质下降 - mel_for_vocoder = mel_gen.to(device) - with torch.no_grad(): - audio_gen = vocoder.decode(mel_for_vocoder) - audio_gen_np = audio_gen.cpu().numpy().squeeze() - sr_gen = SOTA_CONFIG.sample_rate - - # 保存音频 - out_wav = f"outputs/1111_clone_test_{i+1}.wav" - import soundfile as sf - sf.write(out_wav, audio_gen_np, sr_gen) - print(f" 生成音频 shape: {audio_gen_np.shape}") - print(f" 保存到: {out_wav}") - print(f" 合成耗时: {gen_time:.2f}s") - - # v10.3: 综合评估 (SECS + WER + UTMOS + RTF + SIM-o) - gen_wav, gen_sr = torchaudio.load(out_wav) - if gen_wav.shape[0] > 1: - gen_wav = gen_wav.mean(dim=0, keepdim=True) - if gen_sr != 16000: - gen_wav = torchaudio.functional.resample(gen_wav, gen_sr, 16000) - - # 判断语言 (英文用英文评估, 中文用中文) - lang = "english" if all(ord(c) < 128 for c in text if c.isalpha()) else "chinese" - metrics = evaluator.evaluate_all( - gen_wav=gen_wav, - ref_wav=audio_ref_16k, - reference_text=text, - sr=16000, - gen_time_sec=gen_time, - language=lang, - ) - # v10.4.6: 基于实验数据的合理目标 (F5 基线均值) - # F5 基线 SECS 方差测试结果: 中文短1=0.51±0.06, 英文短=0.31±0.03 - # PV 目标: 匹配或超过 F5 基线均值 - is_long = laag_info.get('chunks', 1) > 1 - is_english = all(ord(c) < 128 for c in text if c.isalpha()) - if is_long: - secs_target = 0.55 # 长文本 (chunked + FiLM) 目标 0.55 - elif is_english: - secs_target = 0.30 # 英文短文本 (F5 基线均值 0.31) - else: - secs_target = 0.50 # 中文短文本 (F5 基线均值 0.51) - - print(f" ┌─── 综合评估 ───") - print(f" │ SECS: {metrics['secs']:.4f} (目标 >= {secs_target:.2f}, F5基线)") - print(f" │ WER: {metrics['wer']:.4f} (目标 <= 0.10)") - print(f" │ UTMOS: {metrics['utmos']:.4f} (目标 >= 4.0)") - print(f" │ SIM-o: {metrics['sim_o']:.4f} (目标 >= 0.50)") - print(f" │ RTF: {metrics['rtf']:.4f} (目标 <= 1.0)") - print(f" │ 时长: {metrics['audio_duration']:.2f}s") - print(f" │ ASR: '{metrics['recognized_text'][:60]}'") - print(f" └────────────────") - - print("\n" + "=" * 70) - print("验证完成! 检查 outputs/ 目录") - print("=" * 70) - - -if __name__ == "__main__": - main() diff --git a/personavoice/__init__.py b/personavoice/__init__.py deleted file mode 100644 index ff0873c..0000000 --- a/personavoice/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -"""PersonaVoice: 1秒极限语音克隆核心架构 (v10.4.8 满血 SOTA). - -本模块是 PersonaVoice 项目的入口, 暴露 v10.4.8 满血 SOTA 核心架构组件. - -v10.4.8 核心架构 (LAAG + OES + FiLM + F5 官方流程): -1. F5-TTS 预训练骨干 (冻结) + FiLM Adapter (persona/emotion 注入) -2. OES (正交环境子流形): env_scale=0.1 渐进初始化 (解决 1111.mp3 SECS 灾难) -3. CEAG (交叉熵注意力引导): 带 padding mask, v10.4.7 实验验证对当前基线无显著效果, disabled -4. LAAG (长度自适应生成): 动态 Chunking + 动态 CFG + 动态 FiLM 激活 -5. Silero VAD + RMS 归一化: 前端精准预处理 (替代老旧 WebRTC VAD) -6. Persona 管线: BERT 编码聊天记录 → Big Five → persona_emb - -v10.3+ 时长计算: 使用 F5-TTS 官方 infer_batch_process 内部公式 -v10.4.8 诚实定位: SECS SOTA (+47.6% vs XTTS v2), 短文本 WER 为 Future Work - -架构位置: 顶层包入口, 对外暴露核心骨干与声码器. -""" - -from personavoice.tts_backbone.f5_pretrained_backbone import ( - F5TTSPretrainedBackbone, - PersonaEmotionFiLM, - load_pretrained_f5tts_backbone, -) -from personavoice.tts_backbone.vocoder import VocosVocoder -from personavoice.microaug.cross_manifold_refiner import ( - OrthogonalEnvironmentSubmanifold, -) -from personavoice.config import SOTA_CONFIG, get_config, get_inference_kwargs - -__version__ = "10.4.8" - -__all__ = [ - "F5TTSPretrainedBackbone", - "PersonaEmotionFiLM", - "load_pretrained_f5tts_backbone", - "VocosVocoder", - "OrthogonalEnvironmentSubmanifold", - "SOTA_CONFIG", - "get_config", - "get_inference_kwargs", -] diff --git a/personavoice/common/__init__.py b/personavoice/common/__init__.py deleted file mode 100644 index 89d166d..0000000 --- a/personavoice/common/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -"""共享基础设施: 本地模型路径管理、离线模式配置. - -架构位置: 全局基础设施层, 在所有模型加载之前调用, 确保离线运行环境. -""" - -from personavoice.common.local_models import ( - LOCAL_MODEL_PATHS, - setup_offline_mode, - get_hf_cache_dir, - is_model_cached, - get_local_model_path, - list_cached_models, - print_cache_status, -) - -__all__ = [ - "LOCAL_MODEL_PATHS", - "setup_offline_mode", - "get_hf_cache_dir", - "is_model_cached", - "get_local_model_path", - "list_cached_models", - "print_cache_status", -] diff --git a/personavoice/common/local_models.py b/personavoice/common/local_models.py deleted file mode 100644 index dcbca1d..0000000 --- a/personavoice/common/local_models.py +++ /dev/null @@ -1,222 +0,0 @@ -"""本地模型路径管理器: 强制使用本地 HuggingFace 缓存, 避免联网. - -架构位置: 全局基础设施层, 在所有模型加载之前调用, 确保离线运行环境. - -本地缓存位置: C:\\Users\\\\.cache\\huggingface\\hub -已缓存的模型: - - Qwen/Qwen2.5-0.5B (LLM / Tokenizer) - - SWivid/F5-TTS (TTS 骨干) - - speechbrain/spkrec-ecapa-voxceleb (说话人编码器) - - charactr/vocos-mel-24khz (声码器) - - bert-base-chinese (中文 BERT) - - openai/whisper-large-v3-turbo (Whisper ASR) - -使用方法: - from personavoice.common.local_models import ( - get_local_model_path, - setup_offline_mode, - LOCAL_MODEL_PATHS, - ) - - # 1. 全局开启离线模式 (推荐在程序入口调用) - setup_offline_mode() - - # 2. 获取本地模型路径 - path = get_local_model_path("qwen2.5_0.5b") - tokenizer = AutoTokenizer.from_pretrained(path) -""" -import os -import logging -from pathlib import Path -from typing import Optional, Dict - -logger = logging.getLogger(__name__) - - -# ========================================================================= -# 本地模型路径映射 -# ========================================================================= -# 所有 HuggingFace 模型 ID 与本地缓存路径的映射 -# 优先使用本地缓存, 若不存在则回退到在线模式 (但会打印警告) -LOCAL_MODEL_PATHS: Dict[str, str] = { - # ── LLM / Tokenizer ── - "qwen2.5_0.5b": "Qwen/Qwen2.5-0.5B", - "qwen3_0.6b": "Qwen/Qwen3-0.6B", - "qwen3_4b": "Qwen/Qwen3-4B", - # ── TTS Backbone ── - "f5_tts": "SWivid/F5-TTS", - # ── Speaker Encoder ── - "ecapa_voxceleb": "speechbrain/spkrec-ecapa-voxceleb", - # ── Vocoder ── - "vocos_mel_24khz": "charactr/vocos-mel-24khz", - # ── Text Encoder ── - "bert_chinese": "bert-base-chinese", - "minilm_l6": "sentence-transformers/all-MiniLM-L6-v2", - # ── ASR ── - "whisper_large_v3_turbo": "openai/whisper-large-v3-turbo", -} - - -# ========================================================================= -# 离线模式配置 -# ========================================================================= -_OFFLINE_MODE_CONFIGURED = False - - -def setup_offline_mode() -> None: - """全局开启 HuggingFace 离线模式. - - 设置环境变量: - - HF_HUB_OFFLINE=1 禁止网络请求 - - TRANSFORMERS_OFFLINE=1 transformers 库离线模式 - - HF_DATASETS_OFFLINE=1 datasets 库离线模式 - - 调用后所有 transformers / huggingface_hub 调用都只从本地缓存读取, - 不会发起任何网络请求, 避免连接超时. - """ - global _OFFLINE_MODE_CONFIGURED - - if _OFFLINE_MODE_CONFIGURED: - return - - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - os.environ["HF_DATASETS_OFFLINE"] = "1" - # 减少 transformers 日志噪音 - os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") - os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") - - _OFFLINE_MODE_CONFIGURED = True - logger.info( - "[local_models] HuggingFace 离线模式已启用 " - "(HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1)" - ) - - -def get_hf_cache_dir() -> Path: - """获取 HuggingFace 本地缓存目录. - - Returns: - Path: 缓存目录路径 (如 C:\\Users\\\\.cache\\huggingface\\hub) - """ - # 优先使用环境变量 - cache_env = os.environ.get("HF_HOME") or os.environ.get("HUGGINGFACE_HUB_CACHE") - if cache_env: - cache_path = Path(cache_env) - if cache_path.name == "hub": - return cache_path - return cache_path / "hub" - - # 默认位置: ~/.cache/huggingface/hub - try: - from huggingface_hub import constants - return Path(constants.HF_HUB_CACHE) - except ImportError: - # 回退到默认位置 - home = Path.home() - return home / ".cache" / "huggingface" / "hub" - - -def is_model_cached(model_id: str) -> bool: - """检查模型是否已在本地缓存. - - Args: - model_id: HuggingFace 模型 ID (如 "Qwen/Qwen2.5-0.5B") - - Returns: - bool: 模型是否已缓存 - """ - cache_dir = get_hf_cache_dir() - # HuggingFace 缓存目录格式: models---- - # 例如: models--Qwen--Qwen2.5-0.5B - cache_name = "models--" + model_id.replace("/", "--") - model_cache_path = cache_dir / cache_name - - # 必须存在 snapshots 子目录且非空 - snapshots_dir = model_cache_path / "snapshots" - if not snapshots_dir.exists(): - return False - - # 检查是否有至少一个 snapshot - snapshots = list(snapshots_dir.iterdir()) if snapshots_dir.exists() else [] - return len(snapshots) > 0 - - -def get_local_model_path(key: str, allow_network_fallback: bool = False) -> str: - """获取本地模型路径. - - 优先返回 HuggingFace 缓存中的模型路径, 若未缓存: - - allow_network_fallback=True: 返回原始 model_id (允许联网下载) - - allow_network_fallback=False: 抛出 FileNotFoundError - - Args: - key: 模型别名 (如 "qwen2.5_0.5b", "f5_tts", "ecapa_voxceleb") - allow_network_fallback: 未缓存时是否允许联网回退 - - Returns: - str: 模型路径 (可直接传给 AutoTokenizer.from_pretrained 等) - - Raises: - KeyError: key 不在 LOCAL_MODEL_PATHS 中 - FileNotFoundError: 模型未缓存且不允许联网回退 - """ - if key not in LOCAL_MODEL_PATHS: - raise KeyError( - f"未知模型别名: '{key}'. 可用别名: {list(LOCAL_MODEL_PATHS.keys())}" - ) - - model_id = LOCAL_MODEL_PATHS[key] - - # 检查本地缓存 - if is_model_cached(model_id): - # 返回 model_id, transformers 会自动从缓存加载 - # (配合 setup_offline_mode() 不会联网) - logger.debug(f"[local_models] 使用本地缓存: {model_id}") - return model_id - - # 未缓存 - if allow_network_fallback: - logger.warning( - f"[local_models] 模型 '{model_id}' 未在本地缓存, 将尝试联网下载. " - f"建议预先下载: huggingface-cli download {model_id}" - ) - return model_id - - raise FileNotFoundError( - f"模型 '{model_id}' (别名 '{key}') 未在本地缓存. " - f"请先下载: huggingface-cli download {model_id} " - f"或设置 allow_network_fallback=True" - ) - - -def list_cached_models() -> Dict[str, bool]: - """列出所有模型的缓存状态. - - Returns: - Dict[别名, 是否已缓存] - """ - return {key: is_model_cached(model_id) for key, model_id in LOCAL_MODEL_PATHS.items()} - - -def print_cache_status() -> None: - """打印所有模型的缓存状态 (用于调试).""" - print("\n" + "=" * 60) - print(" HuggingFace 本地模型缓存状态") - print("=" * 60) - cache_dir = get_hf_cache_dir() - print(f" 缓存目录: {cache_dir}") - print(f" 离线模式: {'已启用' if _OFFLINE_MODE_CONFIGURED else '未启用'}") - print() - for key, model_id in LOCAL_MODEL_PATHS.items(): - cached = is_model_cached(model_id) - status = "[已缓存]" if cached else "[未缓存]" - print(f" {status:10s} {key:25s} → {model_id}") - print("=" * 60 + "\n") - - -# ========================================================================= -# 自动初始化: 模块导入时自动启用离线模式 -# ========================================================================= -# 在导入本模块时自动启用离线模式, 避免后续所有 HF 调用联网 -# 用户可通过 setup_offline_mode() 再次确认 (幂等) -setup_offline_mode() diff --git a/personavoice/config.py b/personavoice/config.py deleted file mode 100644 index f1dbd4d..0000000 --- a/personavoice/config.py +++ /dev/null @@ -1,318 +0,0 @@ -"""PersonaVoice v10.0 统一 SOTA 配置 (唯一配置源). - -架构位置: 全局基础设施层, 所有模块 (骨干/前端/评估) 必须从此处读取配置, -消除 v9.x 时代 api_server.py / eval_200_samples.py / f5_pretrained_backbone.py -三处配置互相矛盾的隐患. - -v10.0 核心变更 (相对 v9.1): - 1. IEAG → CEAG (Cross-Entropy Attention Guidance): - 从最小化全自注意力熵升级为最小化文本-音频交叉注意力熵, - 直击短文本对齐崩塌 (Manifold Collapse under Information Asymmetry). - 2. 新增 Duration Predictor: BERT-based 句级时长预测, 取代 seconds_per_word 启发式. - 3. OES env_scale 从 1.0 改为 0.1 渐进初始化: - 未训练时不破坏音色 (解决 1111.mp3 SECS=0.03 灾难). - 4. 前端 Silero VAD + RMS 归一化: 提升 1s 样本参考纯度与生成响度一致性. - -设计原则: - - frozen dataclass: 运行时不可变, 防止意外修改 - - 单一配置源: 前端 Demo / 评估脚本 / 骨干模型三处完全一致 - - 8GB 显存可行: 所有配置在 RTX 4070 8GB 上验证 -""" - -from dataclasses import dataclass, field -from typing import Tuple - - -@dataclass(frozen=True) -class SOTAConfig: - """v10.0 SOTA 配置 (frozen, 全局唯一). - - 所有推理参数的权威来源. 修改此处的值会自动影响: - - personavoice/demo/api_server.py (前端 Demo) - - personavoice/experiment/eval_200_samples.py (评估) - - personavoice/tts_backbone/f5_pretrained_backbone.py (骨干) - """ - - # ── ODE 积分 ── - # v10.1: 96 → 32 (F5-TTS 官方默认, 3x 速度提升, 质量无损) - # v10.3.1: 48 实测 SECS 下降, 回退到 32 (最佳平衡点) - steps: int = 32 - - # Sway Sampling (F5-TTS 官方默认 -1.0, 偏向早期时间步) - sway_sampling_coef: float = -1.0 - - # ── 静态 CFG (替代无效的 TD-CFG, 消融 p=0.41/0.74) ── - # v10.1: 3.0 → 2.0 (F5-TTS 官方默认, 减少计算量) - # v10.4: 2.0 → 2.5 (参数搜索验证: SECS 0.5601→0.6086, +8.6%) - # 搜索结果: cfg=2.5 在中文短/长文本上均优于 cfg=2.0 - cfg_strength: float = 2.5 # v10.4.5: CFG=2.5 (短/长文本最均衡) - - # ── OES (正交环境子流形, 核心 A) ── - # env_weight=0.0: 纯净录音棚音质 (SECS SOTA 配置) - # env_weight=1.0: 完美复现 1s 样本的麦克风质感 - env_weight: float = 0.0 - - # OES env_scale 初始化 (v10.0: 0.1 渐进, 未训练时不破坏音色) - # v9.x 的 1.0 会导致未训练的随机 env_basis 破坏 1111.mp3 等难样本 - oes_env_scale_init: float = 0.1 - - # ── CEAG (Cross-Entropy Attention Guidance, 核心 B, v10.0 升级) ── - # 从 IEAG (全自注意力熵) 升级为 CEAG (文本-音频交叉注意力熵) - # 直击短文本对齐崩塌: 强制每个 mel 帧聚焦于特定 text token - # v10.2.2: 临时关闭以验证 F5-TTS 官方基线 - use_ceag: bool = False - - # CEAG 引导强度 (v10.1: 0.25 → 0.20, 配合动态 LAAG) - ceag_lambda_max: float = 0.20 - - # CEAG 激活窗口 (v10.1: [0.05,0.5] → [0.1,0.4], 减少激活步数 3x 速度提升) - # t∈[0,1], 早期 t≈0 是噪声, 晚期 t≈1 是干净 mel, 中期是对齐关键期 - # 32 步 × [0.1,0.4] = 10 步激活 (vs 96 步 × [0.05,0.5] = 43 步) - ceag_t_start: float = 0.1 - ceag_t_end: float = 0.4 - - # CEAG 提取 attention 的 DiT block 索引 (负数=倒数) - # v10.1: (-3,-2,-1) → (-2,-1), 减少层数 1.5x 速度提升 - ceag_layers: Tuple[int, ...] = (-2, -1) - - # CEAG 梯度裁剪 (逐层, 防止某层梯度淹没其他层) - ceag_grad_max_norm: float = 1.0 - - # ── 时长计算 (v10.1: 删除未训练的 Duration Predictor, 改用 F5-TTS 官方机制) ── - # F5-TTS 官方公式: duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed) - # 1s 克隆场景: 无 ref_text, 基于文本长度和正常语速计算 - # 这不是备用方案, 这是 1s 克隆场景下的正确时长计算方式 - speed: float = 1.0 # 语速因子 (1.0 = 正常语速, F5-TTS 官方默认) - seconds_per_char: float = 0.3 # 每字符秒数 (中文每字, 英文每词约 0.3s) - - # 时长范围限制 (帧数, 94帧/秒 @ 24kHz/hop=256) - duration_min_frames: int = 94 # 最少 1s - duration_max_frames: int = 3094 # 最多 ~33s - - # ── 前端音频预处理 (v10.0 新增) ── - # Silero VAD (替代老旧 WebRTC VAD, 对呼吸声/辅音切分更精准) - use_silero_vad: bool = True - silero_vad_threshold: float = 0.5 - silero_vad_min_speech_duration_ms: int = 100 - silero_vad_min_silence_duration_ms: int = 50 - - # RMS 能量归一化 (生成音频响度对齐参考音频) - use_rms_normalize: bool = True - rms_target: float = 0.1 # 目标 RMS (生成音频略低于参考) - - # 参考音频时长 (1s 极限场景) - ref_duration_sec: float = 1.0 - - # 音频采样率 / Mel 参数 (与 F5-TTS / Vocos 一致) - sample_rate: int = 24000 - hop_length: int = 256 - n_mels: int = 100 - - # ── 随机种子 (0=不设置, 与 F5 官方基线一致, 自然随机性更稳定) ── - # v10.4.2: seed=42 会导致某些文本 SECS 严重下降 (CUDA 非确定性) - # F5 官方基线不设置种子, 两个测试文本都稳定达标 (0.57-0.59) - seed: int = 0 - - # ── LAAG (Length-Aware Adaptive Generation, v10.1 核心创新) ── - # 基于文本长度动态调整生成策略, 解决 1s 参考下长短文本效果不均 - # 理论: 1s 参考(94帧)信息量固定, 文本长度变化导致信息瓶颈 - # - 短文本: 信息过剩, CEAG 过度聚焦 → WER↑ - # - 长文本: 信息稀释, 音色漂移 → SECS↓ - # LAAG 三大机制: - # A. 动态 Chunking: 长文本切分, 每 chunk 充分利用 1s 参考 - # B. 动态 CEAG λ: 基于文本长度自适应引导强度 - # C. 动态 CFG: 短文本增强引导, 长文本保持稳定 - use_laag: bool = True - - # A. 动态 Chunking (继承 F5-TTS 官方 chunk_text 机制) - # max_chars 公式: 基于 ref_text 长度和参考音频时长动态计算 - # F5-TTS 官方: max_chars = int(ref_text_len / ref_audio_sec * (22 - ref_audio_sec)) - # PersonaVoice 1s 参考: ref_audio_sec=1, 简化为固定 max_chars - laag_chunk_max_chars: int = 135 # F5-TTS 官方默认 - laag_cross_fade_duration: float = 0.15 # 交叉淡化时长(秒) - laag_min_chunk_chars: int = 10 # 最小 chunk 长度, 短于此不切分 - - # B. 动态 CEAG λ (基于文本长度的自适应引导) - # 公式: λ(L) = λ_base × clamp(1 + α × (L - L_ref) / L_ref, λ_min_ratio, λ_max_ratio) - # L = 文本 token 数, L_ref = 参考文本 token 数 (1s ≈ 10 token) - # 短文本 (LL_ref): λ 提高, 强化对齐 - laag_ceag_lambda_base: float = 0.25 # 基础 λ (与 ceag_lambda_max 一致) - laag_ceag_lambda_alpha: float = 0.5 # 动态调整系数 α - laag_ceag_lambda_min_ratio: float = 0.4 # λ 最小比例 (短文本) - laag_ceag_lambda_max_ratio: float = 2.0 # λ 最大比例 (长文本) - laag_ceag_ref_token_count: int = 10 # 参考文本 token 数 (1s ≈ 10 token) - - # C. 动态 CFG (基于文本长度的自适应引导强度) - # 短文本: CFG 增强 (更强文本引导, 降低 WER) - # 长文本: CFG 保持 (避免过度引导导致音色漂移) - # v10.3: 基础值对齐官方 F5-TTS (2.0), 动态范围收窄避免音色漂移 - # v10.3.1: 进一步收窄动态范围, 短文本 CFG 过高导致音色漂移 (SECS 下降) - # v10.4: 基础值 2.0→2.5 (参数搜索验证), 动态范围调整 - # v10.4.5: alpha=0.0, CFG=2.5 (短/长文本最均衡, 多次测试验证) - # LAAG 动态 CFG 机制保留 (alpha 可调), 但当前配置使用固定 CFG=2.5 - laag_cfg_base: float = 2.5 # 基础 CFG (多次测试验证最均衡) - laag_cfg_alpha: float = 0.0 # 动态调整系数 (0=固定 CFG) - laag_cfg_min: float = 2.5 # CFG 最小值 (与 base 一同, 固定 CFG) - laag_cfg_max: float = 2.5 # CFG 最大值 (与 base 一同, 固定 CFG) - - # v10.4.5: Best-of-N 多样本选择 - # v10.4.6: 禁用 (内部/外部 SECS 不一致) - # v10.4.7: 实验验证无效 (WER 0.6964, CFR 70%), 回滚到禁用状态 - # 根因: Flow Matching 流形崩塌是架构性限制, Best-of-N 无法解决 - # (所有样本都崩塌时, 选最优也选不出好的) - # 论文故事线改为 Trade-off: 承认短文本限制, 作为 Future Work - best_of_n: int = 1 # v10.4.7: 禁用 (实验证明无效) - best_of_n_max_chars: int = 0 # 不触发 - - # ── v10.4.7 短文本 CFG 增强 (已废弃, 实验证明恶化) ── - # 实验结果: CFG=3.5 导致 WER 从 0.4047 恶化到 0.6964 - # 原因: 过强 CFG 扰动 ODE 积分轨迹, 加剧流形崩塌 - # 保留字段以保持向后兼容, 但不启用 - short_text_cfg_boost: float = 2.5 # 等于基础 CFG (不启用增强) - short_text_byte_threshold: int = 0 # 阈值=0 永不触发 - - # ── 版本标识 ── - version: str = "10.4.8" - - -# 全局单例 (frozen, 不可变) -SOTA_CONFIG = SOTAConfig() - - -def get_config() -> SOTAConfig: - """获取全局 SOTA 配置单例. - - Returns: - SOTAConfig: frozen 配置实例 - """ - return SOTA_CONFIG - - -def get_inference_kwargs() -> dict: - """获取推理参数 kwargs (直接传给 backbone.synthesize). - - 确保 api_server.py / eval_200_samples.py / 任何推理代码使用完全相同的参数. - - Returns: - dict: 可直接 **unpack 的推理参数 - """ - cfg = SOTA_CONFIG - return { - "steps": cfg.steps, - "cfg_strength": cfg.cfg_strength, - "use_ceag": cfg.use_ceag, - "ceag_lambda_max": cfg.ceag_lambda_max, - "ceag_t_start": cfg.ceag_t_start, - "ceag_t_end": cfg.ceag_t_end, - "ceag_layers": cfg.ceag_layers, - "sway_sampling_coef": cfg.sway_sampling_coef, - "seed": cfg.seed, - # v10.1 LAAG 参数 - "use_laag": cfg.use_laag, - "laag_chunk_max_chars": cfg.laag_chunk_max_chars, - "laag_cross_fade_duration": cfg.laag_cross_fade_duration, - "laag_min_chunk_chars": cfg.laag_min_chunk_chars, - "laag_ceag_lambda_base": cfg.laag_ceag_lambda_base, - "laag_ceag_lambda_alpha": cfg.laag_ceag_lambda_alpha, - "laag_ceag_lambda_min_ratio": cfg.laag_ceag_lambda_min_ratio, - "laag_ceag_lambda_max_ratio": cfg.laag_ceag_lambda_max_ratio, - "laag_ceag_ref_token_count": cfg.laag_ceag_ref_token_count, - "laag_cfg_base": cfg.laag_cfg_base, - "laag_cfg_alpha": cfg.laag_cfg_alpha, - "laag_cfg_min": cfg.laag_cfg_min, - "laag_cfg_max": cfg.laag_cfg_max, - } - - -def compute_dynamic_ceag_lambda(text_token_count: int, cfg: "SOTAConfig" = None) -> float: - """v10.1 LAAG: 基于文本长度动态计算 CEAG λ. - - 公式: λ(L) = λ_base × clamp(1 + α × (L - L_ref) / L_ref, λ_min_ratio, λ_max_ratio) - - 短文本 (LL_ref): λ 提高, 强化对齐 (SECS↑) - - Args: - text_token_count: 文本 token 数 - cfg: 配置实例 (默认用全局 SOTA_CONFIG) - - Returns: - 动态 CEAG λ - """ - if cfg is None: - cfg = SOTA_CONFIG - L = max(1, text_token_count) - L_ref = cfg.laag_ceag_ref_token_count - ratio = 1.0 + cfg.laag_ceag_lambda_alpha * (L - L_ref) / L_ref - ratio = max(cfg.laag_ceag_lambda_min_ratio, min(cfg.laag_ceag_lambda_max_ratio, ratio)) - return cfg.laag_ceag_lambda_base * ratio - - -def compute_dynamic_cfg(text_token_count: int, cfg: "SOTAConfig" = None) -> float: - """v10.1 LAAG: 基于文本长度动态计算 CFG 强度. - - 短文本: CFG 增强 (更强文本引导, 降低 WER) - 长文本: CFG 降低 (避免过度引导导致音色漂移, 保持 SECS) - - 公式: CFG(L) = clamp(CFG_base × (1 + α × (L_ref - L) / L_ref), CFG_min, CFG_max) - - Args: - text_token_count: 文本 token 数 - cfg: 配置实例 (默认用全局 SOTA_CONFIG) - - Returns: - 动态 CFG 强度 - """ - if cfg is None: - cfg = SOTA_CONFIG - L = max(1, text_token_count) - L_ref = cfg.laag_ceag_ref_token_count - # 短文本 L0 → CFG 增大 - # 长文本 L>L_ref: (L_ref-L)<0 → CFG 减小 - factor = 1.0 + cfg.laag_cfg_alpha * (L_ref - L) / L_ref - cfg_val = cfg.laag_cfg_base * factor - return max(cfg.laag_cfg_min, min(cfg.laag_cfg_max, cfg_val)) - - -def compute_target_length(text: str, cfg: "SOTAConfig" = None) -> int: - """v10.1: 基于文本长度计算目标 mel 帧数 (F5-TTS 官方机制). - - F5-TTS 官方公式: duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed) - 1s 克隆场景: 无 ref_text, 基于文本长度和正常语速计算. - - 这不是备用方案, 这是 1s 克隆场景下的正确时长计算方式: - - 中文: 每字约 0.3 秒 - - 英文: 每词约 0.3 秒 - - 帧数 = 秒数 × (sample_rate / hop_length) = 秒数 × 94 - - Args: - text: 目标文本 - cfg: 配置实例 (默认用全局 SOTA_CONFIG) - - Returns: - target_length: 目标 mel 帧数 - """ - import re - if cfg is None: - cfg = SOTA_CONFIG - - # 统计文本单元 (中文字 + 英文词) - chinese_chars = len(re.findall(r"[\u4e00-\u9fff]", text)) - english_words = len(re.findall(r"[a-zA-Z]+", text)) - numbers = len(re.findall(r"\d+", text)) - - # 文本单元数 (中文每字 1 单元, 英文每词 1 单元, 数字每串 1 单元) - text_units = max(1, chinese_chars + english_words + numbers) - - # 基于正常语速计算秒数 - target_sec = text_units * cfg.seconds_per_char / cfg.speed - - # 转换为帧数 (94 帧/秒 @ 24kHz/hop=256) - frames_per_sec = cfg.sample_rate / cfg.hop_length - target_length = int(target_sec * frames_per_sec) - - # 安全限制 - target_length = max(cfg.duration_min_frames, min(cfg.duration_max_frames, target_length)) - - return target_length diff --git a/personavoice/demo/__init__.py b/personavoice/demo/__init__.py deleted file mode 100644 index 6d5fc7c..0000000 --- a/personavoice/demo/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""PersonaVoice 演示模块. - -架构位置: 项目演示层, 提供 Web API 与静态资源, 用于展示 PersonaVoice -端到端语音克隆能力, 不参与训练/推理主流程. -""" diff --git a/personavoice/demo/api_server.py b/personavoice/demo/api_server.py deleted file mode 100644 index 3aa6acd..0000000 --- a/personavoice/demo/api_server.py +++ /dev/null @@ -1,1203 +0,0 @@ -"""v10.4.8 PersonaVoice 交互式演示后端 API. - -架构位置: 演示模块的 Web API 服务端, 集成 F5-TTS 预训练模型做实际语音合成, -对外提供声音克隆/人格分析/架构状态查询接口. - -核心功能: -- 声音克隆 (clone_voice): F5-TTS + LAAG + 动态 FiLM + OES + 韵律迁移 (v10.4.8) -- 人格分析 (analyze_persona): BERT 编码聊天记录 → Big Five 人格特征 → persona_emb -- 人格校准 (calibrate_persona): 基于用户反馈微调人格表示 -- 架构状态查询 - -启动: python -m personavoice.demo.api_server 或直接运行此文件 -""" - -from __future__ import annotations - -import sys -import io -import base64 -import tempfile -import os -import time -from pathlib import Path - -# v10.4.8: 确保 personavoice 模块可被导入 (直接运行时需要) -# 当作为模块运行 (python -m personavoice.demo.api_server) 时不需要 -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent # personavoice/demo/api_server.py -> voice/ -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -# 所有模型已缓存,设置完全离线模式,避免 HuggingFace 网络检查超时 -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" -os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1" -os.environ["SENTENCE_TRANSFORMERS_OFFLINE"] = "1" - -# 配置 ffmpeg 路径(Whisper ASR 需要,使用 imageio-ffmpeg 自带的二进制) -try: - import imageio_ffmpeg - import shutil - _ffmpeg_src = imageio_ffmpeg.get_ffmpeg_exe() - # transformers 的 ffmpeg_read 直接调用 "ffmpeg" 命令,需要 ffmpeg.exe 在 PATH 中 - # 将 imageio-ffmpeg 的二进制复制为 ffmpeg.exe 放到一个目录并加入 PATH - _ffmpeg_bin_dir = os.path.join(tempfile.gettempdir(), "personavoice_ffmpeg") - os.makedirs(_ffmpeg_bin_dir, exist_ok=True) - _ffmpeg_dst = os.path.join(_ffmpeg_bin_dir, "ffmpeg.exe") - if not os.path.exists(_ffmpeg_dst): - shutil.copy2(_ffmpeg_src, _ffmpeg_dst) - os.environ["FFMPEG_BINARY"] = _ffmpeg_dst - os.environ["IMAGEIO_FFMPEG_EXE"] = _ffmpeg_dst - if _ffmpeg_bin_dir not in os.environ.get("PATH", ""): - os.environ["PATH"] = _ffmpeg_bin_dir + os.pathsep + os.environ.get("PATH", "") - print(f"[Demo] ffmpeg 路径: {_ffmpeg_dst}") -except Exception as _e: - print(f"[Demo] 警告: 未找到 ffmpeg ({_e}), Whisper ASR 可能无法工作") - -from typing import Dict, List, Optional - -import numpy as np -import torch -from fastapi import FastAPI, File, Form, UploadFile, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse -from pydantic import BaseModel - - -# ───────────────────────────────────────────────────────────────────── -# PersonaVoice 训练好的骨干 (sota_v50) -# ───────────────────────────────────────────────────────────────────── -_pv_backbone = None -_pv_backbone_loading = False -_pv_backbone_error: Optional[str] = None -_pv_tokenizer = None -_pv_vocoder = None # Vocos 声码器, 用于将 mel 转换为 wav - - -def get_personavoice_backbone(): - """获取训练好的 PersonaVoice 骨干单例 (sota_v50 + FiLM Adapter). - - 加载流程: - 1. 加载 F5-TTS 预训练骨干 (含 FiLM Adapter) - 2. 加载 sota_v50/best_model.pt 训练权重 - 3. 加载 F5-TTS tokenizer - 4. 加载 Vocos 声码器 (mel → wav) - - Returns: - (backbone, tokenizer, vocoder) 或 (None, None, None) - """ - global _pv_backbone, _pv_backbone_loading, _pv_backbone_error - global _pv_tokenizer, _pv_vocoder - - if _pv_backbone is not None: - return _pv_backbone, _pv_tokenizer, _pv_vocoder - - if _pv_backbone_loading: - return None, None, None - - try: - _pv_backbone_loading = True - _pv_backbone_error = None - - # 强制离线模式 - try: - from personavoice.common.local_models import setup_offline_mode - setup_offline_mode() - except ImportError: - pass - - import torch - device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"[Demo] 加载 PersonaVoice 训练骨干 (device={device})...") - - # 1. 加载 F5-TTS 预训练骨干 + FiLM Adapter - from personavoice.tts_backbone.f5_pretrained_backbone import ( - load_pretrained_f5tts_backbone, - ) - backbone, _ = load_pretrained_f5tts_backbone(device=device) - - # 2. 加载 sota_v50 训练权重 - from pathlib import Path - ckpt_path = Path("checkpoints/sota_v50/best_model.pt") - if ckpt_path.exists(): - ckpt = torch.load( - str(ckpt_path), map_location=device, weights_only=False - ) - backbone.load_state_dict(ckpt["backbone_state_dict"], strict=False) - print(f"[Demo] 已加载训练权重: {ckpt_path} (step={ckpt.get('step', '?')}, SECS={ckpt.get('secs', '?'):.4f})") - else: - print(f"[Demo] 警告: 未找到训练权重 {ckpt_path}, 将使用基础 F5-TTS") - - backbone = backbone.to(device) - backbone.eval() - _pv_backbone = backbone - - # 3. 加载 F5-TTS tokenizer (使用项目工具函数) - from personavoice.experiment.utils import load_tokenizer - _pv_tokenizer = load_tokenizer() - print(f"[Demo] Tokenizer 加载完成") - - # 4. 加载 Vocos 声码器 - from personavoice.tts_backbone.vocoder import VocosVocoder - _pv_vocoder = VocosVocoder(device=device, input_n_mels=100) # backbone 输出 100 mel bins - print(f"[Demo] Vocos 声码器加载完成") - - print(f"[Demo] PersonaVoice 训练骨干加载完成") - return _pv_backbone, _pv_tokenizer, _pv_vocoder - - except Exception as e: - _pv_backbone_error = str(e) - print(f"[Demo] PersonaVoice 骨干加载失败: {e}") - import traceback - traceback.print_exc() - return None, None, None - finally: - _pv_backbone_loading = False - - -_vocos_mel_extractor = None - - -def _get_vocos_mel_extractor(device: str = "cpu"): - """获取 Vocos 自带的 MelSpectrogramFeatures 提取器 (SOTA标准). - - SOTA关键: Vocos 声码器训练时使用自己的 mel 提取器 (100-bin, 24kHz), - 其 mel scale 和数值范围与 torchaudio.transforms.MelSpectrogram 不同. - 必须使用 Vocos 自带的提取器才能保证 mel -> wav 的正确重建. - """ - global _vocos_mel_extractor - if _vocos_mel_extractor is None: - from vocos.feature_extractors import MelSpectrogramFeatures - _vocos_mel_extractor = MelSpectrogramFeatures( - sample_rate=24000, - n_fft=1024, - hop_length=256, - n_mels=100, - padding="center", - ).to(device) - _vocos_mel_extractor.eval() - print("[Demo] Vocos MelSpectrogramFeatures 提取器初始化完成 (100-bin, 24kHz)") - return _vocos_mel_extractor - - -def _extract_mel_ref_from_audio( - audio_path: str, - target_sr: int = 24000, - ref_duration_sec: float = 1.0, - n_mels: int = 100, -) -> "torch.Tensor": - """从参考音频中提取 1s mel 频谱 (Vocos 兼容格式). - - SOTA修正: 使用 Vocos 自带的 MelSpectrogramFeatures 提取器 (100-bin, 24kHz), - 确保 mel -> Vocos -> wav 的重建质量. - 旧版使用 torchaudio MelSpectrogram 导致 Vocos 解码失败 (SECS ~0.01). - - Returns: - mel_ref: (T_ref, n_mels) 参考mel, T_ref ≈ 94 (1s @ 24kHz/256hop) - """ - import torchaudio - wav, sr = torchaudio.load(audio_path) - if sr != target_sr: - resampler = torchaudio.transforms.Resample(sr, target_sr) - wav = resampler(wav) - if wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - - # 取前 1s - ref_samples = int(target_sr * ref_duration_sec) - wav_ref = wav[:, :ref_samples] - - # 使用 Vocos 自带的 MelSpectrogramFeatures 提取器 (SOTA标准) - mel_extractor = _get_vocos_mel_extractor() - with torch.no_grad(): - mel_ref = mel_extractor(wav_ref) # (1, n_mels, T_ref) - mel_ref = mel_ref.squeeze(0).T # (T_ref, n_mels) - # Vocos mel 已经是 log-mel 格式, 范围约 [-80, 20] - # 安全范围裁剪 (Vocos 期望 log-mel) - mel_ref = torch.clamp(mel_ref, min=-80, max=20) - return mel_ref - - -def _extract_speaker_emb(audio_path: str) -> "torch.Tensor": - """从参考音频中提取 ECAPA 说话人嵌入 (192维).""" - import torchaudio - wav, sr = torchaudio.load(audio_path) - if sr != 16000: - resampler = torchaudio.transforms.Resample(sr, 16000) - wav = resampler(wav) - if wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - - # 使用 ECAPA-TDNN 提取嵌入 - from speechbrain.inference.speaker import EncoderClassifier - encoder = EncoderClassifier.from_hparams( - source="speechbrain/spkrec-ecapa-voxceleb", - savedir="./models/ecapa_tdnn", - run_opts={"device": "cpu"}, - ) - emb = encoder.encode_batch(wav).squeeze(0) # (192,) - return emb - - -# ───────────────────────────────────────────────────────────────────── -# FastAPI 应用 -# ───────────────────────────────────────────────────────────────────── -app = FastAPI( - title="PersonaVoice Demo API", - description="v10.4.8 交互式声音克隆 (Trade-off: LAAG + 动态 FiLM + OES + 人格分析)", - version="10.4.8", -) - -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -# ───────────────────────────────────────────────────────────────────── -# 请求/响应模型 -# ───────────────────────────────────────────────────────────────────── -class CloneRequest(BaseModel): - text: str = "你好,这是 PersonaVoice 的声音克隆演示。" - emotion: Optional[str] = None - emotion_intensity: float = 0.5 - - -# ───────────────────────────────────────────────────────────────────── -# 工具函数 -# ───────────────────────────────────────────────────────────────────── -def _ensure_wav(input_path: str, target_sr: int = 24000) -> str: - """使用 ffmpeg 将任意格式音频转换为单声道 WAV. - - 解决前端上传 m4a/mp3/webm 等格式时 soundfile/torchaudio 无法识别的问题. - 返回新的临时 WAV 文件路径,调用方负责删除. - """ - import subprocess - import shutil - - ext = os.path.splitext(input_path)[1].lower() - # 已经是 WAV 且无需改采样率时直接返回(节省一次转码) - if ext == ".wav": - return input_path - - wav_path = input_path + ".converted.wav" - ffmpeg_exe = shutil.which("ffmpeg") or os.environ.get("FFMPEG_BINARY", "ffmpeg") - cmd = [ - ffmpeg_exe, "-y", "-hide_banner", "-loglevel", "error", - "-i", input_path, - "-ar", str(target_sr), "-ac", "1", "-sample_fmt", "s16", - wav_path, - ] - try: - subprocess.run(cmd, check=True) - return wav_path - except Exception as e: - raise RuntimeError(f"ffmpeg 音频转换失败: {e}") - - -def _tensor_to_base64_wav( - waveform: torch.Tensor, - sample_rate: int = 16000, -) -> str: - """将 waveform tensor 编码为 base64 WAV 字符串.""" - import wave - - if waveform.dim() > 1: - waveform = waveform.squeeze(0) - wav_array = (waveform.cpu().numpy() * 32767).clip(-32768, 32767).astype(np.int16) - - buf = io.BytesIO() - with wave.open(buf, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sample_rate) - wf.writeframes(wav_array.tobytes()) - return base64.b64encode(buf.getvalue()).decode("ascii") - - -def _numpy_wav_to_base64(wav: np.ndarray, sample_rate: int) -> str: - """将 numpy waveform 编码为 base64 WAV 字符串 (F5-TTS 输出格式).""" - import wave - - wav_np = np.asarray(wav).astype(np.float32).squeeze() - if wav_np.ndim > 1: - wav_np = wav_np.mean(axis=0) - wav_array = (wav_np * 32767).clip(-32768, 32767).astype(np.int16) - - buf = io.BytesIO() - with wave.open(buf, "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sample_rate) - wf.writeframes(wav_array.tobytes()) - return base64.b64encode(buf.getvalue()).decode("ascii") - - -def _waveform_to_png_base64(wav: np.ndarray, sample_rate: int) -> str: - """v3.3: 生成音频波形 PNG 图片 (base64). - - 类似 Audacity 的波形显示: 深色背景 + 居中对称波形 + 渐变色. - 用户一眼就能看出这是"声音的形状", 比 Mel 频谱直观得多. - """ - from PIL import Image, ImageDraw - - # 1. 参数 - W, H = 800, 160 # 输出尺寸 - bg_color = (12, 17, 28) # 深色背景 #0c111c - wave_color = (80, 200, 180) # 青色主波形 #50c8b4 - wave_fill = (40, 100, 90, 60) # 填充色(半透明) - grid_color = (30, 40, 55) # 网格线 - text_color = (100, 115, 140) # 轴标签 - center_line = (50, 65, 85) # 中心零线 - - # 2. 下采样到目标宽度 - n_samples = len(wav) - if n_samples == 0: - img = Image.new("RGB", (W, H), bg_color) - buf = io.BytesIO() - img.save(buf, format="PNG") - return base64.b64encode(buf.getvalue()).decode("utf-8") - - # 每列像素对应多少个采样点 - samples_per_col = max(1, n_samples // W) - # 重采样: 取每列的最大最小值(包络) - n_cols = min(W, n_samples) - envelope_max = np.zeros(n_cols, dtype=np.float32) - envelope_min = np.zeros(n_cols, dtype=np.float32) - - for i in range(n_cols): - start = i * samples_per_col - end = min(start + samples_per_col, n_samples) - chunk = wav[start:end] - if len(chunk) > 0: - envelope_max[i] = float(np.max(np.abs(chunk))) - envelope_min[i] = -envelope_max[i] # 对称 - - # 3. 归一化到画布高度 (留边距) - margin = 20 - draw_h = H - 2 * margin - peak = max(float(envelope_max.max()), 1e-6) - scale = (draw_h / 2 - 4) / peak # -4 防止触边 - - # 4. 创建图像 - img = Image.new("RGB", (W, H), bg_color) - draw = ImageDraw.Draw(img) - - cy = H // 2 # 中心 Y 坐标 - - # 5. 绘制网格线 - for gy in [margin, cy, H - margin]: - draw.line([(0, gy), (W, gy)], fill=grid_color, width=1) - # 中心零线稍亮 - draw.line([(0, cy), (W, cy)], fill=center_line, width=1) - - # 6. 绘制波形 (填充 + 描边) - points_top = [] - points_bottom = [] - for i in range(n_cols): - x = int(i * W / n_cols) - y_top = int(cy - envelope_max[i] * scale) - y_bot = int(cy - envelope_min[i] * scale) - points_top.append((x, y_top)) - points_bottom.append((x, y_bot)) - - # 填充区域 (上半部 + 反转下半部) - fill_points = points_top + list(reversed(points_bottom)) - if len(fill_points) >= 3: - draw.polygon(fill_points, fill=(30, 80, 70)) - - # 描边 (上下两条线) - if len(points_top) >= 2: - draw.line(points_top, fill=wave_color, width=1) - draw.line(points_bottom, fill=wave_color, width=1) - - # 7. 时间轴标签 - duration = n_samples / sample_rate - draw.text((4, H - 14), f"0.0s", fill=text_color) - draw.text((W - 36, H - 14), f"{duration:.1f}s", fill=text_color) - draw.text((W // 2 - 16, H - 14), f"{duration/2:.1f}s", fill=text_color) - - # 8. 编码为 PNG - buf = io.BytesIO() - img.save(buf, format="PNG") - return base64.b64encode(buf.getvalue()).decode("utf-8") - - -def _mel_to_png_base64(mel_np: np.ndarray) -> str: - """v3.3: 将 Mel 频谱渲染为 PNG 图片 (base64), 替代前端 Canvas 渲染. - - 使用 PIL + Viridis colormap 在服务端生成高质量频谱图, - 解决浏览器 Canvas 渲染不一致、colormap 失真等问题. - """ - from PIL import Image, ImageDraw, ImageFont - - # 1. 归一化: percentile clip → [0, 255] - p1 = float(np.percentile(mel_np, 1)) - p99 = float(np.percentile(mel_np, 99)) - mel_clipped = np.clip(mel_np, p1, p99) - mel_uint8 = ((mel_clipped - p1) / max(p99 - p1, 1e-6) * 255).astype(np.uint8) - - n_mels, n_frames = mel_uint8.shape - - # 2. Viridis colormap LUT (256 级, 与 matplotlib 一致) - _viridis_pts = [ - [0.00,(68,1,84)],[0.04,(71,22,104)],[0.08,(69,51,122)],[0.12,(62,78,138)], - [0.16,(55,102,152)],[0.20,(48,125,163)],[0.24,(42,146,172)],[0.28,(35,165,179)], - [0.32,(31,183,183)],[0.36,(34,199,197)],[0.40,(47,214,208)],[0.44,(67,226,216)], - [0.48,(91,237,222)],[0.52,(118,247,225)],[0.56,(148,253,225)],[0.60,(180,255,221)], - [0.64,(212,253,213)],[0.68,(243,249,200)],[0.72,(269,242,184)],[0.76,(292,232,166)], - [0.80,(313,220,147)],[0.84,(332,205,127)],[0.88,(350,187,107)],[0.92,(366,167,87)], - [0.96,(380,145,68)],[1.00,(393,123,50)] - ] - # 构建 256 级 LUT - lut_r, lut_g, lut_b = [], [], [] - for i in range(256): - v = i / 255.0 - # 找到 v 所在的区间并插值 - lo = 0 - for j in range(len(_viridis_pts)): - if _viridis_pts[j][0] <= v: - lo = j - else: - break - hi = min(lo + 1, len(_viridis_pts) - 1) - t = (v - _viridis_pts[lo][0]) / max(_viridis_pts[hi][0] - _viridis_pts[lo][0], 1e-8) - r = min(255, int(_viridis_pts[lo][1][0] * (1-t) + _viridis_pts[hi][1][0] * t)) - g = min(255, int(_viridis_pts[lo][1][1] * (1-t) + _viridis_pts[hi][1][1] * t)) - b = min(255, int(_viridis_pts[lo][1][2] * (1-t) + _viridis_pts[hi][1][2] * t)) - lut_r.append(r); lut_g.append(g); lut_b.append(b) - - lut_r = np.array(lut_r, dtype=np.uint8) - lut_g = np.array(lut_g, dtype=np.uint8) - lut_b = np.array(lut_b, dtype=np.uint8) - - # 3. 通过 LUT 映射到 RGB - r_img = lut_r[mel_uint8] - g_img = lut_g[mel_uint8] - b_img = lut_b[mel_uint8] - rgb = np.stack([r_img, g_img, b_img], axis=2) # (n_mels, n_frames, 3) - - # 4. PIL Image: Y 轴翻转 (低频在下) - img = Image.fromarray(rgb[::-1].copy(), mode="RGB") - - # 5. 缩放到合理尺寸 (最大宽度 600px, 保持宽高比) - max_w = 600 - if n_frames > max_w: - scale = max_w / n_frames - new_size = (max_w, max(int(n_mels * scale), 80)) - img = img.resize(new_size, Image.BILINEAR) - - # 6. 编码为 PNG base64 - buf = io.BytesIO() - img.save(buf, format="PNG") - return base64.b64encode(buf.getvalue()).decode("utf-8") - - -def _preprocess_ref_audio(audio_path: str, target_sr: int = 24000) -> str: - """v10.0 前端音频预处理: Silero VAD + RMS 归一化 (修复地雷3). - - 替代老旧的 WebRTC VAD / 能量阈值法, 使用 Silero VAD 神经网络 - 精准去除前后静音, 对呼吸声和辅音 (/s/, /f/) 切分极其精准. - - Pipeline: - 1. Silero VAD 精准语音端点检测 → 去除前后静音 - 2. 重采样到 target_sr (F5-TTS 要求 24kHz) - 3. RMS 能量归一化 → 响度一致性 - 4. 补 50ms 尾部静音 (F5-TTS 官方要求) - - Args: - audio_path: 音频文件路径 - target_sr: 目标采样率 (默认 24000) - - Returns: - 预处理后的 wav 文件路径 - """ - # v10.0: 使用新的 audio_preprocess 模块 (Silero VAD + RMS 归一化) - from personavoice.demo.audio_preprocess import preprocess_ref_audio - from personavoice.config import SOTA_CONFIG - - waveform, sr = preprocess_ref_audio( - audio_path, - target_sr=target_sr, - use_silero_vad=SOTA_CONFIG.use_silero_vad, - use_rms_normalize=SOTA_CONFIG.use_rms_normalize, - vad_threshold=SOTA_CONFIG.silero_vad_threshold, - rms_target=SOTA_CONFIG.rms_target, - ) - - # 保存为 wav - import torchaudio - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: - out_path = f.name - torchaudio.save(out_path, waveform, sr) - return out_path - - -def _get_ref_text(audio_path: str) -> str: - """用 F5-TTS 的 Whisper ASR 转写参考音频(需要 ffmpeg,已配置).""" - from f5_tts.infer.utils_infer import transcribe - ref_text = transcribe(audio_path) - # 确保 ref_text 以句号结尾(F5-TTS 要求) - if not ref_text.endswith(". ") and not ref_text.endswith("。"): - if ref_text.endswith("."): - ref_text += " " - else: - ref_text += ". " - return ref_text - - -def _save_upload_to_tensor(upload: UploadFile) -> torch.Tensor: - """将上传的音频文件转为 waveform tensor.""" - import torchaudio - - suffix = os.path.splitext(upload.filename or "audio.wav")[1] or ".wav" - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: - content = upload.file.read() - tmp.write(content) - tmp_path = tmp.name - - try: - waveform, sr = torchaudio.load(tmp_path) - if sr != 16000: - resampler = torchaudio.transforms.Resample(sr, 16000) - waveform = resampler(waveform) - if waveform.shape[0] > 1: - waveform = waveform.mean(dim=0, keepdim=True) - return waveform.squeeze(0) - finally: - os.unlink(tmp_path) - - -# ───────────────────────────────────────────────────────────────────── -# 语音合成函数 (训练模型 / 基础 F5-TTS) -# ───────────────────────────────────────────────────────────────────── - -async def _synthesize_with_personavoice( - ref_audio_path: str, - text: str, - emotion: Optional[str], - emotion_intensity: float, - env_weight: float = 1.0, - persona_emb: Optional["torch.Tensor"] = None, -) -> tuple: - """使用训练好的 PersonaVoice 模型合成语音 (sota_v50 + FiLM + OES + IEAG). - - Args: - persona_emb: 人格嵌入 (1, 64), 若为 None 则使用零向量 - - Returns: - (wav_np, sample_rate) 或 (None, None) - """ - backbone, tokenizer, vocoder = get_personavoice_backbone() - if backbone is None: - raise HTTPException( - status_code=503, - detail=f"PersonaVoice 模型未加载成功: {_pv_backbone_error or '正在加载中,请稍后重试'}", - ) - - import torch - # v10.4.6: 移除固定种子 (与 F5 官方基线行为一致, 自然随机性产生更均衡的 SECS) - # F5 基线不设置种子, 设置种子反而导致某些文本 SECS 严重下降 - device = next(backbone.parameters()).device - - print(f"[Demo] PersonaVoice 合成中 (v10.3 SOTA: F5-TTS官方流程+LAAG+OES+FiLM): text={text[:50]}...") - - with torch.no_grad(): - # v10.3: 加载原始音频波形 (F5-TTS 官方 cond=audio, 不做 VAD/RMS 预处理) - import torchaudio as _ta - audio_ref_raw, sr_raw = _ta.load(ref_audio_path) - if audio_ref_raw.shape[0] > 1: - audio_ref_raw = audio_ref_raw.mean(dim=0, keepdim=True) - # 重采样到 24kHz (F5-TTS 要求) - if sr_raw != 24000: - audio_ref_raw = _ta.functional.resample(audio_ref_raw, sr_raw, 24000) - audio_ref_b = audio_ref_raw.to(device).float() # (1, T_samples) - print(f"[Demo] audio_ref shape: {audio_ref_b.shape}, sr=24000") - - # v10.3: 用 Whisper 转录参考音频得到 ref_text (F5-TTS 官方要求) - from transformers import pipeline as hf_pipeline - import numpy as np - audio_ref_16k = _ta.functional.resample(audio_ref_raw, 24000, 16000) - audio_ref_np = audio_ref_16k.squeeze(0).cpu().numpy().astype(np.float32) - # 懒加载 ASR pipeline (首次使用时加载) - global _asr_pipe - if '_asr_pipe' not in globals() or _asr_pipe is None: - _asr_pipe = hf_pipeline( - "automatic-speech-recognition", - model="openai/whisper-large-v3-turbo", - torch_dtype=torch.float16 if "cuda" in str(device) else torch.float32, - device=device, - ) - asr_result = _asr_pipe(audio_ref_np, generate_kwargs={"language": "chinese"}) - ref_text = asr_result["text"].strip() - print(f"[Demo] ref_text: '{ref_text}'") - - # 1. 提取 1s 参考 mel (Vocos 兼容: 100-bin, 24kHz, 用于签名兼容) - mel_ref = _extract_mel_ref_from_audio( - ref_audio_path, target_sr=24000, ref_duration_sec=1.0, n_mels=100 - ).to(device).float() # (T_ref, 100) - mel_ref_b = mel_ref.unsqueeze(0) # (1, T_ref, 100) - print(f"[Demo] mel_ref shape: {mel_ref_b.shape}") - - # 2. 提取 ECAPA 说话人嵌入 (192维) - v10.4: 移除参考音频增强 (伤害 SECS) - # 直接从原始 1s 音频提取 (与 F5-TTS 官方基线一致, SECS 更高) - from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator - _ecapa_local = ECAPAEvaluator(device=str(device), vocoder_device=str(device)) - speaker_emb = _ecapa_local.extract_embedding(audio_ref_16k, sr=16000).to(device).float() - # 确保是 (1, 192) 格式 - if speaker_emb.dim() == 1: - speaker_emb_b = speaker_emb.unsqueeze(0) # (1, 192) - elif speaker_emb.dim() == 2 and speaker_emb.shape[0] == 1: - speaker_emb_b = speaker_emb # 已经是 (1, 192) - else: - speaker_emb_b = speaker_emb.reshape(1, -1) # 强制 (1, 192) - print(f"[Demo] speaker_emb shape: {speaker_emb_b.shape}, norm={speaker_emb_b.norm():.4f}") - - # 3. 编码文本 - text_tokens = tokenizer.encode(text) - if isinstance(text_tokens, torch.Tensor): - text_tokens_b = text_tokens.unsqueeze(0).to(device) - else: - text_tokens_b = torch.tensor([text_tokens], device=device) - print(f"[Demo] text_tokens shape: {text_tokens_b.shape}") - - # 4. 人格/情感嵌入 - # v10.4.6: persona_emb 传递给 LAAG, 由 LAAG 动态决定是否激活 FiLM - # - 短文本: FiLM off (稳定基线) - # - 长文本: FiLM on (speaker_emb 作为 persona_emb, 防音色漂移) - # emotion_emb 应为 4 维 (one-hot, 会被 emotion_proj 投影到 64) - if persona_emb is not None: - persona_emb_b = persona_emb.to(device).float() - if persona_emb_b.dim() == 1: - persona_emb_b = persona_emb_b.unsqueeze(0) - else: - # v10.4.6: 用 speaker_emb 作为 persona_emb (LAAG 会动态决定是否激活 FiLM) - # 这样长文本生成时 FiLM 能用 speaker_emb 防止音色漂移 - persona_emb_b = speaker_emb_b.clone() - emotion_emb_b = torch.zeros(1, 4, device=device) - - # v10.1: 删除 Duration Predictor, 用 F5-TTS 官方时长公式 (由 LAAG 层计算) - # LAAG 层通过 compute_target_length() 自动计算目标长度 - - # 6. 合成 mel (v10.1 LAAG: 长度自适应生成, 满血架构无备用方案) - # v10.1 核心创新: LAAG (Length-Aware Adaptive Generation) - # - 动态 Chunking: 长文本切分, 每 chunk 充分利用 1s 参考 - # - 动态 CEAG λ: 基于文本长度自适应引导强度 - # - 动态 CFG: 短文本增强引导, 长文本保持稳定 - # - F5-TTS 官方时长公式: 基于文本长度计算目标帧数 - t_start = time.time() - # v10.0: 启用 OES 环境解缠 (env_weight: 0=录音棚纯净 SOTA, 1=原始质感) - if hasattr(backbone, "set_env_weight"): - backbone.set_env_weight(float(env_weight)) - - # v10.1: LAAG 合成入口 (长度自适应生成, 无备用方案) - # v10.3: 传入 ref_text + audio_ref (F5-TTS 官方流程) - print(f"[Demo] v10.3 LAAG 启用 (F5-TTS 官方流程 + 长度自适应生成)") - mel_gen, laag_info = backbone.laag_synthesize( - text_str=text, - mel_ref=mel_ref_b, - speaker_emb=speaker_emb_b, - persona_emb=persona_emb_b, - emotion_emb=emotion_emb_b, - tokenizer=tokenizer, - ref_text=ref_text, # v10.3: F5-TTS 官方 ref_text - audio_ref=audio_ref_b, # v10.3: F5-TTS 官方 cond=audio - ) - print(f"[Demo] LAAG 策略: {laag_info.get('strategy', 'unknown')}, " - f"chunks: {laag_info.get('chunks', 1)}") - t_elapsed = time.time() - t_start - print(f"[Demo] mel 合成完成: shape={mel_gen.shape}, 耗时={t_elapsed:.2f}s") - - # v10.3: 优先用官方流程生成的 audio (避免重复 vocoder 转换) - if hasattr(backbone, '_last_audio_gen') and backbone._last_audio_gen is not None: - wav_np = backbone._last_audio_gen.cpu().numpy().squeeze() - sr = backbone._last_sr_gen - print(f"[Demo] v10.3 使用官方流程生成的 audio: shape={wav_np.shape}, sr={sr}") - return wav_np, sr - - # 备选: 用 vocoder 转换 mel (理论上不会走到这里) - # Vocos 期望 log-mel 范围 [-80, 20] (SOTA 标准) - print(f"[Demo] mel range before clamp: min={mel_gen.min():.3f}, max={mel_gen.max():.3f}, mean={mel_gen.mean():.3f}") - mel_gen = torch.clamp(mel_gen, min=-80, max=20) - print(f"[Demo] mel range after clamp: min={mel_gen.min():.3f}, max={mel_gen.max():.3f}") - - # 7. mel → wav (Vocos 声码器, 满血架构) - # v10.4.6 修复: mel_gen 已经是 (B, 100, T) = (B, n_mels, T), 无需转置 - # 之前错误地 transpose(1,2) 变成 (B, T, 100), 导致 Vocos 收到错误形状 - if mel_gen.dim() == 2: - mel_gen_vocos = mel_gen.unsqueeze(0) - else: - mel_gen_vocos = mel_gen # (1, 100, T) - 直接使用 - - wav_t = vocoder.mel_to_waveform(mel_gen_vocos.cpu()) # (1, T_samples) - wav_np = wav_t.squeeze(0).cpu().numpy() if wav_t.dim() > 1 else wav_t.cpu().numpy() - sr = 24000 # Vocos 输出 24kHz - - print(f"[Demo] wav 合成完成: shape={wav_np.shape}, sr={sr}, " - f"duration={len(wav_np) / sr:.2f}s") - - return wav_np, sr - - -# ───────────────────────────────────────────────────────────────────── -# API 端点 -# ───────────────────────────────────────────────────────────────────── -@app.get("/api/health") -async def health(): - """健康检查 + backbone 状态.""" - try: - backbone, tokenizer, vocoder = get_personavoice_backbone() - return { - "status": "ok", - "version": "10.4.8", - "personavoice_loaded": backbone is not None, - "personavoice_loading": _pv_backbone_loading, - "personavoice_error": _pv_backbone_error, - "modules": { - "f5_backbone": True, - "film_adapter": True, - "oes": True, - "ceag": True, - "laag": True, - "vocoder": True, - "personavoice_trained": backbone is not None, - }, - } - except Exception as e: - return {"status": "error", "message": str(e)} - - -@app.post("/api/clone") -async def clone_voice( - audio: UploadFile = File(...), - text: str = Form("你好,这是 PersonaVoice 的声音克隆演示。"), - emotion: Optional[str] = Form(None), - emotion_intensity: float = Form(0.5), - env_weight: float = Form(1.0), - chat_history: Optional[str] = Form(None), - structured_recalls: Optional[str] = Form(None), -): - """端到端声音克隆. - - 使用训练好的 PersonaVoice 模型 (v10.4.8: F5-TTS + LAAG + 动态 FiLM + OES). - 接收参考音频 + 文本 + 聊天记录(可选), 返回合成音频(base64) + 波形可视化. - 若提供 chat_history, 会提取 persona_emb 注入 FiLM 影响语音风格. - """ - try: - # 1. 保存上传音频到临时文件, 并统一转换为 WAV (兼容 m4a/mp3/webm 等) - suffix = os.path.splitext(audio.filename or "audio.wav")[1] or ".wav" - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: - content = await audio.read() - tmp.write(content) - raw_audio_path = tmp.name - - ref_audio_path = _ensure_wav(raw_audio_path, target_sr=24000) - - try: - # 2. 人格提取 (若提供聊天记录) - persona_emb = None - if chat_history: - import json as _json - try: - chat_list = _json.loads(chat_history) - if isinstance(chat_list, list) and len(chat_list) > 0: - # v10.4.7: 兼容两种聊天记录格式 - # - List[str]: ["你好", "我很好"] - # - List[Dict]: [{"role":"user","content":"你好"}, ...] - if isinstance(chat_list[0], dict): - chat_str_list = [ - m.get("content", "") for m in chat_list - if isinstance(m, dict) and m.get("content") - ] - else: - chat_str_list = [str(s) for s in chat_list if s] - if chat_str_list: - from personavoice.persona.extractor import extract_persona_emb - recalls = None - if structured_recalls: - try: - recalls = _json.loads(structured_recalls) - except Exception: - pass - persona_emb = extract_persona_emb(chat_str_list, recalls) - print(f"[Demo] persona_emb extracted: shape={persona_emb.shape}, norm={persona_emb.norm():.4f}") - else: - print("[Demo] persona extraction skipped: 聊天记录为空") - except Exception as e: - print(f"[Demo] persona extraction skipped: {e}") - - # 3. 语音合成 - print(f"[Demo] === 收到合成请求 (PersonaVoice v10.4.8) ===") - print(f"[Demo] text: {repr(text[:100])}") - print(f"[Demo] emotion: {emotion}") - print(f"[Demo] persona: {'yes' if persona_emb is not None else 'no'}") - - wav, sr = await _synthesize_with_personavoice( - ref_audio_path, text, emotion, emotion_intensity, - env_weight=env_weight, - persona_emb=persona_emb, - ) - used_engine = "personavoice-trained" - - if wav is None: - raise HTTPException( - status_code=500, - detail="PersonaVoice 合成失败", - ) - - # 3. 编码音频为 base64 WAV - wav_debug = np.asarray(wav).astype(np.float32).squeeze() - print(f"[Demo] WAV debug: shape={wav_debug.shape}, min={wav_debug.min():.6f}, max={wav_debug.max():.6f}, " - f"mean={wav_debug.mean():.6f}, std={wav_debug.std():.6f}, " - f"has_nan={np.isnan(wav_debug).any()}, has_inf={np.isinf(wav_debug).any()}") - audio_b64 = _numpy_wav_to_base64(wav, sr) - print(f"[Demo] audio_b64 length: {len(audio_b64)}") - - # 4. 生成音频波形可视化 - wave_b64 = None - wave_duration = 0.0 - if wav is not None: - wave_b64 = _waveform_to_png_base64(wav_debug, sr) - wave_duration = float(len(wav_debug)) / sr - - # 5. 计算指标 (供前端展示) - import torch as _torch - # speaker_embedding_norm: ECAPA 嵌入 L2 范数 (参考音频) - try: - spk_emb = _extract_speaker_emb(ref_audio_path) - spk_norm = float(spk_emb.norm()) if spk_emb is not None else 0.0 - except Exception: - spk_norm = 0.0 - # persona_representation_norm: persona_emb L2 范数 - if persona_emb is not None: - persona_norm = float(persona_emb.norm()) - else: - persona_norm = 0.0 - # z_emotion_norm: 情感嵌入范数 (当前为零向量) - emotion_norm = 0.0 - - # 条件嵌入统计 (供前端"条件嵌入"面板展示) - conditions = { - "speaker_emb (ECAPA 192-d)": { - "shape": [192], "mean": round(spk_norm / 192, 6), - "std": 0.0, "norm": round(spk_norm, 3) - }, - "persona_emb (Big Five → 64-d)": { - "shape": [64], "mean": round(persona_norm / 64, 6), - "std": 0.0, "norm": round(persona_norm, 3) - }, - "emotion_emb (4-class → 64-d)": { - "shape": [64], "mean": round(emotion_norm / 64, 6), - "std": 0.0, "norm": round(emotion_norm, 3) - }, - } - - return { - "success": True, - "engine": used_engine, - "synthesized_text": text, - "audio_base64": audio_b64, - "wave_base64": wave_b64, - "sample_rate": sr, - "wave_duration": round(wave_duration, 2), - "ref_audio_duration": 1.0, # 1s 极限克隆 - "ref_audio_used": "1s (极限克隆截取)", - "speaker_embedding_norm": round(spk_norm, 3), - "persona_representation_norm": round(persona_norm, 3), - "z_emotion_norm": round(emotion_norm, 3), - "conditions": conditions, - } - finally: - # 清理临时文件(包括原始上传文件和 ffmpeg 转换后的 WAV) - paths_to_clean = {ref_audio_path, raw_audio_path} - for p in paths_to_clean: - if p and os.path.exists(p): - try: - os.unlink(p) - except OSError: - pass - except HTTPException: - raise - except Exception as e: - import traceback - raise HTTPException(status_code=500, detail=f"{str(e)}\n{traceback.format_exc()}") - - -# ───────────────────────────────────────────────────────────────────── -# 人格分析端点 -# ───────────────────────────────────────────────────────────────────── - -class PersonaAnalyzeRequest(BaseModel): - chat_history: List[str] - structured_recalls: Optional[Dict] = None - - -class PersonaCalibrateRequest(BaseModel): - feedback: str - chat_history: Optional[List[str]] = None - structured_recalls: Optional[Dict] = None - - -@app.post("/api/persona/analyze") -async def analyze_persona(req: PersonaAnalyzeRequest): - """从聊天记录提取人格特征. - - Pipeline: BERT 编码聊天记录 → Big Five 人格特征 → persona_emb (64维) - 返回 trait_radar (雷达图数据) + persona_representation (统计信息). - - v10.4.7: 兼容 List[str] 和 List[Dict] 两种聊天记录格式 - """ - try: - from personavoice.persona.extractor import ( - extract_persona_emb, - extract_big_five_traits, - ) - - chat_list = req.chat_history - if not chat_list: - raise HTTPException(status_code=400, detail="聊天记录不能为空") - - # v10.4.7: 兼容 List[Dict] 格式 (前端可能发送 [{"role":"user","content":"..."}]) - if isinstance(chat_list[0], dict): - chat_list = [ - m.get("content", "") for m in chat_list - if isinstance(m, dict) and m.get("content") - ] - else: - chat_list = [str(s) for s in chat_list if s] - - if not chat_list: - raise HTTPException(status_code=400, detail="聊天记录解析后为空") - - # 提取 persona_emb (64维) - persona_emb = extract_persona_emb(chat_list, req.structured_recalls) - - # 提取 Big Five 人格特征 (用于雷达图) - traits = extract_big_five_traits(chat_list, req.structured_recalls) - - return { - "success": True, - "persona_representation": { - "shape": list(persona_emb.shape), - "norm": float(persona_emb.norm()), - "mean": float(persona_emb.mean()), - "std": float(persona_emb.std()), - }, - "trait_radar": { - "labels": list(traits.keys()), - "values": list(traits.values()), - "labels_zh": ["开放性", "尽责性", "外向性", "宜人性", "情绪稳定性"], - "max_value": 1.0, - }, - } - except HTTPException: - raise - except Exception as e: - import traceback - raise HTTPException(status_code=500, detail=f"{str(e)}\n{traceback.format_exc()}") - - -@app.post("/api/persona/calibrate") -async def calibrate_persona(req: PersonaCalibrateRequest): - """基于用户反馈校准人格表示. - - 用户输入自然语言反馈 (如 "让外向性更高"), 系统解析反馈并微调人格特征. - """ - try: - from personavoice.persona.extractor import calibrate_traits - - feedback = req.feedback.strip() - if not feedback: - raise HTTPException(status_code=400, detail="校准反馈不能为空") - - chat_list = req.chat_history or [] - recalls = req.structured_recalls - - # 解析反馈并校准 - result = calibrate_traits(feedback, chat_list, recalls) - - return { - "success": True, - "result": result, - } - except HTTPException: - raise - except Exception as e: - import traceback - raise HTTPException(status_code=500, detail=f"{str(e)}\n{traceback.format_exc()}") - - -@app.get("/api/architecture") -async def get_architecture(): - """返回架构信息 (用于前端可视化).""" - return { - "version": "10.4.8", - "modules": [ - { - "id": "oes", - "name": "OES", - "title": "正交环境子流形 (v10.0 核心)", - "description": "环境子空间分解 + 正交约束, 分离录音环境与音色 (env_scale=0.1 渐进初始化, env_weight=0.0: 纯净录音棚音质)", - "color": "#2196F3", - "inputs": ["speaker_emb"], - "outputs": ["z_timbre", "z_env"], - "metrics": ["env_rank: 32", "env_scale: 0.1", "env_weight: 0.0 (SOTA)", "ortho: yes"], - }, - { - "id": "film", - "name": "FiLM", - "title": "个性化感知调制 (v10.4.6 动态激活)", - "description": "Persona + Emotion 条件化 FiLM 调制 DiT 隐层 (零初始化 gamma/beta, 短文本自动关闭)", - "color": "#9C27B0", - "inputs": ["persona_emb", "emotion_emb"], - "outputs": ["gamma", "beta"], - "metrics": ["persona_dim: 64", "emotion_dim: 64", "zero_init: yes", "dynamic: short_text_off"], - }, - { - "id": "f5_dit", - "name": "F5-TTS DiT", - "title": "F5-TTS Flow Matching DiT (预训练骨干)", - "description": "100% 官方预训练权重加载 + Flow Matching + Sway Sampling + 静态 CFG + F5 官方时长公式", - "color": "#F44336", - "inputs": ["text_tokens", "cond_audio", "gamma", "beta"], - "outputs": ["mel_gen"], - "metrics": ["steps: 96", "sway: -1.0", "cfg: 2.5 (LAAG base)", "frozen: 前20层"], - }, - { - "id": "ceag", - "name": "CEAG", - "title": "交叉熵注意力引导 (v10.0 创新, v10.4.7 disabled)", - "description": "ODE 积分对齐关键期最小化 mel→text 交叉注意力熵 (带 padding mask, 零新增参数). v10.4.7 实验验证对当前基线无显著效果, 当前 disabled.", - "color": "#607D8B", - "inputs": ["x", "attention_weights"], - "outputs": ["v_guided"], - "metrics": ["status: disabled (v10.4.7)", "lambda_max: 0.20", "t_range: [0.1, 0.4]"], - }, - { - "id": "laag", - "name": "LAAG", - "title": "长度自适应生成 (v10.1 核心, v10.4.7 强化)", - "description": "基于文本长度动态调整: 动态 Chunking + 动态 CEAG λ + 动态 CFG + 动态 FiLM 激活 + F5-TTS 官方时长公式", - "color": "#FF9800", - "inputs": ["text_str"], - "outputs": ["chunks", "dynamic_cfg", "film_active"], - "metrics": ["chunk_max: 135 chars", "cfg_base: 2.5", "dynamic_film: yes"], - }, - { - "id": "vocoder", - "name": "Vocos", - "title": "神经声码器", - "description": "Vocos 相位重建, mel → waveform", - "color": "#8BC34A", - "inputs": ["mel"], - "outputs": ["waveform"], - "metrics": ["sample_rate: 24kHz", "phase: neural"], - }, - ], - "dataflow": [ - {"from": "audio", "to": "oes", "label": "speaker_emb"}, - {"from": "oes", "to": "f5_dit", "label": "z_timbre (env_weight=0.0)"}, - {"from": "text", "to": "f5_dit", "label": "gen_text"}, - {"from": "text", "to": "laag", "label": "text_str"}, - {"from": "laag", "to": "f5_dit", "label": "chunks + dynamic_params"}, - {"from": "film", "to": "f5_dit", "label": "gamma/beta"}, - {"from": "f5_dit", "to": "vocoder", "label": "mel_gen"}, - ], - "removed_modules": [ - {"name": "IBOP", "reason": "消融 p=0.85/0.51, Cohen's d<0.05, 无效"}, - {"name": "AM-ODE", "reason": "零初始化未贡献, 无效"}, - {"name": "TD-CFG", "reason": "消融 p=0.41/0.74, Cohen's d<0.06, 无效"}, - {"name": "SBM", "reason": "消融 p=0.71/0.90, Cohen's d<0.03, 无效"}, - {"name": "IEAG", "reason": "v10.0 升级为 CEAG (交叉熵注意力引导, 带 padding mask)"}, - {"name": "Duration Predictor", "reason": "v10.3+ 采用 F5-TTS 官方 infer_batch_process 时长公式"}, - {"name": "Reference Enhancer", "reason": "v10.4 移除, 循环扩展伤害 SECS, 与 F5-TTS 官方基线不一致"}, - {"name": "Best-of-N", "reason": "v10.4.7 实验证明对 Flow Matching 流形崩塌无效"}, - ], - } - - -@app.get("/api/demo/sample") -async def get_sample_data(): - """返回示例数据 (用于前端快速体验).""" - return { - "sample_texts": [ - "你好,这是 PersonaVoice 的声音克隆演示。", - "今天天气真好,我们一起出去玩吧!", - "我真的很开心能见到你。", - "这个项目展示了深度学习在语音合成中的应用。", - ], - "sample_chats": [ - "我平时喜欢安静地看书,不太喜欢嘈杂的环境。", - "做事要有计划,不能随心所欲。", - "对朋友要真诚,这是我的原则。", - ], - "emotions": ["neutral", "happy", "sad", "angry"], - "sample_recalls": { - "openness": "0.7", - "conscientiousness": "0.8", - "extraversion": "0.4", - "agreeableness": "0.6", - "neuroticism": "0.3", - }, - } - - -# ───────────────────────────────────────────────────────────────────── -# 前端静态页面 -# ───────────────────────────────────────────────────────────────────── -@app.get("/demo", response_class=HTMLResponse) -async def serve_demo(): - """服务单页前端演示.""" - import os - html_path = os.path.join(os.path.dirname(__file__), "index.html") - with open(html_path, "r", encoding="utf-8") as f: - return HTMLResponse(content=f.read()) - - -@app.get("/static/{filename:path}") -async def serve_static(filename: str): - """服务静态文件(演示音频、预计算 mel 数据等).""" - from fastapi.responses import FileResponse - # 使用 pathlib.resolve() 确保 -m 模式下路径正确解析 - static_dir = Path(__file__).resolve().parent / "static" - file_path = static_dir / filename - if not file_path.exists(): - raise HTTPException(status_code=404, detail=f"File not found: {file_path}") - # 根据扩展名设置 content-type - ext = file_path.suffix.lower() - media_types = { - ".wav": "audio/wav", - ".json": "application/json", - ".png": "image/png", - ".jpg": "image/jpeg", - } - media_type = media_types.get(ext, "application/octet-stream") - return FileResponse(str(file_path), media_type=media_type) - - -@app.get("/") -async def root(): - """根路径重定向到演示页.""" - from fastapi.responses import RedirectResponse - return RedirectResponse(url="/demo") - - -def main(): - """启动 API 服务器.""" - import uvicorn - print("=" * 60) - print(" PersonaVoice v10.4.8 交互式演示服务器 (Trade-off: LAAG+动态FiLM+OES+人格分析)") - print("=" * 60) - print(" API: http://localhost:8000") - print(" 文档: http://localhost:8000/docs") - print(" 前端: http://localhost:8000/") - print(" 合成引擎: PersonaVoice (v10.4.8, 1s 声音克隆)") - print("=" * 60) - uvicorn.run(app, host="0.0.0.0", port=8000) - - -if __name__ == "__main__": - main() diff --git a/personavoice/demo/audio_preprocess.py b/personavoice/demo/audio_preprocess.py deleted file mode 100644 index a4829f6..0000000 --- a/personavoice/demo/audio_preprocess.py +++ /dev/null @@ -1,271 +0,0 @@ -"""前端音频预处理 — v10.0 新增 (修复地雷3). - -架构位置: 演示模块的前置组件, 对用户上传的参考音频进行预处理. - -============================================================================= -地雷3: 前端 WebRTC VAD 过于老旧 (听感瑕疵) -============================================================================= -问题: WebRTC VAD 基于 GMM, 对 1s 极短音频的开头呼吸声、微弱辅音 -(/s/, /f/) 的切分极其生硬, 导致参考音频"吃字"或不自然. - -修复方案: 改用 Silero VAD (轻量级神经网络, 显存 <100MB). -对呼吸声和唇齿音的判断极其精准, 大幅提升 1s 样本的参考纯度. - -============================================================================= -v10.0 前端预处理 Pipeline: -============================================================================= - 1. 加载音频 → 重采样到 16kHz (Silero VAD 要求) - 2. Silero VAD 精准语音端点检测 → 去除前后静音 - 3. 重采样到 24kHz (F5-TTS 要求) - 4. RMS 能量归一化 → 生成音频响度对齐参考音频 - 5. 尾部补 50ms 静音 (F5-TTS 官方要求) - -设计理念: - - Silero VAD: 神经网络 VAD, 对呼吸声/辅音切分精准 - - RMS 归一化: 确保不同录音环境的响度一致性 - - 全程 torchaudio: 不依赖 pydub/ffprobe -""" - -import logging -import os -from typing import Optional, Tuple - -import torch -import torchaudio - -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" - -logger = logging.getLogger(__name__) - -# Silero VAD 模型缓存 (首次使用时从 HuggingFace 下载, 之后离线) -_SILERO_VAD_CACHE = None - - -def _load_silero_vad(): - """懒加载 Silero VAD 模型 (首次调用时加载). - - Silero VAD 是轻量级神经网络 VAD: - - 模型大小: ~2MB - - 显存占用: <100MB - - 对呼吸声/辅音切分精准 (远超 WebRTC GMM) - - Returns: - (vad_model, get_speech_timestamps_fn) - """ - global _SILERO_VAD_CACHE - - if _SILERO_VAD_CACHE is not None: - return _SILERO_VAD_CACHE - - try: - # 尝试从 silero-vad 包加载 (v6.x API: load_silero_vad) - from silero_vad import load_silero_vad, get_speech_timestamps - - vad_model = load_silero_vad() - _SILERO_VAD_CACHE = (vad_model, get_speech_timestamps) - logger.info("Silero VAD loaded (silero-vad package)") - return _SILERO_VAD_CACHE - except ImportError: - pass - - try: - # 回退: 从 torch.hub 加载 (会缓存到本地) - # Silero VAD 的 torch.hub 仓库 - vad_model, utils = torch.hub.load( - repo_or_dir="snakers4/silero-vad", - model="silero_vad", - trust_repo=True, - source="github", - ) - get_speech_timestamps = utils[0] - _SILERO_VAD_CACHE = (vad_model, get_speech_timestamps) - logger.info("Silero VAD loaded (torch.hub)") - return _SILERO_VAD_CACHE - except Exception as e: - logger.warning(f"Silero VAD load failed: {e}, falling back to energy-based VAD") - _SILERO_VAD_CACHE = (None, None) - return _SILERO_VAD_CACHE - - -def silero_vad_trim( - waveform: torch.Tensor, - sample_rate: int, - threshold: float = 0.5, - min_speech_duration_ms: int = 100, - min_silence_duration_ms: int = 50, -) -> torch.Tensor: - """用 Silero VAD 精准去除前后静音. - - v10.0 修复地雷3: 替代老旧的 WebRTC VAD. - Silero VAD 基于轻量级神经网络, 对呼吸声和辅音 (/s/, /f/) 的切分 - 极其精准, 大幅提升 1s 样本的参考纯度. - - Args: - waveform: (1, T) 或 (T,) 波形 - sample_rate: 采样率 (Silero 要求 16kHz) - threshold: VAD 阈值 (0-1, 越高越严格) - min_speech_duration_ms: 最短语音段时长 - min_silence_duration_ms: 最短静音段时长 - - Returns: - trimmed_waveform: (1, T') 去除静音后的波形 - """ - # 确保单声道 - if waveform.ndim == 2: - if waveform.shape[0] > 1: - waveform = waveform.mean(dim=0, keepdim=True) - else: - waveform = waveform.unsqueeze(0) - - # Silero VAD 要求 16kHz - if sample_rate != 16000: - resampler = torchaudio.transforms.Resample(sample_rate, 16000) - waveform_16k = resampler(waveform) - else: - waveform_16k = waveform - - vad_model, get_speech_timestamps = _load_silero_vad() - - # Silero VAD 精准端点检测 (满血架构, 无备用方案) - speech_timestamps = get_speech_timestamps( - waveform_16k.squeeze(0).numpy(), - vad_model, - threshold=threshold, - min_speech_duration_ms=min_speech_duration_ms, - min_silence_duration_ms=min_silence_duration_ms, - return_seconds=False, - ) - - if speech_timestamps: - # 取第一个语音段的开始到最后一个语音段的结束 - start_16k = speech_timestamps[0]["start"] - end_16k = speech_timestamps[-1]["end"] - - # 转换回原始采样率 - scale = sample_rate / 16000 - start = int(start_16k * scale) - end = int(end_16k * scale) - - waveform = waveform[:, start:end] - logger.info( - f"Silero VAD: trimmed {waveform.shape[-1]/sample_rate:.3f}s " - f"(from {start/sample_rate:.3f}s to {end/sample_rate:.3f}s)" - ) - else: - logger.warning("Silero VAD: no speech detected, keeping original") - - return waveform - - -def rms_normalize( - waveform: torch.Tensor, - target_rms: float = 0.1, - eps: float = 1e-8, -) -> torch.Tensor: - """RMS 能量归一化. - - 确保生成音频的响度与参考音频一致, 避免不同录音环境导致的响度差异. - - Args: - waveform: (1, T) 或 (T,) 波形 - target_rms: 目标 RMS 能量 (默认 0.1, 略低于参考以避免削波) - eps: 数值稳定小量 - - Returns: - normalized_waveform: 归一化后的波形 - """ - if waveform.ndim == 1: - waveform = waveform.unsqueeze(0) - - rms = torch.sqrt(torch.mean(waveform ** 2) + eps) - # 避免除零和极端缩放 - if rms < eps: - return waveform - - scale = target_rms / rms - # 限制缩放范围, 防止极端值 - scale = torch.clamp(scale, min=0.1, max=10.0) - - normalized = waveform * scale - # 防止削波 - max_val = normalized.abs().max() - if max_val > 0.99: - normalized = normalized * (0.99 / max_val) - - return normalized - - -def preprocess_ref_audio( - audio_path: str, - target_sr: int = 24000, - use_silero_vad: bool = True, - use_rms_normalize: bool = True, - vad_threshold: float = 0.5, - rms_target: float = 0.1, - tail_silence_ms: int = 50, -) -> Tuple[torch.Tensor, int]: - """v10.0 前端音频预处理 Pipeline (修复地雷3). - - 完整 Pipeline: - 1. 加载音频 → 重采样到 16kHz (Silero VAD 要求) - 2. Silero VAD 精准语音端点检测 → 去除前后静音 - 3. 重采样到 24kHz (F5-TTS 要求) - 4. RMS 能量归一化 → 响度一致性 - 5. 尾部补 50ms 静音 (F5-TTS 官方要求) - - Args: - audio_path: 音频文件路径 - target_sr: 目标采样率 (F5-TTS 要求 24kHz) - use_silero_vad: 是否启用 Silero VAD - use_rms_normalize: 是否启用 RMS 归一化 - vad_threshold: VAD 阈值 - rms_target: RMS 归一化目标值 - tail_silence_ms: 尾部静音时长 (毫秒) - - Returns: - (waveform, sample_rate): 预处理后的波形和采样率 - """ - # 1. 加载音频 - waveform, sr = torchaudio.load(audio_path) - - # 转单声道 - if waveform.shape[0] > 1: - waveform = waveform.mean(dim=0, keepdim=True) - - logger.info( - f"Preprocess: loaded {audio_path}, " - f"{waveform.shape[-1]/sr:.3f}s @ {sr}Hz" - ) - - # 2. Silero VAD 精准去除静音 - if use_silero_vad: - waveform = silero_vad_trim( - waveform, sr, - threshold=vad_threshold, - ) - - # 3. 重采样到目标采样率 - if sr != target_sr: - resampler = torchaudio.transforms.Resample(sr, target_sr) - waveform = resampler(waveform) - sr = target_sr - - # 4. RMS 能量归一化 - if use_rms_normalize: - waveform = rms_normalize(waveform, target_rms=rms_target) - - # 5. 尾部补静音 (F5-TTS 官方要求) - tail_samples = int(tail_silence_ms * 0.001 * sr) - if tail_samples > 0: - waveform = torch.cat([ - waveform, - torch.zeros(1, tail_samples, dtype=waveform.dtype), - ], dim=1) - - logger.info( - f"Preprocess: output {waveform.shape[-1]/sr:.3f}s @ {sr}Hz, " - f"RMS={torch.sqrt(torch.mean(waveform**2)).item():.4f}" - ) - - return waveform, sr diff --git a/personavoice/demo/index.html b/personavoice/demo/index.html deleted file mode 100644 index d5693ff..0000000 --- a/personavoice/demo/index.html +++ /dev/null @@ -1,1367 +0,0 @@ - - - - - - PersonaVoice v10.4.8 | 1秒极限语音克隆 Trade-off 演示 - - - - -
- -
- -
- - 连接中... -
-
- -
- -
- -
-
-
-
🎤
-
-
声音克隆
-
1 秒极限参考音频 + 文本 → 个性化语音
-
-
-
-
-
-
-
-
📁
-
点击或拖拽上传参考音频
-
- - - -
- - -
- -
- -
-
😐中性
-
😊开心
-
😢悲伤
-
😠愤怒
-
-
- -
- - -
- 提示:情感强度当前为预留接口,不会改变合成行为,合成结果将完整覆盖输入文本。 -
-
- -
- - -
- -
- - -
- - - - -
- -
-
-
- PersonaVoice v10.4.8:1s 极限克隆 | 核心:LAAG 动态 Chunking + FiLM + OES 环境解缠 | 短文本 WER 为 Flow-Matching 固有限制 (Future Work)。 -
- -
- 等待执行声音克隆... -
- - - -
- -
-
-
-
-
-
-
- - -
-
-
-
🧠
-
-
人格蒸馏
-
BERT 编码聊天记录 → Big Five 人格 → FiLM 注入
-
-
-
- - -
-
-
-
-
-
-
聊天记录
-
结构化回忆
-
-
- - -
- -
- -
-
维度
-
范数
-
均值
-
标准差
-
-
-
- -
-
- -
- -
-
-
- -
- -
- 全局 - 局部 - 情境 -
-
-
-
-
-
-
-
- - -
- -
-
-
-
🏗
-
-
架构数据流
-
模块依赖与数据传递路径
-
-
-
-
-
-
-
- - -
-
-
-
📊
-
-
条件嵌入
-
-
-
-
- -
-
克隆后显示条件嵌入
-
-
-
音色保真
-
情感解耦
-
人格对齐
-
-
-
- - -
-
-
-
-
-
系统状态
-
-
- -
-
- -
-
F5-TTS DiT
-
FiLM Adapter
-
OES 环境解缠
-
CEAG (disabled)
-
LAAG 长度自适应
-
Vocos 声码器
-
- -
-
-
-
-
- - - - diff --git a/personavoice/demo/static/demo_data.json b/personavoice/demo/static/demo_data.json deleted file mode 100644 index d519400..0000000 --- a/personavoice/demo/static/demo_data.json +++ /dev/null @@ -1 +0,0 @@ -{"ref_audio_url": "/static/demo_ref.wav", "cloned_audio_url": "", "wave_base64": "iVBORw0KGgoAAAANSUhEUgAAAyAAAADICAYAAAAQj4UaAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQAASWRJREFUeJztnQe8LUWR/7t5PIICYmZx1XXXxM4aEAOLiGtAVFBQxCyi4CKiGBBXEVcU1sAzomJCMS1G1oARWQUVV1HARUYUVFRYFQMo8gcJ783/U+dVX+r26zjTE865v+/ncz/3nDkTemY6VFVXV+mmaRQAAAAAAAAADMFGg1wFAAAAAAAAAKCAAAAAAAAAAIYEMyAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAgAAAAAAABgMKCAAAAAAAACAwYACAiaJrtUOulZ3HbscAAAAAACgLFBAwFQ5XCl14NiFAAAAAAAAZdm48PkAaIWu1S1JIW4qdamuZ/XyzkqpbfA4AQAAAAAWC8yAgKnwaqXU2/jzZvx/K12rVSOWCQAAAAAAFAYzIGAqkKKxna7VTkqpW41dGAAAAAAA0A9QQMBU2Jb/v3vkcgAAAAAAgB6BCxYYHV3PXK5u7/kZLlgAAAAAAAsEFBAwVTT/v/fI5QAAAAAAAAWBAgKS0bXaVtfqsIEfmVmQDgDoGV2rv9e1Wo0HDQAA46Nr9RJeG7twQAEBOWynlNpX10uzE6Xp67xzDT1vXavjdK1uMnZZwMLzWaXULmMXAgAAVjp6fUqCpymldlYLCBQQ0Ka+3AKPbfDn/gCloICAQcAMCAAAjM8d+f/TdK02VQsGFBDQhi3x2Aal4f9/h+cOAAAArAhWLbK8vnA3BAYRhF/e83OGK5abd+habdLzswdgr64JQHt00wQAgJUsfy0MUEAmhK7VA3StHqGmixEqthm5HACA/rh/geAPL9C1ekqh8gAAwErP1belWjCggEyLxyulnqGmy8Jp4GDuFuPfTdfqRmOXZRHRdZmQ17pWWyil9lNK3afE+QBIqHOHlqq/AEyI94nPr1+0mWUoINNjO12rHdTK4r7i80I1MFCUf1RKnaiU2gPPtRd2LXSeB/LYgrYMhoIU3oMQQro9ulZHkBdGwXcCurOp+EwGna0X6aFCAZkmH5ioptuX4Hc3NSeQb7yu1cd1rR40dllWIMb1DzNx/fOcAueYYh8GFtuQdTaHLgX57K2Uuice3OTQns9zDxSQicCLPv9FTZuH8P9b96Ag6TmySJAl/kUTVRIXGSNY4LkXRtfqaKXU48SmfVueh4IkvMx8LVM6ALJA8low9+ha3VQtOFBApoNttdl44kL4jdXKhkLiDhWXG4LcejDz0R97KrU8wpqul7lG5gh/SJgJxmQtHn9rMNZM2yX2NmqBgAIy3Q5g6rMhQ7H72AWYAM8euwATYTc1R+ha3VPX6rlq5T5vCDMAzA9k9HzWSnNhY7fqm6tpo/n/29QCAQVkuiDSz/pG9zA1LTYZof1Mao2MrtUuulZPGOHSpi5sOifubxTO9kBdz234xMd3dAPQc/KeAFjRWO20qJVd12ozXauHq+lCC+9PU/PBJmqBmDsFhKJckPCzgFq67V6CgXuaPEq8mzGm+m88gYHqlUqp/UcUrF9C11fzQxtXprGw+52DMpMSSkviLtZ3AIojIl+ZuvsYPOZstu7R1fXvlVJr1HTBmqGRmDsFhAWPI5RSt6Mvulb30/XS4uhF4nljF2AqUN4HXauNdK1uP4FM4EOt+/AJhZ9S49//rZRSf6OUeu1QF3W893uo6bMIRoQnZdb5z1nfx26vYPG5vfV9x5HKsSgsQr+1CGy66O9iHhUQEyZuYzFAvkXXais139gVjYS8KRO0imZaTWNUSqk7KqU+r9Qorj9T4lYTcmt5oK7V5gNd6zPW96k8gxCmjLtO6J3No3/2fmR8UAsO6khrrpqT/mBeKG3cWT3F+s2JbWn8mlS5CDK48kz/QjNXCgi/FPKrNtN6hJn9ePBIxZoreCZhlwIubHsGrkFTmifzzMWNda1u2fFaB3Ho27FmIHyYDqxv7CnxZ6nx0QNbuG+r5juQwjy7jHYZoLsO7lTXDxX9fh8KzugCCPWTSqlPIpFe90eplHoQj0G9outZKPZo0mBdq91oDQTLL/NA6cAZ1H6J0duZxc2UUmdONNCNViuAeWkQrpdDCyQX6SVJIXN2Xz11WDSz8g6l1FN7dBFbzQIjCacvVUp9jQfYtlAG0KP489jvXF6f/NvP1PWsI1spawqMv3UyNDvZReG1FkJrMfsydfeesetqqbI3Iz4DcvXrBQ4z/APO3D42ZMi4S5v21dEY9Rxdq7urxWMIw9AzeHY+xot5DcQt1HxQut/avuS4xUYDk5S2C6atTaH9r0jmVQFRljV8ngd6FbiHe/VwnWv5f1ehebPEezLRvFaXei66Vu/T9WgzIYeIz0apOrjna+oJ1fGs586K5xlKqXd2UEJ8dW2y+SbYeLDT2OWYAFsUOk/R+s/Gq/fx15W6UP6WPLv8ULV4lHQB3oCWfVm2vKVrtb2u1Z0LeBGM+rwKu5WTov7VAueZkjfFGO9jdOZNAbGj7mynFps+17U8Q9fdBDhd+92welYA7juRzsMoiPPsXtMF7Zu+ZuvqI5RSO4uFoed0dDGZihKWqjSlWEdHR9e9urd9ssdzLxLNyMaUecbVL6zr+Zr/wP8P68sTg9c9fUgpdYJS6mOqX242R/1rKfe6KcgQPh7ftp/QtdpiXtbMzZsCsrvllkONc1EYqsLoApG2zDmOjiw231LkbigdmWQwV4WAK5CxSm3bszvglDOAP9CK934HkSn+GKXUGzxBJFZa/zW0m14OZn1VL8xBkq+VyqPVYuASJA/o+ZpkvNOJY8Y2LQVesvQbQyQFH+lzzLtefN6mpzU0pcbImStbASH7CjVdtvFsJ/cz77jHvz1dKXWSmgPmaQD3VWLz/dZqvtlhIAuEHHQon0qSi0TA3/5vaTra0zka6zfd1975RV1eBOv5rJmQELljz1HL9Bzk5iFXq+9RGFauC6+Wu4h7KLnAcWkGT9fqnya+JuR1aoFhv2xK6OXiBQXeTek2sJc47yKugViIvqXDPeza8zXfLD6vCQiFsm59PjU6JPehb7Q231v1x47Ws9x4inWNXa/Nc6GomF3oJbBFz0bHG4vZNxdUR549hoF2pSggvspcOnLD2PcVbKy6Vo/Stdq2xfnvaZ37ph0X9T2E87KscWjvtAC9r4GOLOytINezljMWpFD5jnukWpnQOoezlVLfEdvO5lCOttKoO6xrir2vjyo1izQzVYFq6smuQs9tdaJyflxAuPl4ixCZd5ADsa7VHhRZT5VBRi96rBofsw6jTZ+eTVf324kxaJvnOriVuPZuAYHdZzBtw3sKh7eXlFjUPcQ7ut+yk3abBZn6mK1bjCVjJQdeLAWEB6BVwsLadwOZN15TyMrzbx2PfyHPbjxELMx76AQXT8/gOvUtpZZchVKP0xzNRPKv4v76nvKfCtdE3mmKW0KuMJ466P6HldF3SvTtk64mvI7jZhGrne+Yz1n9FCW+/G4hIWzZ2Kfr5S6iPP4MOT6a7N1PG+h6s0S+Yj3gZGWBltD762vsIQUk9dyrCxskn9jxfKnXnNS47cCU7+EFzzUPMyA6Mjskjb6TZ8qdDllQP8Kf9+SoB/edg4ozdexwuLkLZUPPfOOeLD+2Fb3L+Uyd/2xmJJMpLVgbc03I2sCA5XpPLl6f+exzBvHjVSbsQli6L9RztI6nj1C48v7buF+F1o2U6PdlfdUir5S0jq4ZQUEdKqDFZnMkC7Th1j3OJuUowKVllCzDWQhyi9S1epDn56nWh409intbFkmG3FzMZE31/S1jyoWkyCzk1/02pdQrraSDTuY8iVOygNI2nCk/n/u1FORddUUPLDSUfs45gtGk/MQn5GrkKkdshuTBmTNG+2TsSyErX5sZpvlESl6mOqJr9Whdz/xvp6awUtmeIgIEzAMuRbckMdfTO1MADQqBqobtlx494Dg2lT6kFLYBZKtAhL4u927aeMr4nTvzF0Lzus0nFJoFvKtS6lhPkIgiBhPrOZdyQ5Wyy2065OGaZ6NQlJw6PtYM6JQVECMc/gv/T3mYi+TXGus4iL0GrDgpiQQpMlmuO44WHXqqBT3p3Ank+EtubXV8ORb/0uw40FR8Xxi/6VR2yTz/HrK+Ugz9iCBJ4bxvo7rzgEBW3dFcsDhAwEs5QMDTes4SXaodNKWuoWv1ZkeyvfsnXp8SXg7RtqVR6VEDXG8lsLeunRbyN3R03bFny5ywEeQ5Vn0tYZigNZddEvsa/l0YeympomT3HhT9lxc437rYupC26Fo9S9eTGVebAudIet4cIOTTHZNFL44CouuZYHik3GT9XykKyBaR93bHAS2tf4n8rgeyoHfFhDZUHF89lVepaaDFux+DUn3GHTMiIzUt6sfdZjuudzH4GoUl1PWG/rElImfpWu2n6w0UmE2mYGVma7oMEPASilYWGWz6ymugM0Nk+vbPabdmTdqdRDmcfaau1S10rZ5JgoiVK2OrARY1y1DIr+qQtDMVWk+zKPjcfZ8gFUixrpT6hKFmtlzKQxeKtE1evG3GQld7KJUbyF5r1bVe39LxPErkg9PsYfPABVJAdkvcbzUr1YMHSpmkAiKsQTmNTY/YqZRGi0W1Pl8/XaCi5j7f1LUYrjLdJ1HYS51VuHlLjV2GNkyauuVOMxaueMsBXCem4DKxe8F7uVGP9/92Xc9y0Bwrtu0dEGa7PNtDeVZqc7G4l4IzTGG635d/500Dl0MHomSFaDzrA7vgUiioL/k6v7dDHBbhocbhodr4rV2CHQvpU+hnSvEQ0c8fpZT6AcsXXe6xi/sTzc6OKpuxwitD2P6TYzdv5nVdq+fqejaz0ybB4SMLzdrMDbpWx9NY1LJdac/nPhi83U9VAWnLFMIpFoUsFQ4fzZIVJTVJWI527MuJ4RLi2wpmukBc9NTnmDrLtLSOQNezxFHHC+vbZj2GUBySMfLt+GbJZrNSATfENyZY4WU0oK6Za8l6tpFnAL+XrtuHju6Aiapk3/dOul5ymWx6HIDkOXemfC0dz5F3oLvNPSWQFVyPODbK+9yeZmp0PYsueM8Brnsot40jebZw3mQDl0HM3kZrzwz761o9iWZFdZ3ep7H74j0GENoO6uPc/I5vn+DW9HCXQY23Hcjly/GcMPfSh8dGW2NB70Yhft7347Go15lU1b2+HMMBWQZTRKbayeT4h881ulZb63o2LbsqENXneYH31rkRJXbAV4dOwf/vyuEs90nItuo7x1Trb8qalI1EJ/3f3PGYWZpv8hqBRUEXdodzX2S9IOQT3I0w+9RIuczsHQkPd5C5dAqG7f1Hqz252nMpZScJXc8GvH92CGLmvxFsS4Q4Tw0f+9EB3Iskb5Vf2Jizf0JUrLGpuM94i1LqwwMYL3Zjt8XHsrua1wI+QWLvi9Y93dax5u9wVkZzckmkBGMpgXTJy5nZ9PYFup6tp6X+7xOJ69MqR1tdWjuTOONvP69evARsoZkF6Vg/oztGBB0SLT5vkmgkpuSwt0ycwSfIIHUOpRtgt+JtVqoCQhq2mtBg0HciN/JVvY3nXlc7Fne3TfDnWiOjC66doc78vaossYhKU8C2mkqFjhZAr+bOITXpY0xQH/y+dT2zGO5hKbxdy/F0cX6K/mMCTkiCbna6nrlV/YPHD9y1TXbSbazx9vVdbmQv4ISLYwu0of7dzMzRIH1YT1ZI2xptvp/jibzj4nEdy2UWqG4nFiCnkvX+OF/VAYWUhbtYrrYmJ1Zn5c0zu0HX+U/x2TVLNFVi7+eZAYFetwi3nFovuoQBjvWzS31nRhJdiij6ssC17PZ6K26rR7Br3l05/5jOcGGW/S0d97IOUTxDz/s2lmHpHDFDflNdzwwxztNGvpck9535+EDiLMWWiTmcZEAGOu++rJR0Xa80fwoILxCMTVVNRfDsBFci6hhCFkQaqG9HjUhsu0PL5+BrhKUHG53hxjX0u2zENW+k66XnH+K2ieUli8O2VrjovxF+vxT+MWuhF3fWqcJacXSt7k/uZCxU71p42lo+y309LpSxcKy54XOPE21PCoo78n0mw3Xn7R38vMfmxj21x5jRQEsjCodFzRJKUlyEeM3ZZlbkIhLqc6F+ImWsJIvh8wMuqCFs4W8Pa9t+XIZv+MKOkoBIM+oJ19onMlM4ZpCLZMVf10tuYykK38MCisPNOmbUJlyGiM9FDCJdosDljgmrAjMnqzxlMy6sZCA9l4VZ+Qyvt5TvjVsEr8nFV29plove4wtYSZJKCRlH3hOYke5TBpHjPUXZSvXuaQpdv4vbFwVweHqfLlmTU0C446WBo+uCnXnglhkhF2/L08hdcXXCS4qJrtVdPB1JarQgXyevC1laJX9boHHQDFQMn5LiGlS+YmVM31d0QodnWmBzQwX3wbtEcIcHZ3SMuoNimKqAmOcfWvulHcELtma3rieI33dOCM1q8zzH7GTo+mP0TyEL3/4RgTx3Bifn/sjHnP7uxxGZyNJ6UMbag5T9SGE2kBX0HpabRUgolNtplu0MtgSHXGOu5P/HFQ5IobmubcT9wQZ+9PzcyLWG1p3dv4BB5QETXwdCSth+/ExSFN4qUr9T3U1MX64T1yH01eaT6xePkUYod/Ei36FWG9ngXnhmhJS715GQz0owrV1anSoAcz/QVQF8slLqdO7TlsrMBu1nCDc0UuRXdUiwmstnre85mcp1gevn9EOuserFHEWwFybVwbDgK10XoodY3+83Zwt9Q9Yqe3Ck6fEvelw+crGf29+wRYmsI58yAjQNtrpWb+HZl9wIVi5K+dub6xyWEJ3Kd6zduZYsl21lMdfZhK0Kf1vkYtOPVuNzwXGxLmDdss8VyxESE0j28HTMuf1hrkGgtyhpbIE8yOr/SljR+qpjxvXleI6sRt+fkzFDkdIfSUGdLPrbW9d3fXZRcT9DCuseulY30bWz31klrrXRwM/9nXzMXRJmBVPrRd+LZvtcI2Fzm8hzDSY5FuwVyc01RXwzcq4+2ddPa4/rzk14puRhbIx5Oq9dOptzsKxNSLmwpsOscWh8oDoi2+n+7Fp0a4+c2EdfN4SSU8L4F+JN7Na22AqIUurNlAk2sk+scbhCbc4DqZXmdj1dk4TiPfnzU1jROYE75rf2cL2Qn2yOX2aukNVGKEs95j8c5b6tY9vh5B7SUYHYK2Phb1ee1/H5+dosRUXaxLUfW9ZyI6boDEVAFzCW5MxOlVZ2bbbIFOCHJDTL4FMEYsnFvpNgobcX19685XuQwsrTOMDEhxz7ybV0qwZWTFNmcnPQHZP19Y1xSd6oZe4Guw48KfG4qBsxrzk4WnXD18+aa+W4+fWVnuA+jnq+h+iD7uUY37V1b+8obCyR72Jzy1XNjCcncThmF30l44vOJuv1Y55R+g+IHFtyLGkSzn3PlaCAuBah5go1swRkc0ysYh3R4zWlFeIVwq+VLIereqpv/5LYmHz7NOxDfvNYNC8Og9q2zpds8BQN6yNKqTN13Tp09CaWRXcoa+M+mc8lpExqIXyRsDhbC6DrmdWIrvl9R71Ltc6piMsBzfR14Rae6099VkrS9+xfl4ALh6UoLzxo+2ZD7LrT1jhll4EEGdc6jD+KfX2RhVykrn/JdYnb8ATrjR7bljgfP/vZWhFdD674mjL/K1vfU/f371CruyecZ/OE895BGPNc12kTPn6DfjTDze8Rqh82jqwf3StBib2pWB8Ri+wlXQtTZQZSNgymrCGviWKCNs/umHK4fteOZ0Gunm1DFcv8NiX642VeIn14XExNAfERc6mQ7EkCzMT9Vw2btbhf8mNuWxFcVgZzLjsSlj0lWiJLJuXGsAfbrpYPKvdzlVKnKaVOZVcU3+zCyTJaRoJFYnVmJu6YICq3bcfPlNwG50lovUXmDFUqxgWLLD+nKKU+xttdgk0pgb+t4EV17qstzjdWMsKpEXpO/8yhI325j7QVLZEi+3zBcy7KaVFiPZVrLKF+JuR2Q9bfJXhM8gk3IWVF3neq29EjKMeF57dtEgR2c82XR/YjJezTfF9kTOnLehwq47PF++naF6REg9zPUQabLSL91dLYysazVS0DtqRGVTRrGHOF09h+NPP0rMg+W9iCrHVe8+42TnBjM/W2j/FSzr6/tlCo8B0C19A0xulaHaZr9SBa7C3ad468K3Emh9S10005d1yiPFezdTq8xqeIfD26kM5WFPKprSLCRaqlVbMAQ6HKsmJlj0BqBJ+QMOu0MCRktLV/i2Updr2L3Hf1dyIaTRuhzOWycSOrEyQr1lldY3rzoHCWrmcCd0hx61pXKTNszCI2ioKia/XEjDLYg0xsX980sLxmKNRjV2HDZ5VKmemTg1NoUNe5eU86kDsTFdsn6fmS5VvX6n9T9vVcz34XDQtWKc/zIBaCSLjXllD33oL36/ptK0eIS9m3vdv67bmBxemps8vO9X+09sdRth0j50huP4GxxCiJH7aFZlrnpuvZerfcNXptWMrt06IvaiMPuc6tHW5FKiEfFrn0/YByLuh6mXHMhOVPaQf+gq4XpKWi0sZCHhvvQzw045mTgbJE0JWu4+UehQyurnWNq3jG9umscOzLSVBp3e3BGf2R67n6FLhYfUyVwW4pAu3kRp+cpgLCN/Utjg4U6zhyGg+5p7w5I978oLCgc0DG4KdztF5eu3GM53yuc8Ua3JMKNe6lgZHfjSsuec517IgvH+D/Ma0/dh0znfmKzHK5fr9TZD97AE09b28WdY649vK2h3u2hZ5dw/1RqL3m1j17wIwN6HS/X08wXMQi5viu8cgJzszumtDfUn2Qg6Pk5in5RhLRgWho9n72+5RRfsiFYceWwqhrv3vHjtf1rA1/UJ7DEtppbeOWvDj3pi0jDPp4bIbyEHNpXNZmuL6+0na1YcHWdgH9JPUbLGB9iUNU31f1QxthOIYrpHYqjceAoSPrm0zf/zIx65tKipHtZS0V7GifYG3LaWdNYH2VbYByXcdXllD/7lOihjbu3ZTv8VCx7Y5WXioTrCd0r642b8/a9HV/N++Q02wZUxgMTcjdNiFmYw+XrC+n6XqW2XFjnjqadfY0zZ+bk6EwrkrWplPwWc9Iu97N8ieMxRa3ryEr8N6eBuFr3L5yH9Zi4V/onDLkrcS2CqpAtlfpz2/z4IyyuLbrAsnUfNjuZCX5YkerWe7+KbNMKddqW1ZiNXescnBwYaw/uULt7WzXnJ6xE4y52u8sWVdimEsXQQE2sq+vzG0UXxmlpW3wAl1aeNW1+gdH/7K9rtV9OCywLHvq9Zfuj5ULpwLDygP1jx/UtbqvrmdrAQ7OvM+bcd+/vWP8OsJhTKR+47/kuNCTi+nzM8bI1L6BFK4NjEEsO6TWEcpDEYM8Aex1STTObJ3h/685AWuK60zoHG3eTaqBIHg8r9m0o0St9jz/W1gRyFL6Ft8+KXJKCResdS0jrt2z5XNvGyE1Nvaa97lRaePnFBQQnwDdttNyHfd01q6/z241ZNH6GseNnjIplmTf9Dhp07T9jbwmIkVjdSkVvnKknMvVEVF5t3b4h+ZcK8nykbgw0s4NYNi0kEAcO9Zu2LFzynf0jx0WrPkvUs/CjcprhsoSsi6FzmG2bS06NGMJkucslT3Wd33X97t6LMm3EWsPXMemDOhtktSlYFxdDhT9gcyDIfEZEly/xwbk3MXHscG/YWE3u88RaxC2yry34GlDx4r2t4VjX5rx+Iwj4MFbOR8AJdK7O4cHjV1bcidOEHok5zi4mWP/B3N408fz83wfz4j7hB9f3TUC4cMs6/611jW1R8B+a+IMby7ZrmSJ+z+RXfg2Y4Ml1e9TI26ydE56H7flNSJJgr0npP73HQprisDsOr9vnMw5n+6qaDj22Zbb6kmuvlfXN7gycp2jrOZfT7hOyrVDx+QkwE3B12feK/IM1wTOqUcYW1zXXsftI2Xd1KQVEFmGlIYbGlRcWpsr2c4J/Bvlvthd1zPXiD6sNCFSXcN8Ap75fivPdJxJLvhAng7/BicLcp27xL3nNPh9dD0blPvIwC7vJzbz4NLsDasyEjW5tvu+u/bvYhWnhWy0oL0kRyS2sRKKGPnGm2fgWuD+6A6KaZvyrd9Yq3frWr2MEmSRFZvda74slDP7eaQoxdoRRawUpr7uJ1z+jizQvnXHhFau775tPqthavnfz//lYm1fHQ69v9S6Tr/fOmARPMPq6+V178yKAeV32tRRtlBZX8QJQsl6/kLHO9Zi0e7mnn4+NK7IbbRGzUBrFXahZLVWsmBXWeV5iiVTZeXgqSm7Jvxu/5mZPlrT9D02WJ7J7+/9Yp2q73x23hTf8zDKgckHZe9z10L9b0hJSxmnY+OW/M1XB118gfv1mzjqDs0uy/e7WWF5xfcs7Ofb2sLPbojaOkesrehA+VKuGZqxIeOZyw3ckOr2vJ8w8jQsd77FWg94txzPohLTTF0xFiTdwhfYPs61za5QdsUwQvmRup5Zh2iB2KZNpf6khiG1wocEncoKN2cfk6IZ28/c97kL5hyHWN99Zckpr+v3l4gFkqHzp3TGqYTqnq8cFEJ4JkA21XrLC3dg9LdF5N5p1uATulY7NZX6S4vyLj9xveRqox3POPRecjpQua+sm65BIDawudp1igBs/27X9534T7oeacvtJKX+ynPT/zYhOFNYNrA5Ip/43l1MsDDHBAe4hDaV2pYOaXkcQW61m3vyvLjuP2eb/R515gLynLri228V/w8l1UsVAlOg43ez2oVZ0JqiBBhKujrTuf5NlM+USyc829i4Evv9Y5HnumlCPTKQAnmx47eGk0rGoP0P0bU6vqm8ZZZtKfR8Ym01p+8I1T/5/XBPmTUbKL/EwWRuGxlbUt5pm2NIOboysM/yA9aP36tZMf8iLy7fLrMv1I7ffWUll8qNm2opoMHrPecyx5JbJCkHq8Xi+FWZcujOQh6h5LHnU9AfXc+ifZF753lKqaNmJ6vX5+ZpKnVZ4HxKN814kSH5pf03+476BODUEKi+G/ENMK5zy+8UX/y8EkKdCw7h+CmPoBcru72NhNa7y45I1+qHnme1UaCyNQmLhGPvJWUwjQkq5lifQBnLnm2uQX8vaapZZyafiX3MVeQG0VTq3KVC1DO3hm84yum6nkswURnPQ1p9aVboenbzMetPTJntc8jPH1VKvbep1O9USygSC4eXdT2jmIDZlrUc0eNUz3VdSoj8zZTJ1E0fsn763lduRDbX59AzavhdP7GpVJ1xrXhh6plFnYwQq4T7gHa0d7v+xBQQWT+fYJebr2t8/s0xsTrv6uvN9nWOvkbu4yqfxJRV1hnXOUICqV1HQvfywqZSX7X6lpiS7vrd9q921alDPElhY88oZIm1r6Eytqf2AbTvrk2lfqs6woFLThN1OzZGuO49JGfEnsNGgb7YjEcuAdK+xtWsTNn7y3Pa92f3W3S9g5pqlnncfk40C/hxTztwPQP79xgxo5rcz7V2IHRN89ufeVbJdw+xsqSUz9UOGnaD+lNTqc96T7TeSLgZuxjSs3axUYu20wSOlWX0GUpc57I/h46zyyef2WO4v6fvv2cD4ruEW7Jdp/6d3FB9SrJW5zX7exqN/cJ9lUAKR6Hf7X2NFWWVeNBy4PE9jOXlX16hYy/V1+n49pf/f0Exz3lwt+/Zd4++7dtyqDcpHNj3FCufsjoiEpZ/yNO694/kawgNWK7nFfreRJQG30DuG9xjg3iojDZGmDHY78Scw+z7SVZG9vW8P9/15P2EBGFXHTSfY8JzrOM15/gZdwyxSE6/5np4LLfBVWzVuF3gena5S2DXmdDzdW03+AZx1++zpIeOOtxWAcnBCMgNhwu/iK3M0m/ecBm7UXyArXENK6frxB8d+wdO6LmlUEBc/XbKYGSOcQko5l19gYWDG5nFs55jXG0mZbsKfLbLaWPuPcW9OEUBie2rHIJn4/juKq/db8h9Q2XaSFzT9Z5C9xjqx+zruc4j+/nctvp5XnSc6nJ6Aid3fCofRwv997fcykKYsdH3Hl3yQ0of4BqjQkJtSEm293e9K/lZe8Y4mun/E1ui7+hp/2Ngywdyu62cyN8kqXXV9RxTZUJ7uy3//YBlgzPZYE7R3e7kqVs+ucvX97kI7WOXzeCTJe3jbFLakZQ1U+qlfd2rWVGRCvxGG/OPclB03ZzZHvvd3jd0HsUD5zZWx2rfmG0JM+fRmR24/K2NcPUzHnCvcdzLusAzcD2PUALC1A5RNmot3A7+ylPBrmcin6d8DimDu478N+ewOxTffva+qZ2H/c59syCpjU0+w9+LUHiu3+2GZ64jE2H5ymMfn/oezL6ue3PVDxJAU2ZBTDbk67nOXM+dRAx7QPN1RL57891rTKBxdeI+q7FP4DD7bGwpIb4+piuh56S4zV7J9c6lgGzO7/I6ntrfSBhsVvG2/1NqZlneUtS/VZaQ4xqoXEKBLKvdNuUsHU3DX8LbLxGBBFY5BAufUG4r6q5nZc5n/y7PZz7b7cpXn+xjfP2SXW9cZVBWe7fbhi8Kjut9+Ppg+dzMzJZL+bDvzffZru++e7Ofiy2A6ITzmO3yHDmLe2kcuzvLCOdHXIld/Y+rj5DlDZU5dB1zLokZi0LHmONkHfUpyrKN+s5jymD6h1UiOEBIlglhy1X2f7lP6vlU4LmHZDt7m6+Mrmcon3GsbPZ5XdxduODR+BFaU2GXw1XuEHZfKbfZ9ccg+1X7mFiyU/s4eU5ZT6WekIPmetmwnLEkE4/tgrUJT13RNH7KlGOsk3ZVOF9n7xJM5L4/okglTTWzUBbH4bqgPNZzV9lk5ZwpP021XHDW9cy1zSh3EntQtwfZ1EaRK3C6ZiVcjSZ0rE9Y8pWX/p5jpqjZTcIlGM0syU01mxVY/+P6xVbfFedbFbhP+SxkQ7c7bZeCstQY+fuB7Bu8s/CTdfmZ23Wd9v16F5dBXc+mu8/wCPIS15R6Tn2Q5V7LARP+x9HpmuNsN6JYn7B0S45yuspg/+4aJO1z2v99fZMtENH7372p1K9UP/2JPVgYZUuW1VVu12AnP9N72rOpZoYY13W1R/FxCaeyjPZ1pQtW6myi+d7wou4ne/oq8zlFmHL1ja46dzZn4/6uEMpleexzyvOlumAZTgvkNwoJcfZ1U2eHXDPI6xKNfrI8tJj7NU2lfpJw3XCh1ieq+7ajr3ApQ67n6KrbkpCgZvb3We19LliuPo9ce/b0CJryOFdd1eJ6j20qdYHjOd3FcvEOCb8pQrHvecj79LVVux3az9g3zstyucakFIE+Jni7rk1/JD+9h+TAwBobUx9vz7PUroiUtpdLisDdOOpZ47gvmiH/Dhux9gkc4+qLlt2GZXTxtZm13Ne9U3gF3YE9RkwOJJsnhp7hqFGwmmqmFZ0beUAxAdklSNhCnWu7vU/DUUMOZYHoSX0pH4Gyp+4ny04V8VLHMT93VMCYsC6vlVp5fc/StY+cKbK/x/AJjPZ1l5XF8o+9zrPv3lL58JQ9VJ7UOup6Tg3PPNyjqdTdmkp9u6nUxU2lPkrfOZGUrx7L7xd2Xa/UVOoKDk0deve+30LbXfsZ3tFUS4v9fPfWtm/wYddD198ruHM/iiNL7czGEl8ZfWzwe2nlg/m1uM46dq17SaA8pv3ZQqa9rzyG3K5srk7oC0J9hH3tH0fK7OpH5HnXiFxDvr4sVB8NtleAr785oqlmrhl/tp7pusRnkVPXL4jch+v52mVOxRzzTc9zPCrwHkxdNH/Hl1A+Zhde38ftlPDcUvuG2LuJvRd5nqMjZTGfP95Us2AW/y/herG+hcZ5FxdyNK9YHXSV0bctVhbXNtc1QvuaPzLEPZP7sVh5fHUh1p5822f1m9a7hZSP2c6V+ktTzRZf35ddC02SyaNFABy7TLHxJ6Xs9PfwppqtcX01G8xD53kuR93bk1MP3J1DeLuC9LiON/2hkTGO5dQWL+Ywyduz6//hvI70ATRmxp7hFMLwvk18btMI7WPt/WON4DXsm0YhW3/bVOoUEoiaKqo5lyLUyfk6Dtuv9iCPYCD3v5CtzE1mgw7tG+t0fOW/hvM7XNmiI/S9W9dxZHlTkcgbdBwJ3jbS+npG4sDiupcmoRN6QqC+/SLSEVHSoocrVWaA57CeynMfOe0xBZ9Q67qG6/q++iU7zdR+QW7736aaLZx7VlPNhIWzmkr9ualmg4pcBBwbIFzbqL0Wh5W4mkPvvqmp1HuVUpcntJ2LE94n/X5sU81cZm02S2y3KrFPvtazX6h9mW002NEU/1WJdTWlr/H1ifT/eqFMns4KayNmK2LXSimH/H+mZz9ffTuG/1+U0Xfbv7/Z2vY4DlF7nqfse3F44PeLNvgtVRBhaEmtc77+KzTeuSy5rkiT8hqfDlxTbjdJXo/0lP15gevIc/9IREFavsP68eR5iW3Rd37ffTSJf6HrnRQ49zlNpc5oqlk9Oy3Qh6WWObZ9w5O5+zov9LybSl1Hxgg2JJKx6t2JZZeEnp3chwx31M/JpM4X+c7TVOp0qitNpX7OZSWvme831ayPUIkG4obXZBF0r7RI/8t8Ljr3FU2lTm4q9SL+zTe2T0oBIStAzoMPbXdVQiUUjTfwZ1psS3yP/fqOIwtNTNstTI4vrKsTM1zhmoJVSn1IfP4UN/hSoYVDjdjX6Zr/pCXv0lTq7KZaylXS9pq+a5vtX1r2Y6W+7DjvVZ7oLLQuwjVzEiujXdZYeUn5CXV2vw09cwrb21Tq/0opzNSJilCbqQN8q0s5nrPZLu/1kITzpA5KrvPbx7/Ko9DLhbGu+4gNvGYf2S5LQxHU/quplspo/G1d5WhERmmzTf6X+zWBpK22FVcea1/Lt4/8bMI2+oQZuc0IMWY7WfSUNVPgOpevj0oRYMw9K8tF89UipPvJXG8vEwtY7Ws8ho0HdoZx1/XM/1+Kz0dyIBOp6Mhyf4zd0XbhSDQvZgXVV0d97/9CK1/Tr5tKPbOpZhZXOyEnle8Stgq/ie/vPgOPqzahvtf8l58/zMaxml1N6D6P5uhvJkGc635OYo+OlLKYSHJfcZRBsSLrKlvWc2yqmQFCXjfW9qKnbNH/u9ov1d3/8OxrG6Uv5PEo1seXuEezj8sgmY0QwGPjUBNpl67tn7OutY6NY6E+1lvUyJ/Z5wO85rAYU1BAQi4AoQ7Et5+P/+KstCey1YqUkZez9jZGB2kGsTadgPzs8388i0MJK9bETww8a98gb18ztD1UXnncNZbmnnJ86LyuchlcIfTsxuVzWzICPfmTSyuTT3AJDeo/9Rx3IitjV5SyxBTi3bIIkX27dvhniIHmR47ffR1eqKN07ef6bm9/LgvwXlc2ztPymsR73aBuNNVsMW0vsDIqy5IiEJnFgcr6b9+Ty82TMG3ZdZ7YtV1/RvmNnWd/fg/HsqD9GuEOFrW8ZWDKQO4sSswKULShg5d2qmazujSbdLwYVx7IwvvLNzhppX7K7+t6VlhSMPf1laaaCby/ZEXnyY78Ml/g81/eVDOL8pfZir+jwxDl67tO4Pr0Q1bqHmW1jbNZkDTC5D78HMw9rpXfeyJFKE5tqzTbvIaMY+zX/h5+fxQd0QSF8PUdZrb9lNh4ap4JP1vaX0LKzl9b9KUlnleb7W3G7t+zsvZFx770/L+3tHH9M7omUbBOGQ98x/f5vO3PsVnwZwb629k5PC7jMq+MJCf5bUieub60rDx6IkJqjJSJnKdtD8g9PGHxFIXwuyk3evp7LV/Xt2hmKK6I3IOsCPYiQlkJ5IB9w8HrK8r/6Fr9s/Gx1/WyCF6zTZGGJ78fxZY0Vzl9iyvte/ukY0D6FS/iChGq9Kfw2h2bixMHP5M92ccfM6LZyOcqt13m2e91iQ06VgeKQmXS9bLnaq4vr+1btGyXMbYInSJAEddzxCeZxdp1fvszTXXvwFFJUgedJ7Bl+GDrfDpxJuksz327yutz0RgCsy5Bedroa6wZB9+9pFgFac2Mmc5XGf2E/fl3LOzfI7J48yJu3+RqlkLTYtGtOeYNrORQmf7Agv+GO68Xqpbl6TC5hXQ9E9xNRvBZsi7BBzgz9GxXTxkbYbQ6W5zfKAh0DVISKCjAG5pqadZFlmV2Dl0H24e85pdFf/A4u69iQfljup5FRXtUgsJbkh+LjOE29rgpt4fePz03I+C57iVFsZXJ61xtrwnsfwEbSk1/SEF6VOAefOs/JOSaSwvSU9tErO/0jXGxgCDyWGNoovWNt+b+e/0OlXOG+JOcXuDxCeeW333ySEyhIlfbkshnRm3qfJ69dHE+t29XAuJcTF12rauTPNHKY7I3J4B8nXCxpbI73f3mfQaEHtDF7GqSquWGfvsTd+jm3OTPbfvjjg7PArwj55A2goFY4Et8ooPg6vIjTvkut7vCw34w45z2toOEW51NSkjZxko22MYq4Ntf8ifHb18sYE0ote7DhS1M9qbwiPNr4VtuMG4ErrKQDzQNFtd5fpff5bbreZbHrAlq2ApN0XVSSFlE7npeRaevE5DPxVV/SXCVs5F237vUhgN1la5Bguj3Asc7zylmZM8RrhmXe4Q4e/bjD4nurSn9kutzI5QFmhW7ntcCOZWPBJ4iZgrMrJ8L1zNsrPHaV//IdYjcWj8SKcu7HPcp/5u/pT4/1Ffxmsmn8OzgUJzqK07CZ9d34jcJ7qk+V8TGs+7Qdy2Dmf36MAe+MW3WrKEJnYc8OmJ8O6EMMYE85ZicsXEms7Cx56Xitw1mCnm/K7ntnOroQ2Q5c8prHx9z1W7LG63rXeKpIydwwIc13I5uFChj09KbyIeUvT/BAW0oCtiDhMxGv11QWh6YhALCfIKjCJzlqGS+zlJitlFW5bfwtJMcyKbIR0TZr04UursQipJEQkCI0LNMKV/jGRi/0ULxMJ9/JyzoUnBeG1HuTJ2iRauXJJY9p3w2Zkrd7HOBZzYpVoaQYlOUpvJOAS/bzfE5VhdiHelVVkf318Cx5zvWZdD2bwae2W+4g6XtXxPbyW0myYLLlt83Be7BLq8pR9/uKNkhKNkFyIRV9L1Hr+W3qWZW+Z0CCr+vrZwsIr89jf+TW9E6hyLYcFu/VMx+hATiay2XJLPviSLqoqtM9r3TeoZPdI0ux2WiMn+Mo9AEBV0PX4sJ+KwIXJmgCPw8oV/7a1MlGXHGQi5sDy0OT4XuPaXPM+5AzrZGi3AdAQjM+TfcWC0Jo19K7X/E+ZZmwgKQu+DlGYJ6sLwWZkFyCDs607LZJbn+sqmWr2tYVpj1/YJZMxPDXOvzLRXRktDMqYSSaV4irvsMTuZKM6cHC/cznztViJCreIi/SFlCzAJeyTObFLHt+U01i26lOBDFSQulgJgoAsKvNrWi2NrrNdwBk3WYfHAnizU7Ya+LCHF2Yidi4/P9TZl2tNeshPCVy7ZUK+701ya+3w0+swD1YA6T+is+372baplwKZH+l6mxz0Plk+V6gbXNbP+o2H4aR70qIYzG3Me68vrAbzkWL5/Ad61DkPw5C4k7sT+9K1KawbxjY7TwKctyAFwKC8jC894cJjS3LX3McQ3534WsB2PSWPU/Fg77wODJbrDaplwzZQbQDKTyd3Kd3ZXy+jhcGl1lusYhUJ7ksLKG3leb3Aixcs2i0FibY+5tDQsuBAkBXcPD/yhyrYZnmabMVUJ42sDdrAW0ViZ37ZBLaVViZtVeK+bjgMBstq0wNDntjttY6H37rpkCRdWMYQIzhPg4K+ZtMM9mHw4te08RfryNcpFyT205k9dFUXu/D+dVokhUv3es33P1xzljbjKsEL7dEQBg5qnTVOqzUp7igE1XLpQCYmChwBfiL8ZSI+ZFgEP6pLZFCsquCue6B1kZcp5RyHJ/Oeed8Lk0ySQ7IWUht/I3vEi7DbMBgxvwnzmyzKm+0ITM0Yn139zHtUJwDD2/7/K0ZdRSFClfMpQzpMR5Auf/iGemshR2jHfNA9bRHEnnrZ76bbZd6bHEherhmmU7VuoCuk5uwdk6bCydLiHknY5jhp4BcbmvKVFWY+G2XQLsgTB0Htcxrm1mmp+s/++LuLtdKNyUjMvL/3Gf/s0MVx8ZTMEot+taKGh9sy7hec4WnXKIy66ufK61fw27Zp3AhpyY3/jY/NoykoQMHSnGLRN+vA3LrsEzZvcOuIktP7iajR3XJiilKUYOF7aXgOucbfhMgoEsOotGbrSkmCdcLyTPXcD9A/UNX2ZFxAQPSKEZwE32+Zah/OeBfdtEtewi71JEv3dR+Fw1IJNTQBJDb/oaZIo//9Q4JtIp/DjB+loCc82PsKVR+hq/wFo0n3M+w9cDAphvAI5da1nnRp14wiLiJsWXn8+zAwu35yacKySsSDei0EJdHz8ZeNpYcqJlQftU4vVT9tnAn5otQbaAeX3kGvKd0zqC4/jcT5LrwQLrkNoik1zanDrCu1oGtzdfgk367WeWG4jd5prUZ8aGIztmvzyPGdhoMfNbQu2U3RBeJ479SUuB2LYQr7VmcpsRraFZ8PMtdS6fkEG5byiHzO5TN95x3d6erLMeZcq2sJ/kmVEw23ztxOZ4X5Ec5bvasc+SEBrBvi8bn6HQhSvZnE1un96wofhFgf0/5XiuMVfvEKdHxmvz2Sgi51ljNwUfMRznOo0qjykXjaE5BqgmsN1nMLs0UD/DF6vUL5oqa03yQisgPp/bWAVZpeYMYc26uWcXOSV2vuUycW6JIlD4URPulRuuETjIXeElbNk3EVxi+KaxQ9PbMumfr4w2lKioa7xuuQBuw4uuV2h8C3DtbUYBoTCXkiulex0HXMjlQnFNc13fbEtp1nBISpc1zeceEGun5pjfWAOTL0TtYcGTVcsyNs+iFDXVrE6fx2s1KEzpyZzvJ8VdKJWLPG5V9M6vGFP5SMFSAnxlPSdjxu5FgXPRtQ7MaLNrhVFpluxK5SMNUjQzKmd9Qtbyq1sIeV2w87X4PpdE5jAxpKyBmAyiXsaeEa0JlbmeGkcggNRZNVfOKGIDhY3dVP4zwaAYk81cClbOQmmfy6I877mBuuczTNDY+FURTtfkNyHewTlj7PZOoY5bwbKJqxxHeva/TrhcftBaW3i6596KznpymWkpwJMzx54mUK9/H7iW7XpOazgmyyQVEF7k+Ux20XBZLJTn++hhhTtwpWfbxcIPmxYDPVFU5C6NRT7TXzoEE5pCpghiX4rUFfke9m+q2VqML2SWRbpjxN5zMTI7hC+llMdhqXw6W8P2EgmmSjBIxBmO/iMtNzfz7Wr9tz/b2/5snXeNxyrlCmNp/uwkhufbWZdZgfwhD0TkYlIMttS7rEZfmbAxxDz/axL3uzizPflmNl6V6TJork8RqNpa9S4TYXpDM6P2+GIME0NFdSIXNzu7t1m8q7jfL40dtYzeT/EwmxPg7VayTBMtjBIzKt62NuKmlMIsFLIDe7bpihYJms1/+nsWr5fIKS+d65KMuqASxuPfinxeL2UjjwkvvWtTzdx5XO2nRMJc2+gVWj/7ezY+vZHr95s52axPESw+80f9UIsIcevkKVqWkdZVt1b4VqwCIgZ3e5GtL4LJWGEuS/IXx72tE5WNLJHk51h3fH8uZW6D0JLso5g6K2HOGRrAQsqSK1pFjK6KSe7CxR9HyhIMzdpUM3cXkxiyDYMpZpmzkymzHa7PpzqCUKzLrDu/tWbYvB19jwlHfUqsFPDHnAnxWdKebX0nAcLl420WJ6bycM/2FB/v0oIKvfdjHeHPzSJhm8ey6xcZCnYbKgoU18vrHOtXHseGi1yDTgprW4SWnkf+yM9X1idyNyHj3tf5e04QCl9Akusiigkdc2jq4l02zphIe6acVB++6wlkEDrX1WLtmx2VibjMYVhqLBcuezbwatvIw7LJTpEkejO3z6Egg2BTqcNF4JH3N5X6VOD5LesvRuSSFsquzavVxJn6jME69uH+KA+Ym3MyKB99dNRDsA+vs9hWbGtY8LqMMzTb1l6iSH6TjM7XlEl+l7gSP6UkKHKdO7ZvV1/+kO9qKPKYr4zSL3MX13qkwgLwWEKtncwyVC9CCSpzO/qYG9xRAy4avqEAlbpKS5PADdgL48eyRPkUQDuSkivGO7lQ5IaMvcb1rnlWe0wui7jRUKSXC4W7Y05Uwj4UxW/0PCNxCo855rpdI2uNSW5fuAXPOhm34OTFyuRSxO09abxqKnWG6B9Ca8ZcXMQW/DUcBrVLn/9NTn53I85JY8tZlHjOdf4dzHogXc+Ul/dwBCfnWsZIyOrTEnLUtMEkhu1KsehOBfg85+GQmPeTNJtCYc3VxJnsDIjQrGnqjAbCf3dMU9ufV6s5JJSpkhflnu7pfC7oeumMgadrBJ+u7mI2b+14rlwL60WRcsl1HpeLhX8psdLbMHRW7R+KdRYufK6SLt7J7jFdWWuFCyy2ULcr3F7PEM/j02MVxbHtCHugZYvr6QX9oek8v+kwzvyJ21CJmRBSfuT9mlllWV8pMteYLHtPA7hDXSyvSZEE1WLzv8KV9teWx0SXma7Ufo/6vKzrUALlplIPpnWDXY1X5B7JSTRJ1tjN+pnGK5N/QnKUlbPjeg4ZvHtG0lbJbwoY4Q52uPuGkpLmQEFLpkbj+B9Ski4Ua3Imz6QVEAfGn1c5Gv13EsNFThXXgBMSXl/o6TRUZobg1EVxxhc81IGEBJYUS6rLUu5zPWqlEPEgQB1orgB8RSTzuH3vJjleLxaVvkPweuri5YmRSSiK2uMDsxa0IDAHV5CCQ0skiOuJXzmUDpfrwxgDmAosYJX7XNqy7sp2WbcdZ1ihPKbQrOHjLXfe8wKLiceiiMtZKj25I06NxvTZTaXO5G3kvrRWyBM7tAhmssGzS+iLhp5RCxl1f82KhMFXdtcsPrnK/irHBaxFpLEQZ1ufH9SiLD6GWvPViSZQ13im2RfMZXLMmwJyXaByv6fFQp8pYfw9k6I0NdUsqkuWAMb+zyYef+i6rmN9i6Jomu/aBAUkFAow153q9C4dOnegWQMw708L2CSyDOscAvvehXI/TEFYerHDp7RhK+Kh1n1fKvyA7dnKbxSYqaCsweRCMhWOZqXLrFEzoX/PHzEHiI81gbLIqHp7cwjmLPjdmoy5l2e6VvYCZyFf6it5nJjMbBmzEhSCvvg7xzbj8/9Dn8zD3gVtFh1f08LFmkLSTonfWcZPV1sv7TZZIqz1VeL9PsdE71zANtd0KN+U72uuFZC/Bh7uqINcAVyzHSVDhvpmWrpakV/HitK5Ypo712/RhBV24fJNp1j1YzUy1+L7N9kDGVuKurrIGQaPz+0JSuBa/6HZt/ddHMVqtuCPXawon88vrc70FS0ubwRZw6SEx6ZSH2+qmZL4bamsk6LL5d4gKeFI/CLig/0+4aZyRQc3oHOFG9W7HIkOp4DPjXcsZBnsELF94VuMP2/8LFFgezLnBOmKSX5o+HjMxXpCawtkX2AyhlP73GAlW8GkdK5Z2HYnWt+3zoymBQxZLiPZVFjb8b7mYk3X1Beh24QqSgmf8qndZx8uJtJSf1nHRkz5Fv7KFk9j9VQtGrW0tNgWU6MkS3/l3Ig6pfipVRYSMLfr+6IkCFoLnWViwLFYWmjOytcGSlJTzaK3fF3Xs+RbFGXlmDb1jTIF63o2y0DrwIipJkmjxazG1UMmv8pdyF0SCjlpwtgeFEkAuI7r2Yc6Kvjm2OMmHNr1yow8RUNQKit3m+uafm1e8dXVf1u204azIW35sXVdE2FqbqD2reulmfWhZmen5qFy1BSMex6+30GBO2XAPGErSgFRluvPw0SCvLGE0tIUsxZ4ICvt7VwddCYkSBax8JMAawnY8t4nk0+hqdSlVjk/zQrIvM++dYEy9gZpqtYBA5adZgruPCGaaubWsMydsKlG98c9X9TV1DDlXZ+xcX+ZqvJhhI+lKDGFgiKUYmgFu0T7HBPfWGkbA0ohPROaCblXgjwGCbPdhqZSf/ZEV0w5dt3Qa8pWiguW7HCusaZCpziNNkXkAuA2Wbmlu8bhI9TRscN5Ssyi7JVc94aynpoOdSU/67bCEg20pRZ0LwrGvWQqz2SMcow5M1eSviIN+jCJDeeda0XfulmP17l4asbEibphphikn6EWiLlSQNh942dixqPUlOpKQk6DdrH6/S4jtv81HRaN29nD76/G5RzLjeMxSqlPDFyGMV2QbMuKHkHR+dpA15x7OOrb/TN80A+IRHtLYR6sb1cVMsSUwggY/zNgMJVS+RPGZtDkdnOe8NiVm2itNRtQWhgvnfuD1jd8t8B5pqYQdU1QOHfMlQJiRVP60gJZcHz0unCtY6bfZZmsLezF6JTFPUUoaTyd13uWdmgXtaQvaHHhTwfOP3HNwD7iG0wNW5luBxE2m2pZnVoUwWkQKAdAxr7fLdDGfjJ1JZEVMhOZiBLyjU3jWeDc3wVvmBGbrFtjIoMGpViUmUQetx4pkm/2ermC56IomAeuoCUIjVpQ5lEB+Twvsrl6Yu44xafbes5zIC35bQgJnqe0uJZPqL4iI1fJ0IyxqG7XjqEHS2CSln0Z/s/Apqlm+YmmINTHMLlZrp1Q3z+0hZMiQw2dU6g08v3JaFiTyL0xZTjJ4dpALqPJuetylMWpLWgfisvVAjF3CgiHAz2UM3s2JtzYolglprR4uAON5Z+bEoZU5iCQ0CAypXf7jpGncKegdD+XE7tN2soNxmMe+mOavVRK7TsRt4ZRgqhQZKgFNSKcwXmv+mTydTwTqbz9Z2Gh+RcFEwaW4jxHbq8pc8KitdW5U0AckCY8paRk88IQ0+6vYZ/mJjGC0Aab1bT9jadgOR2cplJ/bCr15qaauUECMLc01cw9dArRul47dgHmFTG+THG8mBdW9SgfnMLJGCcFuxObpLFT5Huc5JYUJeLdasGYFx+4lUgfU6Bm4fnFnqSBOaT4lZ+cmfiJwmI+zto2BeFAIge7Pl3kgJ+pWdIA6EofSWdXEkMrH0fyX4nEhlNA95C1fAa7S82Dy9TUFNgDuEy0Vm3jRZv9WBQF5DvWothFgaIrFYcrMS0868KPIlZD05mRr2aOsPhTO9eDlYSPEtpNAbNGZ8gO62TOObKSBRVaOPzGCU7lAwCmASVge8gAbnV/HsCVeUxZcFEifcWgMfxEStY6NbfR5oZxbl4UuBWpgByjFgsj3K5E954TI8kRpzYbMhhNpQ7X9Szr+KQ6yYHZZSXXAbDQXMWz0qjf3TiWFZC3q345TSn1ugXIIm/YxPo+75HRkuDxFO6PIzH3CsgCCWTyPqbeIJq+NH5dzyxLN7E6QHO9X6rxGa2+LVBd7z2cLADzBIcFfurY5ZhzyGXld6wU9B3C/vrCC7VXZBAEsLJZhEXoi8JfRDzzebc+UNSL/VuGQfyC74emmoQCshJnpgAAYMo8VCn1fVbk9p5IYIG5gZ/XQoV4BdMHCshEYOs25bwgfq2mTRObyWgqdWZLi73JztpY63wOUhMgI6M0AACAAWgqdanJDYE1Yq3pujYUgCyggEwLI3SfoaYJhYSzc2GU5jPic20ytjaV+laP1wQAAABWMsbFldzYFi7iEpgec78GZEFZO3EF6azeLlCpq0XUq3UTXuj4m7ELAQAAABTmlyt9vSEYBsyATAtq9CebqeQJR+i6uufrHCFylkyOplLvbSr1+bHLAQAAABQG62fAIGAGZFq8cOLJ7YayipjQhm8c6HoAAADASmbdgqY2ABMFCsiEaCp1/thlmAg/4f9tomgBAAAAIDPMua7VfZVSf8WDA0MABQTkxgr/dt+PjLOfUzKpP/R9LQAAAAAM4l4NwBK6abDWCKSDbNwAAAAAAKALUEAAAAAAAAAAg4EoWAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAGAwoIAAAAAAAAIDBgAICAAAAAAAAgAICAAAAAAAAWDwwAwIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAAAYDCggAAAAAAAAgMGAAgIAAAAAAABQQ/H/AVGIu/U/EZGGAAAAAElFTkSuQmCC", "wave_duration": 10.4, "ref_text": "大家好,欢迎来到PersonaVoice语音合成演示平台,今天我将为大家展示个性化声音克隆技术。", "gen_text": "你好,这是PersonaVoice的个性化声音克隆演示,通过短短几秒的参考音频,就能复制出相似的音色和说话风格。", "sample_rate": 24000, "trait_radar": {"labels": ["热情", "支配性", "情绪稳定", "尽责性", "开放性", "宜人性", "外向性", "安慰方式", "幽默类型", "冲突处理", "问候方式", "告别方式", "昵称使用", "表达频率", "细节程度", "与家人", "与朋友", "压力下", "开心时", "难过时", "生气时", "语速", "音调范围", "音量倾向", "停顿模式", "清晰度"], "values": [-0.011020803824067116, -0.02304561249911785, -0.06663818657398224, -0.04549655318260193, -0.06418636441230774, -0.014245625585317612, 0.01244845986366272, 0.06743951886892319, 0.10047459602355957, 0.015310566872358322, 0.027534686028957367, 0.11408227682113647, 0.08736889809370041, 0.10849438607692719, 0.10601066052913666, -0.07131894677877426, 0.05187293142080307, -0.11000701785087585, 0.12122133374214172, -0.11115194112062454, 0.07636027038097382, -0.007268454879522324, -0.0994938462972641, -0.12548322975635529, -0.0032881200313568115, -0.005256921052932739]}, "conditions": {"speaker_emb": {"shape": [1, 192], "mean": 0.0, "std": 1.0026094913482666, "norm": 13.856338500976562}, "persona_emb": {"shape": [1, 64], "mean": -9.313225746154785e-09, "std": 1.0079002380371094, "norm": 7.999960422515869}, "emotion_emb": {"shape": [1, 64], "mean": 9.778887033462524e-09, "std": 1.0079001188278198, "norm": 7.9999589920043945}}, "z_emotion_norm": 1.512, "speaker_embedding_norm": 2.923, "persona_representation_norm": 3.245} \ No newline at end of file diff --git a/personavoice/demo/static/demo_ref.wav b/personavoice/demo/static/demo_ref.wav deleted file mode 100644 index 3a3510e..0000000 Binary files a/personavoice/demo/static/demo_ref.wav and /dev/null differ diff --git a/personavoice/experiment/__init__.py b/personavoice/experiment/__init__.py deleted file mode 100644 index 09991f6..0000000 --- a/personavoice/experiment/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -"""实验模块: SOTA 评估脚本集合. - -架构位置: 与核心库代码分离的实验脚本集合, 引用 personavoice 主架构, -不包含任何架构本体的实现, 仅负责评估. - -核心评估工具 (复用): -- ecapa_evaluator: 真实 ECAPA SECS 评估 (Vocos/Griffin-Lim 声码器) -- wer_evaluator: Whisper-based WER 评估 -- utils: 共享工具函数 (数据加载, tokenizer, logger) - -SOTA 评估 (顶会 Table 1): -- eval_200_samples: 200 样本统计显著性评估 (PV vs F5-TTS Baseline) -- baseline_external: 外部 SOTA 对比 (CosyVoice / XTTS v2, 1s 极限克隆) -- cfr_analysis: 灾难失败率分析 (CFR 长尾截断) - -顶会可视化 (Figure 4/5/6): -- visualize: Pareto 前沿 + 波形/频谱对比 + 注意力热力图 -""" diff --git a/personavoice/experiment/baseline_external.py b/personavoice/experiment/baseline_external.py deleted file mode 100644 index 85e6ecc..0000000 --- a/personavoice/experiment/baseline_external.py +++ /dev/null @@ -1,599 +0,0 @@ -"""外部 SOTA 对比实验: CosyVoice (Zero-Shot) vs XTTS v2 vs PersonaVoice. - -架构位置: 顶会论文 Table 1 的外部基线对比部分. 不引用本项目架构本体, -仅调用第三方开源模型的官方推理 API, 然后用本项目的 ECAPA/Whisper 评估器 -计算 SECS 与 WER, 确保评估链路一致. - -零成本方案 (8GB 显存可运行): - - 靶子 A: CosyVoice (阿里通义实验室 2024, 中文零样本克隆天花板) - 官方仓库: https://github.com/FunAudioLLM/CosyVoice - 推理 API: cosyvoice.inference_zero_shot(tts_text, prompt_text, prompt_speech_16k) - - - 靶子 B: XTTS v2 (Coqui, 经典 3 秒克隆基线) - HuggingFace: coqui/XTTS-v2 - 推理 API: TTS.tts_to_file(text, speaker_wav, language, file_path) - -预期结果 (1s 极限参考音频): - - CosyVoice SECS ≈ 0.30-0.40, WER ≈ 30%-50% (设计假设 ≥3s 参考) - - XTTS v2 SECS ≈ 0.30-0.40, WER ≈ 30%-50% (设计假设 ≥3s 参考) - - PersonaVoice SECS ≈ 0.4832, WER ≈ 24% (本项目, 1s 极限优化) - -执行步骤: - 1. 加载 200 个 LibriTTS 测试样本 (1s 参考音频 + 目标文本) - 2. 对每个样本, 用 CosyVoice 和 XTTS v2 生成 Wav - 3. 用 ECAPAEvaluator 计算真实 SECS (Vocos + ECAPA-TDNN) - 4. 用 WEREvaluator 计算真实 WER (Whisper-large-v3-turbo) - 5. 输出对比 JSON, 直接填入论文 Table 1 - -使用方法: - python -m personavoice.experiment.baseline_external \\ - --n_samples 200 --seed 42 \\ - --models cosyvoice xtts \\ - --output results/baseline_external.json - -依赖: - # CosyVoice (clone 并安装) - git clone https://github.com/FunAudioLLM/CosyVoice.git - cd CosyVoice && pip install -r requirements.txt - - # XTTS v2 (pip 一键安装) - pip install TTS -""" - -import os -import sys -import json -import time -import argparse -import logging -from pathlib import Path -from typing import Dict, List, Optional - -import numpy as np -import torch -import torchaudio -import soundfile as sf - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - -# 添加 CosyVoice 和 Matcha-TTS 到 Python 路径 -COSYVOICE_ROOT = PROJECT_ROOT / "CosyVoice" -MATCHA_ROOT = COSYVOICE_ROOT / "third_party" / "Matcha-TTS" -if COSYVOICE_ROOT.exists(): - sys.path.insert(0, str(COSYVOICE_ROOT)) -if MATCHA_ROOT.exists(): - sys.path.insert(0, str(MATCHA_ROOT)) - -# 强制离线模式, 避免联网 -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" -os.environ["COQUI_TOS_AGREED"] = "1" - -from personavoice.experiment.utils import setup_logger -from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator -from personavoice.experiment.wer_evaluator import WEREvaluator -from personavoice.experiment.eval_200_samples import ( - load_200_test_samples, -) - -logger = setup_logger("baseline_external") - - -# ============================================================ -# 参考音频文本提取 (CosyVoice 需要 prompt_text) -# ============================================================ - -def extract_reference_text(sample: Dict, max_chars: int = 80) -> str: - """从样本中获取参考音频对应的文本 (作为 CosyVoice prompt_text). - - CosyVoice 需要 prompt_text (参考音频对应的文本内容). - 在 LibriTTS 中, 参考音频就是目标文本对应语音的开头 1s, - 所以直接使用 target_text 作为 prompt_text. - """ - ref_text = sample.get("target_text", "") - if len(ref_text) > max_chars: - ref_text = ref_text[:max_chars] - return ref_text - - -# ============================================================ -# CosyVoice 推理封装 -# ============================================================ - -class CosyVoiceBaseline: - """CosyVoice (Zero-Shot) 基线封装. - - 使用官方 inference_zero_shot API, 喂入 1s 参考音频. - 需要预先 clone CosyVoice 仓库并下载预训练权重. - """ - - def __init__( - self, - model_dir: str = "pretrained_models/CosyVoice-300M", - device: str = "cuda", - fp16: bool = True, - ): - """初始化 CosyVoice. - - Args: - model_dir: CosyVoice 预训练模型目录 - device: 推理设备 - fp16: 是否使用半精度 - """ - logger.info(f"Loading CosyVoice from {model_dir}...") - try: - from cosyvoice.cli.cosyvoice import CosyVoice - self.model = CosyVoice( - model_dir=model_dir, - load_jit=False, - load_trt=False, - fp16=fp16, - ) - self.sample_rate = self.model.sample_rate - self.available = True - logger.info(f"CosyVoice loaded, sample_rate={self.sample_rate}") - except ImportError: - logger.warning( - "CosyVoice not installed. " - "Please: git clone https://github.com/FunAudioLLM/CosyVoice.git && " - "cd CosyVoice && pip install -r requirements.txt" - ) - self.available = False - except Exception as e: - logger.warning(f"CosyVoice load failed: {e}") - self.available = False - - def generate( - self, - text: str, - prompt_wav_path: str, - prompt_text: str = "", - ) -> Optional[np.ndarray]: - """生成语音. - - Args: - text: 目标文本 - prompt_wav_path: 参考音频文件路径 (CosyVoice 需要文件路径) - prompt_text: 参考音频对应文本 - - Returns: - wav_gen (T,) 或 None - """ - if not self.available: - return None - - try: - # CosyVoice inference_zero_shot 返回 generator - # 注意: 官方 API 参数名为 prompt_wav, 且必须是文件路径 - for chunk in self.model.inference_zero_shot( - tts_text=text, - prompt_text=prompt_text, - prompt_wav=prompt_wav_path, - stream=False, - speed=1.0, - ): - wav_gen = chunk["tts_speech"].numpy().flatten() - return wav_gen - return None - except Exception as e: - logger.warning(f"CosyVoice generate failed: {e}") - return None - - -# ============================================================ -# XTTS v2 推理封装 -# ============================================================ - -class XTTSv2Baseline: - """XTTS v2 (Coqui) 基线封装. - - 使用官方 TTS API, 喂入 1s 参考音频. - pip install TTS 即可使用. - """ - - def __init__( - self, - model_name: str = "tts_models/multilingual/multi-dataset/xtts_v2", - device: str = "cuda", - ): - """初始化 XTTS v2. - - Args: - model_name: 模型名称 - device: 推理设备 - """ - logger.info(f"Loading XTTS v2: {model_name}...") - try: - from TTS.api import TTS - self.tts = TTS(model_name=model_name).to(device) - self.sample_rate = 24000 # XTTS v2 固定 24kHz - self.available = True - logger.info(f"XTTS v2 loaded, sample_rate={self.sample_rate}") - except ImportError: - logger.warning( - "TTS library not installed. Please: pip install TTS" - ) - self.available = False - except Exception as e: - logger.warning(f"XTTS v2 load failed: {e}") - self.available = False - - def generate( - self, - text: str, - prompt_wav_path: str, - language: str = "en", - ) -> Optional[np.ndarray]: - """生成语音. - - Args: - text: 目标文本 - prompt_wav_path: 参考音频文件路径 (XTTS 需要文件路径, 不是张量) - language: 语言代码 - - Returns: - wav_gen (T,) 或 None - """ - if not self.available: - return None - - try: - # XTTS v2: tts_to_file 直接保存到文件 - tmp_path = prompt_wav_path + ".xtts_tmp.wav" - self.tts.tts_to_file( - text=text, - speaker_wav=[prompt_wav_path], - language=language, - file_path=tmp_path, - ) - wav_gen, sr = sf.read(tmp_path) - os.remove(tmp_path) - return wav_gen - except Exception as e: - logger.warning(f"XTTS v2 generate failed: {e}") - return None - - -# ============================================================ -# 评估循环 -# ============================================================ - -def evaluate_baseline( - baseline_name: str, - baseline, - samples: List[Dict], - ecapa_eval: ECAPAEvaluator, - wer_eval: WEREvaluator, - tmp_dir: Path, - max_samples: Optional[int] = None, -) -> Dict: - """对单个基线模型在 200 样本上评估 SECS 和 WER. - - Args: - baseline_name: 基线名称 (cosyvoice / xtts) - baseline: 基线模型实例 - samples: 测试样本列表 - ecapa_eval: ECAPA 评估器 - wer_eval: WER 评估器 - tmp_dir: 临时文件目录 - max_samples: 最大评估样本数 (None=全部) - - Returns: - 评估结果字典 - """ - if not baseline.available: - logger.warning(f"{baseline_name} not available, skipping.") - return { - "baseline": baseline_name, - "available": False, - "n_samples": 0, - "results": [], - } - - n = len(samples) if max_samples is None else min(max_samples, len(samples)) - logger.info(f"Evaluating {baseline_name} on {n} samples...") - - # 加载完整 wav 张量 (从预处理数据中, 16kHz 3s) - data_path = PROJECT_ROOT / "data" / "libritts_processed" / "libritts_devclean_processed.pt" - logger.info(f"Loading wav tensors from {data_path}...") - data = torch.load(str(data_path), weights_only=False) - wav_all = data["tensors"]["wav"] # (N, 48000) 16kHz - - results = [] - t_start = time.time() - - for i, sample in enumerate(samples[:n]): - idx = sample.get("idx", i) - text = sample["target_text"] - - # 从完整 wav 中截取前 1s 作为参考 - wav_full = wav_all[idx] # (48000,) 16kHz, 3s - wav_ref_16k = wav_full[:16000].unsqueeze(0) # (1, 16000) 1s - - # 保存 1s 参考音频到临时文件 (XTTS 需要) - ref_wav_path = tmp_dir / f"ref_{idx}.wav" - torchaudio.save( - str(ref_wav_path), wav_ref_16k, 16000, - encoding="PCM_S", bits_per_sample=16, - ) - - # 生成语音 - t_gen_start = time.time() - if baseline_name == "cosyvoice": - prompt_text = extract_reference_text(sample) - wav_gen = baseline.generate(text, str(ref_wav_path), prompt_text) - elif baseline_name == "xtts": - wav_gen = baseline.generate(text, str(ref_wav_path), language="en") - else: - wav_gen = None - gen_time_ms = (time.time() - t_gen_start) * 1000 - - if wav_gen is None: - logger.warning(f"[{i+1}/{n}] {baseline_name} idx={idx}: generation failed") - results.append({ - "idx": idx, - "text_len": len(text.split()), - "secs": 0.0, - "wer": 1.0, - "time_ms": gen_time_ms, - "status": "failed", - }) - continue - - # 保存生成音频 - gen_wav_path = tmp_dir / f"gen_{baseline_name}_{idx}.wav" - sf.write(str(gen_wav_path), wav_gen, baseline.sample_rate) - - # 计算真实 SECS (Vocos + ECAPA-TDNN) - try: - wav_ref_np = wav_ref_16k.numpy().flatten() - wav_gen_np, sr_gen = sf.read(str(gen_wav_path)) - # 统一采样率到 16kHz - if sr_gen != 16000: - wav_gen_np = torchaudio.functional.resample( - torch.from_numpy(wav_gen_np).float(), sr_gen, 16000 - ).numpy() - - secs = ecapa_eval.compute_secs( - torch.from_numpy(wav_gen_np).float(), - torch.from_numpy(wav_ref_np).float(), - sr=16000, - ) - except Exception as e: - logger.warning(f"SECS compute failed for idx={idx}: {e}") - secs = 0.0 - - # 计算真实 WER (Whisper) - try: - wer = wer_eval.compute_wer( - torch.from_numpy(wav_gen_np).float(), - text, - sr=16000, - ) - except Exception as e: - logger.warning(f"WER compute failed for idx={idx}: {e}") - wer = 1.0 - - # 清理临时文件 - try: - gen_wav_path.unlink() - ref_wav_path.unlink() - except Exception: - pass - - results.append({ - "idx": idx, - "text_len": len(text.split()), - "secs": float(secs), - "wer": float(wer), - "time_ms": gen_time_ms, - "gen_length_sec": len(wav_gen) / baseline.sample_rate, - "status": "ok", - }) - - if (i + 1) % 20 == 0: - elapsed = time.time() - t_start - eta = elapsed / (i + 1) * (n - i - 1) - logger.info( - f"[{i+1}/{n}] {baseline_name}: " - f"SECS={secs:.4f}, WER={wer:.4f}, " - f"elapsed={elapsed:.1f}s, ETA={eta:.1f}s" - ) - - # 释放 wav_all - del wav_all, data - torch.cuda.empty_cache() - - # 统计 - secs_vals = [r["secs"] for r in results if r["status"] == "ok"] - wer_vals = [r["wer"] for r in results if r["status"] == "ok"] - time_vals = [r["time_ms"] for r in results if r["status"] == "ok"] - - stats = { - "baseline": baseline_name, - "available": True, - "n_samples": len(results), - "n_success": len(secs_vals), - "n_failed": len(results) - len(secs_vals), - "secs_mean": float(np.mean(secs_vals)) if secs_vals else 0.0, - "secs_std": float(np.std(secs_vals)) if secs_vals else 0.0, - "secs_median": float(np.median(secs_vals)) if secs_vals else 0.0, - "wer_mean": float(np.mean(wer_vals)) if wer_vals else 1.0, - "wer_std": float(np.std(wer_vals)) if wer_vals else 0.0, - "wer_median": float(np.median(wer_vals)) if wer_vals else 1.0, - "avg_time_ms": float(np.mean(time_vals)) if time_vals else 0.0, - "total_time_sec": time.time() - t_start, - "results": results, - } - - logger.info( - f"{baseline_name} done: " - f"SECS={stats['secs_mean']:.4f}±{stats['secs_std']:.4f}, " - f"WER={stats['wer_mean']:.4f}±{stats['wer_std']:.4f}, " - f"n={stats['n_success']}/{stats['n_samples']}" - ) - - return stats - - -# ============================================================ -# 主函数 -# ============================================================ - -def main(): - parser = argparse.ArgumentParser( - description="External SOTA baseline comparison (CosyVoice / XTTS v2)" - ) - parser.add_argument( - "--n_samples", type=int, default=200, - help="Number of test samples (default: 200)" - ) - parser.add_argument( - "--seed", type=int, default=42, - help="Random seed for sample selection (default: 42)" - ) - parser.add_argument( - "--models", nargs="+", default=["cosyvoice", "xtts"], - choices=["cosyvoice", "xtts"], - help="Baselines to evaluate" - ) - parser.add_argument( - "--cosyvoice_dir", type=str, - default="pretrained_models/CosyVoice-300M", - help="CosyVoice model directory" - ) - parser.add_argument( - "--output", type=str, default="results/baseline_external.json", - help="Output JSON path" - ) - parser.add_argument( - "--device", type=str, default="cuda", - help="Device for baseline inference" - ) - parser.add_argument( - "--eval_device", type=str, default="cpu", - help="Device for ECAPA/Whisper evaluation (cpu saves GPU for baseline)" - ) - args = parser.parse_args() - - # 加载 200 个测试样本 - logger.info(f"Loading test samples (seed={args.seed})...") - samples = load_200_test_samples(seed=args.seed) - if args.n_samples < len(samples): - samples = samples[:args.n_samples] - logger.info(f"Loaded {len(samples)} samples") - - # 初始化评估器 (与 PersonaVoice 评估使用相同的评估链路) - logger.info("Initializing ECAPA evaluator (Vocos + ECAPA-TDNN)...") - ecapa_eval = ECAPAEvaluator( - device=args.eval_device, - prefer_vocoder="vocos", - ) - logger.info("Initializing WER evaluator (Whisper-large-v3-turbo)...") - wer_eval = WEREvaluator(device=args.eval_device) - - # 临时文件目录 - tmp_dir = PROJECT_ROOT / "results" / "baseline_tmp" - tmp_dir.mkdir(parents=True, exist_ok=True) - - all_results = { - "metadata": { - "n_samples": args.n_samples, - "seed": args.seed, - "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), - "eval_pipeline": "Vocos + ECAPA-TDNN (SECS) + Whisper-large-v3-turbo (WER)", - "ref_duration_sec": 1.0, - "baselines": args.models, - }, - "personavoice_reference": { - # 从 eval_200_statistics_merged.json 读取 PersonaVoice 的结果 - "secs_mean": 0.4832, - "wer_mean": 0.2405, - "n_samples": 200, - "note": "PersonaVoice v8.0 result from eval_200_statistics_merged.json" - }, - "results": {}, - } - - # 评估每个基线 - for model_name in args.models: - logger.info(f"\n{'='*60}\nEvaluating {model_name}\n{'='*60}") - - if model_name == "cosyvoice": - baseline = CosyVoiceBaseline( - model_dir=args.cosyvoice_dir, - device=args.device, - ) - elif model_name == "xtts": - baseline = XTTSv2Baseline(device=args.device) - else: - logger.warning(f"Unknown baseline: {model_name}") - continue - - result = evaluate_baseline( - baseline_name=model_name, - baseline=baseline, - samples=samples, - ecapa_eval=ecapa_eval, - wer_eval=wer_eval, - tmp_dir=tmp_dir, - ) - all_results["results"][model_name] = result - - # 释放显存 - del baseline - torch.cuda.empty_cache() - - # 清理临时目录 - try: - for f in tmp_dir.glob("*"): - f.unlink() - tmp_dir.rmdir() - except Exception: - pass - - # 保存结果 - output_path = PROJECT_ROOT / args.output - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - json.dump(all_results, f, indent=2, ensure_ascii=False) - logger.info(f"Results saved to {output_path}") - - # 打印对比表格 - print("\n" + "=" * 70) - print("External SOTA Baseline Comparison (Table 1)") - print("=" * 70) - print(f"{'Model':<20} {'SECS↑':<15} {'WER↓':<15} {'N':<10}") - print("-" * 70) - - # PersonaVoice - pv = all_results["personavoice_reference"] - print( - f"{'PersonaVoice v8':<20} " - f"{pv['secs_mean']:.4f} " - f"{pv['wer_mean']:.4f} " - f"{pv['n_samples']}" - ) - - # Baselines - for name, res in all_results["results"].items(): - if res.get("available", False): - print( - f"{name:<20} " - f"{res['secs_mean']:.4f}±{res['secs_std']:.4f} " - f"{res['wer_mean']:.4f}±{res['wer_std']:.4f} " - f"{res['n_success']}/{res['n_samples']}" - ) - else: - print(f"{name:<20} {'N/A':<15} {'N/A':<15} {'N/A':<10}") - - print("=" * 70) - print( - "Conclusion: PersonaVoice achieves SOTA on 1s extreme voice cloning, " - "outperforming CosyVoice and XTTS v2 (designed for ≥3s reference)." - ) - - -if __name__ == "__main__": - main() diff --git a/personavoice/experiment/cfr_analysis.py b/personavoice/experiment/cfr_analysis.py deleted file mode 100644 index 6fde71c..0000000 --- a/personavoice/experiment/cfr_analysis.py +++ /dev/null @@ -1,251 +0,0 @@ -"""CFR (Catastrophic Failure Rate) 计算. - -创新点: 在1s极限信息缺失下,创新模块的主要贡献不是均值优化, -而是截断长尾灾难分布,显著提升系统鲁棒性. - -灾难边界定义: - - 语义灾难边界 (Semantic Disaster): WER > 50% - - 音色崩塌边界 (Timbre Collapse): SECS < 0.25 - -Usage: - python -m personavoice.experiment.cfr_analysis -""" - -import json -import logging -import numpy as np -from pathlib import Path -from typing import Dict, List, Tuple - -logger = logging.getLogger(__name__) - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -RESULTS_DIR = PROJECT_ROOT / "results" - -# 灾难边界 -WER_DISASTER_THRESHOLD = 0.50 # WER > 50% = 语义灾难 -SECS_DISASTER_THRESHOLD = 0.25 # SECS < 0.25 = 音色崩塌 - - -def load_ablation_data() -> Dict[str, List[Dict]]: - """加载消融实验数据.""" - path = RESULTS_DIR / "ablation_200_samples.json" - with open(str(path), "r", encoding="utf-8") as f: - data = json.load(f) - - # 配置名映射 (中文友好) - config_map = { - "full_v8": "Full v8.0 (完整)", - "wo_td_cfg": "-TD-CFG (无时间衰减引导)", - "wo_ibop": "-IBOP (无正交投影)", - "wo_sbm": "-SBM (无薛定谔桥)", - } - - result = {} - for key, label in config_map.items(): - if key in data: - result[label] = data[key] - return result - - -def compute_cfr(samples: List[Dict]) -> Dict[str, float]: - """计算单个配置的CFR指标. - - Args: - samples: 样本列表,每个含wer和secs字段 - - Returns: - dict: CFR指标 - """ - n = len(samples) - if n == 0: - return {} - - wer_values = [s["wer"] for s in samples] - secs_values = [s["secs"] for s in samples] - - # 灾难率 - wer_disaster = sum(1 for w in wer_values if w > WER_DISASTER_THRESHOLD) - secs_disaster = sum(1 for s in secs_values if s < SECS_DISASTER_THRESHOLD) - combined_disaster = sum( - 1 for w, s in zip(wer_values, secs_values) - if w > WER_DISASTER_THRESHOLD or s < SECS_DISASTER_THRESHOLD - ) - - # 均值 - wer_mean = np.mean(wer_values) - secs_mean = np.mean(secs_values) - - # 标准差 (鲁棒性指标) - wer_std = np.std(wer_values) - secs_std = np.std(secs_values) - - # 分位数 (长尾分析) - wer_p90 = np.percentile(wer_values, 90) - wer_p95 = np.percentile(wer_values, 95) - secs_p10 = np.percentile(secs_values, 10) - secs_p05 = np.percentile(secs_values, 5) - - return { - "n_samples": n, - "wer_mean": float(wer_mean), - "wer_std": float(wer_std), - "wer_p90": float(wer_p90), - "wer_p95": float(wer_p95), - "wer_disaster_rate": float(wer_disaster / n * 100), - "wer_disaster_count": wer_disaster, - "secs_mean": float(secs_mean), - "secs_std": float(secs_std), - "secs_p10": float(secs_p10), - "secs_p05": float(secs_p05), - "secs_disaster_rate": float(secs_disaster / n * 100), - "secs_disaster_count": secs_disaster, - "combined_disaster_rate": float(combined_disaster / n * 100), - "combined_disaster_count": combined_disaster, - } - - -def run_cfr_analysis(): - """运行CFR分析.""" - logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") - logger.info("=" * 70) - logger.info("CFR (Catastrophic Failure Rate) Analysis") - logger.info("灾难边界: WER>50%% (语义灾难), SECS<0.25 (音色崩塌)") - logger.info("=" * 70) - - data = load_ablation_data() - if not data: - logger.error("No ablation data found!") - return - - # 计算每个配置的CFR - results = {} - for config_name, samples in data.items(): - results[config_name] = compute_cfr(samples) - - # 打印对比表 - logger.info("\n" + "=" * 90) - logger.info("CFR对比表 (200 samples)") - logger.info("=" * 90) - header = f"{'Config':<30} {'WER Mean':<10} {'WER CFR%':<10} {'SECS Mean':<10} {'SECS CFR%':<10} {'Combined CFR%':<14}" - logger.info(header) - logger.info("-" * 90) - - for config, metrics in results.items(): - line = ( - f"{config:<30} " - f"{metrics['wer_mean']:<10.4f} " - f"{metrics['wer_disaster_rate']:<10.1f} " - f"{metrics['secs_mean']:<10.4f} " - f"{metrics['secs_disaster_rate']:<10.1f} " - f"{metrics['combined_disaster_rate']:<14.1f}" - ) - logger.info(line) - - logger.info("-" * 90) - - # 长尾分析 - logger.info("\n" + "=" * 90) - logger.info("长尾分布分析 (分位数)") - logger.info("=" * 90) - header = f"{'Config':<30} {'WER P90':<10} {'WER P95':<10} {'SECS P10':<10} {'SECS P05':<10}" - logger.info(header) - logger.info("-" * 90) - - for config, metrics in results.items(): - line = ( - f"{config:<30} " - f"{metrics['wer_p90']:<10.4f} " - f"{metrics['wer_p95']:<10.4f} " - f"{metrics['secs_p10']:<10.4f} " - f"{metrics['secs_p05']:<10.4f}" - ) - logger.info(line) - - logger.info("-" * 90) - - # 灾难率改善 (相对Full v8.0) - if "Full v8.0 (完整)" in results: - full_cfr = results["Full v8.0 (完整)"]["combined_disaster_rate"] - logger.info("\n" + "=" * 90) - logger.info("灾难率改善 (相对Full v8.0)") - logger.info("=" * 90) - for config, metrics in results.items(): - if config == "Full v8.0 (完整)": - continue - delta = metrics["combined_disaster_rate"] - full_cfr - logger.info( - f" {config:<30} CFR={metrics['combined_disaster_rate']:.1f}% " - f"(vs Full={full_cfr:.1f}%, delta=+{delta:.1f}%)" - ) - - # 保存结果 - output_path = RESULTS_DIR / "cfr_analysis.json" - with open(str(output_path), "w", encoding="utf-8") as f: - json.dump(results, f, indent=2, ensure_ascii=False) - logger.info(f"\nCFR results saved to: {output_path}") - - # 生成可视化 - try: - _plot_cfr_comparison(results) - except Exception as e: - logger.warning(f"Plot failed: {e}") - - return results - - -def _plot_cfr_comparison(results: Dict): - """生成CFR对比图.""" - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - - configs = list(results.keys()) - wer_cfrs = [results[c]["wer_disaster_rate"] for c in configs] - secs_cfrs = [results[c]["secs_disaster_rate"] for c in configs] - combined_cfrs = [results[c]["combined_disaster_rate"] for c in configs] - - fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - - # 简化配置名 - short_names = [c.split("(")[0].strip() for c in configs] - - # WER CFR - bars1 = axes[0].bar(short_names, wer_cfrs, color=["#2ecc71", "#e74c3c", "#e67e22", "#9b59b6"]) - axes[0].set_title("Semantic Disaster Rate (WER > 50%)", fontsize=12, fontweight="bold") - axes[0].set_ylabel("Disaster Rate (%)") - axes[0].set_ylim(0, max(wer_cfrs) * 1.2 + 5) - for bar, val in zip(bars1, wer_cfrs): - axes[0].text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 1, - f"{val:.1f}%", ha="center", va="bottom", fontweight="bold") - axes[0].tick_params(axis="x", rotation=30) - - # SECS CFR - bars2 = axes[1].bar(short_names, secs_cfrs, color=["#2ecc71", "#e74c3c", "#e67e22", "#9b59b6"]) - axes[1].set_title("Timbre Collapse Rate (SECS < 0.25)", fontsize=12, fontweight="bold") - axes[1].set_ylabel("Disaster Rate (%)") - axes[1].set_ylim(0, max(secs_cfrs) * 1.2 + 5) - for bar, val in zip(bars2, secs_cfrs): - axes[1].text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 1, - f"{val:.1f}%", ha="center", va="bottom", fontweight="bold") - axes[1].tick_params(axis="x", rotation=30) - - # Combined CFR - bars3 = axes[2].bar(short_names, combined_cfrs, color=["#2ecc71", "#e74c3c", "#e67e22", "#9b59b6"]) - axes[2].set_title("Combined Disaster Rate", fontsize=12, fontweight="bold") - axes[2].set_ylabel("Disaster Rate (%)") - axes[2].set_ylim(0, max(combined_cfrs) * 1.2 + 5) - for bar, val in zip(bars3, combined_cfrs): - axes[2].text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 1, - f"{val:.1f}%", ha="center", va="bottom", fontweight="bold") - axes[2].tick_params(axis="x", rotation=30) - - plt.tight_layout() - output_path = RESULTS_DIR / "cfr_comparison.png" - plt.savefig(str(output_path), dpi=150, bbox_inches="tight") - plt.close() - logger.info(f"CFR plot saved to: {output_path}") - - -if __name__ == "__main__": - run_cfr_analysis() diff --git a/personavoice/experiment/comprehensive_evaluator.py b/personavoice/experiment/comprehensive_evaluator.py deleted file mode 100644 index 1f6d60d..0000000 --- a/personavoice/experiment/comprehensive_evaluator.py +++ /dev/null @@ -1,367 +0,0 @@ -"""PersonaVoice v10.3 综合评估器 (SOTA 多指标). - -核心指标 (对标顶会论文): - 1. SECS (Speaker Encoder Cosine Similarity): 音色相似度, ECAPA-TDNN - 2. WER (Word Error Rate): 语音可懂度, Whisper ASR - 3. UTMOS: 客观语音质量 MOS 预测, speechbrain - 4. RTF (Real-Time Factor): 推理速度, 生成时长/音频时长 - 5. SIM-o: WavLM-based 说话人相似度 (CosyVoice3 评测指标) - -设计原则: - - 满血架构, 无备用方案 - - 所有指标使用真实模型评估, 不用随机投影 - - 支持批量评估和单样本评估 -""" -import os -# v10.4: 禁用 wandb (避免 speechbrain lazy import 冲突) -os.environ.setdefault("WANDB_DISABLED", "true") -os.environ.setdefault("WANDB_MODE", "disabled") -os.environ.setdefault("HF_HUB_OFFLINE", "1") -os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") - -import sys -import time -import torch -import torchaudio -import numpy as np -from pathlib import Path -from typing import Dict, Optional, Tuple - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - - -class ComprehensiveEvaluator: - """综合评估器: SECS + WER + UTMOS + RTF + SIM-o. - - 满血架构, 所有指标使用真实模型. - """ - - def __init__(self, device: str = "cpu", vocoder_device: str = "cpu"): - self.device = device - self.vocoder_device = vocoder_device - - # 1. ECAPA SECS 评估器 (复用现有) - from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator - self.ecapa = ECAPAEvaluator(device=device, vocoder_device=vocoder_device) - self.secs_available = self.ecapa.available - - # 2. Whisper ASR (WER 评估) - self._asr_pipe = None - self._wer_available = False - - # 3. UTMOS (语音质量 MOS) - self._utmos_predictor = None - self._utmos_available = False - - # 4. WavLM (SIM-o 评估, CosyVoice3 指标) - self._wavlm_model = None - self._sim_available = False - - print(f"ComprehensiveEvaluator: SECS={'on' if self.secs_available else 'off'}, " - f"WER=lazy, UTMOS=lazy, SIM-o=lazy") - - def _ensure_asr(self, device: str): - """懒加载 Whisper ASR (避免初始化时内存峰值).""" - if self._asr_pipe is not None: - return - from transformers import pipeline as hf_pipeline - print("Loading Whisper ASR for WER evaluation...") - self._asr_pipe = hf_pipeline( - "automatic-speech-recognition", - model="openai/whisper-large-v3-turbo", - torch_dtype=torch.float16 if "cuda" in device else torch.float32, - device=device, - ) - self._wer_available = True - print(" Whisper ASR loaded.") - - def _ensure_utmos(self, device: str): - """懒加载 UTMOS 预测器.""" - if self._utmos_predictor is not None: - return - try: - from speechbrain.inference.speaker import SpeakerRecognition - # UTMOS 使用 speechbrain 的 metric 模型 - # 基于域名自适应的 MOS 预测 - import torchaudio - # 尝试加载 UTMOS 模型 (speechbrain/spkrec-ecapa-voxceleb) - # 如果不可用, 用 DNSMOS 替代 - self._utmos_predictor = "utmos_placeholder" - self._utmos_available = True - print(" UTMOS predictor loaded (placeholder).") - except Exception as e: - print(f" UTMOS not available: {e}") - self._utmos_available = False - - def _ensure_wavlm(self, device: str): - """懒加载 WavLM (SIM-o 指标, CosyVoice3 评测). - - v10.4.6: 缓存失败状态, 避免重复尝试加载 (网络不通时每次都会超时). - """ - # v10.4.6: 已尝试过 (成功或失败), 不重复尝试 - if hasattr(self, "_wavlm_tried") and self._wavlm_tried: - return - self._wavlm_tried = True - - if self._wavlm_model is not None: - return - try: - from transformers import Wav2Vec2FeatureExtractor, WavLMForXVector - print("Loading WavLM for SIM-o evaluation...") - self._wavlm_extractor = Wav2Vec2FeatureExtractor.from_pretrained( - "microsoft/wavlm-base-plus-sv" - ) - self._wavlm_model = WavLMForXVector.from_pretrained( - "microsoft/wavlm-base-plus-sv" - ).to(device) - self._wavlm_model.eval() - self._sim_available = True - print(" WavLM loaded.") - except Exception as e: - print(f" WavLM not available (用 ECAPA 替代 SIM-o): {type(e).__name__}") - self._sim_available = False - - @torch.no_grad() - def compute_secs( - self, gen_wav: torch.Tensor, ref_wav: torch.Tensor, sr: int = 16000 - ) -> float: - """SECS: 说话人编码余弦相似度 (ECAPA-TDNN).""" - if not self.secs_available: - return 0.0 - return self.ecapa.compute_secs(gen_wav, ref_wav, sr=sr) - - @torch.no_grad() - def compute_wer( - self, gen_wav: torch.Tensor, reference_text: str, sr: int = 16000, language: str = "chinese" - ) -> Tuple[float, str]: - """WER: 词错误率 (Whisper ASR). - - Args: - gen_wav: 生成音频 (B, T) 或 (T,) - reference_text: 参考文本 (正确文本) - sr: 采样率 - language: 语言 - - Returns: - (wer, recognized_text) - """ - self._ensure_asr(self.device) - if not self._wer_available: - return 0.0, "" - - if gen_wav.dim() > 1: - gen_wav = gen_wav.squeeze(0) - if sr != 16000: - gen_wav = torchaudio.functional.resample(gen_wav, sr, 16000) - - audio_np = gen_wav.cpu().numpy().astype(np.float32) - result = self._asr_pipe(audio_np, generate_kwargs={"language": language}) - recognized = result["text"].strip() - - # 计算中文字符级 WER (中文无空格分词) - if language == "chinese": - # 字符级 CER - ref_chars = list(reference_text.replace(" ", "").replace(",", "").replace("。", "").replace("?", "").replace("!", "")) - hyp_chars = list(recognized.replace(" ", "").replace(",", "").replace("。", "").replace("?", "").replace("!", "")) - wer = self._char_error_rate(ref_chars, hyp_chars) - else: - # 英文词级 WER - ref_words = reference_text.lower().split() - hyp_words = recognized.lower().split() - wer = self._word_error_rate(ref_words, hyp_words) - - return wer, recognized - - def _char_error_rate(self, ref: list, hyp: list) -> float: - """计算字符错误率 (编辑距离).""" - if len(ref) == 0: - return 1.0 if len(hyp) > 0 else 0.0 - # 编辑距离 - m, n = len(ref), len(hyp) - dp = [[0] * (n + 1) for _ in range(m + 1)] - for i in range(m + 1): - dp[i][0] = i - for j in range(n + 1): - dp[0][j] = j - for i in range(1, m + 1): - for j in range(1, n + 1): - if ref[i - 1] == hyp[j - 1]: - dp[i][j] = dp[i - 1][j - 1] - else: - dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) - return dp[m][n] / len(ref) - - def _word_error_rate(self, ref: list, hyp: list) -> float: - """计算词错误率.""" - if len(ref) == 0: - return 1.0 if len(hyp) > 0 else 0.0 - m, n = len(ref), len(hyp) - dp = [[0] * (n + 1) for _ in range(m + 1)] - for i in range(m + 1): - dp[i][0] = i - for j in range(n + 1): - dp[0][j] = j - for i in range(1, m + 1): - for j in range(1, n + 1): - if ref[i - 1] == hyp[j - 1]: - dp[i][j] = dp[i - 1][j - 1] - else: - dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]) - return dp[m][n] / len(ref) - - @torch.no_grad() - def compute_sim_o( - self, gen_wav: torch.Tensor, ref_wav: torch.Tensor, sr: int = 16000 - ) -> float: - """SIM-o: 说话人相似度 (CosyVoice3 评测指标). - - v10.4.6: WavLM 无法本地下载 (网络限制), 用 ECAPA x-vector 替代. - ECAPA 和 WavLM 都是 x-vector 提取器, 功能等价. - 网络恢复后可切换回 WavLM (取消注释 _ensure_wavlm 路径). - """ - # v10.4.6: 优先尝试 WavLM, 失败则用 ECAPA - self._ensure_wavlm(self.device) - if self._sim_available: - # WavLM 路径 - if gen_wav.dim() == 1: - gen_wav = gen_wav.unsqueeze(0) - if ref_wav.dim() == 1: - ref_wav = ref_wav.unsqueeze(0) - - if sr != 16000: - gen_wav = torchaudio.functional.resample(gen_wav, sr, 16000) - ref_wav = torchaudio.functional.resample(ref_wav, sr, 16000) - - def extract(wav): - inputs = self._wavlm_extractor( - wav.squeeze(0).cpu().numpy(), sampling_rate=16000, return_tensors="pt" - ) - inputs = {k: v.to(self.device) for k, v in inputs.items()} - outputs = self._wavlm_model(**inputs) - emb = outputs.embeddings # (1, 1024) - return emb - - emb_gen = extract(gen_wav) - emb_ref = extract(ref_wav) - sim = torch.nn.functional.cosine_similarity(emb_gen, emb_ref).item() - return sim - - # v10.4.6: ECAPA 替代路径 (WavLM 不可用时) - # ECAPA x-vector 与 WavLM x-vector 功能等价, 都是说话人嵌入 - if not self.secs_available: - return 0.0 - return self.ecapa.compute_secs(gen_wav, ref_wav, sr=sr) - - @torch.no_grad() - def compute_utmos(self, gen_wav: torch.Tensor, sr: int = 16000) -> float: - """UTMOS: 客观 MOS 预测 (1-5 分). - - 基于 WavLM 特征的 MOS 预测. - 若 UTMOS 不可用, 返回基于能量的近似质量分. - """ - self._ensure_utmos(self.device) - if not self._utmos_available: - return 0.0 - - if gen_wav.dim() == 1: - gen_wav = gen_wav.unsqueeze(0) - if sr != 16000: - gen_wav = torchaudio.functional.resample(gen_wav, sr, 16000) - - # 基于 WavLM 特征的 MOS 预测 - # 使用能量和 SNR 作为质量的近似指标 - wav_np = gen_wav.squeeze(0).cpu().numpy() - # RMS 能量 - rms = np.sqrt(np.mean(wav_np ** 2)) - # 过零率 - zcr = np.mean(np.abs(np.diff(np.sign(wav_np))) > 0) - # 静音比例 - silence_ratio = np.mean(np.abs(wav_np) < 0.01) - - # 启发式 MOS 估计 (1-5) - # 理想: rms 0.05-0.15, zcr 0.1-0.3, silence < 0.3 - mos = 3.0 # 基础分 - if 0.03 < rms < 0.2: - mos += 0.5 - if zcr < 0.5: - mos += 0.3 - if silence_ratio < 0.3: - mos += 0.5 - if rms > 0.001: # 非静音 - mos += 0.2 - - return min(5.0, max(1.0, mos)) - - def compute_rtf(self, gen_time_sec: float, audio_duration_sec: float) -> float: - """RTF: 实时率 (生成时间/音频时长). - - RTF < 1.0 表示比实时快. - """ - if audio_duration_sec <= 0: - return float("inf") - return gen_time_sec / audio_duration_sec - - @torch.no_grad() - def evaluate_all( - self, - gen_wav: torch.Tensor, - ref_wav: torch.Tensor, - reference_text: str, - sr: int = 16000, - gen_time_sec: float = 0.0, - language: str = "chinese", - ) -> Dict[str, float]: - """综合评估所有指标. - - Args: - gen_wav: 生成音频 (B, T) 或 (T,) - ref_wav: 参考音频 (B, T) 或 (T,) - reference_text: 参考文本 - sr: 采样率 - gen_time_sec: 生成耗时 (秒) - language: 语言 - - Returns: - dict: { - "secs": float, # 说话人相似度 (0-1, 越高越好) - "wer": float, # 词/字符错误率 (0-1, 越低越好) - "utmos": float, # 客观 MOS (1-5, 越高越好) - "sim_o": float, # WavLM 相似度 (0-1, 越高越好) - "rtf": float, # 实时率 (<1 快, >1 慢) - "audio_duration": float, # 音频时长 (秒) - "recognized_text": str, # ASR 识别结果 - } - """ - if gen_wav.dim() == 1: - gen_wav = gen_wav.unsqueeze(0) - if ref_wav.dim() == 1: - ref_wav = ref_wav.unsqueeze(0) - - # 音频时长 - audio_duration = gen_wav.shape[-1] / sr - - # 1. SECS - secs = self.compute_secs(gen_wav, ref_wav, sr=sr) - - # 2. WER - wer, recognized = self.compute_wer(gen_wav, reference_text, sr=sr, language=language) - - # 3. UTMOS - utmos = self.compute_utmos(gen_wav, sr=sr) - - # 4. SIM-o - sim_o = self.compute_sim_o(gen_wav, ref_wav, sr=sr) - - # 5. RTF - rtf = self.compute_rtf(gen_time_sec, audio_duration) - - return { - "secs": secs, - "wer": wer, - "utmos": utmos, - "sim_o": sim_o, - "rtf": rtf, - "audio_duration": audio_duration, - "recognized_text": recognized, - } diff --git a/personavoice/experiment/ecapa_evaluator.py b/personavoice/experiment/ecapa_evaluator.py deleted file mode 100644 index 1741635..0000000 --- a/personavoice/experiment/ecapa_evaluator.py +++ /dev/null @@ -1,386 +0,0 @@ -"""PersonaVoice v4.0 真实 ECAPA SECS 评估模块 (SOTA 修正版). - -架构位置: 实验模块中的核心评估工具, 通过真实声码器 + ECAPA-TDNN 计算真实 SECS, -被所有训练/消融/验证脚本复用. - -核心修复 (P0): - 旧版使用 nn.Linear(80, 192) 随机投影伪造 ECAPA 嵌入, 导致 SECS≈0.01. - 本版通过真实声码器 (Vocos/Griffin-Lim) 将生成 mel 转换为波形, - 再用 speechbrain/spkrec-ecapa-voxceleb 提取真实 192 维嵌入, - 计算真实 SECS. - -评估链路: - generated_mel (80, T) ──► Vocos/Griffin-Lim ──► wav (16kHz) - │ - ▼ - ECAPA-TDNN - │ - ▼ - emb_gen (192) - │ - emb_ref (192, 真实) ──────────────────────► cosine_similarity ──► SECS - -声码器选择策略: - 1. Vocos (首选): 高质量神经声码器, 需要100-bin/24kHz mel - - 若输入mel为80-bin, 自动插值到100-bin - - 若输入mel已归一化, 自动反归一化到log-mel范围 - 2. Griffin-Lim (回退): 通用, 兼容任何mel格式, 质量较低但诚实 -""" -import os -import sys -import torch -import torchaudio -import numpy as np -from pathlib import Path -from typing import Dict, Optional, Tuple - -# v10.4: 禁用 wandb (避免 speechbrain lazy import 冲突) -os.environ.setdefault("WANDB_DISABLED", "true") -os.environ.setdefault("WANDB_MODE", "disabled") -os.environ.setdefault("HF_HUB_OFFLINE", "1") -os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - - -class ECAPAEvaluator: - """真实ECAPA-TDNN说话人相似度评估器 (SOTA修正版). - - 通过真实声码器将mel转换为波形, 再提取ECAPA嵌入, 计算真实SECS. - """ - - # Vocos charactr/vocos-mel-24khz 的 mel 配置 - VOCOS_SAMPLE_RATE = 24000 - VOCOS_N_MELS = 100 - VOCOS_N_FFT = 1024 - VOCOS_HOP_LENGTH = 256 - - # 项目原始 mel 配置 (SOTA: Vocos兼容 100-bin) - PROJECT_SAMPLE_RATE = 16000 - PROJECT_N_MELS = 100 # SOTA: 与 Vocos 一致 - - def __init__( - self, - device: str = "cpu", - cache_dir: str = "./models/ecapa_tdnn", - vocoder_device: Optional[str] = None, - prefer_vocoder: str = "auto", - ): - """初始化ECAPA评估器. - - Args: - device: ECAPA编码器设备 (建议cpu, 评估时不占GPU显存) - cache_dir: ECAPA模型缓存目录 - vocoder_device: 声码器设备 (默认与device相同) - prefer_vocoder: "vocos" / "griffin_lim" / "auto" - """ - print("Loading ECAPA-TDNN evaluator (SOTA rectified)...") - self.device = device - self.vocoder_device = vocoder_device or device - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(parents=True, exist_ok=True) - - # 加载 ECAPA 编码器 - try: - from speechbrain.inference.speaker import EncoderClassifier - - self.encoder = EncoderClassifier.from_hparams( - source="speechbrain/spkrec-ecapa-voxceleb", - savedir=str(self.cache_dir), - run_opts={"device": device}, - ) - self.embedding_dim = 192 - self.available = True - print(f" ECAPA loaded, embedding_dim={self.embedding_dim}") - except Exception as e: - print(f" [WARN] Failed to load ECAPA: {e}") - self.available = False - self.embedding_dim = 192 - - # 加载声码器 - self._vocoder = None - self._vocoder_type = None - self._griffin_lim = None - self._inverse_mel = None - self._mel_interp = None # 80->100 mel 插值 (可学习, 但评估时用线性插值) - self._load_vocoder(prefer_vocoder) - - def _load_vocoder(self, prefer: str = "auto"): - """加载声码器 (满血架构, 只用 Vocos, 无备用方案).""" - from vocos import Vocos - - self._vocoder = Vocos.from_pretrained( - "charactr/vocos-mel-24khz" - ).to(self.vocoder_device) - self._vocoder.eval() - self._vocoder_type = "vocos" - print(f" Vocoder: Vocos (24kHz, 100 mel bins) on {self.vocoder_device}") - - # ----------------------------------------------------------------- - # 嵌入提取 - # ----------------------------------------------------------------- - @torch.no_grad() - def extract_embedding(self, wav: torch.Tensor, sr: int = 16000) -> torch.Tensor: - """从波形提取真实ECAPA嵌入. - - Args: - wav: (B, T) 或 (T,) 波形, 16kHz - sr: 采样率 - - Returns: - emb: (B, 192) 嵌入 - """ - if not self.available: - return torch.zeros( - wav.shape[0] if wav.dim() > 1 else 1, self.embedding_dim - ) - - if wav.dim() == 1: - wav = wav.unsqueeze(0) - - wav = wav.to(self.device) - # ECAPA 期望 16kHz - if sr != 16000: - wav = torchaudio.functional.resample(wav, sr, 16000) - - # ECAPA 需要足够长的音频 (至少 ~1秒) - min_len = 8000 - if wav.shape[-1] < min_len: - wav = torch.nn.functional.pad(wav, (0, min_len - wav.shape[-1])) - - emb = self.encoder.encode_batch(wav) - if emb.dim() == 3: - emb = emb.squeeze(1) - - return emb.cpu() - - # ----------------------------------------------------------------- - # 声码器: mel -> wav - # ----------------------------------------------------------------- - @torch.no_grad() - def mel_to_waveform( - self, - mel: torch.Tensor, - mel_sample_rate: int = 16000, - mel_n_mels: int = 100, - is_normalized: bool = False, - ) -> Tuple[torch.Tensor, int]: - """将mel频谱转换为波形 (真实声码器). - - Args: - mel: (B, n_mels, T) 或 (n_mels, T) mel频谱 - mel_sample_rate: mel提取时的采样率 - mel_n_mels: mel的bin数 - is_normalized: mel是否经过per-sample归一化 (SOTA标准: False, 使用raw log-mel) - - Returns: - wav: (B, T_samples) 波形 - sr: 波形采样率 - """ - squeeze = False - if mel.dim() == 2: - mel = mel.unsqueeze(0) - squeeze = True - - mel = mel.to(self.vocoder_device) - - # Vocos 声码器解码 (满血架构, 无备用方案) - wav = self._vocos_decode( - mel, mel_sample_rate, mel_n_mels, is_normalized - ) - sr = self.VOCOS_SAMPLE_RATE - # 重采样到 16kHz (ECAPA 需要) - wav = torchaudio.functional.resample( - wav, self.VOCOS_SAMPLE_RATE, 16000 - ) - sr = 16000 - - if squeeze: - wav = wav.squeeze(0) - - return wav.cpu(), sr - - def _vocos_decode( - self, - mel: torch.Tensor, - mel_sample_rate: int, - mel_n_mels: int, - is_normalized: bool, - ) -> torch.Tensor: - """Vocos 声码器解码. - - 处理 mel 格式转换: - 1. 反归一化 (若需要): normalized mel -> log-mel - 2. mel bin 插值: 80 -> 100 - 3. Vocos 解码: mel -> 24kHz wav - """ - # Step 1: 反归一化 (仅当输入是归一化的) - # SOTA标准: 使用 raw log-mel (is_normalized=False), 无需反归一化 - if is_normalized: - # 旧版归一化数据: mel = (mel - mean) / std - # 反归一化到 log-mel 范围 [-80, 20] - mel = mel * 12.0 - 30.0 - - # 确保 mel 在合理范围 (Vocos 期望 log-mel) - mel = torch.clamp(mel, min=-80, max=20) - - # Step 2: mel bin 插值 (80 -> 100 或其他 -> 100) - if mel_n_mels != self.VOCOS_N_MELS: - # 线性插值: (B, 80, T) -> (B, 100, T) - mel = mel.transpose(1, 2) # (B, T, n_mels) - mel = torch.nn.functional.interpolate( - mel, size=self.VOCOS_N_MELS, mode="linear", align_corners=False - ) - mel = mel.transpose(1, 2) # (B, 100, T) - - # Step 3: Vocos 解码 - wav = self._vocoder.decode(mel) - return wav - - # ----------------------------------------------------------------- - # SECS 计算 - # ----------------------------------------------------------------- - @torch.no_grad() - def compute_secs( - self, - wav_generated: torch.Tensor, - wav_reference: torch.Tensor, - sr: int = 16000, - ) -> float: - """计算真实SECS (从波形). - - Args: - wav_generated: (B, T) 或 (T,) 生成的波形 - wav_reference: (B, T) 或 (T,) 参考波形 - sr: 采样率 - - Returns: - secs: 平均余弦相似度 - """ - if not self.available: - return 0.0 - - emb_gen = self.extract_embedding(wav_generated, sr) - emb_ref = self.extract_embedding(wav_reference, sr) - - secs = torch.nn.functional.cosine_similarity(emb_gen, emb_ref, dim=-1) - return secs.mean().item() - - @torch.no_grad() - def compute_secs_from_mel( - self, - mel_generated: torch.Tensor, - emb_reference: torch.Tensor, - mel_sample_rate: int = 16000, - mel_n_mels: int = 100, - is_normalized: bool = False, - ) -> float: - """从生成mel计算真实SECS (SOTA修正版). - - 关键修复: 通过真实声码器将mel转换为波形, 再提取ECAPA嵌入, - 而非使用 nn.Linear(80, 192) 随机投影. - - Args: - mel_generated: (B, n_mels, T) 生成的mel (raw log-mel) - emb_reference: (B, 192) 参考音频的真实ECAPA嵌入 - mel_sample_rate: mel提取时的采样率 - mel_n_mels: mel的bin数 - is_normalized: mel是否经过per-sample归一化 (SOTA标准: False) - - Returns: - secs: 真实SECS (余弦相似度) - """ - if not self.available: - return 0.0 - - # mel -> wav (真实声码器) - wav_gen, sr = self.mel_to_waveform( - mel_generated, - mel_sample_rate=mel_sample_rate, - mel_n_mels=mel_n_mels, - is_normalized=is_normalized, - ) - - # wav -> ECAPA 嵌入 - emb_gen = self.extract_embedding(wav_gen, sr=sr) - - # 与参考嵌入计算余弦相似度 - emb_ref = emb_reference.to(emb_gen.device) - if emb_ref.dim() == 1: - emb_ref = emb_ref.unsqueeze(0) - - secs = torch.nn.functional.cosine_similarity(emb_gen, emb_ref, dim=-1) - return secs.mean().item() - - @torch.no_grad() - def compute_secs_mel_vs_mel( - self, - mel_generated: torch.Tensor, - mel_reference: torch.Tensor, - mel_sample_rate: int = 16000, - mel_n_mels: int = 100, - is_normalized: bool = False, - ) -> float: - """从两个mel计算SECS (都通过声码器转换为波形). - - Args: - mel_generated: (B, n_mels, T) 生成的mel (raw log-mel) - mel_reference: (B, n_mels, T) 参考mel (raw log-mel) - mel_sample_rate: mel采样率 - mel_n_mels: mel bin数 - is_normalized: 是否归一化 (SOTA标准: False) - - Returns: - secs: 真实SECS - """ - if not self.available: - return 0.0 - - wav_gen, sr = self.mel_to_waveform( - mel_generated, mel_sample_rate, mel_n_mels, is_normalized - ) - wav_ref, _ = self.mel_to_waveform( - mel_reference, mel_sample_rate, mel_n_mels, is_normalized - ) - - return self.compute_secs(wav_gen, wav_ref, sr=sr) - - -def test_ecapa_evaluator(): - """测试ECAPA评估器 (SOTA修正版).""" - print("\n" + "=" * 60) - print("Testing ECAPA Evaluator (SOTA Rectified)") - print("=" * 60) - - device = "cuda" if torch.cuda.is_available() else "cpu" - evaluator = ECAPAEvaluator(device="cpu", vocoder_device=device) - - # 测试1: 相同波形的SECS应接近1.0 - print("\n[Test 1] Same waveform SECS (should be ~1.0):") - wav1 = torch.randn(1, 16000) * 0.1 - secs_same = evaluator.compute_secs(wav1, wav1) - print(f" SECS (same): {secs_same:.4f}") - - # 测试2: 不同波形的SECS应较低 - print("\n[Test 2] Different waveform SECS (should be < 1.0):") - wav2 = torch.randn(1, 16000) * 0.1 - secs_diff = evaluator.compute_secs(wav1, wav2) - print(f" SECS (diff): {secs_diff:.4f}") - - # 测试3: mel -> wav -> ECAPA 链路 - print("\n[Test 3] mel -> wav -> ECAPA pipeline:") - mel_fake = torch.randn(2, 80, 126) # 模拟生成的mel (归一化) - emb_ref = torch.randn(2, 192) # 模拟参考ECAPA嵌入 - secs_mel = evaluator.compute_secs_from_mel( - mel_fake, emb_ref, - mel_sample_rate=16000, mel_n_mels=80, is_normalized=True - ) - print(f" SECS (mel->wav->ECAPA): {secs_mel:.4f}") - print(f" Vocoder type: {evaluator._vocoder_type}") - - print("\n[DONE] ECAPA evaluator test complete.") - - -if __name__ == "__main__": - test_ecapa_evaluator() diff --git a/personavoice/experiment/eval_200_samples.py b/personavoice/experiment/eval_200_samples.py deleted file mode 100644 index 7fc02a9..0000000 --- a/personavoice/experiment/eval_200_samples.py +++ /dev/null @@ -1,923 +0,0 @@ -"""PersonaVoice v9.1 200样本统计显著性评估 (SOTA P0 级任务). - -架构位置: 实验模块中的大规模统计评估脚本, 基于 v9.1 精简配置在 200 个样本上 -进行统计显著性验证, 解决 v7.7 报告"20样本标准差过大、缺乏 p-value"的学术软肋. - -v9.1 精简架构 (基于 200 样本消融实验): - - 保留: F5-TTS 预训练骨干 + FiLM + OES (env_weight=0.0) + IEAG - - 移除: IBOP/AM-ODE/TD-CFG/SBM (消融实验证实无效) - -核心目标 (对应 SOTA_VERIFICATION_REPORT.md 第 14 节): - 1. 暴力扩充测试集: 20 → 200 样本 (P0 级任务) - 2. 计算统计显著性: p-value, 95%置信区间, 效应量 (Cohen's d) - 3. 同时评估 F5-TTS Baseline 与 PersonaVoice v9.1 精简配置 - 4. 按文本长度分层分析 (短/长文本) - 5. 为信息熵衰减定律 (Phase 4) 收集 (target_length, SECS) 数据点 - -v9.1 最佳配置 (基于 SOTA_VERIFICATION_REPORT.md 14.5 节): - - 静态 CFG: cfg_strength=2.5 (替代无效的 TD-CFG) - - IEAG: lambda_max=0.15, t∈[0.1, 0.4] (有效, WER 改善主要贡献者) - - OES: env_weight=0.0 (纯净录音棚音质, SECS SOTA 配置) - - 动态长度: seconds_per_word=0.30 - - ODE 步数: 64 (v9.0 SOTA 配置) - - Sway Sampling: coef=-1.0 (F5-TTS 官方默认) - -输出: - - results/eval_200_samples.json: 完整 200 样本评估数据 - - results/eval_200_statistics.json: 统计显著性分析 - - results/eval_200_by_length.json: 按文本长度分层分析 - - results/entropy_decay_data.json: 信息熵衰减定律数据点 -""" - -import json -import sys -import time -from pathlib import Path -from typing import Dict, List, Tuple, Optional - -import numpy as np -import torch -import torchaudio -from scipy import stats as scipy_stats - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - -from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone -from personavoice.experiment.utils import load_tokenizer, setup_logger -from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator -from personavoice.experiment.wer_evaluator import WEREvaluator - - -def estimate_target_length(text: str, seconds_per_word: float = 0.35) -> int: - """动态目标长度估计 (v9.0 优化). - - 根据文本词数估计目标音频长度 (帧数), 解决数据集 3s 截断导致的长文本 OOD 问题. - - Args: - text: 目标文本 - seconds_per_word: 每词秒数 (v9.0 最佳配置: 0.35s/word, 10样本WER=0.2748) - - Returns: - target_length: 目标 mel 帧数 (94帧/秒) - """ - n_words = max(1, len(text.split())) - target_sec = n_words * seconds_per_word + 1.0 # 加 1s 缓冲 - target_sec = max(1.0, min(target_sec, 30.0)) # 限制 1-30s - return int(target_sec * 94) # 94帧/秒 @ 24kHz/hop=256 - - -def estimate_target_length_f5_official( - text: str, - ref_text: str, - ref_mel_frames: int = 94, - speed: float = 1.0, -) -> int: - """F5-TTS官方语速推算 (v9.0 借鉴XTTS自回归时长自适应). - - XTTS自回归自动决定时长, F5-TTS需要预设. - F5-TTS官方公式 (utils_infer.py:493): - duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed) - - 这比简单的 n_words * 0.35 更准确, 因为它基于参考音频的实际语速. - - Args: - text: 生成文本 - ref_text: 参考音频对应文本 (1s参考的文本内容) - ref_mel_frames: 参考音频mel帧数 (1s = 94帧) - speed: 语速因子 (1.0=正常, <1.0=慢, >1.0=快) - - Returns: - target_length: 目标 mel 帧数 - """ - ref_text_len = max(1, len(ref_text.encode("utf-8"))) - gen_text_len = max(1, len(text.encode("utf-8"))) - # F5-TTS官方公式: 总时长 = 参考时长 + 参考语速推算的生成时长 - duration = ref_mel_frames + int(ref_mel_frames / ref_text_len * gen_text_len / speed) - # 限制在合理范围 (1s-30s) - duration = max(94, min(duration, 30 * 94)) - return duration - -logger = setup_logger("eval_200_samples") - - -# ============================================================================= -# 数据加载: 200样本 -# ============================================================================= - -def load_200_test_samples(seed: int = 42) -> List[Dict]: - """加载200个测试样本 (从387样本中随机抽取, 确保统计显著性). - - 策略: - - 从387样本中随机抽取200个 (无放回) - - 固定seed保证可复现 - - 每个样本包含1s参考mel + 完整mel + 目标文本 + ECAPA嵌入 - - Args: - seed: 随机种子 - - Returns: - samples: 200个样本列表 - """ - data_path = PROJECT_ROOT / "data" / "libritts_processed" / "libritts_devclean_processed.pt" - data = torch.load(str(data_path), weights_only=False) - - mel = data["tensors"]["mel"] # (N, 100, T) - wav = data["tensors"]["wav"] # (N, T_samples) v10.4.6: 加载原始波形 - ecapa_emb = data["tensors"]["ecapa_emb"] # (N, 192) - texts = data["metadata"]["texts"] - speaker_names = data["metadata"]["speaker_names"] - sample_rate = data["metadata"].get("sample_rate", 24000) - - n_total = mel.shape[0] - n_samples = min(200, n_total) - logger.info(f"Loading {n_samples} samples from {n_total} total (seed={seed})") - - rng = np.random.RandomState(seed) - indices = rng.choice(n_total, n_samples, replace=False) - indices = sorted(indices.tolist()) - - samples = [] - for idx in indices: - idx = int(idx) - mel_full = mel[idx] # (100, T) - if mel_full.dim() == 2: - mel_full = mel_full.transpose(0, 1) # (T, 100) - - # 1s参考 (94帧 @ 24kHz/hop=256) - mel_ref = mel_full[:94, :] # (94, 100) - - # v10.4.6: 提取 1s 参考音频波形 (F5-TTS 官方 cond=audio 要求) - # 数据集 wav 是 16kHz, F5-TTS 要求 24kHz - # 1s @ 16kHz = 16000 samples, 重采样到 24kHz = 24000 samples - ref_samples_1s_16k = 16000 # 1s @ 16kHz - wav_full = wav[idx] # (T_samples,) - if wav_full.dim() > 1: - wav_full = wav_full.squeeze(0) - audio_ref_1s_16k = wav_full[:ref_samples_1s_16k] # (16000,) @ 16kHz - - # 重采样到 24kHz (F5-TTS 要求) - import torchaudio as _ta - audio_ref_1s = _ta.functional.resample(audio_ref_1s_16k, 16000, 24000) # (24000,) @ 24kHz - - # v10.4.6: ref_text 由 Whisper ASR 转录得到 (与前端流程一致) - # F5-TTS 官方要求 ref_text 与 ref_audio 内容一致 - # 数据集中 wav 和 text 是同一句话, 但 1s 截取只包含前几个词 - # 用 Whisper 转录 1s 截取得到准确的 ref_text - target_text = texts[idx] if idx < len(texts) else "" - - samples.append({ - "mel_ref": mel_ref, - "mel_full": mel_full, - "audio_ref": audio_ref_1s, # v10.4.6: 1s 参考波形 @ 24kHz - "audio_ref_16k": audio_ref_1s_16k, # v10.4.6: 1s 参考波形 @ 16kHz (Whisper 转录用) - "target_text": target_text, - "speaker_emb": ecapa_emb[idx], - "speaker_name": speaker_names[idx] if idx < len(speaker_names) else "", - "idx": idx, - }) - - logger.info(f"Loaded {len(samples)} test samples (ref=1.0s, ref_samples=94)") - return samples - - -# ============================================================================= -# PersonaVoice v9.1 评估 -# ============================================================================= - -@torch.no_grad() -def evaluate_v91_sample( - sample: Dict, - backbone, - tokenizer, - ecapa_evaluator: ECAPAEvaluator, - wer_evaluator: WEREvaluator, - seconds_per_word: float = 0.30, - cfg_strength: float = 3.0, - steps: int = 96, - device: str = "cuda", - use_oes: bool = True, - env_weight: float = 0.0, - use_ceag: bool = True, - ceag_lambda_max: float = 0.25, - ceag_t_start: float = 0.05, - ceag_t_end: float = 0.5, - ceag_layers: Tuple[int, ...] = (-3, -2, -1), - sway_sampling_coef: float = -1.0, -) -> Dict: - """评估单个样本 (v10.1 满血配置: LAAG + CEAG). - - v10.1 配置 (满血架构, 无备用方案): - - LAAG: 长度自适应生成 (动态 Chunking + 动态 CEAG λ + 动态 CFG + F5-TTS 官方时长公式) - - CEAG: 交叉注意力熵引导 (升级自 IEAG, 带 padding mask) - - OES env_scale=0.1: 渐进初始化 (解决 1111.mp3 SECS 灾难) - - 静态 CFG=2.0, steps=32, Sway Sampling=-1.0 - - 移除: IBOP/AM-ODE/TD-CFG/SBM/Duration Predictor (消融实验证实无效或未训练) - """ - mel_ref = sample["mel_ref"].to(device).float() - target_text = sample["target_text"] - speaker_emb = sample["speaker_emb"].to(device).float() - # v10.4.6: 加载 1s 参考音频波形 (F5-TTS 官方 cond=audio 要求, 24kHz) - audio_ref = sample.get("audio_ref") - if audio_ref is not None: - audio_ref = audio_ref.to(device).float() - audio_ref_16k = sample.get("audio_ref_16k") - - # v10.4.6: 用 Whisper ASR 转录 1s 参考音频得到 ref_text (与前端流程一致) - # F5-TTS 官方要求 ref_text 与 ref_audio 内容一致 - # 1s 截取只包含前几个词, 用 Whisper 转录得到准确内容 - global _asr_pipe_eval - if '_asr_pipe_eval' not in globals() or _asr_pipe_eval is None: - from transformers import pipeline as hf_pipeline - _asr_pipe_eval = hf_pipeline( - "automatic-speech-recognition", - model="openai/whisper-large-v3-turbo", - torch_dtype=torch.float16 if "cuda" in str(device) else torch.float32, - device=device, - ) - audio_ref_np = audio_ref_16k.numpy().astype("float32") - asr_result = _asr_pipe_eval(audio_ref_np, generate_kwargs={"language": "english"}) - ref_text = asr_result["text"].strip() - if not ref_text: - ref_text = "Hello." # 空转录兜底 - - mel_ref_b = mel_ref.unsqueeze(0) - speaker_emb_b = speaker_emb.unsqueeze(0) - # v10.0 修复: persona_emb 应为 192 维 (会被 persona_proj 投影到 64) - # emotion_emb 应为 4 维 (one-hot, 会被 emotion_proj 投影到 64) - persona_emb_b = torch.zeros(1, 192, device=device) - emotion_emb_b = torch.zeros(1, 4, device=device) - - n_words = len(target_text.split()) - - # v9.1: OES 环境权重设置 (默认 0.0: 纯净录音棚音质, SOTA 配置) - if use_oes and hasattr(backbone, "set_env_weight"): - backbone.set_env_weight(env_weight) - elif hasattr(backbone, "set_env_weight"): - backbone.set_env_weight(0.0) - - # 编码文本 - text_tokens = tokenizer.encode(target_text) - if isinstance(text_tokens, torch.Tensor): - text_tokens_b = text_tokens.unsqueeze(0).to(device) - else: - text_tokens_b = torch.tensor([text_tokens], device=device) - - t_start = time.time() - try: - # v10.1 LAAG: 长度自适应生成 (满血架构, 无备用方案) - # v10.4.6: 传入 audio_ref + ref_text (F5-TTS 官方 cond=audio 流程) - mel_gen, laag_info = backbone.laag_synthesize( - text_str=target_text, - mel_ref=mel_ref_b, - speaker_emb=speaker_emb_b, - persona_emb=persona_emb_b, - emotion_emb=emotion_emb_b, - tokenizer=tokenizer, - ref_text=ref_text, - audio_ref=audio_ref, - ) - except Exception as e: - return { - "secs": 0.0, "wer": 1.0, "time_ms": 0.0, - "error": str(e), "target_length": 0, - "text_len": n_words, "gen_length_sec": 0.0, - } - - t_elapsed = (time.time() - t_start) * 1000 - - # v10.4.6 修复: backbone.synthesize 返回 (B, n_mels=100, T) 格式 - # ECAPA compute_secs_from_mel 期望 (B, n_mels, T) - 无需转置! - # 之前错误地做了 transpose(1,2) 变成 (B, T, 100), 导致短文本 SECS=0 - mel_gen_vocos = mel_gen # (B, 100, T) - 直接使用 - - # SECS评估 - try: - secs = ecapa_evaluator.compute_secs_from_mel( - mel_generated=mel_gen_vocos.cpu(), - emb_reference=speaker_emb_b.cpu(), - mel_sample_rate=24000, - mel_n_mels=100, - is_normalized=False, - ) - except Exception: - secs = 0.0 - - # WER评估 - try: - wav_gen, sr_wav = ecapa_evaluator.mel_to_waveform( - mel_gen_vocos.cpu(), mel_sample_rate=24000, mel_n_mels=100, is_normalized=False - ) - if wav_gen.dim() == 1: - wav_gen = wav_gen.unsqueeze(0) - if sr_wav != 16000: - wav_gen = torchaudio.functional.resample(wav_gen, sr_wav, 16000) - wer = wer_evaluator.compute_wer(wav_gen.squeeze(0), target_text, sr=16000) - except Exception: - wer = 1.0 - - # 生成音频长度 (秒): mel 形状 (B, 100, T), T 是时间帧 - gen_length_sec = mel_gen_vocos.shape[-1] / 94.0 # 94帧/秒 - - # v10.4.6: target_length 由 LAAG 内部 F5-TTS 官方公式计算, 这里仅记录实际生成长度 - actual_target_length = int(mel_gen_vocos.shape[-1]) - - return { - "secs": float(secs), - "wer": float(wer), - "time_ms": t_elapsed, - "idx": sample.get("idx", -1), - "text": target_text[:80], - "text_len": n_words, - "target_length": actual_target_length, - "gen_length_sec": float(gen_length_sec), - "speaker_name": sample.get("speaker_name", ""), - } - - -# ============================================================================= -# F5-TTS Baseline 评估 -# ============================================================================= - -@torch.no_grad() -def evaluate_baseline_f5tts( - sample: Dict, - f5tts_obj, - ecapa_evaluator: ECAPAEvaluator, - wer_evaluator: WEREvaluator, - device: str = "cuda", -) -> Dict: - """评估F5-TTS官方原版基线 (不加载微调权重).""" - import torchaudio - from f5_tts.infer.utils_infer import infer_process - - mel_ref = sample["mel_ref"].cpu() # (T_ref, 100) - target_text = sample["target_text"] - speaker_emb = sample["speaker_emb"].cpu() # (192,) - n_words = len(target_text.split()) - - # mel_ref -> wav (作为F5-TTS参考) - mel_ref_vocos = mel_ref.transpose(0, 1).unsqueeze(0) # (1, 100, T_ref) - try: - ref_wav_16k, _ = ecapa_evaluator.mel_to_waveform( - mel_ref_vocos, mel_sample_rate=24000, mel_n_mels=100, is_normalized=False - ) - if ref_wav_16k.dim() == 1: - ref_wav_16k = ref_wav_16k.unsqueeze(0) - except Exception as e: - return { - "secs": 0.0, "wer": 1.0, "time_ms": 0.0, - "error": str(e), "text_len": n_words, "gen_length_sec": 0.0, - } - - # 保存参考wav到临时文件 - tmp_dir = PROJECT_ROOT / "tmp_eval200" - tmp_dir.mkdir(exist_ok=True) - ref_path = tmp_dir / f"ref_baseline_{sample.get('idx', 0)}.wav" - torchaudio.save(str(ref_path), ref_wav_16k, 16000) - - t_start = time.time() - try: - ref_text = target_text.strip() - if not ref_text.endswith("."): - ref_text += ". " - else: - ref_text += " " - - wav_gen, sr_gen, _ = infer_process( - ref_audio=str(ref_path), - ref_text=ref_text, - gen_text=target_text, - model_obj=f5tts_obj.ema_model, - vocoder=f5tts_obj.vocoder, - mel_spec_type=f5tts_obj.mel_spec_type, - show_info=lambda x: None, - nfe_step=32, - cfg_strength=2, - sway_sampling_coef=-1, - speed=1.0, - device=f5tts_obj.device, - ) - t_elapsed = (time.time() - t_start) * 1000 - - if isinstance(wav_gen, np.ndarray): - wav_gen = torch.from_numpy(wav_gen).float() - if wav_gen.dim() == 1: - wav_gen = wav_gen.unsqueeze(0) - - if sr_gen != 16000: - wav_gen = torchaudio.functional.resample(wav_gen, sr_gen, 16000) - - except Exception as e: - try: - ref_path.unlink(missing_ok=True) - except Exception: - pass - return { - "secs": 0.0, "wer": 1.0, "time_ms": 0.0, - "error": str(e), "text_len": n_words, "gen_length_sec": 0.0, - } - - try: - ref_path.unlink(missing_ok=True) - except Exception: - pass - - # ECAPA SECS - secs = 0.0 - try: - emb_gen = ecapa_evaluator.extract_embedding(wav_gen, sr=16000) - secs = torch.nn.functional.cosine_similarity( - emb_gen.squeeze(0), speaker_emb, dim=-1 - ).item() - except Exception: - secs = 0.0 - - # WER - wer = 1.0 - try: - wer = wer_evaluator.compute_wer(wav_gen.squeeze(0), target_text, sr=16000) - except Exception: - wer = 1.0 - - # 生成音频长度 (秒) - gen_length_sec = wav_gen.shape[-1] / 16000.0 - - return { - "secs": float(secs), - "wer": float(wer), - "time_ms": t_elapsed, - "idx": sample.get("idx", -1), - "text_len": n_words, - "gen_length_sec": float(gen_length_sec), - } - - -# ============================================================================= -# 统计显著性分析 -# ============================================================================= - -def compute_statistics(scores: List[float]) -> Dict: - """计算统计指标: 均值, 标准差, 95%CI, 标准误差.""" - arr = np.array(scores) - n = len(arr) - mean = float(np.mean(arr)) - std = float(np.std(arr, ddof=1)) if n > 1 else 0.0 - sem = std / np.sqrt(n) if n > 0 else 0.0 - - # 95% 置信区间 (t分布) - if n > 1: - t_val = scipy_stats.t.ppf(0.975, df=n-1) - ci_low = mean - t_val * sem - ci_high = mean + t_val * sem - else: - ci_low = ci_high = mean - - return { - "mean": mean, - "std": std, - "sem": float(sem), - "ci_95_low": float(ci_low), - "ci_95_high": float(ci_high), - "n": n, - "min": float(np.min(arr)), - "max": float(np.max(arr)), - "median": float(np.median(arr)), - } - - -def compute_paired_t_test( - scores_a: List[float], - scores_b: List[float], - alpha: float = 0.05, -) -> Dict: - """配对样本t检验 (PersonaVoice vs Baseline). - - H0: 两个系统性能无差异 - H1: PersonaVoice 优于 Baseline (单侧检验) - - Args: - scores_a: Baseline 分数 - scores_b: PersonaVoice 分数 - alpha: 显著性水平 - - Returns: - 统计检验结果 - """ - n = min(len(scores_a), len(scores_b)) - a = np.array(scores_a[:n]) - b = np.array(scores_b[:n]) - - # 配对差异 - diff = b - a - mean_diff = float(np.mean(diff)) - std_diff = float(np.std(diff, ddof=1)) if n > 1 else 0.0 - - # t统计量 - if std_diff > 0: - t_stat = mean_diff / (std_diff / np.sqrt(n)) - else: - t_stat = 0.0 - - # 单侧p-value (H1: b > a) - if n > 1: - p_value_one_sided = float(scipy_stats.t.sf(t_stat, df=n-1)) - else: - p_value_one_sided = 1.0 - - # 双侧p-value - if n > 1: - p_value_two_sided = float(2 * scipy_stats.t.sf(abs(t_stat), df=n-1)) - else: - p_value_two_sided = 1.0 - - # Cohen's d 效应量 - if std_diff > 0: - cohen_d = mean_diff / std_diff - else: - cohen_d = 0.0 - - # 效应量解释 - if abs(cohen_d) < 0.2: - effect_size = "negligible" - elif abs(cohen_d) < 0.5: - effect_size = "small" - elif abs(cohen_d) < 0.8: - effect_size = "medium" - else: - effect_size = "large" - - return { - "n_pairs": n, - "mean_diff": mean_diff, - "std_diff": std_diff, - "t_statistic": float(t_stat), - "p_value_one_sided": p_value_one_sided, - "p_value_two_sided": p_value_two_sided, - "cohen_d": float(cohen_d), - "effect_size": effect_size, - "significant_at_alpha": bool(p_value_one_sided < alpha), - "alpha": alpha, - } - - -def analyze_by_text_length(results: List[Dict], threshold: int = 8) -> Dict: - """按文本长度分层分析 (短文本 vs 长文本). - - Args: - results: 评估结果列表 - threshold: 长短文本分界 (词数) - - Returns: - 分层分析结果 - """ - short = [r for r in results if r["text_len"] < threshold] - long = [r for r in results if r["text_len"] >= threshold] - - short_secs = [r["secs"] for r in short] - short_wer = [r["wer"] for r in short] - long_secs = [r["secs"] for r in long] - long_wer = [r["wer"] for r in long] - - return { - "threshold_words": threshold, - "short_text": { - "n_samples": len(short), - "secs": compute_statistics(short_secs), - "wer": compute_statistics(short_wer), - }, - "long_text": { - "n_samples": len(long), - "secs": compute_statistics(long_secs), - "wer": compute_statistics(long_wer), - }, - } - - -def collect_entropy_decay_data(results: List[Dict]) -> Dict: - """收集信息熵衰减定律数据点 (target_length, SECS). - - 用于Phase 4: 拟合 SECS vs 生成音频长度 的衰减曲线, - 定义"时序漂移边界 (Temporal Drift Boundary)". - - Returns: - data: 包含 (length, secs) 数据点, 可用于拟合 - """ - data_points = [] - for r in results: - if r.get("gen_length_sec", 0) > 0: - data_points.append({ - "gen_length_sec": r["gen_length_sec"], - "secs": r["secs"], - "wer": r["wer"], - "text_len": r["text_len"], - "idx": r.get("idx", -1), - }) - - # 按生成长度分桶计算平均SECS - bins = [0, 3, 5, 7, 10, 15, 20, 30] - bin_stats = [] - for i in range(len(bins) - 1): - low, high = bins[i], bins[i + 1] - bucket = [d for d in data_points if low <= d["gen_length_sec"] < high] - if bucket: - bin_stats.append({ - "length_range": [low, high], - "n_samples": len(bucket), - "secs_mean": float(np.mean([d["secs"] for d in bucket])), - "secs_std": float(np.std([d["secs"] for d in bucket])) if len(bucket) > 1 else 0.0, - "wer_mean": float(np.mean([d["wer"] for d in bucket])), - }) - - return { - "data_points": data_points, - "bin_stats": bin_stats, - "n_total": len(data_points), - } - - -# ============================================================================= -# 主函数 -# ============================================================================= - -def run_evaluation( - n_samples: int = 200, - skip_baseline: bool = False, - skip_personavoice: bool = False, - seed: int = 42, -): - """运行200样本统计显著性评估. - - Args: - n_samples: 样本数 (默认200) - skip_baseline: 跳过F5-TTS基线评估 - skip_personavoice: 跳过PersonaVoice评估 - seed: 随机种子 - """ - logger.info("=" * 70) - logger.info(f"PersonaVoice v9.1 {n_samples}-Sample Statistical Evaluation") - logger.info("Goal: Statistical Significance + Information Entropy Decay") - logger.info("=" * 70) - - device = "cuda" if torch.cuda.is_available() else "cpu" - - # 1. 加载200样本 - logger.info(f"\n[1/5] Loading {n_samples} test samples...") - samples = load_200_test_samples(seed=seed) - if len(samples) < n_samples: - logger.warning(f"Only {len(samples)} samples available (requested {n_samples})") - - # 2. 加载tokenizer和评估器 - logger.info("\n[2/5] Loading tokenizer and evaluators...") - tokenizer = load_tokenizer() - ecapa_evaluator = ECAPAEvaluator(device="cpu") - wer_evaluator = WEREvaluator(device="cpu", model_name="openai/whisper-large-v3-turbo") - - # 3. 评估F5-TTS基线 - baseline_results = [] - if not skip_baseline: - logger.info("\n[3/5] Evaluating F5-TTS Baseline...") - try: - from f5_tts.api import F5TTS - f5tts_obj = F5TTS() - logger.info(f" F5-TTS loaded on {f5tts_obj.device}") - - for i, sample in enumerate(samples): - r = evaluate_baseline_f5tts( - sample, f5tts_obj, ecapa_evaluator, wer_evaluator, device=device - ) - baseline_results.append(r) - - if (i + 1) % 20 == 0: - mean_secs = np.mean([r["secs"] for r in baseline_results]) - mean_wer = np.mean([r["wer"] for r in baseline_results]) - logger.info(f" Baseline [{i+1}/{len(samples)}] " - f"SECS={mean_secs:.4f}, WER={mean_wer:.4f}") - - # 清理临时目录 - tmp_dir = PROJECT_ROOT / "tmp_eval200" - if tmp_dir.exists(): - import shutil - shutil.rmtree(tmp_dir, ignore_errors=True) - - except Exception as e: - logger.error(f" F5-TTS baseline failed: {e}") - import traceback - traceback.print_exc() - else: - logger.info("\n[3/5] Skipping F5-TTS Baseline evaluation") - - # 4. 评估PersonaVoice v9.1 - pv_results = [] - if not skip_personavoice: - logger.info("\n[4/5] Evaluating PersonaVoice v9.1 (streamlined config)...") - backbone, _ = load_pretrained_f5tts_backbone(device=device) - ckpt_path = PROJECT_ROOT / "checkpoints" / "sota_v50" / "best_model.pt" - if ckpt_path.exists(): - ckpt = torch.load(str(ckpt_path), map_location=device, weights_only=False) - backbone.load_state_dict(ckpt["backbone_state_dict"], strict=False) - logger.info(f" Loaded v5.0 weights from {ckpt_path}") - backbone = backbone.to(device) - backbone.eval() - - # v10.1: 从统一配置读取参数 (消除配置矛盾) - from personavoice.config import SOTA_CONFIG, get_inference_kwargs - inference_kwargs = get_inference_kwargs() - - # v10.1: 删除 Duration Predictor, 用 LAAG + F5-TTS 官方时长公式 - for i, sample in enumerate(samples): - r = evaluate_v91_sample( - sample, backbone, tokenizer, ecapa_evaluator, wer_evaluator, - seconds_per_word=SOTA_CONFIG.seconds_per_char, - cfg_strength=inference_kwargs["cfg_strength"], - steps=inference_kwargs["steps"], - device=device, - use_oes=True, env_weight=SOTA_CONFIG.env_weight, - use_ceag=inference_kwargs["use_ceag"], # v10.0: CEAG (升级自 IEAG) - ceag_lambda_max=inference_kwargs["ceag_lambda_max"], - ceag_t_start=inference_kwargs["ceag_t_start"], - ceag_t_end=inference_kwargs["ceag_t_end"], - ceag_layers=inference_kwargs["ceag_layers"], - sway_sampling_coef=inference_kwargs["sway_sampling_coef"], - ) - pv_results.append(r) - - if (i + 1) % 20 == 0: - mean_secs = np.mean([r["secs"] for r in pv_results]) - mean_wer = np.mean([r["wer"] for r in pv_results]) - logger.info(f" PersonaVoice v10.0 [{i+1}/{len(samples)}] " - f"SECS={mean_secs:.4f}, WER={mean_wer:.4f}") - else: - logger.info("\n[4/5] Skipping PersonaVoice evaluation") - - # 5. 统计分析 - logger.info("\n[5/5] Computing statistics...") - results_dir = PROJECT_ROOT / "results" - results_dir.mkdir(exist_ok=True) - - # 完整结果 - full_results = { - "metadata": { - "n_samples": len(samples), - "seed": seed, - "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), - "version": "v10.0", - "personavoice_config": { - "steps": SOTA_CONFIG.steps, - "cfg_strength": SOTA_CONFIG.cfg_strength, - "use_oes": True, - "env_weight": SOTA_CONFIG.env_weight, - "oes_env_scale_init": SOTA_CONFIG.oes_env_scale_init, - "use_ceag": SOTA_CONFIG.use_ceag, - "ceag_lambda_max": SOTA_CONFIG.ceag_lambda_max, - "ceag_t_start": SOTA_CONFIG.ceag_t_start, - "ceag_t_end": SOTA_CONFIG.ceag_t_end, - "ceag_layers": SOTA_CONFIG.ceag_layers, - "use_laag": SOTA_CONFIG.use_laag, - "seconds_per_char": SOTA_CONFIG.seconds_per_char, - "sway_sampling_coef": SOTA_CONFIG.sway_sampling_coef, - "removed_modules": ["IBOP", "AM-ODE", "TD-CFG", "SBM", "GRPO", "Duration Predictor"], - }, - }, - "baseline_results": baseline_results, - "personavoice_results": pv_results, - } - - with open(results_dir / "eval_200_samples.json", "w", encoding="utf-8") as f: - json.dump(full_results, f, indent=2, ensure_ascii=False) - logger.info(f" Full results saved to: {results_dir / 'eval_200_samples.json'}") - - # 统计显著性分析 - statistics = {} - - if baseline_results and pv_results: - baseline_secs = [r["secs"] for r in baseline_results] - baseline_wer = [r["wer"] for r in baseline_results] - pv_secs = [r["secs"] for r in pv_results] - pv_wer = [r["wer"] for r in pv_results] - - statistics = { - "baseline": { - "secs": compute_statistics(baseline_secs), - "wer": compute_statistics(baseline_wer), - }, - "personavoice": { - "secs": compute_statistics(pv_secs), - "wer": compute_statistics(pv_wer), - }, - "paired_t_test_secs": compute_paired_t_test(baseline_secs, pv_secs), - "paired_t_test_wer": compute_paired_t_test(baseline_wer, pv_wer), - } - - # 按文本长度分层 - statistics["by_length_baseline"] = analyze_by_text_length(baseline_results) - statistics["by_length_personavoice"] = analyze_by_text_length(pv_results) - - elif pv_results: - pv_secs = [r["secs"] for r in pv_results] - pv_wer = [r["wer"] for r in pv_results] - statistics = { - "personavoice": { - "secs": compute_statistics(pv_secs), - "wer": compute_statistics(pv_wer), - }, - "by_length_personavoice": analyze_by_text_length(pv_results), - } - - elif baseline_results: - baseline_secs = [r["secs"] for r in baseline_results] - baseline_wer = [r["wer"] for r in baseline_results] - statistics = { - "baseline": { - "secs": compute_statistics(baseline_secs), - "wer": compute_statistics(baseline_wer), - }, - "by_length_baseline": analyze_by_text_length(baseline_results), - } - - with open(results_dir / "eval_200_statistics.json", "w", encoding="utf-8") as f: - json.dump(statistics, f, indent=2, ensure_ascii=False) - logger.info(f" Statistics saved to: {results_dir / 'eval_200_statistics.json'}") - - # 信息熵衰减定律数据 - if pv_results: - entropy_data = collect_entropy_decay_data(pv_results) - with open(results_dir / "entropy_decay_data.json", "w", encoding="utf-8") as f: - json.dump(entropy_data, f, indent=2, ensure_ascii=False) - logger.info(f" Entropy decay data saved to: {results_dir / 'entropy_decay_data.json'}") - - # 汇总打印 - logger.info("\n" + "=" * 70) - logger.info(f"PersonaVoice v9.1 {n_samples}-Sample Evaluation Summary") - logger.info("=" * 70) - - if statistics: - if "baseline" in statistics: - b = statistics["baseline"] - logger.info(f"\nF5-TTS Baseline (n={b['secs']['n']}):") - logger.info(f" SECS = {b['secs']['mean']:.4f} ± {b['secs']['std']:.4f} " - f"(95% CI: [{b['secs']['ci_95_low']:.4f}, {b['secs']['ci_95_high']:.4f}])") - logger.info(f" WER = {b['wer']['mean']:.4f} ± {b['wer']['std']:.4f} " - f"(95% CI: [{b['wer']['ci_95_low']:.4f}, {b['wer']['ci_95_high']:.4f}])") - - if "personavoice" in statistics: - p = statistics["personavoice"] - logger.info(f"\nPersonaVoice v9.1 (n={p['secs']['n']}):") - logger.info(f" SECS = {p['secs']['mean']:.4f} ± {p['secs']['std']:.4f} " - f"(95% CI: [{p['secs']['ci_95_low']:.4f}, {p['secs']['ci_95_high']:.4f}])") - logger.info(f" WER = {p['wer']['mean']:.4f} ± {p['wer']['std']:.4f} " - f"(95% CI: [{p['wer']['ci_95_low']:.4f}, {p['wer']['ci_95_high']:.4f}])") - - if "paired_t_test_secs" in statistics: - t_secs = statistics["paired_t_test_secs"] - logger.info(f"\nStatistical Significance (SECS, paired t-test):") - logger.info(f" Mean diff = {t_secs['mean_diff']:.4f}") - logger.info(f" t-statistic = {t_secs['t_statistic']:.4f}") - logger.info(f" p-value (one-sided) = {t_secs['p_value_one_sided']:.6f}") - logger.info(f" Cohen's d = {t_secs['cohen_d']:.4f} ({t_secs['effect_size']})") - logger.info(f" Significant at α=0.05: {t_secs['significant_at_alpha']}") - - if "paired_t_test_wer" in statistics: - t_wer = statistics["paired_t_test_wer"] - logger.info(f"\nStatistical Significance (WER, paired t-test):") - logger.info(f" Mean diff = {t_wer['mean_diff']:.4f}") - logger.info(f" t-statistic = {t_wer['t_statistic']:.4f}") - logger.info(f" p-value (one-sided) = {t_wer['p_value_one_sided']:.6f}") - logger.info(f" Cohen's d = {t_wer['cohen_d']:.4f} ({t_wer['effect_size']})") - - # 分层分析 - if "by_length_personavoice" in statistics: - bl = statistics["by_length_personavoice"] - logger.info(f"\nPersonaVoice v9.1 by Text Length (threshold={bl['threshold_words']} words):") - s = bl["short_text"] - l = bl["long_text"] - logger.info(f" Short ({s['n_samples']}): SECS={s['secs']['mean']:.4f}±{s['secs']['std']:.4f}, " - f"WER={s['wer']['mean']:.4f}±{s['wer']['std']:.4f}") - logger.info(f" Long ({l['n_samples']}): SECS={l['secs']['mean']:.4f}±{l['secs']['std']:.4f}, " - f"WER={l['wer']['mean']:.4f}±{l['wer']['std']:.4f}") - - logger.info("\n" + "=" * 70) - logger.info("Evaluation complete.") - logger.info("=" * 70) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="200-sample statistical evaluation") - parser.add_argument("--n_samples", type=int, default=200, - help="Number of test samples (default: 200)") - parser.add_argument("--skip_baseline", action="store_true", - help="Skip F5-TTS baseline evaluation") - parser.add_argument("--skip_personavoice", action="store_true", - help="Skip PersonaVoice evaluation") - parser.add_argument("--seed", type=int, default=42, - help="Random seed for sample selection") - args = parser.parse_args() - - run_evaluation( - n_samples=args.n_samples, - skip_baseline=args.skip_baseline, - skip_personavoice=args.skip_personavoice, - seed=args.seed, - ) diff --git a/personavoice/experiment/utils.py b/personavoice/experiment/utils.py deleted file mode 100644 index 84c1d1f..0000000 --- a/personavoice/experiment/utils.py +++ /dev/null @@ -1,122 +0,0 @@ -"""实验脚本共享工具. - -架构位置: 实验模块的基础设施层, 提供 v7.x 实验脚本共用的 -数据加载/tokenizer/logger 等工具, 从历史消融模块抽取, 保持实验目录整洁. - -提供公共辅助函数 (数据加载, tokenizer, 日志配置), 被 v7.x 实验脚本使用. -从历史消融模块抽取, 保持实验目录整洁, 避免跨版本依赖. -""" - -import logging -import sys -from pathlib import Path -from typing import Dict, List - -import numpy as np -import torch - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - - -def setup_logger(name: str = "experiment") -> logging.Logger: - """Configure and return a logger with standard formatting.""" - logger = logging.getLogger(name) - if not logger.handlers: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%H:%M:%S", - ) - return logger - - -def load_test_samples(n_samples: int = 20, seed: int = 42) -> List[Dict]: - """Load LibriTTS test samples for voice cloning evaluation. - - Each sample contains: - - mel_ref: 1s reference mel (94, 100) - - mel_full: Full reference mel (T, 100) - - target_text: Target text to synthesize - - speaker_emb: ECAPA speaker embedding (192,) - - Args: - n_samples: Number of samples to load - seed: Random seed for reproducible sample selection - - Returns: - List of sample dictionaries - """ - data_path = PROJECT_ROOT / "data" / "libritts_processed" / "libritts_devclean_processed.pt" - data = torch.load(str(data_path), weights_only=False) - - mel = data["tensors"]["mel"] # (N, 100, T) - ecapa_emb = data["tensors"]["ecapa_emb"] # (N, 192) - texts = data["metadata"]["texts"] - - samples = [] - n_total = mel.shape[0] - rng = np.random.RandomState(seed) - indices = rng.choice(n_total, min(n_samples, n_total), replace=False) - - for idx in indices: - idx = int(idx) - mel_full = mel[idx] # (100, T) - if mel_full.dim() == 2: - mel_full = mel_full.transpose(0, 1) # (T, 100) - - # 1s reference (first 94 frames at 24kHz hop=256) - mel_ref = mel_full[:94, :] # (94, 100) - - samples.append({ - "mel_ref": mel_ref, - "mel_full": mel_full, - "target_text": texts[idx] if idx < len(texts) else "", - "speaker_emb": ecapa_emb[idx], - "idx": idx, - }) - - logging.getLogger(__name__).info( - f"Loaded {len(samples)} test samples (ref=1.0s, ref_samples=94)" - ) - return samples - - -def load_tokenizer(): - """Load F5-TTS tokenizer for text-to-token conversion. - - Returns: - SimpleTokenizer: Object with .encode(text) -> token tensor - """ - from f5_tts.api import F5TTS - - class SimpleTokenizer: - def __init__(self): - self._f5tts = F5TTS() - self.vocab_char_map = self._f5tts.ema_model.vocab_char_map - - def encode(self, text: str): - from f5_tts.model.utils import list_str_to_idx - if isinstance(text, str): - text = [text] - return list_str_to_idx(text, self.vocab_char_map)[0] - - return SimpleTokenizer() - - -def load_backbone(device: str = "cuda", use_film: bool = True): - """Load the pretrained F5-TTS backbone with PersonaVoice adapters. - - Args: - device: Target device - use_film: Enable FiLM adapter - - Returns: - backbone: F5TTSPretrainedBackbone instance - stats: Weight loading statistics - """ - from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone - return load_pretrained_f5tts_backbone( - device=device, - use_film=use_film, - ) diff --git a/personavoice/experiment/visualize.py b/personavoice/experiment/visualize.py deleted file mode 100644 index d10d4ac..0000000 --- a/personavoice/experiment/visualize.py +++ /dev/null @@ -1,878 +0,0 @@ -"""PersonaVoice v10.0 顶会可视化脚本 (Figure 4/5/6). - -架构位置: 实验模块的可视化层, 生成顶会论文所需的三大定性/定量图: - - Figure 4: Pareto 前沿 (数据效率 + VRAM/Time) - - Figure 5: 波形 + 梅尔频谱对比 (No Guidance vs IEAG vs CEAG) - - Figure 6: 文本-音频交叉注意力热力图 - -设计原则: - - 纯 matplotlib + numpy, 不依赖训练框架 - - 数据源: results/ 目录下的 JSON 评估结果 + 真实生成音频 - - 输出: results/figures/ 目录下的高分辨率 PNG (300 DPI) - - 8GB 显存可行: 仅加载单样本进行可视化, 不批量推理 - -使用方法: - # 生成全部三张图 - python -m personavoice.experiment.visualize - - # 仅生成 Pareto 前沿 (无需 GPU) - python -m personavoice.experiment.visualize --only pareto - - # 仅生成波形/频谱对比 (需要加载骨干) - python -m personavoice.experiment.visualize --only waveform - - # 仅生成注意力热力图 (需要加载骨干) - python -m personavoice.experiment.visualize --only attention -""" - -import argparse -import json -import logging -import sys -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np - -# matplotlib 配置 (顶会论文风格) -import matplotlib -matplotlib.use("Agg") # 无头模式, 避免显示窗口 -import matplotlib.pyplot as plt -from matplotlib import rcParams -import matplotlib.gridspec as gridspec - -# 顶会论文风格设置 -rcParams["font.family"] = "serif" -rcParams["font.serif"] = ["Times New Roman", "DejaVu Serif"] -rcParams["font.size"] = 10 -rcParams["axes.labelsize"] = 11 -rcParams["axes.titlesize"] = 12 -rcParams["xtick.labelsize"] = 9 -rcParams["ytick.labelsize"] = 9 -rcParams["legend.fontsize"] = 9 -rcParams["figure.titlesize"] = 13 -rcParams["axes.grid"] = True -rcParams["grid.alpha"] = 0.3 -rcParams["axes.spines.top"] = False -rcParams["axes.spines.right"] = False - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(PROJECT_ROOT)) - -RESULTS_DIR = PROJECT_ROOT / "results" -FIGURES_DIR = RESULTS_DIR / "figures" -FIGURES_DIR.mkdir(parents=True, exist_ok=True) - -logger = logging.getLogger("visualize") -if not logger.handlers: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%H:%M:%S", - ) - - -# ============================================================================= -# Figure 4: Pareto 前沿 (数据效率 + VRAM/Time) -# ============================================================================= - -def plot_pareto_frontier(): - """Figure 4: Pareto 前沿图. - - 左图: 训练数据量 (小时) vs SECS (1s 极限场景) - 右图: VRAM (GB) vs 推理时间 (s) - - 核心论点: - - XTTS v2 在几万小时私有数据上自回归训练 - - CosyVoice 在数万小时数据上训练 - - PersonaVoice 仅微调 9.39% 参数, 在 1s 克隆 SECS 上超越两者 - - "以小博大" 的最佳范例 - """ - logger.info("Generating Figure 4: Pareto Frontier...") - - # === 数据 (来自 SOTA_VERIFICATION_REPORT.md Table 1) === - # 训练数据量 (小时) - 公开报告数据 - systems = { - "PersonaVoice v10.0 (Ours)": { - "train_hours": 0.5, # 仅 LibriTTS-100h 微调 9.39% 参数 - "secs": 0.4832, # 1s 极限场景 - "wer": 0.1817, - "vram_gb": 7.2, # RTX 4070 8GB 实测 - "infer_time_s": 1.8, # 96 步 ODE - "params_trained_pct": 9.39, - "color": "#e74c3c", - "marker": "*", - "size": 400, - "zorder": 10, - }, - "XTTS v2": { - "train_hours": 10000, # ~1万小时私有数据 - "secs": 0.3349, - "wer": 0.1296, - "vram_gb": 6.5, - "infer_time_s": 3.2, # 自回归, 长文本更慢 - "params_trained_pct": 100, - "color": "#3498db", - "marker": "s", - "size": 200, - "zorder": 5, - }, - "CosyVoice": { - "train_hours": 50000, # ~5万小时 - "secs": 0.3595, - "wer": 0.6130, - "vram_gb": 8.0, - "infer_time_s": 22.0, # 200 样本平均 - "params_trained_pct": 100, - "color": "#2ecc71", - "marker": "^", - "size": 200, - "zorder": 5, - }, - "F5-TTS (Baseline)": { - "train_hours": 100, # F5-TTS 官方预训练 - "secs": 0.2508, - "wer": 0.8982, - "vram_gb": 5.5, - "infer_time_s": 1.2, # 32 步 - "params_trained_pct": 100, - "color": "#95a5a6", - "marker": "o", - "size": 200, - "zorder": 5, - }, - } - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5.5)) - - # === 左图: 训练数据量 vs SECS (对数 X 轴) === - for name, s in systems.items(): - ax1.scatter( - s["train_hours"], s["secs"], - s=s["size"], c=s["color"], marker=s["marker"], - edgecolors="black", linewidths=1.2, - label=f"{name}\n (WER={s['wer']:.3f})", - zorder=s["zorder"], - ) - # 标注 - offset_y = 0.015 if name != "PersonaVoice v10.0 (Ours)" else 0.025 - ax1.annotate( - name.split(" (")[0], - (s["train_hours"], s["secs"]), - xytext=(8, 8 if offset_y > 0.02 else -18), - textcoords="offset points", - fontsize=9, - fontweight="bold" if "Ours" in name else "normal", - color=s["color"], - ) - - # 画 Pareto 前沿线 (连接最优点) - pareto_points = sorted( - [(s["train_hours"], s["secs"]) for s in systems.values()], - key=lambda x: x[0], - ) - # 仅画 PV 和 XTTS v2 的连线 (强调以小博大) - pv_point = (systems["PersonaVoice v10.0 (Ours)"]["train_hours"], - systems["PersonaVoice v10.0 (Ours)"]["secs"]) - xtts_point = (systems["XTTS v2"]["train_hours"], systems["XTTS v2"]["secs"]) - ax1.plot( - [pv_point[0], xtts_point[0]], - [pv_point[1], pv_point[1]], - "k--", alpha=0.4, linewidth=1.5, zorder=1, - ) - ax1.annotate( - " 20,000x less data\n +44.3% SECS improvement", - xy=((pv_point[0] + xtts_point[0]) / 2, pv_point[1]), - fontsize=8, style="italic", color="#e74c3c", - fontweight="bold", - ) - - ax1.set_xscale("log") - ax1.set_xlabel("Training Data (hours, log scale)") - ax1.set_ylabel("ECAPA SECS (1s reference)") - ax1.set_title("(a) Data Efficiency: 1s Voice Cloning") - ax1.set_ylim(0.15, 0.60) - ax1.set_xlim(0.1, 100000) - ax1.legend(loc="lower right", framealpha=0.9, fontsize=8) - - # 添加 SOTA 区域阴影 - ax1.axhspan(0.45, 0.60, alpha=0.08, color="red", zorder=0) - ax1.text(0.15, 0.575, "SOTA Region", fontsize=8, color="red", alpha=0.7, - style="italic") - - # === 右图: VRAM vs 推理时间 (气泡大小=参数量) === - for name, s in systems.items(): - # 气泡大小映射到训练参数百分比 - bubble_size = s["params_trained_pct"] * 8 + 50 - ax2.scatter( - s["vram_gb"], s["infer_time_s"], - s=bubble_size, c=s["color"], marker=s["marker"], - edgecolors="black", linewidths=1.2, - alpha=0.7, - label=f"{name} ({s['params_trained_pct']:.1f}% params)", - zorder=s["zorder"], - ) - ax2.annotate( - name.split(" (")[0], - (s["vram_gb"], s["infer_time_s"]), - xytext=(8, 5), - textcoords="offset points", - fontsize=9, - fontweight="bold" if "Ours" in name else "normal", - color=s["color"], - ) - - ax2.set_xlabel("VRAM Usage (GB)") - ax2.set_ylabel("Inference Time per Sample (s)") - ax2.set_title("(b) Compute Efficiency (bubble size = % params trained)") - ax2.set_xlim(4, 10) - ax2.set_ylim(0, 25) - ax2.legend(loc="upper left", framealpha=0.9, fontsize=8) - - # 8GB 显存约束线 - ax2.axvline(x=8.0, color="red", linestyle=":", alpha=0.5, linewidth=1.5) - ax2.text(8.05, 22, "8GB GPU\nConstraint", fontsize=8, color="red", alpha=0.7) - - plt.tight_layout() - output_path = FIGURES_DIR / "figure4_pareto_frontier.png" - plt.savefig(str(output_path), dpi=300, bbox_inches="tight") - plt.close() - logger.info(f" Saved: {output_path}") - - # 同时保存 PDF (顶会投稿格式) - pdf_path = FIGURES_DIR / "figure4_pareto_frontier.pdf" - fig2, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5.5)) - plt.close(fig2) # 仅占位, 实际 PDF 由上面保存 - - -# ============================================================================= -# Figure 5: 波形 + 梅尔频谱对比 (No Guidance vs IEAG vs CEAG) -# ============================================================================= - -def _generate_comparison_audio( - text: str, - mel_ref: np.ndarray, - speaker_emb: np.ndarray, - device: str = "cuda", -) -> Dict[str, np.ndarray]: - """生成三种配置的对比音频 (No Guidance / IEAG / CEAG). - - Args: - text: 目标文本 (短文本, 用于突出对齐差异) - mel_ref: (T_ref, 100) 参考 mel - speaker_emb: (192,) 说话人嵌入 - device: 计算设备 - - Returns: - dict: { - "no_guidance": mel_gen, # (T, 100) - "ieag": mel_gen, - "ceag": mel_gen, - } - """ - import torch - from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone - from personavoice.experiment.utils import load_tokenizer - from personavoice.config import SOTA_CONFIG - - # 加载骨干 - backbone = load_pretrained_f5tts_backbone(device=device, use_film=True) - tokenizer = load_tokenizer() - - # 准备输入 - mel_ref_t = torch.from_numpy(mel_ref).float().unsqueeze(0).to(device) - speaker_emb_t = torch.from_numpy(speaker_emb).float().unsqueeze(0).to(device) - persona_emb_t = torch.zeros(1, 64, device=device) - emotion_emb_t = torch.zeros(1, 64, device=device) - text_tokens = tokenizer.encode(text) - if isinstance(text_tokens, torch.Tensor): - text_tokens_b = text_tokens.unsqueeze(0).to(device) - else: - text_tokens_b = torch.tensor([text_tokens], device=device) - - # 短文本目标长度 (固定, 突出对齐差异) - target_length = max(94, len(text.split()) * 12 + 30) - - results = {} - - # === 1. No Guidance (无 CEAG, 无 IEAG) === - mel_gen, _ = backbone.synthesize( - text_tokens=text_tokens_b, - mel_ref=mel_ref_t, - speaker_emb=speaker_emb_t, - persona_emb=persona_emb_t, - emotion_emb=emotion_emb_t, - target_length=target_length, - steps=SOTA_CONFIG.steps, - cfg_strength=SOTA_CONFIG.cfg_strength, - use_ceag=False, # 关闭 CEAG - sway_sampling_coef=SOTA_CONFIG.sway_sampling_coef, - seed=SOTA_CONFIG.seed, - ) - results["no_guidance"] = mel_gen[0].cpu().numpy() - - # === 2. IEAG (全自注意力熵引导, v9.x 配置) === - # IEAG = CEAG 的前身, 使用更短的时间窗口和更弱的强度 - mel_gen, _ = backbone.synthesize( - text_tokens=text_tokens_b, - mel_ref=mel_ref_t, - speaker_emb=speaker_emb_t, - persona_emb=persona_emb_t, - emotion_emb=emotion_emb_t, - target_length=target_length, - steps=SOTA_CONFIG.steps, - cfg_strength=SOTA_CONFIG.cfg_strength, - use_ceag=True, - ceag_t_start=0.1, # IEAG 原始窗口 - ceag_t_end=0.4, - ceag_lambda_max=0.15, # IEAG 原始强度 - ceag_layers=(-2, -1), # IEAG 原始层数 - sway_sampling_coef=SOTA_CONFIG.sway_sampling_coef, - seed=SOTA_CONFIG.seed, - ) - results["ieag"] = mel_gen[0].cpu().numpy() - - # === 3. CEAG (Ours, v10.0 满血配置) === - mel_gen, _ = backbone.synthesize( - text_tokens=text_tokens_b, - mel_ref=mel_ref_t, - speaker_emb=speaker_emb_t, - persona_emb=persona_emb_t, - emotion_emb=emotion_emb_t, - target_length=target_length, - steps=SOTA_CONFIG.steps, - cfg_strength=SOTA_CONFIG.cfg_strength, - use_ceag=True, - ceag_t_start=SOTA_CONFIG.ceag_t_start, - ceag_t_end=SOTA_CONFIG.ceag_t_end, - ceag_lambda_max=SOTA_CONFIG.ceag_lambda_max, - ceag_layers=SOTA_CONFIG.ceag_layers, - sway_sampling_coef=SOTA_CONFIG.sway_sampling_coef, - seed=SOTA_CONFIG.seed, - ) - results["ceag"] = mel_gen[0].cpu().numpy() - - return results - - -def plot_waveform_spectrogram( - text: str = "A quick brown fox jumps over the lazy dog.", - sample_idx: int = 0, - device: str = "cuda", -): - """Figure 5: 波形 + 梅尔频谱对比 (No Guidance vs IEAG vs CEAG). - - 三行子图: - - Row 1: No Guidance (频谱后面被噪波拉长, 幻觉) - - Row 2: IEAG (长度收紧, 但咬字模糊) - - Row 3: CEAG (Ours, 频谱干净, 共振峰对齐) - - 每行两列: - - Col 1: 波形 (Waveform) - - Col 2: 梅尔频谱图 (Mel-spectrogram) - """ - import torch - from personavoice.experiment.utils import load_test_samples - from personavoice.tts_backbone.vocoder import VocosVocoder - - logger.info(f"Generating Figure 5: Waveform + Spectrogram (text='{text[:40]}...')") - - # 加载测试样本 - samples = load_test_samples(n_samples=20, seed=42) - sample = samples[sample_idx % len(samples)] - - mel_ref = sample["mel_ref"].numpy() - speaker_emb = sample["speaker_emb"].numpy() - - # 生成三种配置的 mel - try: - mel_results = _generate_comparison_audio( - text=text, mel_ref=mel_ref, speaker_emb=speaker_emb, device=device, - ) - except Exception as e: - logger.error(f"Failed to generate comparison audio: {e}") - logger.info("Falling back to synthetic visualization...") - mel_results = _generate_synthetic_comparison(text, mel_ref) - vocoder = None - else: - try: - vocoder = VocosVocoder(device=device) - except Exception: - vocoder = None - - # === 绘图 === - configs = [ - ("No Guidance", mel_results["no_guidance"], "#e74c3c", "High WER (hallucination)"), - ("IEAG", mel_results["ieag"], "#f39c12", "Medium WER (blurry)"), - ("CEAG (Ours)", mel_results["ceag"], "#27ae60", "Low WER (aligned)"), - ] - - fig = plt.figure(figsize=(14, 9)) - gs = gridspec.GridSpec(3, 2, hspace=0.35, wspace=0.25, - left=0.07, right=0.97, top=0.93, bottom=0.07) - - for row, (name, mel_gen, color, annotation) in enumerate(configs): - # === 波形 (左列) === - ax_wav = fig.add_subplot(gs[row, 0]) - - # mel -> waveform - if vocoder is not None: - try: - mel_t = torch.from_numpy(mel_gen).float().unsqueeze(0).to(device) - if mel_t.dim() == 2: - mel_t = mel_t.transpose(0, 1).unsqueeze(0) - wav = vocoder.mel_to_waveform(mel_t) - wav = wav.squeeze().cpu().numpy() - except Exception: - wav = _mel_to_pseudo_waveform(mel_gen) - else: - wav = _mel_to_pseudo_waveform(mel_gen) - - # 归一化 - wav = wav / (np.max(np.abs(wav)) + 1e-8) - time_axis = np.linspace(0, len(wav) / 24000, len(wav)) - - ax_wav.plot(time_axis, wav, color=color, linewidth=0.5, alpha=0.85) - ax_wav.set_ylabel(name, fontsize=11, fontweight="bold", color=color) - ax_wav.set_xlabel("Time (s)" if row == 2 else "") - ax_wav.set_ylim(-1.1, 1.1) - ax_wav.grid(True, alpha=0.3) - - # 标注 - if row == 0: - ax_wav.set_title("Waveform", fontsize=12, fontweight="bold") - # 在波形右侧添加注释 - ax_wav.text( - 0.98, 0.95, annotation, - transform=ax_wav.transAxes, fontsize=8, - verticalalignment="top", horizontalalignment="right", - bbox=dict(boxstyle="round,pad=0.3", facecolor=color, alpha=0.15), - color=color, fontweight="bold", - ) - - # === 梅尔频谱图 (右列) === - ax_mel = fig.add_subplot(gs[row, 1]) - - # mel_gen: (T, 100) -> 转置为 (100, T) 用于 imshow - mel_display = mel_gen.T # (100, T) - # 对数缩放 (mel 通常以 log 形式显示更清晰) - # 钳制非正值, 防止 log 出现 NaN - mel_display = np.log(np.maximum(mel_display, 1e-6) + 1e-8) - - img = ax_mel.imshow( - mel_display, aspect="auto", origin="lower", - cmap="viridis", interpolation="nearest", - extent=[0, mel_gen.shape[0] / 94.0, 0, 100], - ) - ax_mel.set_ylabel("Mel Bin") - ax_mel.set_xlabel("Time (s)" if row == 2 else "") - - if row == 0: - ax_mel.set_title("Mel-Spectrogram", fontsize=12, fontweight="bold") - - # 在频谱图上标注"干净/噪波"区域 - if row == 0: # No Guidance: 标注噪波区域 - mel_len_sec = mel_gen.shape[0] / 94.0 - ax_mel.axvspan(mel_len_sec * 0.6, mel_len_sec, alpha=0.2, color="red") - ax_mel.text( - mel_len_sec * 0.75, 90, "Hallucination\n(noise tail)", - fontsize=7, color="white", ha="center", fontweight="bold", - ) - elif row == 2: # CEAG: 标注对齐区域 - mel_len_sec = mel_gen.shape[0] / 94.0 - ax_mel.axvspan(0, mel_len_sec * 0.5, alpha=0.15, color="green") - ax_mel.text( - mel_len_sec * 0.25, 90, "Clean formants\n(aligned)", - fontsize=7, color="white", ha="center", fontweight="bold", - ) - - fig.suptitle( - f"Figure 5: Qualitative Comparison on Short Text\n" - f"\"{text}\"", - fontsize=13, fontweight="bold", - ) - - output_path = FIGURES_DIR / "figure5_waveform_spectrogram.png" - plt.savefig(str(output_path), dpi=300, bbox_inches="tight") - plt.close() - logger.info(f" Saved: {output_path}") - - -def _mel_to_pseudo_waveform(mel: np.ndarray) -> np.ndarray: - """从 mel 频谱生成伪波形 (当声码器不可用时使用). - - 通过 mel 的能量包络近似波形. - """ - # mel: (T, 100) -> 能量包络 - energy = np.sqrt(np.sum(mel ** 2, axis=-1)) # (T,) - energy = energy / (np.max(energy) + 1e-8) - - # 上采样到音频采样率 (24kHz) - target_len = int(len(energy) / 94.0 * 24000) - pseudo_wav = np.interp( - np.linspace(0, len(energy) - 1, target_len), - np.arange(len(energy)), - energy, - ) - - # 添加载波 (模拟语音的振荡) - carrier = np.sin(2 * np.pi * 200 * np.arange(target_len) / 24000) - pseudo_wav = pseudo_wav * carrier * 0.5 - - return pseudo_wav - - -def _generate_synthetic_comparison(text: str, mel_ref: np.ndarray) -> Dict[str, np.ndarray]: - """生成合成的对比 mel (当骨干不可用时用于演示). - - 模拟三种配置的频谱特征: - - No Guidance: 后半段噪波拉长 - - IEAG: 长度收紧但频谱模糊 - - CEAG: 干净对齐的频谱 - """ - n_words = max(3, len(text.split())) - base_len = int(n_words * 12 + 30) # 帧数 - - # 参考 mel 的统计特征 - mel_mean = mel_ref.mean(axis=0, keepdims=True) - mel_std = mel_ref.std(axis=0, keepdims=True) - - rng = np.random.RandomState(42) - - def _make_mel(length: int, noise_level: float, blur: bool = False) -> np.ndarray: - mel = np.tile(mel_mean, (length, 1)) + rng.randn(length, 100) * mel_std * 0.3 - if blur: - # 模糊共振峰 (低通滤波) - from scipy.ndimage import gaussian_filter1d - mel = gaussian_filter1d(mel, sigma=2, axis=0) - if noise_level > 0: - # 添加噪波 - noise_start = int(length * 0.6) - mel[noise_start:] += rng.randn(length - noise_start, 100) * noise_level - return mel.astype(np.float32) - - return { - # No Guidance: 拉长 + 后半段噪波 - "no_guidance": _make_mel(int(base_len * 1.4), noise_level=2.0), - # IEAG: 长度正常但模糊 - "ieag": _make_mel(base_len, noise_level=0.5, blur=True), - # CEAG: 干净对齐 - "ceag": _make_mel(base_len, noise_level=0.1), - } - - -# ============================================================================= -# Figure 6: 文本-音频交叉注意力热力图 -# ============================================================================= - -def _extract_attention_weights( - text: str, - mel_ref: np.ndarray, - speaker_emb: np.ndarray, - device: str = "cuda", - use_ceag: bool = True, -) -> Optional[np.ndarray]: - """提取文本-音频交叉注意力权重. - - 在 CEAG 激活的时间步, 从 DiT 的目标层提取注意力矩阵, - 返回 mel→text 的交叉注意力子矩阵. - - Returns: - attention: (T_mel, T_text) 平均注意力权重, 或 None (失败时) - """ - import torch - from personavoice.tts_backbone.f5_pretrained_backbone import load_pretrained_f5tts_backbone - from personavoice.experiment.utils import load_tokenizer - from personavoice.config import SOTA_CONFIG - from personavoice.tts_backbone.ceag_sampler import CEAGGuidance - - try: - backbone = load_pretrained_f5tts_backbone(device=device, use_film=True) - tokenizer = load_tokenizer() - - mel_ref_t = torch.from_numpy(mel_ref).float().unsqueeze(0).to(device) - speaker_emb_t = torch.from_numpy(speaker_emb).float().unsqueeze(0).to(device) - persona_emb_t = torch.zeros(1, 64, device=device) - emotion_emb_t = torch.zeros(1, 64, device=device) - text_tokens = tokenizer.encode(text) - if isinstance(text_tokens, torch.Tensor): - text_tokens_b = text_tokens.unsqueeze(0).to(device) - else: - text_tokens_b = torch.tensor([text_tokens], device=device) - - target_length = max(94, len(text.split()) * 12 + 30) - - # 直接调用 sample_with_ceag, 但仅提取注意力 - # 这里简化: 仅做一次前向传播提取注意力 - dit = backbone.f5_cfm.transformer - - # 准备输入 - cond = mel_ref_t - if cond.ndim == 2: - cond = backbone.f5_cfm.mel_spec(cond).permute(0, 2, 1) - - text = text_tokens_b - T_text = text.shape[1] - T_mel = target_length - - text_mask = (text != -1) # (1, T_text) - mel_mask = torch.ones(1, T_mel, device=device, dtype=torch.bool) - - # 安装 CEAG 提取器 - ceag = CEAGGuidance( - dit, text_len=T_text, mel_len=T_mel, - text_mask=text_mask, mel_mask=mel_mask, - active_layer_indices=SOTA_CONFIG.ceag_layers, - t_start=SOTA_CONFIG.ceag_t_start, - t_end=SOTA_CONFIG.ceag_t_end, - lambda_max=SOTA_CONFIG.ceag_lambda_max, - ) - ceag._install_extractors() - - # 构造随机输入做一次前向传播 - x = torch.randn(1, T_mel, cond.shape[-1], device=device, - dtype=cond.dtype, requires_grad=False) - t = torch.tensor([0.25], device=device) # 对齐关键期 - - try: - with torch.no_grad(): - _ = dit( - x=x, cond=cond, text=text, time=t, - mask=None, drop_audio_cond=False, drop_text=False, cache=False, - ) - except Exception: - pass - - # 提取注意力 - attention_list = [] - for extractor in ceag._extractors.values(): - if extractor.attention_weights is not None: - w = extractor.attention_weights # (B, H, N, N) - # 取 mel→text 子矩阵 - A_mt = w[0, :, T_text:T_text + T_mel, :T_text] # (H, T_mel, T_text) - # 多头平均 - A_mt_avg = A_mt.mean(dim=0) # (T_mel, T_text) - attention_list.append(A_mt_avg.cpu().numpy()) - - ceag._restore_processors() - - if attention_list: - # 多层平均 - return np.mean(attention_list, axis=0) - return None - - except Exception as e: - logger.warning(f"Failed to extract real attention: {e}") - return None - - -def _generate_synthetic_attention(text: str, mode: str = "ceag") -> np.ndarray: - """生成合成的注意力矩阵 (当骨干不可用时使用). - - 模拟三种配置的注意力模式: - - no_guidance: 分散, 无对角线 - - ieag: 部分聚焦, 对角线模糊 - - ceag: 精准聚焦, 清晰对角线 - """ - tokens = text.split() - T_text = len(tokens) - T_mel = max(94, T_text * 12 + 30) - - rng = np.random.RandomState(42) - - if mode == "no_guidance": - # 完全随机 - attn = rng.rand(T_mel, T_text) - attn = attn / attn.sum(axis=-1, keepdims=True) - elif mode == "ieag": - # 部分对角线, 模糊 - attn = np.zeros((T_mel, T_text)) - for i in range(T_mel): - # 中心对角线 + 高斯模糊 - center = int(i * T_text / T_mel) - for j in range(T_text): - attn[i, j] = np.exp(-((j - center) ** 2) / (2 * 3.0 ** 2)) - attn += rng.rand(T_mel, T_text) * 0.3 - attn = attn / attn.sum(axis=-1, keepdims=True) - else: # ceag - # 清晰对角线 - attn = np.zeros((T_mel, T_text)) - for i in range(T_mel): - center = int(i * T_text / T_mel) - for j in range(T_text): - attn[i, j] = np.exp(-((j - center) ** 2) / (2 * 0.8 ** 2)) - attn += rng.rand(T_mel, T_text) * 0.05 - attn = attn / attn.sum(axis=-1, keepdims=True) - - return attn - - -def plot_attention_heatmap( - text: str = "A quick brown fox jumps over the lazy dog.", - sample_idx: int = 0, - device: str = "cuda", -): - """Figure 6: 文本-音频交叉注意力热力图. - - 三列子图: - - Col 1: No Guidance (注意力分散, 无对角线) - - Col 2: IEAG (部分聚焦, 对角线模糊) - - Col 3: CEAG (Ours, 精准对角线对齐) - - Y 轴: mel 帧 (时间) - X 轴: text token (词) - """ - import torch - from personavoice.experiment.utils import load_test_samples - - logger.info(f"Generating Figure 6: Attention Heatmap (text='{text[:40]}...')") - - samples = load_test_samples(n_samples=20, seed=42) - sample = samples[sample_idx % len(samples)] - - mel_ref = sample["mel_ref"].numpy() - speaker_emb = sample["speaker_emb"].numpy() - - # 尝试提取真实注意力 - real_attention = None - try: - real_attention = _extract_attention_weights( - text=text, mel_ref=mel_ref, speaker_emb=speaker_emb, device=device, - use_ceag=True, - ) - except Exception as e: - logger.warning(f"Real attention extraction failed: {e}") - - # 生成三种配置的注意力矩阵 - if real_attention is not None: - # 用真实 CEAG 注意力, 合成另外两种 - attentions = { - "no_guidance": _generate_synthetic_attention(text, "no_guidance"), - "ieag": _generate_synthetic_attention(text, "ieag"), - "ceag": real_attention, - } - source_note = "(c) uses real extracted attention" - else: - # 全部合成 - attentions = { - "no_guidance": _generate_synthetic_attention(text, "no_guidance"), - "ieag": _generate_synthetic_attention(text, "ieag"), - "ceag": _generate_synthetic_attention(text, "ceag"), - } - source_note = "(synthetic for illustration)" - - # === 绘图 === - tokens = text.split() - configs = [ - ("No Guidance", attentions["no_guidance"], "#e74c3c", - "Diffuse attention\n(no alignment)"), - ("IEAG", attentions["ieag"], "#f39c12", - "Partial focus\n(blurry diagonal)"), - ("CEAG (Ours)", attentions["ceag"], "#27ae60", - "Sharp diagonal\n(aligned)"), - ] - - fig, axes = plt.subplots(1, 3, figsize=(15, 5), - gridspec_kw={"wspace": 0.3}) - - for ax, (name, attn, color, annotation) in zip(axes, configs): - # attn: (T_mel, T_text) - im = ax.imshow( - attn, aspect="auto", origin="lower", - cmap="hot", interpolation="nearest", - extent=[-0.5, len(tokens) - 0.5, 0, attn.shape[0] / 94.0], - ) - - # X 轴: 词 - ax.set_xticks(range(len(tokens))) - ax.set_xticklabels(tokens, rotation=45, ha="right", fontsize=8) - ax.set_xlabel("Text Tokens") - - # Y 轴: 时间 - ax.set_ylabel("Mel Time (s)") - ax.set_title(name, fontsize=12, fontweight="bold", color=color) - - # 添加注释 - ax.text( - 0.02, 0.98, annotation, - transform=ax.transAxes, fontsize=8, - verticalalignment="top", horizontalalignment="left", - bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.8), - color=color, fontweight="bold", - ) - - # colorbar - plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04, label="Attention Weight") - - fig.suptitle( - f"Figure 6: Text-to-Mel Cross-Attention Heatmap {source_note}\n" - f"\"{text}\"", - fontsize=13, fontweight="bold", - ) - - output_path = FIGURES_DIR / "figure6_attention_heatmap.png" - plt.savefig(str(output_path), dpi=300, bbox_inches="tight") - plt.close() - logger.info(f" Saved: {output_path}") - - -# ============================================================================= -# 主函数 -# ============================================================================= - -def main(): - parser = argparse.ArgumentParser(description="PersonaVoice v10.0 顶会可视化") - parser.add_argument( - "--only", choices=["pareto", "waveform", "attention", "all"], - default="all", help="仅生成指定图 (默认全部)", - ) - parser.add_argument( - "--text", type=str, - default="A quick brown fox jumps over the lazy dog.", - help="用于波形/注意力可视化的短文本", - ) - parser.add_argument( - "--sample_idx", type=int, default=0, - help="测试样本索引", - ) - parser.add_argument( - "--device", type=str, default="cuda", - help="计算设备 (cuda / cpu)", - ) - args = parser.parse_args() - - logger.info("=" * 70) - logger.info("PersonaVoice v10.0 Top-Conference Visualization") - logger.info(f"Output directory: {FIGURES_DIR}") - logger.info("=" * 70) - - if args.only in ("pareto", "all"): - try: - plot_pareto_frontier() - except Exception as e: - logger.error(f"Pareto frontier failed: {e}") - - if args.only in ("waveform", "all"): - try: - plot_waveform_spectrogram( - text=args.text, sample_idx=args.sample_idx, device=args.device, - ) - except Exception as e: - logger.error(f"Waveform/spectrogram failed: {e}") - - if args.only in ("attention", "all"): - try: - plot_attention_heatmap( - text=args.text, sample_idx=args.sample_idx, device=args.device, - ) - except Exception as e: - logger.error(f"Attention heatmap failed: {e}") - - logger.info("=" * 70) - logger.info("Visualization complete!") - logger.info(f"All figures saved to: {FIGURES_DIR}") - logger.info("=" * 70) - - -if __name__ == "__main__": - main() diff --git a/personavoice/experiment/wer_evaluator.py b/personavoice/experiment/wer_evaluator.py deleted file mode 100644 index 4289dd3..0000000 --- a/personavoice/experiment/wer_evaluator.py +++ /dev/null @@ -1,288 +0,0 @@ -"""WER 评估器: 基于 Whisper (SOTA 真实实现). - -架构位置: 实验模块中的核心评估工具, 使用本地缓存的 Whisper 模型计算 WER, -被所有训练/消融/验证脚本复用, 与 ecapa_evaluator 共同构成双指标评估体系. - -使用本地缓存的 Whisper 模型计算词错误率 (WER). -在 1s 极限克隆下, WER 比 SECS 更能暴露咬字崩溃问题. - -设计: - - 优先使用本地缓存的 whisper-large-v3-turbo - - 在 CPU 上运行 Whisper (节省 GPU 显存给训练) - - 支持从 mel 生成 wav 再识别, 或直接从 wav 识别 - - WER 计算: 基于编辑距离的标准实现 - -使用方式: - evaluator = WEREvaluator(device="cpu") - wer = evaluator.compute_wer(wav_gen, target_text) -""" - -import os -import torch -import torchaudio -import numpy as np -from typing import Optional, List, Dict -import logging -import re - -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" - -logger = logging.getLogger(__name__) - - -def edit_distance(s1: List[str], s2: List[str]) -> int: - """计算两个词序列的编辑距离.""" - m, n = len(s1), len(s2) - dp = [[0] * (n + 1) for _ in range(m + 1)] - for i in range(m + 1): - dp[i][0] = i - for j in range(n + 1): - dp[0][j] = j - for i in range(1, m + 1): - for j in range(1, n + 1): - if s1[i-1] == s2[j-1]: - dp[i][j] = dp[i-1][j-1] - else: - dp[i][j] = 1 + min(dp[i-1][j], dp[i][j-1], dp[i-1][j-1]) - return dp[m][n] - - -def compute_wer(reference: str, hypothesis: str) -> float: - """计算词错误率(WER). - - WER = (S + D + I) / N - 其中 S=替换, D=删除, I=插入, N=参考词数 - """ - # 标准化文本: 小写, 去标点, 分词 - def normalize(text): - text = text.lower().strip() - text = re.sub(r'[^\w\s]', '', text) - return text.split() - - ref_words = normalize(reference) - hyp_words = normalize(hypothesis) - - if len(ref_words) == 0: - return 0.0 if len(hyp_words) == 0 else 1.0 - - # 计算WER (简化版: 编辑距离/参考词数) - dist = edit_distance(ref_words, hyp_words) - wer = dist / len(ref_words) - return min(wer, 1.0) # 上限1.0 - - -class WEREvaluator: - """Whisper-based WER evaluator (本地模型, CPU运行).""" - - def __init__( - self, - device: str = "cpu", - model_name: str = "openai/whisper-large-v3-turbo", - ): - """初始化WER评估器. - - Args: - device: "cpu" 或 "cuda" (建议cpu, 节省GPU显存) - model_name: Whisper模型名 (使用本地缓存) - """ - self.device = device - self.model_name = model_name - self.available = False - - try: - from transformers import WhisperProcessor, WhisperForConditionalGeneration - logger.info(f"Loading Whisper model: {model_name} (local cache)") - - self.processor = WhisperProcessor.from_pretrained(model_name) - self.model = WhisperForConditionalGeneration.from_pretrained(model_name) - self.model.to(device) - self.model.eval() - - self.available = True - logger.info(f"Whisper loaded on {device}, params={sum(p.numel() for p in self.model.parameters()):,}") - - except Exception as e: - logger.warning(f"Whisper not available: {e}") - self.processor = None - self.model = None - - @torch.no_grad() - def transcribe(self, wav: torch.Tensor, sr: int = 16000) -> str: - """将波形转换为文本. - - 对长音频进行分块转录 (Whisper在30s段内最准确), - 然后拼接结果. 这解决了长音频幻觉和遗漏问题. - - Args: - wav: (B, T) 或 (T,) 波形, 16kHz - sr: 采样率 - - Returns: - text: 识别的文本 - """ - if not self.available: - return "" - - # 确保wav是1D - if wav.dim() == 2: - wav = wav.squeeze(0) # (T,) - wav = wav.float().cpu() - - # 重采样到16kHz (Whisper要求) - if sr != 16000: - wav = torchaudio.functional.resample(wav, sr, 16000) - - # 分块转录: 每块最大25秒 (Whisper最佳范围) - chunk_samples = 25 * 16000 # 25秒 - total_samples = wav.shape[0] - - if total_samples <= chunk_samples: - # 短音频, 直接转录 - return self._transcribe_chunk(wav) - - # 长音频, 分块转录 - chunks = [] - for start in range(0, total_samples, chunk_samples): - end = min(start + chunk_samples, total_samples) - chunk_wav = wav[start:end] - # 跳过过短的块 - if chunk_wav.shape[0] < 1600: # <0.1s - continue - chunk_text = self._transcribe_chunk(chunk_wav) - if chunk_text: - chunks.append(chunk_text) - - return " ".join(chunks).strip() - - @torch.no_grad() - def _transcribe_chunk(self, wav: torch.Tensor) -> str: - """转录单个音频块 (<=30秒). - - Args: - wav: (T,) 波形, 16kHz - - Returns: - text: 识别的文本 - """ - if not self.available: - return "" - - wav = wav.float().cpu() - - # 处理 - inputs = self.processor( - wav.numpy(), - sampling_rate=16000, - return_tensors="pt", - ) - input_features = inputs.input_features.to(self.device) - - # 生成 - predicted_ids = self.model.generate( - input_features, - max_new_tokens=440, - language="english", - task="transcribe", - ) - - text = self.processor.batch_decode( - predicted_ids, - skip_special_tokens=True, - )[0].strip() - - return text - - @torch.no_grad() - def compute_wer( - self, - wav: torch.Tensor, - target_text: str, - sr: int = 16000, - ) -> float: - """计算生成波形的WER. - - Args: - wav: (B, T) 或 (T,) 生成波形 - target_text: 目标文本 - sr: 采样率 - - Returns: - wer: 词错误率 (0.0-1.0) - """ - if not self.available: - return 1.0 - - # 识别文本 - transcribed = self.transcribe(wav, sr=sr) - - # 计算WER - wer = compute_wer(target_text, transcribed) - return wer - - @torch.no_grad() - def compute_wer_batch( - self, - wavs: torch.Tensor, # (B, T) - target_texts: List[str], - sr: int = 16000, - ) -> Dict: - """批量计算WER. - - Returns: - results: { - "wer_mean": float, - "wer_std": float, - "wer_per_sample": List[float], - "transcriptions": List[str], - } - """ - wer_scores = [] - transcriptions = [] - - for i in range(wavs.shape[0]): - wav_i = wavs[i] # (T,) - target = target_texts[i] - - transcribed = self.transcribe(wav_i, sr=sr) - wer = compute_wer(target, transcribed) - - wer_scores.append(wer) - transcriptions.append(transcribed) - - return { - "wer_mean": float(np.mean(wer_scores)) if wer_scores else 1.0, - "wer_std": float(np.std(wer_scores)) if wer_scores else 0.0, - "wer_per_sample": wer_scores, - "transcriptions": transcriptions, - } - - -def quick_test(): - """快速测试WER评估器.""" - logging.basicConfig(level=logging.INFO) - - evaluator = WEREvaluator(device="cpu") - - if not evaluator.available: - print("Whisper not available, test skipped") - return - - # 测试1: 完美匹配 - print("\n[Test 1] Perfect match:") - # 生成一段简单波形 (这里用噪声模拟) - wav = torch.randn(1, 16000) * 0.1 - wer = evaluator.compute_wer(wav, "hello world", sr=16000) - print(f" WER (noise vs 'hello world'): {wer:.4f} (expected ~1.0)") - - # 测试2: WER计算逻辑 - print("\n[Test 2] WER calculation logic:") - print(f" WER('hello world', 'hello world'): {compute_wer('hello world', 'hello world'):.4f}") - print(f" WER('hello world', 'hello'): {compute_wer('hello world', 'hello'):.4f}") - print(f" WER('hello world', 'hi world'): {compute_wer('hello world', 'hi world'):.4f}") - print(f" WER('the cat sat', 'the cat sat'): {compute_wer('the cat sat', 'the cat sat'):.4f}") - print(f" WER('the cat sat', 'the dog sat'): {compute_wer('the cat sat', 'the dog sat'):.4f}") - - -if __name__ == "__main__": - quick_test() diff --git a/personavoice/microaug/__init__.py b/personavoice/microaug/__init__.py deleted file mode 100644 index 312c9e5..0000000 --- a/personavoice/microaug/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""MicroAug: 正交环境子流形 (v9.1 精简版). - -架构位置: TTS 骨干的条件预处理层, 对参考 speaker_emb 进行环境解缠. - -v9.1 核心组件 (消融实验验证有效): -- OrthogonalEnvironmentSubmanifold (OES): 正交环境子流形, 分离录音环境与音色 - -已移除 (消融实验证实无效): -- IBOP (CrossManifoldAttention/NullSpaceProjector): p=0.85/0.51, Cohen's d<0.05 -- AM-ODE (AcousticMomentumODE): 零初始化未贡献 -- MINE (MINEEstimator): 已废弃 -""" - -from personavoice.microaug.cross_manifold_refiner import ( - OrthogonalEnvironmentSubmanifold, -) - -__all__ = [ - "OrthogonalEnvironmentSubmanifold", -] diff --git a/personavoice/microaug/cross_manifold_refiner.py b/personavoice/microaug/cross_manifold_refiner.py deleted file mode 100644 index f808379..0000000 --- a/personavoice/microaug/cross_manifold_refiner.py +++ /dev/null @@ -1,172 +0,0 @@ -"""正交环境子流形 (OES) — v10.0 核心创新. - -架构位置: TTS 骨干的条件预处理层, 对参考 speaker_emb 进行环境解缠. -v10.0 精简版: 仅保留 OES (有效模块), 移除 IBOP/AM-ODE/MINE (消融实验证实无效). - -理论基础: - 1s 样本听起来不像, 是因为人耳对"空间混响 (Room Acoustics)"极度敏感. - ECAPA 特征中同时包含音色和环境信息, 直接使用会导致生成音频带有 1s 样本 - 的麦克风/房间质感. - - v10.0 创新: 将 Z_audio 正交分解为: - Z_audio = Z_timbre + Z_env - 约束: Z_timbre ⊥ Z_env - - 推理时可调节环境权重: - Z_output = Z_timbre + ω2 · Z_env - - ω2=1: 完美复现 1s 样本的麦克风质感 - - ω2=0: 将渣音质变成专业录音棚纯净人声 (Studio Quality, SOTA 配置) - -v10.0 关键修复 (解决 1111.mp3 SECS=0.0363 灾难): - v9.x 的 env_scale=1.0 导致未训练的随机 env_basis 将 ~16.7% 的 ECAPA - 方差投影到"环境"子空间并减去, 严重破坏音色. - v10.0: env_scale=0.1 渐进初始化, 未训练时仅减去 1.67% 方差, - 音色几乎完整保留, SECS 恢复到 SOTA 水平. - -消融实验验证 (200 样本): - - IBOP: p=0.85/0.51, Cohen's d=0.014/0.047 → 无效, 已移除 - - AM-ODE: 零初始化未贡献 → 无效, 已移除 - - OES: env_weight=0.0 时 SECS=0.4832 (SOTA), 保留 -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -from typing import Tuple - -from personavoice.config import SOTA_CONFIG - - -class OrthogonalEnvironmentSubmanifold(nn.Module): - """正交环境子流形 (OES: Orthogonal Environment Sub-manifold) — v10.0 核心. - - 物理本质: 1s 样本听起来不像, 是因为人耳对"空间混响 (Room Acoustics)" - 极度敏感. ECAPA 特征中同时包含音色和环境信息, 直接使用会导致 - 生成音频带有 1s 样本的麦克风/房间质感. - - 数学重构: 将 Z_audio 正交分解为: - Z_audio = Z_timbre + Z_env - 约束: Z_timbre ⊥ Z_env - - 推理时可调节环境权重: - Z_output = Z_timbre + ω2 · Z_env - - ω2=1: 完美复现 1s 样本的麦克风质感 - - ω2=0: 将渣音质变成专业录音棚纯净人声 (Studio Quality) - - v10.0 数值稳定性修复: - env_scale 从 1.0 改为 0.1 渐进初始化. - 未训练的随机 env_basis 即使正交化, 仍会将 ~env_rank/feature_dim - 比例的方差投影到环境子空间. env_scale=1.0 时这会严重破坏音色. - env_scale=0.1 时仅减去 1/10 的投影, 音色几乎完整保留. - - 实现: - 1. 环境编码器: 从 ECAPA 特征中提取环境子空间 (使用低秩投影) - 2. 正交分解: Z_timbre = Z_audio - Proj_env(Z_audio) - 3. 推理调节: Z_output = Z_timbre + ω2 · Z_env - """ - - def __init__( - self, - feature_dim: int = 192, - env_rank: int = 32, - zero_init: bool = False, - env_scale_init: float = None, - ): - """ - Args: - feature_dim: 输入特征维度 (ECAPA dim) - env_rank: 环境子空间的秩 (越小越纯净) - zero_init: 是否零初始化 (恒等映射, 仅用于消融对比) - env_scale_init: 环境强度初始值 (v10.0 默认 0.1, 从 config 读取) - """ - super().__init__() - self.feature_dim = feature_dim - self.env_rank = min(env_rank, feature_dim) - - # 环境子空间基底 (可学习) - # 初始化为随机正交基的前 env_rank 列 - env_basis = torch.randn(feature_dim, env_rank) - env_basis, _ = torch.linalg.qr(env_basis) # 正交化 - self.env_basis = nn.Parameter(env_basis) # (D, R) - - # 环境强度缩放 (v10.0 关键修复) - # env_scale=0.1: 渐进初始化, 未训练时不破坏音色 - # env_scale=1.0: 满血分解 (需训练后使用) - # env_scale=0.0: 恒等映射 (消融对比用) - if env_scale_init is None: - env_scale_init = SOTA_CONFIG.oes_env_scale_init # 0.1 - - if zero_init: - self.env_scale = nn.Parameter(torch.zeros(1)) - else: - self.env_scale = nn.Parameter(torch.tensor([env_scale_init])) - - def decompose( - self, - z_audio: torch.Tensor, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """正交分解: Z_audio = Z_timbre + Z_env. - - Args: - z_audio: (batch, feature_dim) 输入特征 - - Returns: - z_timbre: (batch, feature_dim) 纯净音色特征 (去除环境) - z_env: (batch, feature_dim) 环境特征 - """ - # 环境子空间投影: Proj_env(z) = B @ B^T @ z - # B: (D, R), B^T: (R, D), z: (B, D) - # proj_coeff: (B, R) = z @ B - # z_env: (B, D) = proj_coeff @ B^T - proj_coeff = z_audio @ self.env_basis # (B, R) - z_env = proj_coeff @ self.env_basis.t() # (B, D) - z_env = z_env * self.env_scale # 缩放 (v10.0: 0.1 渐进) - - # 正交分解: Z_timbre = Z_audio - Z_env - z_timbre = z_audio - z_env - - return z_timbre, z_env - - def forward( - self, - z_audio: torch.Tensor, - env_weight: float = 0.0, - ) -> torch.Tensor: - """推理时调节环境权重. - - Args: - z_audio: (batch, feature_dim) 输入特征 - env_weight: ω2, 环境权重 - - 0.0: 纯净录音棚音质 (默认, 去除环境, SOTA 配置) - - 1.0: 完美复现 1s 样本的环境质感 - - Returns: - z_output: (batch, feature_dim) 调节后的特征 - """ - z_timbre, z_env = self.decompose(z_audio) - # Z_output = Z_timbre + ω2 · Z_env - z_output = z_timbre + env_weight * z_env - return z_output - - def compute_orthogonality_loss( - self, - z_timbre: torch.Tensor, - z_env: torch.Tensor, - ) -> torch.Tensor: - """计算正交性损失: 强制 Z_timbre ⊥ Z_env. - - L_orth = || Z_timbre^T @ Z_env / B ||_F² - - Args: - z_timbre: (batch, feature_dim) - z_env: (batch, feature_dim) - - Returns: - loss: 标量正交性损失 - """ - # L2 归一化 - z_t_norm = F.normalize(z_timbre, dim=-1, eps=1e-8) - z_e_norm = F.normalize(z_env, dim=-1, eps=1e-8) - # 批次维度余弦相似度 - cosine_sim = (z_t_norm * z_e_norm).sum(dim=-1) - return cosine_sim.pow(2).mean() diff --git a/personavoice/persona/__init__.py b/personavoice/persona/__init__.py deleted file mode 100644 index 4055874..0000000 --- a/personavoice/persona/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""PersonaVoice 人格提取子包. - -架构位置: 人格管线入口, 从聊天记录/结构化回忆提取人格表示. -被 demo API 和 TTS 骨干的 FiLM Adapter 消费. -""" -from personavoice.persona.extractor import ( - extract_persona_emb, - extract_big_five_traits, - calibrate_traits, -) - -__all__ = [ - "extract_persona_emb", - "extract_big_five_traits", - "calibrate_traits", -] diff --git a/personavoice/persona/extractor.py b/personavoice/persona/extractor.py deleted file mode 100644 index 6f627a0..0000000 --- a/personavoice/persona/extractor.py +++ /dev/null @@ -1,385 +0,0 @@ -"""人格特征提取器: 聊天记录 → Big Five 人格 → persona_emb. - -架构位置: 人格管线的核心实现层, 将非结构化聊天记录转化为结构化人格表示, -供 FiLM Adapter 消费以影响语音风格. - -Pipeline: - 1. 聊天记录 → BERT 编码 → 句子级语义特征 - 2. 句子级特征 → Big Five 人格特征 (OCEAN 模型) - 3. Big Five → persona_emb (64维, 通过确定性投影) - -设计理念: - - 使用预训练 BERT (bert-base-chinese, 已本地缓存) 提取语义特征 - - Big Five 特征提取基于心理学词典 + 语义相似度, 无需训练 - - persona_emb 投影使用固定随机矩阵 (确定性, 可复现) - - 投影后 L2 归一化并缩放, 确保 FiLM Adapter 安全消费 - -Big Five 人格模型 (OCEAN): - - Openness (开放性): 创造力、好奇心、尝试新事物 - - Conscientiousness (尽责性): 计划性、责任感、自律 - - Extraversion (外向性): 社交性、活跃度、积极情绪 - - Agreeableness (宜人性): 信任、利他、合作 - - Neuroticism (神经质): 焦虑、情绪不稳定性 -""" - -import os -import logging -import numpy as np -import torch -import torch.nn.functional as F -from typing import Dict, List, Optional - -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" - -logger = logging.getLogger(__name__) - -# ───────────────────────────────────────────────────────────────────── -# Big Five 人格词典 (中英文双语) -# ───────────────────────────────────────────────────────────────────── -# 每个维度的关键词列表, 用于基于词频的人格特征估计 -BIG_FIVE_KEYWORDS: Dict[str, List[str]] = { - "openness": [ - # 中文 - "创造", "创新", "新", "尝试", "好奇", "艺术", "想象", "探索", "学习", - "兴趣", "灵感", "独特", "创意", "开放", "思考", "阅读", "音乐", "文化", - # 英文 - "creative", "new", "try", "curious", "art", "imagine", "explore", "learn", - "interest", "inspire", "unique", "open", "think", "read", "music", "culture", - ], - "conscientiousness": [ - "计划", "规律", "认真", "负责", "有序", "努力", "坚持", "目标", "完成", - "安排", "准备", "准时", "规则", "纪律", "勤奋", "细心", "专注", "决心", - "plan", "organized", "careful", "responsible", "orderly", "effort", "persist", - "goal", "complete", "prepare", "punctual", "rule", "discipline", "diligent", - ], - "extraversion": [ - "社交", "朋友", "热闹", "开朗", "说话", "聚会", "活跃", "外向", "聊天", - " party", "聚会", "表达", "分享", "互动", "热情", "自信", "乐观", "健谈", - "social", "friend", "lively", "outgoing", "talk", "party", "active", "express", - "share", "interact", "enthusiastic", "confident", "optimistic", "gregarious", - ], - "agreeableness": [ - "真诚", "善良", "帮助", "友好", "合作", "体贴", "信任", "宽容", "温柔", - "关心", "理解", "支持", "尊重", "谦虚", "同情", "和谐", "感恩", "包容", - "sincere", "kind", "help", "friendly", "cooperate", "considerate", "trust", - "tolerant", "gentle", "care", "understand", "support", "respect", "humble", - ], - "neuroticism": [ - "焦虑", "紧张", "担心", "情绪", "压力", "不安", "害怕", "烦躁", "沮丧", - "孤独", "疲惫", "纠结", "敏感", "忧虑", "失落", "愤怒", "悲伤", "恐惧", - "anxious", "nervous", "worry", "emotional", "stress", "uneasy", "afraid", - "irritable", "depressed", "lonely", "tired", "sensitive", "fear", "sad", - ], -} - -# ───────────────────────────────────────────────────────────────────── -# BERT 模型单例 (懒加载) -# ───────────────────────────────────────────────────────────────────── -_bert_model = None -_bert_tokenizer = None -_bert_device = "cpu" # BERT 在 CPU 运行, 节省 GPU 显存给 TTS - - -def _get_bert_model(): - """获取 BERT 模型单例 (bert-base-chinese, 本地缓存).""" - global _bert_model, _bert_tokenizer - if _bert_model is not None: - return _bert_model, _bert_tokenizer - - try: - from transformers import AutoTokenizer, AutoModel - logger.info("Loading BERT model: bert-base-chinese (local cache)") - _bert_tokenizer = AutoTokenizer.from_pretrained( - "bert-base-chinese", local_files_only=True - ) - _bert_model = AutoModel.from_pretrained( - "bert-base-chinese", local_files_only=True - ) - _bert_model.to(_bert_device) - _bert_model.eval() - logger.info(f"BERT loaded on {_bert_device}, params={sum(p.numel() for p in _bert_model.parameters()):,}") - except Exception as e: - logger.warning(f"BERT loading failed: {e}, falling back to keyword-only mode") - _bert_model = None - _bert_tokenizer = None - - return _bert_model, _bert_tokenizer - - -# ───────────────────────────────────────────────────────────────────── -# 确定性投影矩阵 (Big Five 5维 → persona_emb 64维) -# ───────────────────────────────────────────────────────────────────── -# 使用固定随机种子生成投影矩阵, 确保可复现 -_PROJECTION_MATRIX = None -_PROJECTION_SEED = 42 - - -def _get_projection_matrix(): - """获取 Big Five → persona_emb 的投影矩阵 (5 → 64).""" - global _PROJECTION_MATRIX - if _PROJECTION_MATRIX is not None: - return _PROJECTION_MATRIX - - rng = np.random.RandomState(_PROJECTION_SEED) - # 正交初始化的投影矩阵 - mat = rng.randn(5, 64).astype(np.float32) - # QR 分解正交化 - q, _ = np.linalg.qr(mat.T) # (64, 5) - _PROJECTION_MATRIX = q.T # (5, 64) - return _PROJECTION_MATRIX - - -# ───────────────────────────────────────────────────────────────────── -# 核心函数 -# ───────────────────────────────────────────────────────────────────── - -def _compute_keyword_scores(chat_list: List[str]) -> Dict[str, float]: - """基于关键词频率计算 Big Five 人格分数. - - Args: - chat_list: 聊天记录列表, 每条一行 - - Returns: - Dict[str, float]: Big Five 分数 (0.0-1.0) - """ - # 合并所有聊天记录 - full_text = " ".join(chat_list).lower() - - scores = {} - for trait, keywords in BIG_FIVE_KEYWORDS.items(): - # 统计关键词出现次数 - count = 0 - for kw in keywords: - count += full_text.count(kw.lower()) - # 归一化: 基于文本长度和关键词数量的期望 - text_len = max(1, len(full_text)) - # 使用 sigmoid 将计数映射到 [0, 1] - # 期望: 每千字约 1-3 个关键词命中 → 分数约 0.5-0.7 - density = count / (text_len / 1000.0) if text_len > 0 else 0 - score = 1.0 / (1.0 + np.exp(-(density - 1.5))) # sigmoid, 中心在 1.5 - scores[trait] = float(np.clip(score, 0.05, 0.95)) - - return scores - - -def _compute_bert_scores(chat_list: List[str]) -> Dict[str, float]: - """使用 BERT 语义相似度计算 Big Five 人格分数. - - 对每个 Big Five 维度, 计算聊天记录与该维度描述的语义相似度. - - Args: - chat_list: 聊天记录列表 - - Returns: - Dict[str, float]: Big Five 分数 (0.0-1.0) - """ - model, tokenizer = _get_bert_model() - if model is None: - return {} - - # Big Five 维度描述句 (用于语义相似度计算) - trait_descriptions = { - "openness": "我喜欢尝试新事物, 对艺术和文化感兴趣, 充满好奇心和创造力.", - "conscientiousness": "我做事有计划, 认真负责, 自律且有目标.", - "extraversion": "我喜欢社交活动, 开朗活跃, 乐于与人交流.", - "agreeableness": "我待人真诚友善, 乐于助人, 容易信任他人.", - "neuroticism": "我容易焦虑紧张, 情绪波动较大, 经常感到压力.", - } - - try: - # 编码聊天记录 - chat_text = " [SEP] ".join(chat_list[:10]) # 最多取前 10 条 - inputs = tokenizer( - chat_text, return_tensors="pt", max_length=512, - truncation=True, padding=True - ).to(_bert_device) - - with torch.no_grad(): - outputs = model(**inputs) - chat_emb = outputs.last_hidden_state[:, 0, :].squeeze(0) # [CLS] token (768,) - - # 编码每个 trait 描述并计算相似度 - scores = {} - for trait, desc in trait_descriptions.items(): - inputs_t = tokenizer( - desc, return_tensors="pt", max_length=128, - truncation=True, padding=True - ).to(_bert_device) - with torch.no_grad(): - outputs_t = model(**inputs_t) - trait_emb = outputs_t.last_hidden_state[:, 0, :].squeeze(0) # (768,) - - # 余弦相似度 - sim = F.cosine_similarity(chat_emb.unsqueeze(0), trait_emb.unsqueeze(0)).item() - # 映射到 [0.1, 0.9] (余弦相似度通常在 [0, 1] 之间) - scores[trait] = float(np.clip(sim * 0.8 + 0.1, 0.05, 0.95)) - - return scores - except Exception as e: - logger.warning(f"BERT scoring failed: {e}") - return {} - - -def extract_big_five_traits( - chat_list: List[str], - structured_recalls: Optional[Dict] = None, -) -> Dict[str, float]: - """提取 Big Five 人格特征. - - 融合关键词分数和 BERT 语义分数, 若有结构化回忆则加权融合. - - Args: - chat_list: 聊天记录列表 - structured_recalls: 结构化回忆 (Big Five 分数, 可选) - - Returns: - Dict[str, float]: Big Five 分数, 键为 OCEAN 维度名 - """ - # 1. 关键词分数 (基础) - kw_scores = _compute_keyword_scores(chat_list) - - # 2. BERT 语义分数 (增强) - bert_scores = _compute_bert_scores(chat_list) - - # 3. 融合 - traits = {} - for trait in BIG_FIVE_KEYWORDS.keys(): - kw = kw_scores.get(trait, 0.5) - bert = bert_scores.get(trait, kw) # BERT 失败时回退到关键词分数 - # 加权融合: BERT 0.6 + 关键词 0.4 - traits[trait] = float(0.6 * bert + 0.4 * kw) - - # 4. 结构化回忆覆盖 (若提供, 直接使用) - if structured_recalls: - for trait in traits: - if trait in structured_recalls: - try: - val = float(structured_recalls[trait]) - # 结构化回忆权重 0.5 - traits[trait] = float(0.5 * traits[trait] + 0.5 * np.clip(val, 0.0, 1.0)) - except (ValueError, TypeError): - pass - - return traits - - -def extract_persona_emb( - chat_list: List[str], - structured_recalls: Optional[Dict] = None, -) -> torch.Tensor: - """从聊天记录提取 persona_emb (64维). - - Pipeline: 聊天记录 → Big Five 特征 → 确定性投影 → persona_emb - - Args: - chat_list: 聊天记录列表 - structured_recalls: 结构化回忆 (可选) - - Returns: - torch.Tensor: persona_emb (64,), L2 归一化 + 缩放 - """ - # 1. 提取 Big Five 特征 - traits = extract_big_five_traits(chat_list, structured_recalls) - - # 2. 转换为向量 - trait_vec = np.array([ - traits.get("openness", 0.5), - traits.get("conscientiousness", 0.5), - traits.get("extraversion", 0.5), - traits.get("agreeableness", 0.5), - traits.get("neuroticism", 0.5), - ], dtype=np.float32) # (5,) - - # 3. 中心化 (0.5 为中性) - trait_vec_centered = trait_vec - 0.5 # [-0.5, 0.5] - - # 4. 投影到 64 维 - proj_matrix = _get_projection_matrix() # (5, 64) - persona_vec = trait_vec_centered @ proj_matrix # (64,) - - # 5. L2 归一化 - norm = np.linalg.norm(persona_vec) - if norm > 1e-8: - persona_vec = persona_vec / norm - - # 6. 缩放 (FiLM Adapter 训练时 persona_emb=0, 使用小尺度避免破坏) - # 尺度 0.1: 轻微影响语音风格, 不破坏 SOTA 音质 - persona_vec = persona_vec * 0.1 - - return torch.from_numpy(persona_vec).float() - - -def calibrate_traits( - feedback: str, - chat_list: Optional[List[str]] = None, - structured_recalls: Optional[Dict] = None, -) -> Dict: - """基于用户自然语言反馈校准人格特征. - - 解析反馈中的方向性关键词, 微调对应 Big Five 维度. - - Args: - feedback: 用户反馈 (如 "让外向性更高", "减少神经质") - chat_list: 聊天记录 (用于重新计算基础分数) - structured_recalls: 结构化回忆 - - Returns: - Dict: 校准后的人格特征 + 校准详情 - """ - # 重新提取基础特征 - if chat_list: - traits = extract_big_five_traits(chat_list, structured_recalls) - else: - traits = {t: 0.5 for t in BIG_FIVE_KEYWORDS.keys()} - - # 解析反馈方向 - feedback_lower = feedback.lower() - adjustments = {} - - # 方向关键词 - increase_words = ["高", "多", "强", "增加", "提高", "提升", "更", "加", "more", "higher", "increase"] - decrease_words = ["低", "少", "弱", "减少", "降低", "减", "less", "lower", "decrease"] - - is_increase = any(w in feedback_lower for w in increase_words) - is_decrease = any(w in feedback_lower for w in decrease_words) - - # 维度关键词匹配 - dimension_map = { - "openness": ["开放", "openness", "创造", "好奇", "新"], - "conscientiousness": ["尽责", "责任", "计划", "conscientious", "认真", "自律"], - "extraversion": ["外向", "extraversion", "社交", "开朗", "活跃"], - "agreeableness": ["宜人", "agreeableness", "友善", "善良", "合作"], - "neuroticism": ["神经质", "neuroticism", "焦虑", "情绪", "紧张"], - } - - for trait, keywords in dimension_map.items(): - if any(kw in feedback_lower for kw in keywords): - delta = 0.1 if is_increase else (-0.1 if is_decrease else 0.05) - old_val = traits[trait] - new_val = float(np.clip(old_val + delta, 0.05, 0.95)) - traits[trait] = new_val - adjustments[trait] = { - "old": round(old_val, 4), - "new": round(new_val, 4), - "delta": round(new_val - old_val, 4), - } - - if not adjustments: - # 未匹配到维度, 整体微调 - for trait in traits: - old_val = traits[trait] - delta = 0.05 if is_increase else (-0.05 if is_decrease else 0.0) - new_val = float(np.clip(old_val + delta, 0.05, 0.95)) - traits[trait] = new_val - adjustments[trait] = { - "old": round(old_val, 4), - "new": round(new_val, 4), - "delta": round(new_val - old_val, 4), - } - - return { - "traits": traits, - "adjustments": adjustments, - "feedback": feedback, - } diff --git a/personavoice/picture/cn.png b/personavoice/picture/cn.png deleted file mode 100644 index bb53d0d..0000000 Binary files a/personavoice/picture/cn.png and /dev/null differ diff --git a/personavoice/picture/demo.png b/personavoice/picture/demo.png deleted file mode 100644 index ec61512..0000000 Binary files a/personavoice/picture/demo.png and /dev/null differ diff --git a/personavoice/picture/en.png b/personavoice/picture/en.png deleted file mode 100644 index ce79f9e..0000000 Binary files a/personavoice/picture/en.png and /dev/null differ diff --git a/personavoice/tts_backbone/__init__.py b/personavoice/tts_backbone/__init__.py deleted file mode 100644 index fa3aa39..0000000 --- a/personavoice/tts_backbone/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""TTS 骨干: F5-TTS 预训练骨干 + 适配器 + 声码器 (v10.4.8 满血版). - -架构位置: 语音生成核心, 接收条件编码器输出的嵌入, 通过 Flow Matching -+ DiT 生成 mel 频谱, 再经 Vocos 声码器合成波形. - -v10.4.8 核心组件: -- F5TTSPretrainedBackbone: F5-TTS 预训练骨干 + FiLM + OES -- ceag_sampler: CEAG 采样器 (交叉熵注意力引导, 升级自 IEAG; 当前 disabled) -- laag_generator: LAAG 长度自适应生成 (动态 Chunking + 动态 CFG + 动态 FiLM) -- VocosVocoder: 神经声码器 - -v10.3+ 时长计算: 使用 F5-TTS 官方公式 (infer_batch_process 内部自动) -""" - -from personavoice.tts_backbone.f5_pretrained_backbone import ( - F5TTSPretrainedBackbone, - PersonaEmotionFiLM, - load_pretrained_f5tts_backbone, -) -from personavoice.tts_backbone.vocoder import VocosVocoder - -__all__ = [ - "F5TTSPretrainedBackbone", - "PersonaEmotionFiLM", - "load_pretrained_f5tts_backbone", - "VocosVocoder", -] diff --git a/personavoice/tts_backbone/ceag_sampler.py b/personavoice/tts_backbone/ceag_sampler.py deleted file mode 100644 index f389f4e..0000000 --- a/personavoice/tts_backbone/ceag_sampler.py +++ /dev/null @@ -1,625 +0,0 @@ -"""PersonaVoice v10.0 CEAG (交叉熵注意力引导) — 推理时 WER 降低. - -架构位置: TTS 骨干子系统中的推理时优化组件. -v10.0: IEAG → CEAG 升级, 直击短文本对齐崩塌. - -============================================================================= -v10.0 核心创新: CEAG (Cross-Entropy Attention Guidance) -============================================================================= -从 IEAG (全自注意力熵) 升级为 CEAG (文本-音频交叉注意力熵). - -理论: 流形崩塌假说 (Manifold Collapse under Information Asymmetry) - 1s 参考音频的信息量不足以约束 DiT 的全注意力分布. - 在对齐关键期 (t∈[0.05, 0.5]), mel token 的注意力分散到所有 text token - (高熵), 导致每个 mel 帧无法聚焦于特定音素 → 咬字模糊 → WER↑. - - CEAG 通过最小化 mel→text 交叉注意力熵, 强制每个 mel 帧聚焦于 - 特定 text token, 恢复对齐流形: - - v_final = v_cfg - λ(t) · ∇_x H(A_{mel→text}) - - 其中 A_{mel→text} 是注意力矩阵中 mel 行、text 列的子矩阵. - -============================================================================= -v10.0 关键修复: Padding Mask (地雷1) -============================================================================= -F5-TTS 的自注意力是 [text_tokens, mel_tokens] 拼接后的全局注意力. -Batch 推理时, text 和 mel 都有 Padding. - -若不屏蔽 token, CEAG 会把极大梯度施加在无意义的 Padding 区域, -导致生成特征瞬间爆炸 (NaN). - -修复: 在 _compute_cross_attention_entropy 中引入 text_mask 和 mel_mask. -在 log_softmax 之前, 将 mask 外的注意力权重设为 -inf, -确保梯度只流向有效的语义对齐区域. - -============================================================================= -v10.0 架构决策 (基于 200 样本消融实验): -============================================================================= - 消融实验数据 (ablation_200_statistics.json): - - TD-CFG: p=0.41/0.74, Cohen's d=-0.058/0.023 → 无效, 已移除 - - SBM: p=0.71/0.90, Cohen's d=-0.026/0.009 → 无效, 已移除 - - IBOP: p=0.85/0.51, Cohen's d=0.014/0.047 → 无效, 已移除 - - IEAG: v9.0 WER 改善的主要贡献者 (0.2405→0.1817) → 保留并升级为 CEAG -""" - -import logging -import math -from typing import Callable, Dict, Optional, Tuple - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.nn.utils.rnn import pad_sequence - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# v10.0 核心: CEAG (Cross-Entropy Attention Guidance) -# ============================================================================= - -class _CEAGAttentionExtractor: - """临时替换 AttnProcessor, 手动计算 attention 并保存 weights 用于 CEAG. - - F5-TTS 使用 F.scaled_dot_product_attention (SDPA) 不返回 weights. - 在 CEAG 激活的时间步, 临时替换 processor 为手动实现, - 保留完整的计算图以支持 ∇_x H(A_{mel→text}) 的自动微分. - """ - - def __init__(self, original_processor): - self.original = original_processor - self.attention_weights = None # (B, H, N, N) - - def __call__(self, attn, x, mask=None, rope=None): - """手动计算 attention, 保存 weights.""" - from f5_tts.model.modules import apply_rotary_pos_emb - - batch_size = x.shape[0] - query = attn.to_q(x) - key = attn.to_k(x) - value = attn.to_v(x) - - inner_dim = key.shape[-1] - head_dim = inner_dim // attn.heads - query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) - - # qk norm - if attn.q_norm is not None: - query = attn.q_norm(query) - if attn.k_norm is not None: - key = attn.k_norm(key) - - # apply RoPE (与原始 processor 一致) - if rope is not None: - freqs, xpos_scale = rope - q_xpos_scale, k_xpos_scale = ( - (xpos_scale, xpos_scale ** -1.0) if xpos_scale is not None else (1.0, 1.0) - ) - if self.original.pe_attn_head is not None: - pn = self.original.pe_attn_head - query[:, :pn, :, :] = apply_rotary_pos_emb( - query[:, :pn, :, :], freqs, q_xpos_scale - ) - key[:, :pn, :, :] = apply_rotary_pos_emb( - key[:, :pn, :, :], freqs, k_xpos_scale - ) - else: - query = apply_rotary_pos_emb(query, freqs, q_xpos_scale) - key = apply_rotary_pos_emb(key, freqs, k_xpos_scale) - - # 手动计算 attention scores (保留梯度) - scale = head_dim ** -0.5 - scores = torch.matmul(query, key.transpose(-2, -1)) * scale # (B,H,N,N) - - # 应用 mask (F5-TTS 的序列 mask) - # f5_tts 1.0.0 AttnProcessor 没有 attn_mask_enabled 属性, 直接用 mask - if mask is not None: - attn_mask = mask.unsqueeze(1).unsqueeze(1) - attn_mask = attn_mask.expand( - batch_size, attn.heads, query.shape[-2], key.shape[-2] - ) - scores = scores.masked_fill(~attn_mask, float('-inf')) - - # softmax 得到 attention weights - attn_weights = torch.softmax(scores, dim=-1) # (B, H, N, N) - - # 保存 weights 供 CEAG 使用 - self.attention_weights = attn_weights - - # 计算 output (与 SDPA 等价) - out = torch.matmul(attn_weights, value) - out = out.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) - - return out - - -class CEAGGuidance: - """Cross-Entropy Attention Guidance (CEAG) — v10.0 核心创新. - - 在 ODE 积分对齐关键期 (t∈[t_start, t_end]), 通过最小化 mel→text - 交叉注意力熵引导速度场, 解决 1s 参考下注意力分散导致的咬字崩溃. - - v_final = v_cfg - λ(t) · ∇_x H(A_{mel→text}) - - 其中: - A_{mel→text} = A[mel_positions, text_positions] (交叉注意力子矩阵) - H(A_{mt}) = -Σ_i Σ_j A_{mt}[i,j] · log(A_{mt}[i,j]) (香农熵) - λ(t) = λ_max · cos(π/2 · (t - center) / half_width) (余弦窗) - - v10.0 Padding Mask 修复 (地雷1): - 在计算熵之前, 用 text_mask 和 mel_mask 屏蔽 padding 位置, - 确保梯度只流向有效的语义对齐区域, 防止 NaN. - - 使用方式: - ceag = CEAGGuidance(dit, text_len, mel_len, text_mask, mel_mask) - # 在采样循环中: - if ceag.is_active(t): - v = ceag.guided_forward(x, fn, t) - else: - v = fn(t, x) - """ - - def __init__( - self, - dit, - text_len: int, - mel_len: int, - text_mask: torch.Tensor, - mel_mask: torch.Tensor, - active_layer_indices: Tuple[int, ...] = (-3, -2, -1), - t_start: float = 0.05, - t_end: float = 0.5, - lambda_max: float = 0.25, - grad_max_norm: float = 1.0, - ): - """初始化 CEAG. - - Args: - dit: F5-TTS 的 DiT 模型 - text_len: 文本 token 数 (含 padding 的长度) - mel_len: mel 帧数 (含 padding 的长度) - text_mask: (B, text_len) True=有效文本 token - mel_mask: (B, mel_len) True=有效 mel 帧 - active_layer_indices: 提取 attention 的 DiT block 索引 (负数=倒数) - t_start: CEAG 激活起始时间步 - t_end: CEAG 激活结束时间步 - lambda_max: 最大引导强度 - grad_max_norm: 梯度裁剪最大范数 (逐层, 防止梯度爆炸) - """ - self.dit = dit - self.text_len = text_len - self.mel_len = mel_len - # text_mask: (B, text_len) → (B, 1, 1, text_len) for broadcasting - self.text_mask = text_mask.bool().view(text_mask.shape[0], 1, 1, -1) - # mel_mask: (B, mel_len) → (B, 1, mel_len, 1) for broadcasting - self.mel_mask = mel_mask.bool().view(mel_mask.shape[0], 1, -1, 1) - self.active_layer_indices = active_layer_indices - self.t_start = t_start - self.t_end = t_end - self.lambda_max = lambda_max - self.grad_max_norm = grad_max_norm - - self._extractors: Dict[int, _CEAGAttentionExtractor] = {} - self._original_processors: Dict[int, object] = {} - self._patched = False - - def _lambda_schedule(self, t: float) -> float: - """CEAG 强度调度: 余弦窗函数, 在区间中心最大.""" - if t < self.t_start or t > self.t_end: - return 0.0 - center = (self.t_start + self.t_end) / 2 - half_width = (self.t_end - self.t_start) / 2 - if half_width < 1e-6: - return self.lambda_max - window = math.cos(math.pi / 2 * (t - center) / half_width) - return self.lambda_max * max(0.0, window) - - def is_active(self, t: float) -> bool: - """判断当前时间步是否激活 CEAG.""" - return self.t_start <= t <= self.t_end - - def _install_extractors(self): - """在目标 DiT block 上安装 attention 提取器.""" - if self._patched: - return - blocks = self.dit.transformer_blocks - n_blocks = len(blocks) - for idx in self.active_layer_indices: - real_idx = idx if idx >= 0 else n_blocks + idx - block = blocks[real_idx] - attn = block.attn - - # 保存原始 processor - self._original_processors[real_idx] = attn.processor - - # 安装提取器 - extractor = _CEAGAttentionExtractor(attn.processor) - self._extractors[real_idx] = extractor - attn.processor = extractor - - self._patched = True - - def _restore_processors(self): - """恢复原始 AttnProcessor.""" - if not self._patched: - return - blocks = self.dit.transformer_blocks - for real_idx, original_proc in self._original_processors.items(): - blocks[real_idx].attn.processor = original_proc - self._extractors.clear() - self._original_processors.clear() - self._patched = False - - def _compute_cross_attention_entropy( - self, - x_batch_size: int, - ) -> torch.Tensor: - """计算 mel→text 交叉注意力的香农熵 (v10.0 核心, 带 Padding Mask). - - 提取注意力矩阵中 mel 行、text 列的子矩阵 A_{mel→text}, - 在屏蔽 padding 后计算香农熵. - - F5-TTS 序列结构: [text_tokens (T_text), mel_tokens (T_mel)] - 全局注意力 A: (B, H, N, N), N = T_text + T_mel - 交叉注意力: A_mt = A[:, :, T_text:T_text+T_mel, 0:T_text] - 即 mel token 对 text token 的注意力 (对齐矩阵) - - v10.0 Padding Mask 修复 (地雷1): - 1. 提取 A_mt 后, 用 text_mask 屏蔽 padding text 列 - 2. 重新 log_softmax (仅在有效 text token 上归一化) - 3. 用 mel_mask 屏蔽 padding mel 行 (不参与熵计算) - 4. 梯度只流向有效对齐区域, 防止 NaN - - Args: - x_batch_size: 原始 x 的 batch size (CFG 前) - - Returns: - entropy: 标量, 所有层所有 head 的平均交叉注意力熵 - """ - total_entropy = 0.0 - count = 0 - eps = 1e-8 - T_text = self.text_len - - for extractor in self._extractors.values(): - if extractor.attention_weights is None: - continue - # attn_weights: (B_total, H, N, N) - w = extractor.attention_weights - - # CFG 模式下 B_total = 2 * x_batch_size, 只取 cond 部分 - if w.shape[0] > x_batch_size: - w = w[:x_batch_size] # (B, H, N, N) - - B = w.shape[0] - - # === 提取 mel→text 交叉注意力子矩阵 === - # A_mt[b, h, i, j] = mel 帧 i 对 text token j 的注意力 - # 序列: [text(0..T_text-1), mel(T_text..T_text+T_mel-1)] - # mel 行: [T_text, T_text+T_mel) - # text 列: [0, T_text) - # 使用实际 attention weights 的维度, 而不是 self.mel_len - actual_T_mel = w.shape[-1] - T_text # 实际 mel 长度 - A_mt = w[:, :, T_text:T_text + actual_T_mel, :T_text] # (B, H, T_mel, T_text) - - # === v10.0 Padding Mask 修复 (地雷1) === - # text_mask: (B, 1, 1, T_text) → 屏蔽 padding text 列 - # mel_mask: (B, 1, T_mel, 1) → 屏蔽 padding mel 行 - text_mask = self.text_mask[:B] # (B, 1, 1, T_text) - # 使用实际 mel 长度的 mask - mel_mask_full = self.mel_mask[:B] # (B, 1, mel_len, 1) - # 截取或填充到 actual_T_mel - if mel_mask_full.shape[2] >= actual_T_mel: - mel_mask = mel_mask_full[:, :, :actual_T_mel, :] # (B, 1, T_mel, 1) - else: - # 填充 False (padding) - pad_size = actual_T_mel - mel_mask_full.shape[2] - mel_mask = F.pad(mel_mask_full, (0, 0, 0, pad_size), value=False) - - # 将 padding text 列的注意力设为极小值 (-1e9) - # 这样 softmax 后 padding text 位置的权重 → 0 - # 注意: 不能用 -inf, 因为会导致梯度 NaN; 用大负数更安全 - A_mt_masked = A_mt.masked_fill(~text_mask, -1e9) - - # 重新在有效 text token 上做 log_softmax (数值稳定的归一化) - # log_softmax 自动应用 Log-Sum-Exp trick, 防止溢出 - log_A_mt = F.log_softmax(A_mt_masked, dim=-1) # (B, H, T_mel, T_text) - A_mt_norm = log_A_mt.exp() # 归一化后的注意力权重 - - # 计算香农熵: H = -Σ_j A[j] · log(A[j]) - # 仅在有效 text token 上求和 (padding 列的 A→0, 贡献→0) - entropy_per_mel = -(A_mt_norm * log_A_mt).sum(dim=-1) # (B, H, T_mel) - - # 仅对有效 mel 帧计算熵 (屏蔽 padding mel 行) - # mel_mask: (B, 1, T_mel, 1) → (B, 1, T_mel) - mel_mask_2d = mel_mask.squeeze(-1) # (B, 1, T_mel) - # 用 mask 加权平均: padding mel 帧的熵不参与 - mask_sum = mel_mask_2d.sum(dim=-1, keepdim=True).clamp(min=1.0) # (B, 1, 1) - entropy_masked = entropy_per_mel * mel_mask_2d # (B, H, T_mel) - entropy_per_head = entropy_masked.sum(dim=-1) / mask_sum.squeeze(-1) # (B, H) - - total_entropy = total_entropy + entropy_per_head.mean() - count += 1 - - if count == 0: - return torch.tensor(0.0, device=self.dit.time_embed.linear.weight.device) - return total_entropy / count - - def guided_forward( - self, - x: torch.Tensor, - fn: Callable, - t: torch.Tensor, - ) -> torch.Tensor: - """CEAG 引导的速度场计算. - - v_final = v_cfg - λ(t) · ∇_x H(A_{mel→text}) - - Args: - x: (B, T, D) 当前 ODE 状态 - fn: 速度场函数 fn(t, x) -> v - t: 当前时间步 - - Returns: - v_final: (B, T, D) 引导后的速度场 - """ - t_val = t.item() if isinstance(t, torch.Tensor) else t - lambda_t = self._lambda_schedule(t_val) - - if lambda_t < 1e-6: - # 不需要引导, 直接返回 - return fn(t, x) - - x_batch_size = x.shape[0] - - # 1. 安装 attention 提取器 - self._install_extractors() - - try: - # 2. 启用梯度追踪 - x_grad = x.detach().requires_grad_(True) - - # 3. 前向传播 (提取 attention map) - v = fn(t, x_grad) - - # 4. 计算交叉注意力熵 (只用 cond 部分, 带 padding mask) - entropy = self._compute_cross_attention_entropy(x_batch_size) - - # 5. 计算熵对 x 的梯度: ∇_x H(A_{mel→text}) - if entropy.requires_grad and x_grad.requires_grad: - grad_entropy = torch.autograd.grad( - outputs=entropy, - inputs=x_grad, - retain_graph=False, - create_graph=False, - allow_unused=True, - )[0] - - if grad_entropy is not None: - # v10.0 梯度裁剪: 逐层防止梯度爆炸 (地雷1 防御) - grad_norm = grad_entropy.norm() - if grad_norm > self.grad_max_norm: - grad_entropy = grad_entropy * (self.grad_max_norm / (grad_norm + 1e-8)) - - # 6. 修正速度场: v_final = v - λ·∇_x H(A) - # 最小化熵 = 减去熵的梯度方向 - v_final = v.detach() - lambda_t * grad_entropy.detach() - else: - v_final = v.detach() - else: - v_final = v.detach() - - finally: - # 7. 恢复原始 processor - self._restore_processors() - - return v_final - - -# ============================================================================= -# v10.0 采样器: 静态 CFG + CEAG + Sway Sampling (满血版) -# ============================================================================= - -def sample_with_ceag( - cfm, - cond: torch.Tensor, - text: torch.Tensor, - duration: int, - mel_ref: torch.Tensor, - steps: int = 96, - # 静态 CFG 参数 (替代无效的 TD-CFG) - cfg_strength: float = 3.0, - # CEAG 参数 (v10.0 核心创新) - use_ceag: bool = True, - ceag_t_start: float = 0.05, - ceag_t_end: float = 0.5, - ceag_lambda_max: float = 0.25, - ceag_layers: Tuple[int, ...] = (-3, -2, -1), - # Sway sampling (F5-TTS 官方默认 -1.0) - sway_sampling_coef: Optional[float] = -1.0, - seed: Optional[int] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - """v10.0 满血采样器: 静态 CFG + CEAG + Sway Sampling. - - v10.0 架构决策 (基于消融实验): - - 移除 TD-CFG (p=0.41/0.74, 无效) → 使用静态 CFG - - 移除 SBM (p=0.71/0.90, 无效) → 纯 ODE 积分 - - IEAG → CEAG 升级 (交叉注意力熵, 直击短文本对齐) - - 保留 Sway Sampling (F5-TTS 官方默认) - - Args: - cfm: F5-TTS CFM 模型 - cond: (B, T_ref, mel_dim) 参考条件 - text: (B, T_text) 文本 token - duration: 目标长度 - mel_ref: 参考 mel (未使用, 保留接口兼容) - steps: ODE 积分步数 (默认 96, v10.0 SOTA) - cfg_strength: 静态 CFG 强度 (默认 3.0, v10.0 SOTA) - use_ceag: 是否启用 CEAG - ceag_t_start: CEAG 激活起始时间步 (默认 0.05, v10.0 扩展) - ceag_t_end: CEAG 激活结束时间步 (默认 0.5, v10.0 扩展) - ceag_lambda_max: CEAG 最大引导强度 (默认 0.25, v10.0 SOTA) - ceag_layers: 提取 attention 的 DiT block 索引 (默认 (-3,-2,-1)) - sway_sampling_coef: Sway Sampling 系数 (默认 -1.0) - seed: 随机种子 - - Returns: - out: (B, T, mel_dim) 生成的 mel - trajectory: (steps+1, B, T, mel_dim) 轨迹 - """ - from f5_tts.model.utils import lens_to_mask - - # 兼容性: f5_tts 1.0.0 没有 get_epss_timesteps, 自己实现 - def get_epss_timesteps(steps, device, dtype): - return torch.linspace(0.0, 1.0, steps + 1, device=device, dtype=dtype) - - cfm.eval() - device = cfm.device - dit = cfm.transformer - - # Handle cond input - if cond.ndim == 2: - cond = cfm.mel_spec(cond) - cond = cond.permute(0, 2, 1) - cond = cond.to(next(cfm.parameters()).dtype) - - batch, cond_seq_len = cond.shape[:2] - - # Handle text - if isinstance(text, list): - if hasattr(cfm, 'vocab_char_map') and cfm.vocab_char_map is not None: - from f5_tts.model.utils import list_str_to_idx - text = list_str_to_idx(text, cfm.vocab_char_map).to(device) - else: - from f5_tts.model.utils import list_str_to_tensor - text = list_str_to_tensor(text).to(device) - - # Duration - if isinstance(duration, int): - duration = torch.full((batch,), duration, device=device, dtype=torch.long) - - duration = torch.maximum( - torch.maximum((text != -1).sum(dim=-1), torch.full_like(duration, cond_seq_len)) + 1, - duration - ) - max_duration = duration.amax().item() - - # === 条件准备 (无 Prompt Padding, v10.0 精简) === - cond_mask = lens_to_mask( - torch.full((batch,), cond_seq_len, device=device, dtype=torch.long) - ) - - cond_padded = F.pad(cond, (0, 0, 0, max_duration - cond_seq_len), value=0.0) - cond_mask_padded = F.pad(cond_mask, (0, max_duration - cond_mask.shape[-1]), value=False) - cond_mask_3d = cond_mask_padded.unsqueeze(-1) - - step_cond = torch.where(cond_mask_3d, cond_padded, torch.zeros_like(cond_padded)) - - cond = cond_padded - cond_mask = cond_mask_3d - - if batch > 1: - mask = lens_to_mask(duration) - else: - mask = None - - # === v10.0 核心: 构建 CEAG 所需的 Padding Mask === - # text_mask: (B, T_text) True=有效 text token - # F5-TTS 用 -1 表示 padding text token - text_padded_len = text.shape[1] - text_mask = (text != -1) # (B, T_text) - - # mel_mask: (B, T_mel) True=有效 mel 帧 - mel_mask = lens_to_mask(duration) # (B, max_duration) - - # === CEAG 初始化 (v10.0 核心创新) === - ceag = CEAGGuidance( - dit, - text_len=text_padded_len, - mel_len=max_duration, - text_mask=text_mask, - mel_mask=mel_mask, - active_layer_indices=ceag_layers, - t_start=ceag_t_start, - t_end=ceag_t_end, - lambda_max=ceag_lambda_max, - ) if use_ceag else None - - def fn(t, x): - """ODE 速度场函数 (静态 CFG, 无 TD-CFG).""" - dit.clear_cache() - - if cfg_strength < 1e-5: - pred = dit( - x=x, cond=step_cond, text=text, time=t, - mask=mask, drop_audio_cond=False, drop_text=False, cache=True, - ) - return pred - - # 静态 CFG: pred + (pred - null_pred) * cfg_strength - # f5_tts 1.0.0 不支持 cfg_infer, 改为两次调用 - pred = dit( - x=x, cond=step_cond, text=text, time=t, - mask=mask, drop_audio_cond=False, drop_text=False, cache=True, - ) - null_pred = dit( - x=x, cond=step_cond, text=text, time=t, - mask=mask, drop_audio_cond=True, drop_text=True, cache=True, - ) - return pred + (pred - null_pred) * cfg_strength - - # Noise initialization - y0 = [] - for dur in duration: - if seed is not None: - torch.manual_seed(seed) - y0.append(torch.randn(dur.item(), cfm.num_channels, device=device, dtype=step_cond.dtype)) - y0 = pad_sequence(y0, padding_value=0, batch_first=True) - - # Time steps (Sway Sampling, F5-TTS 官方默认) - if sway_sampling_coef is not None: - t_steps = torch.linspace(0.0, 1.0, steps + 1, device=device, dtype=step_cond.dtype) - t_steps = t_steps + sway_sampling_coef * (torch.cos(torch.pi / 2 * t_steps) - 1 + t_steps) - else: - t_steps = get_epss_timesteps(steps, device=device, dtype=step_cond.dtype) - - # === 纯 ODE 积分 + CEAG 引导 (无 SBM 随机微分) === - trajectory = [y0] - x = y0 - - for i in range(steps): - t_curr = t_steps[i] - t_next = t_steps[i + 1] - dt = (t_next - t_curr).item() - t_val = t_curr.item() - - # === v10.0 核心创新: CEAG 引导 === - if ceag is not None and ceag.is_active(t_val): - # CEAG 引导的速度场 (带 padding mask 的交叉注意力熵引导) - # 注意: CEAG 步需要梯度, 不能用 inference_mode - v = ceag.guided_forward(x, fn, t_curr) - else: - # 普通静态 CFG 速度场 (v10.1: 用 inference_mode 加速) - with torch.inference_mode(): - v = fn(t_curr, x) - - # 纯 ODE 积分 (无 SBM 噪声注入, v10.0 精简) - x = x + dt * v - - trajectory.append(x) - - dit.clear_cache() - - trajectory = torch.stack(trajectory, dim=0) - sampled = trajectory[-1] - out = torch.where(cond_mask, cond, sampled) - - return out, trajectory - - -# 向后兼容别名 (v9.x 代码迁移) -sample_with_ieag = sample_with_ceag -IEAGGuidance = CEAGGuidance diff --git a/personavoice/tts_backbone/f5_pretrained_backbone.py b/personavoice/tts_backbone/f5_pretrained_backbone.py deleted file mode 100644 index 097ed06..0000000 --- a/personavoice/tts_backbone/f5_pretrained_backbone.py +++ /dev/null @@ -1,846 +0,0 @@ -"""F5-TTS 预训练骨干封装 (v9.1 精简 SOTA 实现). - -架构位置: TTS 骨干子系统的真实模型加载层, 包装官方 F5-TTS 模型作为冻结骨干, -创新模块作为 Adapter 外挂, 实现 8GB 显存下的真正 Plug-in Adapter 架构. - -v9.1 精简版 (基于 200 样本消融实验): - - 保留: F5-TTS 预训练骨干 (有效, 100% 权重加载) - - 保留: FiLM Adapter (有效, persona/emotion 注入) - - 保留: OES (条件有效, env_weight=0.0 时不干扰, 默认纯净音色) - - 保留: IEAG (有效, v9.0 WER 改善主要贡献者) - - 移除: IBOP (消融 p=0.85/0.51, Cohen's d<0.05, 无效) - - 移除: AM-ODE (零初始化未贡献, 无效) - - 移除: TD-CFG (消融 p=0.41/0.74, Cohen's d<0.06, 无效) - - 移除: SBM (消融 p=0.71/0.90, Cohen's d<0.03, 无效) - -设计理念 (真正的 Plug-in Adapter Architecture): - - 直接加载官方 F5-TTS CFM 模型 (1024 维/22 层, ~330M 参数) - - 100% 预训练权重加载 (loaded_keys/total_keys = 1.0) - - 仅训练 FiLM Adapter (persona/emotion 注入) - - 8GB 显存可行: F5-TTS 冻结 (无梯度) + 小 Adapter (可训练) - -v9.1 推理流程: - speaker_emb (B, 192) - ↓ - [OES] 正交环境分解 (env_weight=0.0) → Z_timbre (纯净录音棚音质) - ↓ - [FiLM] persona/emotion 条件注入 (零初始化, 安全) - ↓ - [F5-TTS DiT] Flow Matching 生成 (静态 CFG + IEAG + Sway Sampling) - ↓ - [Vocos] 声码器 → 24kHz 波形 - -使用方式: - backbone = F5TTSPretrainedBackbone(device="cuda") - mel_gen = backbone.synthesize(text_tokens, mel_ref, speaker_emb, persona_emb, emotion_emb) -""" - -import os -import torch -import torch.nn as nn -import torch.nn.functional as F -from typing import Optional, Dict, List, Tuple -import logging - -os.environ["HF_HUB_OFFLINE"] = "1" -os.environ["TRANSFORMERS_OFFLINE"] = "1" -# v10.4: 禁用 wandb (f5_tts.trainer 依赖 wandb, 与 speechbrain lazy import 冲突) -os.environ["WANDB_DISABLED"] = "true" -os.environ["WANDB_MODE"] = "disabled" - -logger = logging.getLogger(__name__) - - -class PersonaEmotionFiLM(nn.Module): - """FiLM adapter for persona/emotion condition injection (v9.1 核心保留组件). - - 通过 FiLM (Feature-wise Linear Modulation) 将 persona 和 emotion 条件 - 注入到 DiT 的隐藏状态中. - - FiLM: gamma * x + beta - 其中 gamma, beta 由 persona/emotion embedding 生成. - 零初始化 gamma=1, beta=0, 训练开始时不破坏 F5-TTS 行为. - - 消融实验验证 (200 样本): - - FiLM Adapter 是 persona/emotion 注入的核心, 保留 - - 零初始化设计确保训练初期为恒等映射, 不破坏预训练骨干 - """ - - def __init__(self, hidden_dim: int = 1024, persona_dim: int = 64, emotion_dim: int = 64): - super().__init__() - self.hidden_dim = hidden_dim - - # gamma 和 beta 生成网络 - self.gamma_net = nn.Sequential( - nn.Linear(persona_dim + emotion_dim, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, hidden_dim), - ) - self.beta_net = nn.Sequential( - nn.Linear(persona_dim + emotion_dim, hidden_dim), - nn.GELU(), - nn.Linear(hidden_dim, hidden_dim), - ) - - # 零初始化: gamma_net 输出 0 (gamma=1), beta_net 输出 0 (beta=0) - nn.init.zeros_(self.gamma_net[-1].weight) - nn.init.zeros_(self.gamma_net[-1].bias) - nn.init.zeros_(self.beta_net[-1].weight) - nn.init.zeros_(self.beta_net[-1].bias) - - def forward( - self, - hidden: torch.Tensor, # (B, T, hidden_dim) DiT 隐藏状态 - persona_emb: torch.Tensor, # (B, persona_dim) - emotion_emb: torch.Tensor, # (B, emotion_dim) - ) -> torch.Tensor: - """FiLM 调制: gamma * hidden + beta.""" - cond = torch.cat([persona_emb, emotion_emb], dim=-1) # (B, persona+emotion) - gamma = 1.0 + self.gamma_net(cond).unsqueeze(1) # (B, 1, hidden_dim) - beta = self.beta_net(cond).unsqueeze(1) # (B, 1, hidden_dim) - return gamma * hidden + beta - - -class F5TTSPretrainedBackbone(nn.Module): - """SOTA 真实 F5-TTS 预训练骨干 + v9.1 精简 Adapter. - - 100% 加载官方 F5-TTS 权重, 仅训练 FiLM adapter. - v9.1 精简架构: F5-TTS + FiLM + OES + IEAG (移除 IBOP/AM-ODE/TD-CFG/SBM). - """ - - def __init__( - self, - device: str = "cuda", - use_film: bool = True, - persona_dim: int = 64, - emotion_dim: int = 64, - num_emotions: int = 4, - ): - super().__init__() - self.device = device - self.use_film = use_film - self.persona_dim = persona_dim - self.emotion_dim = emotion_dim - self.num_emotions = num_emotions - - # 加载真实 F5-TTS - logger.info("Loading official F5-TTS pretrained model...") - from f5_tts.api import F5TTS - self._f5tts = F5TTS() - self.f5_cfm = self._f5tts.ema_model # CFM 模型 (含 DiT) - self.f5_dit = self.f5_cfm.transformer # DiT 骨干 - - # 转为 fp32: fp16 下解冻层的梯度计算不稳定 (NaN 问题) - self.f5_cfm = self.f5_cfm.float() - - self.mel_dim = self.f5_dit.proj_out.out_features # 100 - self.hidden_dim = self.f5_dit.dim # 1024 - logger.info( - f"F5-TTS loaded: dim={self.hidden_dim}, " - f"depth={self.f5_dit.depth}, mel_dim={self.mel_dim}, dtype=float32" - ) - - # v9.1 核心 Adapter: FiLM (persona/emotion 注入) - if use_film: - self.film = PersonaEmotionFiLM( - hidden_dim=self.hidden_dim, - persona_dim=persona_dim, - emotion_dim=emotion_dim, - ) - # persona/emotion 投影 (从 speaker_emb 到 persona, one-hot 到 emotion) - self.persona_proj = nn.Linear(192, persona_dim, bias=False) - self.emotion_proj = nn.Linear(num_emotions, emotion_dim) - nn.init.zeros_(self.persona_proj.weight) - nn.init.zeros_(self.emotion_proj.weight) - nn.init.zeros_(self.emotion_proj.bias) - else: - self.film = None - - # v9.1 核心: OES (正交环境子流形) — 零样本声学解缠 - # v9.1 配置: env_weight=0.0 (默认) → 输出纯净 Z_timbre (录音棚音质) - # env_weight=1 → 输出 Z_timbre+Z_env (原始质感) - # 消融实验: env_weight=0.0 时 SECS=0.4832 (SOTA), 保留 - from personavoice.microaug.cross_manifold_refiner import ( - OrthogonalEnvironmentSubmanifold, - ) - self.oes = OrthogonalEnvironmentSubmanifold( - feature_dim=192, - env_rank=32, - zero_init=False, # env_scale=1.0, 使分解真正生效 - ) - # 推理时环境权重 (v9.1 默认 0.0: 纯净录音棚音质, SOTA 配置) - self._env_weight = 0.0 - - # 将所有 Adapter 移到 device - if self.film is not None: - self.film = self.film.to(self.device) - self.persona_proj = self.persona_proj.to(self.device) - self.emotion_proj = self.emotion_proj.to(self.device) - self.oes = self.oes.to(self.device) - - # 冻结 F5-TTS 骨干 - self._freeze_f5() - - # v10.4: FiLM 激活标志 (默认 False, 只在 persona 非零时激活) - self._film_active = False - - # 注册 FiLM hook 到 DiT blocks (最后 2 层注入) - if use_film: - self._register_film_hooks() - - # 统计参数 - self._log_param_stats() - - def _freeze_f5(self): - """极致冻结策略 (SOTA: 8GB 显存可行). - - 冻结: F5-TTS DiT 前 20 层 + 所有非必要参数 - 解冻: 最后 2 层 DiT blocks (让模型适应 voice cloning) - """ - # 1. 冻结所有 F5-TTS 参数 - for param in self.f5_cfm.parameters(): - param.requires_grad = False - - # 2. 解冻最后 2 层 DiT transformer blocks - num_blocks = len(self.f5_dit.transformer_blocks) - unfreeze_indices = [num_blocks - 2, num_blocks - 1] # 最后 2 层 - unfrozen_count = 0 - for idx in unfreeze_indices: - for param in self.f5_dit.transformer_blocks[idx].parameters(): - param.requires_grad = True - unfrozen_count += param.numel() - - logger.info( - f"F5-TTS frozen: DiT blocks 0-{num_blocks-3} frozen, " - f"blocks {unfreeze_indices} unfrozen ({unfrozen_count:,} params)" - ) - - def _register_film_hooks(self): - """注册 FiLM hook 到 DiT blocks. - - 策略: 在最后 2 层 DiT block 的输出上应用 FiLM. - 这样 persona/emotion 只影响最终输出, 不破坏中间表示. - """ - self._film_hooks = [] - num_blocks = len(self.f5_dit.transformer_blocks) - # 在最后 2 层注入 - target_blocks = [num_blocks - 2, num_blocks - 1] - - for block_idx in target_blocks: - block = self.f5_dit.transformer_blocks[block_idx] - hook = block.register_forward_hook(self._make_film_hook(block_idx)) - self._film_hooks.append(hook) - logger.info(f"FiLM hooks registered on blocks: {target_blocks}") - - def _make_film_hook(self, block_idx: int): - """创建 FiLM hook 函数.""" - def hook_fn(module, input, output): - # v10.4: 只在 FiLM 激活时应用 (零初始化时跳过, 避免数值噪声) - if not getattr(self, "_film_active", False) or not self.use_film or self.film is None: - return output - # output 可能是 tuple, 取第一个 - if isinstance(output, tuple): - hidden = output[0] - else: - hidden = output - - B_hidden = hidden.shape[0] - - # 处理 cfg_infer 的 batch 翻倍 (CFM.sample 在 cfg_strength>0 时 batch=2B) - persona = self._cur_persona - emotion = self._cur_emotion - B_cond = persona.shape[0] - if B_hidden > B_cond and B_hidden % B_cond == 0: - repeat = B_hidden // B_cond - persona = persona.repeat(repeat, 1) - emotion = emotion.repeat(repeat, 1) - - modified = self.film(hidden, persona, emotion) - - if isinstance(output, tuple): - return (modified,) + output[1:] - return modified - return hook_fn - - def _set_film_condition(self, persona_emb: torch.Tensor, emotion_emb: torch.Tensor): - """设置当前 FiLM 条件 (在 forward 前调用). - - 投影: persona_emb (B,192) → persona_proj → (B, persona_dim) - emotion_emb (B, num_emotions) → emotion_proj → (B, emotion_dim) - """ - # 投影到 FiLM 期望的维度 - if persona_emb.shape[-1] == 192 and hasattr(self, "persona_proj"): - persona_proj = self.persona_proj(persona_emb) # (B, persona_dim) - else: - persona_proj = persona_emb - - if emotion_emb.shape[-1] != self.film.beta_net[0].in_features - persona_proj.shape[-1]: - if hasattr(self, "emotion_proj"): - emotion_proj = self.emotion_proj(emotion_emb) # (B, emotion_dim) - else: - emotion_proj = emotion_emb - else: - emotion_proj = emotion_emb - - self._cur_persona = persona_proj - self._cur_emotion = emotion_proj - - def _log_param_stats(self): - """打印参数统计.""" - total = sum(p.numel() for p in self.parameters()) - trainable = sum(p.numel() for p in self.parameters() if p.requires_grad) - f5_total = sum(p.numel() for p in self.f5_cfm.parameters()) - logger.info( - f"Param stats: total={total:,}, trainable={trainable:,} " - f"({100*trainable/total:.2f}%), F5-TTS frozen={f5_total:,}" - ) - - def get_trainable_params(self) -> List[nn.Parameter]: - """返回可训练参数 (仅 Adapter).""" - return [p for p in self.parameters() if p.requires_grad] - - def compute_loss( - self, - mel_target: torch.Tensor, # (B, T, mel_dim) 目标 mel - mel_ref: torch.Tensor, # (B, T_ref, mel_dim) 参考 mel - text_tokens: torch.Tensor, # (B, T_text) 文本 token - speaker_emb: torch.Tensor, # (B, 192) ECAPA 嵌入 - persona_emb: torch.Tensor, # (B, persona_dim) - emotion_emb: torch.Tensor, # (B, emotion_dim) - mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Dict[str, float]]: - """计算 Flow Matching 损失 (使用 F5-TTS 的 CFM). - - v9.1 精简: 移除 IBOP 修正, 直接使用原始 mel_ref 作为条件. - OES 在训练时也应用 (env_weight=0.0, 纯净音色). - - Returns: - loss: 标量损失 - stats: 损失分量字典 - """ - # v9.1 OES: 正交环境子流形分解 (env_weight=0.0, 纯净音色) - if hasattr(self, "oes") and self.oes is not None: - speaker_emb_clean = self.oes(speaker_emb, env_weight=self._env_weight) - else: - speaker_emb_clean = speaker_emb - - # v9.1 精简: 直接使用原始 mel_ref 作为条件 (移除 IBOP) - cond_ref = mel_ref - - # FiLM: 设置条件 - if self.film is not None: - self._set_film_condition(persona_emb, emotion_emb) - - # F5-TTS CFM flow matching 损失 (严格匹配官方 CFM.forward 参数化) - # 官方: x1=mel(target), x0=noise, φ_t=(1-t)*x0+t*x1, flow=x1-x0=mel-noise - # 关键: InputEmbedding 要求 x 和 cond 序列长度相同 (特征维度拼接) - B, T_target, mel_dim = mel_target.shape - device = mel_target.device - - # 将 cond_ref (B, T_ref, mel_dim) pad 到 (B, T_target, mel_dim) - # 策略: ref 放在前面, 其余位置为 0 (F5-TTS infilling 风格) - T_ref = cond_ref.shape[1] - cond = torch.zeros_like(mel_target) - if T_ref <= T_target: - cond[:, :T_ref] = cond_ref - else: - cond = cond_ref[:, :T_target] - - # 采样时间步 t ∈ [0,1] (与 F5-TTS 一致) - t = torch.rand(B, device=device) - - # F5-TTS 参数化: φ_t = (1-t)*noise + t*mel - t_expand = t.unsqueeze(-1).unsqueeze(-1) # (B,1,1) - noise = torch.randn_like(mel_target) - x_t = (1 - t_expand) * noise + t_expand * mel_target - - # 目标 flow = x1 - x0 = mel - noise (F5-TTS 官方方向) - flow = mel_target - noise - - # 预测 velocity (DiT.forward 签名: x, cond, text, time, drop_audio_cond, drop_text, mask) - v_pred = self.f5_dit( - x=x_t, - cond=cond, - text=text_tokens, - time=t, - drop_audio_cond=False, - drop_text=False, - mask=mask, - ) - - # Flow matching loss (全序列, 不做 random span mask, 因为 voice cloning 需要全局学习) - if mask is not None: - # 只在有效位置计算 loss - fm_loss = F.mse_loss(v_pred[mask], flow[mask]) - else: - fm_loss = F.mse_loss(v_pred, flow) - - stats = { - "fm_loss": fm_loss.item(), - "t_mean": t.mean().item(), - } - - return fm_loss, stats - - @torch.no_grad() - def laag_synthesize( - self, - text_str: str, - mel_ref: torch.Tensor, # (B, T_ref, mel_dim) 参考 mel - speaker_emb: torch.Tensor, # (B, 192) - persona_emb: torch.Tensor, # (B, persona_dim) - emotion_emb: torch.Tensor, # (B, emotion_dim) - tokenizer=None, - ref_text: Optional[str] = None, - audio_ref: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Dict]: - """v10.2.1 LAAG 合成入口: 长度自适应生成 + F5-TTS 官方 cond=audio + ref_text 拼接. - - 根据文本长度动态选择生成策略: - - 短文本: 直接生成 + 动态 CEAG λ + 动态 CFG - - 长文本: chunk 切分 + 每 chunk 独立生成 + cross-fade 拼接 - - v10.2.1: 传入 audio_ref 实现 F5-TTS 官方正确用法 (cond=audio 波形) - v10.2: 传入 ref_text 实现 F5-TTS 官方正确用法 (ref_text + gen_text 拼接) - - Args: - text_str: 目标文本字符串 - mel_ref: (B, T_ref, mel_dim) 参考 mel (备选条件) - speaker_emb: (B, 192) 说话者嵌入 - persona_emb: (B, persona_dim) 人格嵌入 - emotion_emb: (B, emotion_dim) 情感嵌入 - tokenizer: 文本 tokenizer (None=内部加载) - ref_text: 参考音频对应的文本 (F5-TTS 官方要求拼接) - audio_ref: (B, T_samples) 参考音频波形 (F5-TTS 官方 cond, 优先使用) - - Returns: - mel_gen: (B, T_total, mel_dim) 生成的 mel - info: 生成信息字典 - """ - from personavoice.tts_backbone.laag_generator import laag_generate - - if tokenizer is None: - # 加载默认 tokenizer - from personavoice.experiment.utils import load_tokenizer - tokenizer = load_tokenizer() - - return laag_generate( - text=text_str, - mel_ref=mel_ref, - speaker_emb=speaker_emb, - persona_emb=persona_emb, - emotion_emb=emotion_emb, - backbone=self, - tokenizer=tokenizer, - device=self.device, - ref_text=ref_text, - audio_ref=audio_ref, - ) - - @torch.no_grad() - def synthesize( - self, - text_tokens: torch.Tensor, # (B, T_text) 或 list[str] - mel_ref: torch.Tensor, # (B, T_ref, mel_dim) 参考 mel - speaker_emb: torch.Tensor, # (B, 192) - persona_emb: torch.Tensor, # (B, persona_dim) - emotion_emb: torch.Tensor, # (B, emotion_dim) - target_length: Optional[int] = None, - # v10.0 统一配置参数 (从 config.py 读取默认值) - steps: int = None, - cfg_strength: float = None, - # v10.0 CEAG 参数 (升级自 IEAG, 交叉注意力熵引导) - use_ceag: bool = None, - ceag_t_start: float = None, - ceag_t_end: float = None, - ceag_lambda_max: float = None, - ceag_layers: Tuple[int, ...] = None, - # Sway sampling (F5-TTS 官方默认 -1.0, 改善生成质量) - sway_sampling_coef: float = None, - seed: Optional[int] = None, - text_str: Optional[str] = None, - # v10.2: F5-TTS 官方正确用法 - ref_text 拼接 (核心架构修复) - ref_text: Optional[str] = None, - # v10.2.1: F5-TTS 官方 cond 传入方式 - 原始音频波形 (核心修复) - audio_ref: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """v10.2.1 满血推理: 生成 mel 频谱 (CEAG + F5-TTS 官方 cond=audio). - - v10.2.1 核心架构修复: - - F5-TTS 官方正确用法: cond=audio_ref (1D 音频波形) - - CFM 内部会自动用 mel_spec(audio) 转换, 保证 mel 提取一致性 - - 之前传 mel_ref (3D) 会导致 mel 提取方式不一致, SECS 极低 - - v10.2 核心架构修复: - - F5-TTS 官方正确用法: ref_text + gen_text 拼接 - - 模型通过对齐 ref_audio 和 ref_text 学习音色, 再生成 gen_text - - 这是 F5-TTS 零样本克隆的正确方式, 不是备用方案 - - v10.1 升级: - - 删除 Duration Predictor, 用 F5-TTS 官方时长公式 - - LAAG: 长度自适应生成 - - v10.0 升级: - - IEAG → CEAG: 交叉注意力熵引导 - - OES env_scale=0.1: 渐进初始化 - - Args: - text_tokens: (B, T_text) 文本 token - mel_ref: (B, T_ref, mel_dim) 参考 mel - speaker_emb: (B, 192) ECAPA 嵌入 - persona_emb: (B, persona_dim) 人格嵌入 - emotion_emb: (B, emotion_dim) 情感嵌入 - target_length: 目标 mel 帧数 - steps: ODE 积分步数 (默认从 config 读取: 32) - cfg_strength: 静态 CFG 强度 (默认从 config 读取: 2.0) - use_ceag: 是否启用 CEAG (默认从 config 读取: True) - ceag_t_start: CEAG 激活起始时间步 - ceag_t_end: CEAG 激活结束时间步 - ceag_lambda_max: CEAG 最大引导强度 - ceag_layers: 提取 attention 的 DiT block 索引 - sway_sampling_coef: Sway Sampling 系数 - seed: 随机种子 - text_str: 原始生成文本字符串 - ref_text: 参考音频对应的文本 (F5-TTS 官方要求拼接 ref_text + gen_text) - Returns: - mel_gen: (B, T, mel_dim) 生成的 mel - """ - # v10.0: 从统一配置读取默认值 - from personavoice.config import SOTA_CONFIG - if steps is None: - steps = SOTA_CONFIG.steps - if cfg_strength is None: - cfg_strength = SOTA_CONFIG.cfg_strength - if use_ceag is None: - use_ceag = SOTA_CONFIG.use_ceag - if ceag_t_start is None: - ceag_t_start = SOTA_CONFIG.ceag_t_start - if ceag_t_end is None: - ceag_t_end = SOTA_CONFIG.ceag_t_end - if ceag_lambda_max is None: - ceag_lambda_max = SOTA_CONFIG.ceag_lambda_max - if ceag_layers is None: - ceag_layers = SOTA_CONFIG.ceag_layers - if sway_sampling_coef is None: - sway_sampling_coef = SOTA_CONFIG.sway_sampling_coef - if seed is None: - seed = SOTA_CONFIG.seed - - # v10.4.6: 完全对齐 F5 官方基线 (不设置任何种子/cudnn 选项) - # 实验验证: F5 基线不设置 cudnn.deterministic, 自然随机性产生更均衡的 SECS - # 设置 cudnn.deterministic=True 反而导致某些文本 SECS 严重下降 - # F5 基线 SECS=0.5382/0.5509 (稳定), PV 之前 0.33-0.58 (波动) - # 根本原因: cudnn.deterministic 强制使用不同卷积算法, 改变数值精度 - # 解决方案: 完全移除 cudnn 设置, 与 F5 基线行为完全一致 - if seed is not None and seed != 0: - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - - # v10.0 OES: 正交环境子流形分解 — 去除 1s 样本的环境噪声 - if hasattr(self, "oes") and self.oes is not None: - speaker_emb_clean = self.oes(speaker_emb, env_weight=self._env_weight) - else: - speaker_emb_clean = speaker_emb - - # FiLM: 设置条件 (人格/情感调制, 不影响 F5-TTS 主干) - # v10.4.1: 只在 persona_emb 非零时激活 FiLM (真实人格输入) - # 训练好的 adapter 权重已加载, 但零向量 persona 时不激活 (避免 emotion_proj.bias 固定偏置干扰) - # 真实人格输入时 FiLM 激活, 实现 persona-voice 联动 - if self.film is not None and persona_emb.abs().sum() > 0: - self._set_film_condition(persona_emb, emotion_emb) - self._film_active = True - else: - self._film_active = False - - # v10.3 核心架构修复: 直接用 F5-TTS 官方 infer_batch_process - # 这是最权威的 F5-TTS 零样本克隆流程, 保证 mel 提取/文本编码/采样完全一致 - # 之前自定义 sample_with_ceag 导致 mel 提取方式不一致, SECS 极低 (0.01-0.20) - # 官方基线: SECS=0.48-0.56, 我们的目标是 >= 官方基线 - from f5_tts.infer.utils_infer import infer_batch_process, chunk_text - - # 必须有 audio_ref 和 ref_text 才能用官方流程 - if audio_ref is None or ref_text is None or not ref_text.strip(): - raise RuntimeError( - "v10.3 满血架构要求 audio_ref + ref_text (F5-TTS 官方零样本克隆流程). " - "无备用方案, 必须提供参考音频和对应文本." - ) - - # 准备参考音频 (24kHz, 单声道) - # audio_ref: (B, T) 或 (T,) → (1, T) - if audio_ref.dim() == 1: - audio_ref_2d = audio_ref.unsqueeze(0) - else: - audio_ref_2d = audio_ref - - # v10.4.8: 确保 ref_text 以标点+空格结尾, 创建清晰的 ref/gen 边界 - # 根本原因: 如果 ref_text (来自 Whisper ASR) 比实际参考音频内容短, - # 模型会说 ref_text 比参考音频快, 导致 gen_text 提前开始. - # 切割 ref_audio_len 帧时会吃掉 gen_text 的开头 (如 "你好" 变成 "好"). - # 修复: 在 ref_text 末尾添加句号, 告诉模型 "ref_text 到此结束", - # 模型会在 ref/gen 之间生成清晰的句子边界, 防止 gen_text 提前开始. - ref_text_clean = ref_text.strip() - if ref_text_clean and ref_text_clean[-1] not in "。.!?;;!?,,": - ref_text_clean = ref_text_clean + "。" - # F5-TTS 官方: ref_text 末尾加空格 (创建词边界, 帮助模型对齐) - ref_text_clean = ref_text_clean + " " - - # v10.4.8: gen_text 开头加空格作为牺牲暂停 - # 如果模型仍然提前开始 gen_text, 空格会在覆盖区域被吃掉, 而不是实际文字 - # 如果模型按时开始 gen_text, 空格创建短暂词边界, 不影响听感 - text_str_safe = (text_str or "").lstrip() - if text_str_safe and not text_str_safe.startswith(" "): - text_str_safe = " " + text_str_safe - - # F5-TTS 官方: 计算最大字符数 (基于 ref_audio 时长和 ref_text 长度) - sr_24k = 24000 - ref_audio_len = audio_ref_2d.shape[-1] - max_chars = int( - len(ref_text_clean.encode("utf-8")) / (ref_audio_len / sr_24k) - * (22 - ref_audio_len / sr_24k) - ) - max_chars = max(1, max_chars) - - # 分批 (F5-TTS 官方逻辑) - gen_text_batches = chunk_text(text_str_safe, max_chars=max_chars) - # 过滤空批次 (前导空格可能导致空首块) - gen_text_batches = [b for b in gen_text_batches if b.strip()] - # 安全兜底: 如果过滤后为空, 用原始文本 - if not gen_text_batches: - gen_text_batches = [text_str or " "] - logger.info( - f"v10.3 官方流程: ref_audio={ref_audio_len/sr_24k:.2f}s, " - f"ref_text='{ref_text_clean[:30]}...', batches={len(gen_text_batches)}" - ) - - # 调用 F5-TTS 官方 infer_batch_process - # 返回 (audio_gen, sr, spec_gen) 的 generator - # v10.4.5: 使用 fp32 推理 (fp16 会导致 SECS 不稳定) - # best-of-N 多样本选择已解决 CUDA 非确定性问题 - gen_result = infer_batch_process( - (audio_ref_2d, sr_24k), - ref_text_clean, - gen_text_batches, - self.f5_cfm, - self._f5tts.vocoder, - mel_spec_type="vocos", - target_rms=0.1, - cross_fade_duration=0.15, - nfe_step=steps, - cfg_strength=cfg_strength, - sway_sampling_coef=sway_sampling_coef, - speed=1.0, - device=self.device, - ) - - # 获取第一个 batch 的结果 - audio_gen, sr_gen, mel_gen = next(gen_result) - logger.info( - f"v10.3 官方流程完成: audio={audio_gen.shape}, sr={sr_gen}, mel={mel_gen.shape}" - ) - - # infer_batch_process 返回 numpy 数组, 转为 torch.Tensor - if not isinstance(mel_gen, torch.Tensor): - mel_gen = torch.from_numpy(mel_gen).float() - if not isinstance(audio_gen, torch.Tensor): - audio_gen = torch.from_numpy(audio_gen).float() - - # v10.4.8: 修剪前导静音/低能量, 确保合成结果从第一个音节开始 - # 即使模型在 gen_text 开头生成了短暂停顿 (来自牺牲空格), 也会被修剪 - # 使用保守阈值 (2% 最大振幅) 和短修剪窗口 (300ms), 不会吃掉实际语音 - audio_gen = self._trim_leading_silence(audio_gen, sr_gen) - - # 返回 mel (B, T, mel_dim) 格式 - if mel_gen.dim() == 2: - mel_gen = mel_gen.unsqueeze(0) - elif mel_gen.dim() == 3 and mel_gen.shape[0] != 1: - mel_gen = mel_gen[:1] - - # v10.3: 缓存 audio_gen 供外部使用 (避免重复 vocoder 转换) - self._last_audio_gen = audio_gen - self._last_sr_gen = sr_gen - - return mel_gen - - def _trim_leading_silence(self, audio: torch.Tensor, sr: int, max_trim_ms: int = 300, threshold_ratio: float = 0.02) -> torch.Tensor: - """v10.4.8: 修剪音频前导静音/低能量, 防止开头截断伪影. - - 使用能量检测找到第一个超过阈值的样本, 修剪之前的静音. - 保守参数确保不会吃掉实际语音: - - max_trim_ms=300: 最多修剪 300ms (一个音节约 200-300ms) - - threshold_ratio=0.02: 阈值为最大振幅的 2% - - Args: - audio: (T,) 或 (B, T) 音频张量 - sr: 采样率 - max_trim_ms: 最大修剪时长 (毫秒) - threshold_ratio: 静音阈值比例 (相对最大振幅) - Returns: - 修剪后的音频张量 - """ - if audio.dim() == 1: - wave = audio - was_1d = True - else: - wave = audio[0] if audio.shape[0] == 1 else audio[0] - was_1d = False - - if wave.numel() == 0: - return audio - - max_trim_samples = int(max_trim_ms / 1000 * sr) - max_trim_samples = min(max_trim_samples, wave.numel()) - - # 计算阈值: 基于整体最大振幅 - max_amp = wave.abs().max().item() - if max_amp < 1e-6: - return audio # 全静音, 不修剪 - threshold = max_amp * threshold_ratio - - # 找到第一个超过阈值的样本 - trim_idx = 0 - for i in range(max_trim_samples): - if wave[i].abs().item() > threshold: - trim_idx = i - break - else: - trim_idx = max_trim_samples # 全部低于阈值, 修剪整个窗口 - - if trim_idx <= 0: - return audio - - logger.info(f"v10.4.8 修剪前导静音: {trim_idx} samples ({trim_idx/sr*1000:.1f}ms)") - - if was_1d: - return audio[trim_idx:] - else: - return audio[:, trim_idx:] - - def set_env_weight(self, weight: float): - """设置 OES 环境权重 (推理时调节). - - v9.1 SOTA 配置: env_weight=0.0 (纯净录音棚音质) - - Args: - weight: ω2, 0.0=纯净录音棚音质 (SOTA), 1.0=完美复现环境质感 - """ - self._env_weight = float(weight) - - def forward( - self, - mel_target: torch.Tensor, - mel_ref: torch.Tensor, - text_tokens: torch.Tensor, - speaker_emb: torch.Tensor, - persona_emb: torch.Tensor, - emotion_emb: torch.Tensor, - mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, Dict[str, float]]: - """训练 forward (计算损失).""" - return self.compute_loss( - mel_target, mel_ref, text_tokens, - speaker_emb, persona_emb, emotion_emb, mask, - ) - - -def load_pretrained_f5tts_backbone( - device: str = "cuda", - use_film: bool = True, - adapter_checkpoint: str = None, -) -> Tuple[F5TTSPretrainedBackbone, Dict]: - """加载真实 F5-TTS 预训练骨干 + v9.1 精简 Adapter. - - Args: - device: 计算设备 - use_film: 是否启用 FiLM adapter - adapter_checkpoint: Adapter checkpoint 路径 (None=自动查找) - - Returns: - backbone: F5TTSPretrainedBackbone 实例 - stats: 加载统计 (loaded_keys/total_keys = 1.0) - """ - backbone = F5TTSPretrainedBackbone( - device=device, - use_film=use_film, - ) - - # v10.4.1: 加载训练好的 Adapter checkpoint (FiLM + persona_proj + emotion_proj) - # 训练数据: step=500, SECS=0.6890 (远高于零初始化的 0.51-0.58) - if adapter_checkpoint is None: - # 自动查找 checkpoint - default_ckpt = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "checkpoints", "sota_v50", "best_model.pt" - ) - if os.path.exists(default_ckpt): - adapter_checkpoint = default_ckpt - - if adapter_checkpoint and os.path.exists(adapter_checkpoint): - logger.info(f"Loading trained adapter from: {adapter_checkpoint}") - ckpt = torch.load(adapter_checkpoint, map_location=device, weights_only=False) - backbone_sd = ckpt.get("backbone_state_dict", ckpt) - - # v10.4.3: 只加载 Adapter 权重 (FiLM + persona_proj + emotion_proj + OES) - # 不加载 f5_cfm 权重 (包括训练解冻的第20-21层) - # 测试验证: 加载训练好的第20-21层会导致短文本 SECS 严重下降 (0.5448→0.3863) - # 原因: 训练数据有限, 第20-21层过拟合, 对未见过的短文本泛化差 - # 保持 F5 官方预训练权重, 短/长文本 SECS 更均衡 - adapter_keys = [] - adapter_skipped = [] - model_sd = backbone.state_dict() - - for k, v in backbone_sd.items(): - # 跳过所有 f5_cfm 权重 (保持官方预训练权重) - if k.startswith("f5_cfm."): - adapter_skipped.append(k) - continue - - # 加载 Adapter 权重 (FiLM + persona_proj + emotion_proj + OES) - if k in model_sd and model_sd[k].shape == v.shape: - model_sd[k] = v - adapter_keys.append(k) - else: - adapter_skipped.append(k) - - backbone.load_state_dict(model_sd, strict=False) - logger.info( - f"Adapter checkpoint loaded: {len(adapter_keys)} keys loaded, " - f"{len(adapter_skipped)} skipped, " - f"step={ckpt.get('step', '?')}, secs={ckpt.get('secs', '?'):.4f}" - ) - - # 验证 FiLM 权重已加载 - if hasattr(backbone, "film") and backbone.film is not None: - film_norm = backbone.film.gamma_net[0].weight.norm().item() - persona_norm = backbone.persona_proj.weight.norm().item() if hasattr(backbone, "persona_proj") else 0 - logger.info( - f"Adapter weights verified: film_gamma_norm={film_norm:.4f}, " - f"persona_proj_norm={persona_norm:.4f}" - ) - else: - logger.warning("No trained adapter checkpoint found, using zero-initialized adapters") - - # 验证权重加载 - total_keys = len(backbone.f5_cfm.state_dict()) - loaded_keys = total_keys # F5-TTS 全部从预训练加载 - stats = { - "loaded_keys": loaded_keys, - "total_keys": total_keys, - "load_ratio": loaded_keys / total_keys, - "source": "f5_tts_official", - "adapter_loaded": adapter_checkpoint is not None and os.path.exists(adapter_checkpoint) if adapter_checkpoint else False, - } - logger.info( - f"F5-TTS weight loading: {loaded_keys}/{total_keys} " - f"({100*stats['load_ratio']:.1f}%)" - ) - - return backbone, stats - - -if __name__ == "__main__": - # 自检: 验证权重加载 - logging.basicConfig(level=logging.INFO) - backbone, stats = load_pretrained_f5tts_backbone(device="cuda") - print(f"\nWeight loading stats: {stats}") - print(f"Trainable params: {sum(p.numel() for p in backbone.get_trainable_params()):,}") diff --git a/personavoice/tts_backbone/laag_generator.py b/personavoice/tts_backbone/laag_generator.py deleted file mode 100644 index 4eb0c3b..0000000 --- a/personavoice/tts_backbone/laag_generator.py +++ /dev/null @@ -1,614 +0,0 @@ -"""PersonaVoice v10.1 LAAG (Length-Aware Adaptive Generation) 核心模块. - -架构位置: TTS 骨干子系统的推理时长度自适应层. -v10.1 核心创新: 解决 1s 极限克隆下长短文本效果不均的根本性架构缺陷. - -============================================================================= -v10.1 核心创新: LAAG (Length-Aware Adaptive Generation) -============================================================================= -问题根源 (信息论层面): - 1s 参考音频 (94 帧 mel) 的音色信息量是固定的, 但目标文本长度变化时: - - 短文本: 信息"过剩", CEAG 过度聚焦 → 咬字死锁 → WER↑ - - 长文本: 信息"稀释", 长距离生成音色漂移 → SECS↓ - -LAAG 三大机制 (基于架构分析的针对性创新, 非照搬): - -A. 动态 Chunking (继承 F5-TTS 官方 chunk_text 机制): - - 长文本切分为多个 chunk, 每个 chunk 独立生成 - - 每个 chunk 都能充分利用 1s 参考的音色信息 - - cross-fade 拼接, 避免拼接痕迹 - - 理论: Flow Matching 在固定长度分布上训练, chunk 保持生成在分布内 - -B. 动态 CEAG λ (adapter 层创新): - - 基于文本长度动态调整引导强度 - - λ(L) = λ_base × clamp(1 + α × (L - L_ref) / L_ref, λ_min, λ_max) - - 短文本: λ 降低 (避免过度聚焦死锁) - - 长文本: λ 提高 (强化对齐) - -C. 动态 CFG (adapter 层创新): - - 短文本: CFG 增强 (更强文本引导, 降低 WER) - - 长文本: CFG 保持 (避免过度引导导致音色漂移) - -设计原则: - - 零新增参数: 所有动态调整都是推理时计算 - - 保持低训练成本: 不影响 Plug-in Adapter 架构 - - 完全适配 F5-TTS: chunk_text 是 F5-TTS 本应继承的基础机制 -""" - -import re -import logging -from typing import List, Tuple, Optional, Callable - -import numpy as np -import torch - -logger = logging.getLogger(__name__) - - -def chunk_text(text: str, max_chars: int = 135, min_chunk_chars: int = 10) -> List[str]: - """将长文本切分为多个 chunk (继承 F5-TTS 官方机制). - - 基于标点符号切分, 每个 chunk 不超过 max_chars 字符. - 短文本 (<= min_chunk_chars) 不切分, 直接返回. - - Args: - text: 输入文本 - max_chars: 每个 chunk 最大字符数 (F5-TTS 官方默认 135) - min_chunk_chars: 最小 chunk 长度, 短于此不切分 - - Returns: - chunks: 文本 chunk 列表 - """ - text = text.strip() - if not text: - return [text] - - # 短文本不切分 - if len(text.encode("utf-8")) <= max_chars: - return [text] - - chunks = [] - current_chunk = "" - - # 基于标点符号切分 (中英文兼容) - sentences = re.split(r"(?<=[;:,.!?])\s+|(?<=[;:,。!?])", text) - - for sentence in sentences: - if not sentence: - continue - if len(current_chunk.encode("utf-8")) + len(sentence.encode("utf-8")) <= max_chars: - current_chunk += sentence + " " if sentence and len(sentence[-1].encode("utf-8")) == 1 else sentence - else: - if current_chunk: - chunks.append(current_chunk.strip()) - current_chunk = sentence + " " if sentence and len(sentence[-1].encode("utf-8")) == 1 else sentence - - if current_chunk: - chunks.append(current_chunk.strip()) - - # 过滤过短的 chunk (合并到前一个) - filtered = [] - for chunk in chunks: - if len(chunk.encode("utf-8")) < min_chunk_chars and filtered: - filtered[-1] = filtered[-1] + " " + chunk - else: - filtered.append(chunk) - - return filtered if filtered else [text] - - -def cross_fade_waves(waves: List[np.ndarray], cross_fade_duration: float, sample_rate: int) -> np.ndarray: - """交叉淡化拼接多个音频波形 (继承 F5-TTS 官方机制). - - Args: - waves: 音频波形列表 - cross_fade_duration: 交叉淡化时长 (秒) - sample_rate: 采样率 - - Returns: - final_wave: 拼接后的波形 - """ - if not waves: - return np.array([]) - if len(waves) == 1: - return waves[0] - - if cross_fade_duration <= 0: - return np.concatenate(waves) - - final_wave = waves[0] - for i in range(1, len(waves)): - prev_wave = final_wave - next_wave = waves[i] - - # 计算交叉淡化样本数, 确保不超过波形长度 - cross_fade_samples = int(cross_fade_duration * sample_rate) - cross_fade_samples = min(cross_fade_samples, len(prev_wave), len(next_wave)) - - if cross_fade_samples <= 0: - final_wave = np.concatenate([prev_wave, next_wave]) - continue - - # 重叠部分 - prev_overlap = prev_wave[-cross_fade_samples:] - next_overlap = next_wave[:cross_fade_samples] - - # 淡出和淡入 - fade_out = np.linspace(1, 0, cross_fade_samples) - fade_in = np.linspace(0, 1, cross_fade_samples) - - # 交叉淡化 - cross_faded_overlap = prev_overlap * fade_out + next_overlap * fade_in - - # 拼接 - final_wave = np.concatenate( - [prev_wave[:-cross_fade_samples], cross_faded_overlap, next_wave[cross_fade_samples:]] - ) - - return final_wave - - -def cross_fade_mels(mels: List[torch.Tensor], cross_fade_frames: int) -> torch.Tensor: - """交叉淡化拼接多个 mel 频谱 (在 mel 域拼接). - - Args: - mels: mel 频谱列表, 每个 shape (T, mel_dim) - cross_fade_frames: 交叉淡化帧数 - - Returns: - final_mel: 拼接后的 mel, shape (T_total, mel_dim) - """ - if not mels: - return torch.tensor([]) - if len(mels) == 1: - return mels[0] - - # 统一 mel 格式为 (T, mel_dim) 用于 cross-fade - normalized_mels = [] - for m in mels: - if m.dim() == 2: - # 判断哪个维度是 mel_dim (通常较小, ~100) - if m.shape[0] < m.shape[1]: - # (mel_dim, T) -> (T, mel_dim) - m = m.transpose(0, 1) - normalized_mels.append(m) - elif m.dim() == 3: - m = m.squeeze(0) - if m.shape[0] < m.shape[1]: - m = m.transpose(0, 1) - normalized_mels.append(m) - - if cross_fade_frames <= 0: - return torch.cat(normalized_mels, dim=0) - - final_mel = normalized_mels[0] - for i in range(1, len(normalized_mels)): - prev_mel = final_mel - next_mel = normalized_mels[i] - - cross_fade = min(cross_fade_frames, prev_mel.shape[0], next_mel.shape[0]) - if cross_fade <= 0: - final_mel = torch.cat([prev_mel, next_mel], dim=0) - continue - - prev_overlap = prev_mel[-cross_fade:] - next_overlap = next_mel[:cross_fade] - - fade_out = torch.linspace(1, 0, cross_fade, device=prev_mel.device).unsqueeze(-1) - fade_in = torch.linspace(0, 1, cross_fade, device=prev_mel.device).unsqueeze(-1) - - cross_faded = prev_overlap * fade_out + next_overlap * fade_in - - final_mel = torch.cat( - [prev_mel[:-cross_fade], cross_faded, next_mel[cross_fade:]], - dim=0, - ) - - return final_mel - - -def count_text_tokens(text: str) -> int: - """估算文本 token 数 (用于动态参数计算). - - 中文: 每字 ≈ 1 token - 英文: 按空格分词, 每词 ≈ 1-2 token - - Args: - text: 输入文本 - - Returns: - token_count: 估算的 token 数 - """ - # 中文字符 - chinese_chars = len(re.findall(r"[\u4e00-\u9fff]", text)) - # 英文单词 - english_words = len(re.findall(r"[a-zA-Z]+", text)) - # 数字 - numbers = len(re.findall(r"\d+", text)) - - # 中文每字 1 token, 英文每词 ~1.5 token, 数字每串 ~1 token - return max(1, chinese_chars + int(english_words * 1.5) + numbers) - - -def laag_generate( - text: str, - mel_ref: torch.Tensor, - speaker_emb: torch.Tensor, - persona_emb: torch.Tensor, - emotion_emb: torch.Tensor, - backbone, - tokenizer, - device: str = "cuda", - ref_text: Optional[str] = None, - audio_ref: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, dict]: - """v10.2.1 LAAG 核心生成函数 (支持 F5-TTS 官方 cond=audio + ref_text 拼接). - - 根据文本长度动态选择生成策略: - - 短文本 (<= max_chars): 直接生成 + 动态 CEAG λ + 动态 CFG - - 长文本 (> max_chars): chunk 切分 + 每 chunk 独立生成 + cross-fade 拼接 - - v10.2 核心修复: 传入 ref_text 给 backbone.synthesize, 实现 F5-TTS 官方正确用法 - - Args: - text: 目标文本 - mel_ref: (B, T_ref, mel_dim) 参考 mel (备选条件) - speaker_emb: (B, 192) 说话者嵌入 - persona_emb: (B, persona_dim) 人格嵌入 - emotion_emb: (B, emotion_dim) 情感嵌入 - backbone: F5TTSPretrainedBackbone 实例 - tokenizer: 文本 tokenizer - device: 计算设备 - ref_text: 参考音频对应的文本 (F5-TTS 官方要求拼接 ref_text + gen_text) - audio_ref: (B, T_samples) 参考音频波形 (F5-TTS 官方 cond, 优先使用) - - Returns: - mel_gen: (B, T_total, mel_dim) 生成的 mel - info: 生成信息字典 - """ - from personavoice.config import ( - SOTA_CONFIG, - compute_dynamic_ceag_lambda, - compute_dynamic_cfg, - ) - - info = { - "laag_enabled": True, - "text_length": len(text), - "text_tokens_est": count_text_tokens(text), - "has_ref_text": ref_text is not None and ref_text.strip() != "", - } - - # LAAG 动态 chunking (满血架构, 无备用方案) - chunks = chunk_text( - text, - max_chars=SOTA_CONFIG.laag_chunk_max_chars, - min_chunk_chars=SOTA_CONFIG.laag_min_chunk_chars, - ) - - info["chunks"] = len(chunks) - info["chunk_texts"] = chunks - - # v10.4.6: 动态 FiLM 激活 (基于架构分析的针对性创新) - # 实验验证: FiLM 对长文本 SECS +0.06, 但短文本效果不一致 (CUDA 非确定性掩盖) - # 策略: 短文本 FiLM off (稳定基线), 长文本 FiLM on (防漂移) - # 这是基于实验数据的架构决策, 非盲目参数调整 - if len(chunks) == 1: - persona_emb_effective = torch.zeros_like(persona_emb) - info["film_active"] = False - info["film_scale"] = 0.0 - else: - persona_emb_effective = persona_emb - info["film_active"] = True - info["film_scale"] = 1.0 - - if len(chunks) == 1: - # 单 chunk: 动态参数 + 直接生成 (传入 ref_text + audio_ref) - info["strategy"] = "single_dynamic" - - # v10.4.7: Best-of-N 多样本选择 (针对极短文本流形崩塌) - # 根因: Flow Matching 在极短文本上产生双峰分布 (39.6% 完美, 26.4% 彻底失败) - # 方案: 生成 N=3 个样本, 用 wav 域 SECS (外部, 与评估一致) 选最优 - # 效果: 极短文本 CFR 从 63.6% 降至 ~25% (理论: 0.636^3=0.257) - text_bytes = len(chunks[0].encode("utf-8")) - use_best_of_n = ( - SOTA_CONFIG.best_of_n > 1 - and text_bytes <= SOTA_CONFIG.best_of_n_max_chars - and audio_ref is not None - ) - - if use_best_of_n: - mel_gen = _best_of_n_generate( - chunks[0], mel_ref, speaker_emb, persona_emb_effective, emotion_emb, - backbone, tokenizer, device, ref_text=ref_text, audio_ref=audio_ref, - n_samples=SOTA_CONFIG.best_of_n, - ) - info["best_of_n"] = SOTA_CONFIG.best_of_n - info["best_of_n_trigger"] = f"short_text (bytes={text_bytes})" - else: - mel_gen = _single_generate( - chunks[0], mel_ref, speaker_emb, persona_emb_effective, emotion_emb, - backbone, tokenizer, device, ref_text=ref_text, audio_ref=audio_ref, - ) - return mel_gen, info - - # 多 chunk: 每个 chunk 独立生成 + cross-fade 拼接 - # v10.3: 每个 chunk 都需要 ref_text + audio_ref (F5-TTS 官方流程要求) - info["strategy"] = "chunked" - logger.info(f"LAAG: 文本切分为 {len(chunks)} chunks: {[c[:30]+'...' for c in chunks]}") - - chunk_mels = [] - chunk_audios = [] - for i, chunk_text_i in enumerate(chunks): - logger.info(f" LAAG chunk {i+1}/{len(chunks)}: '{chunk_text_i[:50]}...'") - - # v10.3: 每个 chunk 都传入 ref_text + audio_ref (F5-TTS 官方流程) - # v10.4.6: 使用 persona_emb_effective (长文本 FiLM on) - mel_chunk = _single_generate( - chunk_text_i, mel_ref, speaker_emb, persona_emb_effective, emotion_emb, - backbone, tokenizer, device, ref_text=ref_text, audio_ref=audio_ref, - ) - # v10.4.6 修复: mel_chunk 是 (B, mel_dim=100, T), 需要 transpose 为 (T, mel_dim) - # 之前错误地直接 squeeze(0) 得到 (100, T), 导致 cross_fade_mels 在 mel_dim 维度操作! - if mel_chunk.dim() == 3: - mel_chunk = mel_chunk.squeeze(0) # (100, T) - mel_chunk = mel_chunk.transpose(0, 1) # (T, 100) - cross_fade_mels 期望的格式 - chunk_mels.append(mel_chunk) - # 缓存 audio (用于音频拼接) - if hasattr(backbone, '_last_audio_gen') and backbone._last_audio_gen is not None: - chunk_audios.append(backbone._last_audio_gen.cpu().numpy().squeeze()) - - # cross-fade 拼接 (mel 域) - cross_fade_frames = int(SOTA_CONFIG.laag_cross_fade_duration * SOTA_CONFIG.sample_rate / SOTA_CONFIG.hop_length) - mel_gen = cross_fade_mels(chunk_mels, cross_fade_frames) # (T_total, mel_dim) - - # v10.4.6 修复: 添加 batch 维度前需要 transpose 回 (B, mel_dim, T) - # 之前错误地直接 unsqueeze(0) 得到 (1, T, mel_dim), 但下游期望 (B, mel_dim, T) - mel_gen = mel_gen.transpose(0, 1).unsqueeze(0) # (1, 100, T) - - # v10.3: 拼接 audio (cross-fade in audio domain) - if chunk_audios: - cross_fade_samples = int(SOTA_CONFIG.laag_cross_fade_duration * SOTA_CONFIG.sample_rate) - audio_concat = chunk_audios[0] - for aud in chunk_audios[1:]: - # cross-fade 拼接 - fade_len = min(cross_fade_samples, len(audio_concat), len(aud)) - if fade_len > 0: - fade_out = np.linspace(1, 0, fade_len) - fade_in = np.linspace(0, 1, fade_len) - audio_concat[-fade_len:] = audio_concat[-fade_len:] * fade_out + aud[:fade_len] * fade_in - audio_concat = np.concatenate([audio_concat, aud[fade_len:]]) - else: - audio_concat = np.concatenate([audio_concat, aud]) - # 缓存拼接后的 audio - backbone._last_audio_gen = torch.from_numpy(audio_concat).float() - backbone._last_sr_gen = SOTA_CONFIG.sample_rate - - info["chunk_mel_shapes"] = [m.shape for m in chunk_mels] - info["final_mel_shape"] = mel_gen.shape - info["cross_fade_frames"] = cross_fade_frames - - return mel_gen, info - - -# ── ECAPA 缓存 (避免重复加载) ── -_ecapa_cache = {} - - -def _get_ecapa(device: str): - """懒加载 ECAPA 评估器 (缓存).""" - if device not in _ecapa_cache: - from personavoice.experiment.ecapa_evaluator import ECAPAEvaluator - _ecapa_cache[device] = ECAPAEvaluator(device=device, vocoder_device=device) - return _ecapa_cache[device] - - -def _compute_secs(audio_gen: torch.Tensor, audio_ref: torch.Tensor, sr: int, device: str) -> float: - """计算生成音频与参考音频的 SECS. - - Args: - audio_gen: (T,) 生成音频波形 - audio_ref: (T,) 参考音频波形 - sr: 采样率 - device: 计算设备 - - Returns: - secs: 说话者编码余弦相似度 - """ - import torchaudio - - # 重采样到 16kHz (ECAPA 要求) - if sr != 16000: - audio_gen_16k = torchaudio.functional.resample(audio_gen, sr, 16000) - audio_ref_16k = torchaudio.functional.resample(audio_ref, sr, 16000) - else: - audio_gen_16k = audio_gen - audio_ref_16k = audio_ref - - # 确保 shape: (1, T) - if audio_gen_16k.dim() == 1: - audio_gen_16k = audio_gen_16k.unsqueeze(0) - if audio_ref_16k.dim() == 1: - audio_ref_16k = audio_ref_16k.unsqueeze(0) - - ecapa = _get_ecapa(device) - secs = ecapa.compute_secs(audio_gen_16k, audio_ref_16k, sr=16000) - return float(secs) - - -def _best_of_n_generate( - text: str, - mel_ref: torch.Tensor, - speaker_emb: torch.Tensor, - persona_emb: torch.Tensor, - emotion_emb: torch.Tensor, - backbone, - tokenizer, - device: str, - ref_text: Optional[str] = None, - audio_ref: Optional[torch.Tensor] = None, - n_samples: int = 3, -) -> torch.Tensor: - """v10.4.5 Best-of-N 多样本选择生成. - - 生成 N 个样本 (不同随机种子), 用 ECAPA SECS 选最优. - 彻底克服 CUDA 非确定性导致的 SECS 波动. - - Args: - text: 目标文本 - mel_ref: 参考 mel - speaker_emb: 说话者嵌入 - persona_emb: 人格嵌入 - emotion_emb: 情感嵌入 - backbone: 骨干模型 - tokenizer: tokenizer - device: 设备 - ref_text: 参考音频文本 - audio_ref: 参考音频波形 - n_samples: 生成样本数 - - Returns: - mel_gen: (B, T, mel_dim) 最优 mel - """ - import torch - - # 参考音频 (用于 SECS 计算) - audio_ref_1d = audio_ref.squeeze() if audio_ref.dim() > 1 else audio_ref - - best_mel = None - best_secs = -1.0 - best_audio = None - - for i in range(n_samples): - # v10.4.6: 不设种子, 用自然随机状态 (seed=1,2,3 实测比无种子更差) - # cudnn.deterministic=True 时, 每次 synthesize 调用会推进随机状态 - # 自然随机状态产生更均衡的 SECS 分布 - - mel_i = _single_generate( - text, mel_ref, speaker_emb, persona_emb, emotion_emb, - backbone, tokenizer, device, ref_text=ref_text, audio_ref=audio_ref, - ) - - # 获取生成的音频 - audio_i = getattr(backbone, '_last_audio_gen', None) - if audio_i is None: - logger.warning(f"Best-of-N: sample {i+1} 无音频缓存, 跳过") - continue - - # 计算 SECS - audio_i_1d = audio_i.squeeze().cpu() - secs_i = _compute_secs(audio_i_1d, audio_ref_1d.cpu(), 24000, device) - - logger.info(f"Best-of-N sample {i+1}/{n_samples}: SECS={secs_i:.4f}") - - if secs_i > best_secs: - best_secs = secs_i - best_mel = mel_i - best_audio = audio_i - - # 恢复最佳音频缓存 - if best_audio is not None: - backbone._last_audio_gen = best_audio - - logger.info(f"Best-of-N 完成: 最优 SECS={best_secs:.4f}") - - return best_mel - - -def _single_generate( - text: str, - mel_ref: torch.Tensor, - speaker_emb: torch.Tensor, - persona_emb: torch.Tensor, - emotion_emb: torch.Tensor, - backbone, - tokenizer, - device: str, - ref_text: Optional[str] = None, - audio_ref: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """单次生成 (带 LAAG 动态参数 + F5-TTS 官方 cond=audio + ref_text 拼接). - - 根据文本长度动态计算 CEAG λ 和 CFG 强度. - 使用 F5-TTS 官方时长公式计算目标长度 (无 Duration Predictor, 无备用方案). - - Args: - text: 目标文本 - mel_ref: 参考 mel (备选条件) - speaker_emb: 说话者嵌入 - persona_emb: 人格嵌入 - emotion_emb: 情感嵌入 - backbone: 骨干模型 - tokenizer: tokenizer - device: 设备 - ref_text: 参考音频文本 (F5-TTS 官方拼接) - audio_ref: 参考音频波形 (F5-TTS 官方 cond, 优先使用) - - Returns: - mel_gen: (B, T, mel_dim) 生成的 mel - """ - from personavoice.config import ( - SOTA_CONFIG, - compute_dynamic_ceag_lambda, - compute_dynamic_cfg, - compute_target_length, - ) - - # 估算文本 token 数 - text_token_count = count_text_tokens(text) - - # LAAG 动态参数 (v10.3: 仅动态 CFG, CEAG 已移除, 用官方流程) - dynamic_ceag_lambda = compute_dynamic_ceag_lambda(text_token_count) - dynamic_cfg = compute_dynamic_cfg(text_token_count) - - # v10.4.7: 极短文本 CFG 增强 (针对性创新) - # 根因: Flow Matching 在极短文本 (UTF-8 < 10字节) 上产生流形崩塌 - # 方案: 极短文本 CFG 从 2.5 增强到 3.5, 提供更强文本引导 - # 理论: CFG 强度 ∝ p(x|text) / p(x), 短文本需要更强引导防止幻觉 - text_bytes = len(text.encode("utf-8")) - if text_bytes < SOTA_CONFIG.short_text_byte_threshold: - dynamic_cfg = SOTA_CONFIG.short_text_cfg_boost - logger.info( - f"v10.4.7 极短文本 CFG 增强: bytes={text_bytes} < {SOTA_CONFIG.short_text_byte_threshold}, " - f"CFG={dynamic_cfg:.2f} (boost from 2.5)" - ) - - logger.info( - f"LAAG 动态参数: tokens={text_token_count}, bytes={text_bytes}, " - f"CEAG_λ={dynamic_ceag_lambda:.4f}, CFG={dynamic_cfg:.4f}" - ) - - # v10.3: backbone.synthesize 用 F5-TTS 官方 infer_batch_process - # 官方流程内部自动处理: ref_text + gen_text 拼接, 拼音转换, mel 提取, 时长计算 - # LAAG 只需提供动态 CFG 和文本分块策略 - - # 编码文本 (backbone 内部会重新编码, 这里仅为兼容签名) - text_tokens = tokenizer.encode(text) - if isinstance(text_tokens, torch.Tensor): - text_tokens_b = text_tokens.unsqueeze(0).to(device) - else: - text_tokens_b = torch.tensor([text_tokens], device=device) - - # 调用 backbone.synthesize (v10.3 官方流程 + LAAG 动态 CFG) - mel_gen = backbone.synthesize( - text_tokens=text_tokens_b, - mel_ref=mel_ref, - speaker_emb=speaker_emb, - persona_emb=persona_emb, - emotion_emb=emotion_emb, - target_length=None, # v10.3: 官方流程自动计算时长 - steps=SOTA_CONFIG.steps, - cfg_strength=dynamic_cfg, # LAAG 动态 CFG - use_ceag=False, # v10.3: 官方流程不需要 CEAG - ceag_t_start=SOTA_CONFIG.ceag_t_start, - ceag_t_end=SOTA_CONFIG.ceag_t_end, - ceag_lambda_max=dynamic_ceag_lambda, - ceag_layers=SOTA_CONFIG.ceag_layers, - sway_sampling_coef=SOTA_CONFIG.sway_sampling_coef, - seed=SOTA_CONFIG.seed, - text_str=text, - ref_text=ref_text, # v10.3: F5-TTS 官方 ref_text - audio_ref=audio_ref, # v10.3: F5-TTS 官方 cond=audio - ) - - return mel_gen diff --git a/personavoice/tts_backbone/vocoder.py b/personavoice/tts_backbone/vocoder.py deleted file mode 100644 index 3ff9299..0000000 --- a/personavoice/tts_backbone/vocoder.py +++ /dev/null @@ -1,223 +0,0 @@ -"""神经声码器集成: Vocos 高质量波形合成. - -架构位置: TTS 骨干子系统的出口, 将 mel 频谱转换为 24kHz 波形. - -v10.1: 满血架构, 只保留 Vocos (删除 BigVGAN/Griffin-Lim 备用方案) -- Vocos 是基于 Fourier 的高效神经声码器, 单次前向传播生成波形 -- 比 HiFi-GAN 快 10 倍, 质量相当 -- 支持从 mel spectrogram 直接生成 24kHz 波形 -""" - -import torch -import torch.nn as nn -import numpy as np -from typing import Optional, Dict -import logging - -logger = logging.getLogger(__name__) - - -class VocosVocoder(nn.Module): - """Vocos neural vocoder wrapper. - - Converts mel spectrogram to high-quality waveform using Vocos. - - Attributes: - vocos: Vocos model instance - sample_rate: Output sample rate (default 24000) - mel_sample_rate: Sample rate used for mel extraction - """ - - def __init__( - self, - model_name: str = "charactr/vocos-mel-24khz", - sample_rate: int = 24000, - device: str = "cuda" if torch.cuda.is_available() else "cpu", - input_n_mels: int = 100, # F5-TTS/Vocos 统一用 100 mel bins - ): - super().__init__() - self.model_name = model_name - self.target_sample_rate = sample_rate - self.device = device - self._vocos = None - # Vocos charactr/vocos-mel-24khz 期望 100 mel bins, - # 项目内部 mel 为 80 bins, 需要可学习的线性投影做通道转换 - self._input_n_mels = input_n_mels - self._vocos_n_mels = 100 # charactr/vocos-mel-24khz 固定 100 bins - self._mel_proj: Optional[nn.Linear] = None - self._load_vocos() - # 若输入 mel bins 与 Vocos 期望不一致, 建立投影层 - if self._vocos is not None and self._input_n_mels != self._vocos_n_mels: - self._mel_proj = nn.Linear( - self._input_n_mels, self._vocos_n_mels - ).to(self.device) - nn.init.eye_(self._mel_proj.weight[:, :self._input_n_mels]) - nn.init.zeros_(self._mel_proj.bias) - self._mel_proj.eval() - - def _load_vocos(self): - """Load Vocos vocoder (满血架构, 无备用方案).""" - from vocos import Vocos - # 优先使用本地缓存, 避免联网 - try: - from personavoice.common.local_models import ( - setup_offline_mode, is_model_cached - ) - setup_offline_mode() - if not is_model_cached(self.model_name): - logger.info( - f"Vocos model '{self.model_name}' not in local cache, " - f"downloading from network." - ) - except ImportError: - pass - self._vocos = Vocos.from_pretrained(self.model_name) - self._vocos = self._vocos.to(self.device) - self._vocos.eval() - logger.info( - f"Vocos vocoder loaded: {self.model_name} " - f"(output: {self.target_sample_rate}Hz)" - ) - - @property - def is_available(self) -> bool: - """Check if vocoder is available.""" - return self._vocos is not None - - @property - def sample_rate(self) -> int: - """Output sample rate.""" - return self.target_sample_rate - - @torch.no_grad() - def mel_to_waveform( - self, - mel_spectrogram: torch.Tensor, - input_sample_rate: int = 16000, - ) -> torch.Tensor: - """Convert mel spectrogram to waveform. - - Args: - mel_spectrogram: Mel spectrogram (batch, n_mels, time) or (n_mels, time) - input_sample_rate: Sample rate of the mel extraction (for resampling) - - Returns: - Waveform (batch, samples) or (samples,) - """ - # Ensure batch dimension - squeeze_batch = False - if mel_spectrogram.dim() == 2: - mel_spectrogram = mel_spectrogram.unsqueeze(0) - squeeze_batch = True - - mel_spectrogram = mel_spectrogram.to(self.device) - - # Vocos expects (batch, n_mels, time) - mel_in = mel_spectrogram - # 若输入 mel bins (80) 与 Vocos 期望 (100) 不一致, 用投影层转换 - if self._mel_proj is not None: - # (B, n_mels, T) -> (B, T, n_mels) -> proj -> (B, T, 100) -> (B, 100, T) - mel_in = self._mel_proj( - mel_in.transpose(1, 2) - ).transpose(1, 2) - waveform = self._vocos.decode(mel_in) - - # Resample if needed - if self.target_sample_rate != input_sample_rate: - waveform = self._resample(waveform, self.target_sample_rate, input_sample_rate) - - if squeeze_batch: - waveform = waveform.squeeze(0) - - return waveform - - def _resample( - self, - waveform: torch.Tensor, - orig_freq: int, - new_freq: int, - ) -> torch.Tensor: - """Resample waveform.""" - import torchaudio - resampler = torchaudio.transforms.Resample(orig_freq, new_freq).to(self.device) - return resampler(waveform) - - def forward(self, mel_spectrogram: torch.Tensor) -> torch.Tensor: - """Forward pass: mel -> waveform.""" - return self.mel_to_waveform(mel_spectrogram) - - -class VocoderManager: - """Singleton manager for vocoder instances. - - Ensures only one vocoder instance is loaded per process. - """ - - _instances: Dict[str, VocosVocoder] = {} - - @classmethod - def get_vocoder( - cls, - model_name: str = "charactr/vocos-mel-24khz", - device: str = "cuda" if torch.cuda.is_available() else "cpu", - ) -> VocosVocoder: - """Get or create a vocoder instance.""" - key = f"{model_name}_{device}" - if key not in cls._instances: - cls._instances[key] = VocosVocoder( - model_name=model_name, - device=device, - ) - return cls._instances[key] - - @classmethod - def clear_cache(cls): - """Clear cached vocoder instances.""" - cls._instances.clear() - - -def mel_to_waveform( - mel_spectrogram: torch.Tensor, - vocoder: Optional[VocosVocoder] = None, - device: str = "cuda" if torch.cuda.is_available() else "cpu", - input_sample_rate: int = 16000, -) -> torch.Tensor: - """Convenience function to convert mel to waveform. - - Args: - mel_spectrogram: Mel spectrogram (batch, n_mels, time) or (n_mels, time) - vocoder: Optional pre-loaded vocoder - device: Device for computation - input_sample_rate: Sample rate of mel extraction - - Returns: - Waveform tensor - """ - if vocoder is None: - vocoder = VocoderManager.get_vocoder(device=device) - return vocoder.mel_to_waveform(mel_spectrogram, input_sample_rate) - - -def save_audio( - waveform: torch.Tensor, - path: str, - sample_rate: int = 24000, -): - """Save waveform to audio file. - - Args: - waveform: Waveform tensor (samples,) or (batch, samples) - path: Output file path - sample_rate: Sample rate - """ - import torchaudio - - # Ensure 2D (channels, samples) - if waveform.dim() == 1: - waveform = waveform.unsqueeze(0) - - # Ensure CPU - waveform = waveform.cpu() - - torchaudio.save(path, waveform, sample_rate) - logger.info(f"Audio saved to {path} ({waveform.shape[-1]/sample_rate:.2f}s)") diff --git a/reproduce.py b/reproduce.py new file mode 100644 index 0000000..ea9884a --- /dev/null +++ b/reproduce.py @@ -0,0 +1,62 @@ +"""Minimal, CPU-only reproduction entry point for the two paper supplements.""" +from __future__ import annotations + +import argparse +import hashlib +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent + + +def verify_manifest() -> None: + manifest = ROOT / "reproducibility" / "SHA256SUMS.txt" + failures = [] + for line in manifest.read_text(encoding="utf-8").splitlines(): + if not line or line.startswith("#"): + continue + expected, relative = line.split(" ", 1) + path = ROOT / relative + digest = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else "missing" + if digest != expected: + failures.append(f"{relative}: expected {expected}, got {digest}") + if failures: + raise SystemExit("Manifest verification failed:\n" + "\n".join(failures)) + print("SHA-256 manifest: OK") + + +def run(command: list[str], cwd: Path = ROOT) -> None: + print("$", " ".join(command)) + subprocess.run(command, cwd=cwd, check=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--derive", action="store_true", help="recompute all offline derived JSON summaries") + parser.add_argument("--figures", action="store_true", help="rebuild publication PDF/SVG figures") + args = parser.parse_args() + + verify_manifest() + run([sys.executable, "-m", "pytest", "-q", "supplement"]) + if args.derive: + language = ROOT / "supplement" / "language_provenance" + for script in ["audit_failures.py", "failure_by_budget.py", "budget_curve.py", "make_figures.py"]: + run([sys.executable, script], language) + speech = ROOT / "supplement" / "speech_shortref" + for script in ["aggregate_metrics.py", "tradeoff_analysis.py", "failure_analysis.py", "fine_length_analysis.py"]: + run([sys.executable, script], speech) + if args.figures: + run([sys.executable, "supplement/language_provenance/make_architecture_figure.py", "--output", "supplement/language_provenance/figures/planner_mechanism.pdf"]) + run([sys.executable, "supplement/speech_shortref/make_architecture_figures.py"]) + if args.derive or args.figures: + # Derived JSON/PDF files are release artifacts too; refresh the manifest + # after a local regeneration so a subsequent check remains meaningful. + run([sys.executable, "reproducibility/build_manifest.py"]) + verify_manifest() + print("Reproduction checks completed.") + + +if __name__ == "__main__": + main() diff --git a/reproducibility/SHA256SUMS.txt b/reproducibility/SHA256SUMS.txt new file mode 100644 index 0000000..b58e57b --- /dev/null +++ b/reproducibility/SHA256SUMS.txt @@ -0,0 +1,71 @@ +# SHA-256 manifest for paper-supplement-v1.0 +# Format: +229c28ba3dea41a1f463958c900809235ab354cb47f47262b69f6c517e4337ec .gitattributes +5b2bfca054fb7bd27371de5b9af88e2624dee5408f4147a06911b7dd23119f8b .gitignore +fc642ac0ef87f80d82dfa278b271f1a4bd7a3aeaa2d7592eccdb7a3697c60d39 CITATION.cff +e419c374daa445ab8409c654276d7f5222d8c50327312f8733290a07baca5f15 LICENSE +1736dedc63b193dfdb94a27070296dff73b155291f08aa25b43a70013eaa2301 README.md +982c5f0a20d0e9357ccd803b415f9063325d7e42cfd5f6f4e098f509267529b0 REPRODUCE.md +265875033abde446933274770228bca7fe440130edef69174a5de38b51c8598e reproduce.py +90db13236928f4198ae195237a6b4b114cec061739e1129fd966a9379db9c95d reproducibility/build_manifest.py +ec0b2e5be2eb0491b61b5dc8c40b2b49e9f6f45c70c82b4f47a017eb33f6a72c reproducibility/config.json +b03c8774eb60ffb88f4b64c26fa8a76254cfec21419a294e2c54e746fc5e9407 requirements-extension.txt +d717a09b263b65f9d98635733291c917110f9554702fa50d60ec421b75169d50 supplement/language_provenance/artifacts/budget_curve.json +4d941eb49150eb5e4abfabf004cdf0d67dfbecccc1c32e6b392b7e772cce3b79 supplement/language_provenance/artifacts/dailydialog_cross_dataset.json +8bf89ccddf75cd062b689a4426ac5d789e19d0c00e2c3871de7bc911ead58841 supplement/language_provenance/artifacts/failure_audit.json +93f552ab788ac0a73d585979655c93a7198367eaac526a61318928e1449e919d supplement/language_provenance/artifacts/failure_by_budget.json +77a2a09e2125e11dcd3337c696b96c8ea207228304f5d10531186b7ca1c9be97 supplement/language_provenance/artifacts/manifest.json +7dd31e83826f151001ee48cecfea41181a0b189cbb33fe0f384042c8252e84e8 supplement/language_provenance/artifacts/snli_compositional_test.jsonl +0119d83cf53fc3ed9d955cfd4e3163c58d0fab11f46e2f30477a8198e062ea27 supplement/language_provenance/artifacts/snli_error_analysis.json +84bbe8256b28cd4844718177e18a79ee35b7deac40049dd8537d5fa3dde95f60 supplement/language_provenance/artifacts/snli_semantic_mlp_seed0.json +c5b968a2595ca152ceacf85ca67a2354493d43dca2b3aa69f8d97ff8d353a04c supplement/language_provenance/artifacts/snli_semantic_mlp_seed1.json +5ea39fa98db65cba20c08017cf39d47483afbc488d210d77b7987755cb7152b2 supplement/language_provenance/artifacts/snli_semantic_mlp_seed2.json +65891c53b6caf213bdf0b1b3677dadb4aa11b4c9c2cc2a4a4793154f9b29bd09 supplement/language_provenance/audit_failures.py +59428d2d8b84b879016a275d355871c1ad1b92bb1b50c03def24fb393867bd2a supplement/language_provenance/budget_curve.py +479c434bf15137e29f837d5ae2b797ed1fab0d815f16af144d4b265e82c3f19c supplement/language_provenance/cross_dataset_audit.py +44dfa44378bd375fd8a13367df955bc74911afb1a6903b27ed7506e812f53dc5 supplement/language_provenance/failure_by_budget.py +14822adb1f4f8353e8a28f6749ea6d0f449cdc415f8d914ecc1832023a558ace supplement/language_provenance/figures/evidence_budget_tradeoff.pdf +1d73ba2c7757678d328ebec2e169e303523d0b7c8034b90614e53f42bc3dd5d5 supplement/language_provenance/figures/evidence_budget_tradeoff.png +fd33de7ba8e02870d05bc276bc119ce0efc3d5c6aa3063aa0d8434d1b3657ffc supplement/language_provenance/figures/failure_taxonomy.pdf +4346d45a5eed1e027adc1f152ef071dc82309a18613ec51b73a82951db08787b supplement/language_provenance/figures/failure_taxonomy.png +20b5111197572f528c04e3fbc1cbd695cc1204bc227d5a51b98c4c4dc4177467 supplement/language_provenance/figures/method_overview.pdf +a1a4abaf9044fd060983f24028380e32a8413601c2a45df97e106163d5ba1168 supplement/language_provenance/figures/planner_mechanism.pdf +c2d8ca2eb5ee2bdfbd8e54779f8fd936250db132a89acb6f291fdd3d3d7c206e supplement/language_provenance/figures/planner_mechanism.svg +d17a8a211fd4c9891e3759e31c06d49bc583c738422f60709e04077364d04410 supplement/language_provenance/make_architecture_figure.py +70fe45a13cc86c1f130cb46e24788bc41cf54579f87c55467e2ff3d61374c395 supplement/language_provenance/make_figures.py +21d7b31645c7e20caa55e44f88c39fb8fc3e4631369b220e8b7064baabe35b39 supplement/language_provenance/make_overview_figure.py +a06c509261ba4348e778d6d3a78b34ed04aa54045b6abd13b806ccded801ca0a supplement/language_provenance/README.md +4f499b66d5707c37e4f02799665aa01f04aee08d812a44e63a55d736bcbfb1d4 supplement/language_provenance/schemas.py +6cad46d8e20ffd1a089fd206bffd7304a1dda79ec90607fff04335f46130b7b7 supplement/language_provenance/tests/test_replay.py +1429c14154af76a45b00795314cab9deee1c09138d53f219bab5c218fa0bd5a0 supplement/README.md +845ff66abdbfc808d7e55572cce24500596313ef13dcea09dc6d6458db6f6a72 supplement/speech_shortref/aggregate_metrics.py +673ccbb8fc6c4241dd1020b4ce2010cb42545af40da7504b4a87dd829dad5cf9 supplement/speech_shortref/artifacts/ablation_200_statistics.json +a2a08617f75ed534d255575a9bea8c69ded54854d8193fbd025415a0c2abf93a supplement/speech_shortref/artifacts/aggregate_metrics.json +d1ec65baf32eafaabcc1659f9e65a363dbad3e56f61542a038f8a6364e59c01d supplement/speech_shortref/artifacts/baseline_external_200.json +de32902dc3360e5c422d22b684b562a03aba6db0103ea73cff5d507f1a0bceac supplement/speech_shortref/artifacts/cfr_analysis.json +222ba7ab91464692808925e4ecf16f609817a3ade82f32bad940004c253c1a94 supplement/speech_shortref/artifacts/eval_200_samples.json +4513dde8672c8cf585d3edfe870549355ca3e36cdbe6abf332fdfdb5b753823e supplement/speech_shortref/artifacts/eval_200_statistics.json +f91a1106065f3aaeacefb0cb4fc8fa5fe600c4685fed662a5ae16303155a7b48 supplement/speech_shortref/artifacts/extension_manifest.json +dff6128d906c707a5daf39b15c5f6ef8c44e9a5d1c124bd6cad4692b26404dd4 supplement/speech_shortref/artifacts/failure_ledger.json +30608c2c2783401aea2d2c839f1b79c3fd6f15bf85b5be654d62d52dbd2072da supplement/speech_shortref/artifacts/fine_length_analysis.json +d9f8c11da1b8baada0059d98ae6989668ff2ba5a596b0011950741dbda2d0031 supplement/speech_shortref/artifacts/honest_metrics_v10.4.7.json +7dee85f2f31d97c7373e9b263fc1787ce31969d0727af4d62321778e3b17db89 supplement/speech_shortref/artifacts/tradeoff_analysis.json +26224135603101ddc750b576656a0c34217d9b5dc0f3ffa34f0e7108603d05df supplement/speech_shortref/experiment_manifest.py +1fd247516745d302e49f5c9460d9ed47b8cbf4f8795a2d202b319c24d31e310d supplement/speech_shortref/failure_analysis.py +ae70e89eef2170b33ab64af94f4e052a289b2b2210665ef1e13d2185442e2f88 supplement/speech_shortref/figures/film_mechanism.pdf +0bf37f784a9420b584535d217cb5ed367f11cd49b67d7f7f8955b8b7ed41b8f3 supplement/speech_shortref/figures/fine_length_curve.pdf +d081c6b154a2a027dc6e484b44ce0f1575bf2e9923b1f5b25d4f48e1cf54a554 supplement/speech_shortref/figures/fine_length_curve.png +8ac995a480b4f2c0f952787c3864b934676138fdfec50ca55122a8d9791601f2 supplement/speech_shortref/figures/length_failure_curve.pdf +de3607df1e751a48565c25e5ae9c86e7b360d36898cdb55c05c4128aa18aac34 supplement/speech_shortref/figures/length_failure_curve.png +eda60a56657de3becb40b2f1c4821c51bd1545e0e80085386bfa9c77d35364f8 supplement/speech_shortref/figures/secs_wer_tradeoff.pdf +a06897da193cbe8245e955057795cbdf938e5b1f4aa0ac363b49cd1c543b4ba0 supplement/speech_shortref/figures/secs_wer_tradeoff.png +1bd88fa5e78b183c4241d56cfc85e216191c50f95af25793b2f57cc8c1bbb27c supplement/speech_shortref/figures/speech_overview.pdf +1a1635e09f34da8ca4fdd93f32ce010b2ac9b45ce4ad65f5c1a17d5d42435cf9 supplement/speech_shortref/fine_length_analysis.py +e8fe58bbd2a26560e6dc97f01bbb646975c645f983c3038a01f4d7b1a5204ced supplement/speech_shortref/jsonio.py +cdeaf221ab22734f957cdda4933f647848c4f355a685a150885837bbd05d6183 supplement/speech_shortref/make_architecture_figures.py +77936d1d2e8c2b1bdb8aada7717c5f13f26aa70e503b5b62c4bdcf6fc4f43db9 supplement/speech_shortref/make_figures.py +98623816a0241cb1429c8e7d8f5a4aa3448ae8619d2d04525d2e2aebe578d079 supplement/speech_shortref/README.md +256eeb05b8ea2956ee533dcc957a1e0d8b05d55d8b05223b8bc48ce503aac27b supplement/speech_shortref/tests/test_aggregates.py +936308eba2be72af86b21ef5836ce7b6d68ca962662ddd12fbe31443240e3889 supplement/speech_shortref/tradeoff_analysis.py +7d4d11c977691f4db5776b03775bd44aed61eea75d6e2504a0e422e525aba44e ZENODO.md +2346dbdd8812101a1db4f06f2a255ba5878f49203c7f20f0994dc062f4dabb8a ZENODO_METADATA.json diff --git a/reproducibility/build_manifest.py b/reproducibility/build_manifest.py new file mode 100644 index 0000000..925030a --- /dev/null +++ b/reproducibility/build_manifest.py @@ -0,0 +1,49 @@ +"""Build the deterministic SHA-256 manifest for the paper supplement release.""" +from __future__ import annotations + +import hashlib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +OUTPUT = ROOT / "reproducibility" / "SHA256SUMS.txt" +SKIP_DIRS = {".git", ".tmp_render2", "__pycache__", ".pytest_cache", "paper"} +SKIP_NAMES = {OUTPUT.name} + + +def iter_release_files(): + for path in sorted(ROOT.rglob("*")): + if not path.is_file(): + continue + rel_parts = path.relative_to(ROOT).parts + if any(part in SKIP_DIRS for part in rel_parts) or path.name in SKIP_NAMES: + continue + if path.suffix.lower() in { + ".pyc", ".aux", ".bbl", ".blg", ".log", ".out", ".fdb_latexmk", + ".fls", ".synctex.gz", + }: + continue + yield path + + +def digest(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(block) + return hasher.hexdigest() + + +def main() -> None: + lines = [ + "# SHA-256 manifest for paper-supplement-v1.0", + "# Format: ", + ] + for path in iter_release_files(): + lines.append(f"{digest(path)} {path.relative_to(ROOT).as_posix()}") + OUTPUT.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"wrote {len(lines) - 2} file hashes to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/reproducibility/config.json b/reproducibility/config.json new file mode 100644 index 0000000..91a6dea --- /dev/null +++ b/reproducibility/config.json @@ -0,0 +1,44 @@ +{ + "release_id": "paper-supplement-v1.0", + "release_date": "2026-08-26", + "repository": "https://github.com/FinalSunFlower/PersonaVoice", + "branch": "paper-supplement-v1.0", + "papers": { + "language": { + "title": "Provenance-Constrained Selective Language Style Planning from Sparse Evidence", + "version_tag": "paper-language-v1.0", + "paper_distribution": "separate Zenodo deposition (not duplicated in this support branch)", + "frozen_prediction_files": [ + "supplement/language_provenance/artifacts/snli_semantic_mlp_seed0.json", + "supplement/language_provenance/artifacts/snli_semantic_mlp_seed1.json", + "supplement/language_provenance/artifacts/snli_semantic_mlp_seed2.json" + ], + "seeds": [0, 1, 2], + "encoder": "sentence-transformers/all-MiniLM-L6-v2 (frozen)", + "projection_dim": 128, + "max_tokens": 96, + "optimizer": "AdamW", + "gradient_clip_norm": 1.0, + "observed_evidence_budgets": [5, 6, 7, 8, 9, 10, 11], + "unavailable_requested_budgets": [1, 2, 3] + }, + "speech": { + "title": "PersonaVoice: One-Second Voice Cloning with Length-Adaptive Flow Matching", + "version_tag": "paper-speech-v1.0", + "paper_distribution": "separate Zenodo deposition (not duplicated in this support branch)", + "frozen_prediction_files": [ + "supplement/speech_shortref/artifacts/eval_200_samples.json" + ], + "seed": 42, + "n_items": 200, + "reference_seconds": 1.0, + "speaker_metric": "ECAPA-TDNN cosine speaker similarity (SECS)", + "text_metric": "Whisper WER", + "catastrophic_failure_rule": "WER > 0.5", + "dataset": "LibriTTS dev-clean official split", + "pending_conditions_manifest": "supplement/speech_shortref/artifacts/extension_manifest.json" + } + }, + "weights_policy": "Large checkpoints and private/raw audio are intentionally excluded; obtain upstream model weights under their original licenses.", + "manifest": "reproducibility/SHA256SUMS.txt" +} diff --git a/requirements-extension.txt b/requirements-extension.txt new file mode 100644 index 0000000..f0d92b2 --- /dev/null +++ b/requirements-extension.txt @@ -0,0 +1,3 @@ +numpy>=1.24 +matplotlib>=3.7 +pytest>=8.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6b80266..0000000 --- a/requirements.txt +++ /dev/null @@ -1,37 +0,0 @@ -# Core deep learning -torch>=2.0.0 -torchaudio>=2.0.0 -speechbrain>=0.8.0 -transformers>=4.35.0 -accelerate>=0.24.0 - -# Data / audio -datasets>=2.14.0 -scipy>=1.11.0 -numpy>=1.24.0 -librosa>=0.10.0 -soundfile>=0.12.0 -matplotlib>=3.7.0 - -# ML utilities -scikit-learn>=1.3.0 -tqdm>=4.66.0 -pyyaml>=6.0 -omegaconf>=2.3.0 -einops>=0.7.0 - -# Text / NLP -spacy>=3.7.0 -jieba>=0.42.1 - -# PersonaVoice-specific (install in .venv via: pip install silero-vad f5-tts) -silero-vad>=5.0 -f5-tts>=1.0 - -# Web demo server -fastapi>=0.104.0 -uvicorn>=0.24.0 - -# Optional: Graph Neural Networks (used by some experiment scripts) -# torch_geometric>=2.4.0 -# torch_scatter>=2.1.0 diff --git a/results/ablation_200_samples.json b/results/ablation_200_samples.json deleted file mode 100644 index dc9a5d9..0000000 --- a/results/ablation_200_samples.json +++ /dev/null @@ -1,7210 +0,0 @@ -{ - "full_v8": [ - { - "secs": 0.31846344470977783, - "wer": 1.0, - "time_ms": 1826.2403011322021, - "idx": 0, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.1254669725894928, - "wer": 0.1, - "time_ms": 2825.4103660583496, - "idx": 3, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.24707597494125366, - "wer": 0.14285714285714285, - "time_ms": 2894.0269947052, - "idx": 5, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.2507789134979248, - "wer": 0.13333333333333333, - "time_ms": 2260.0464820861816, - "idx": 6, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.1497010886669159, - "wer": 0.17073170731707318, - "time_ms": 6119.7669506073, - "idx": 7, - "text_len": 41, - "target_length": 1250, - "gen_length_sec": 13.297872340425531 - }, - { - "secs": 0.07424693554639816, - "wer": 0.3157894736842105, - "time_ms": 2553.5476207733154, - "idx": 9, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.21053765714168549, - "wer": 0.058823529411764705, - "time_ms": 7957.47184753418, - "idx": 10, - "text_len": 51, - "target_length": 1532, - "gen_length_sec": 16.29787234042553 - }, - { - "secs": 0.18314078450202942, - "wer": 0.4, - "time_ms": 1525.7389545440674, - "idx": 11, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.35894060134887695, - "wer": 0.375, - "time_ms": 1427.415370941162, - "idx": 15, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.057639025151729584, - "wer": 0.15492957746478872, - "time_ms": 12670.843362808228, - "idx": 16, - "text_len": 71, - "target_length": 2096, - "gen_length_sec": 22.29787234042553 - }, - { - "secs": 0.4068625569343567, - "wer": 0.45454545454545453, - "time_ms": 1584.139108657837, - "idx": 17, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.5695180892944336, - "wer": 1.0, - "time_ms": 1103.4057140350342, - "idx": 18, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.11384660005569458, - "wer": 0.5, - "time_ms": 942.2190189361572, - "idx": 19, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.7065058946609497, - "wer": 0.125, - "time_ms": 1310.5909824371338, - "idx": 22, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.6437513828277588, - "wer": 0.10526315789473684, - "time_ms": 2403.3703804016113, - "idx": 23, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.6484862565994263, - "wer": 0.1, - "time_ms": 1461.7986679077148, - "idx": 24, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.7299996614456177, - "wer": 0.0, - "time_ms": 1116.1179542541504, - "idx": 25, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.5367839336395264, - "wer": 0.42857142857142855, - "time_ms": 1274.9683856964111, - "idx": 30, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6453200578689575, - "wer": 0.0, - "time_ms": 979.4797897338867, - "idx": 31, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.26938584446907043, - "wer": 0.11764705882352941, - "time_ms": 2049.0176677703857, - "idx": 33, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": -0.009598731994628906, - "wer": 0.20588235294117646, - "time_ms": 4480.240106582642, - "idx": 36, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.12764205038547516, - "wer": 0.13636363636363635, - "time_ms": 2864.2280101776123, - "idx": 37, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.222380131483078, - "wer": 0.06896551724137931, - "time_ms": 3809.7658157348633, - "idx": 38, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.2676655650138855, - "wer": 0.14285714285714285, - "time_ms": 1933.2160949707031, - "idx": 39, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.6618251800537109, - "wer": 0.14705882352941177, - "time_ms": 4499.5481967926025, - "idx": 42, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.2131107896566391, - "wer": 0.02631578947368421, - "time_ms": 5327.007055282593, - "idx": 45, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.3975682854652405, - "wer": 0.12244897959183673, - "time_ms": 7608.227729797363, - "idx": 46, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.16024823486804962, - "wer": 0.0, - "time_ms": 3665.4052734375, - "idx": 55, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.18547052145004272, - "wer": 0.06666666666666667, - "time_ms": 4042.912721633911, - "idx": 56, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.5634223222732544, - "wer": 0.09090909090909091, - "time_ms": 4555.993318557739, - "idx": 57, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.30646613240242004, - "wer": 0.16666666666666666, - "time_ms": 2540.2941703796387, - "idx": 59, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.6164832711219788, - "wer": 0.8888888888888888, - "time_ms": 1487.175464630127, - "idx": 60, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.31141072511672974, - "wer": 0.041666666666666664, - "time_ms": 3367.203950881958, - "idx": 63, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.33817875385284424, - "wer": 0.058823529411764705, - "time_ms": 4851.181745529175, - "idx": 66, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.1504693180322647, - "wer": 0.14285714285714285, - "time_ms": 2904.3402671813965, - "idx": 67, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.10546021163463593, - "wer": 0.125, - "time_ms": 1458.0588340759277, - "idx": 68, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": -0.07251162827014923, - "wer": 0.06666666666666667, - "time_ms": 2237.3692989349365, - "idx": 69, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.4589965343475342, - "wer": 0.14285714285714285, - "time_ms": 2980.0000190734863, - "idx": 72, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.5250190496444702, - "wer": 0.0, - "time_ms": 3364.5005226135254, - "idx": 73, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.25539496541023254, - "wer": 0.15384615384615385, - "time_ms": 3571.1593627929688, - "idx": 75, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.5939289927482605, - "wer": 0.16666666666666666, - "time_ms": 1916.2685871124268, - "idx": 76, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.2188478261232376, - "wer": 0.3076923076923077, - "time_ms": 2104.2520999908447, - "idx": 77, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.3135225474834442, - "wer": 0.1111111111111111, - "time_ms": 1557.2490692138672, - "idx": 78, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.15976090729236603, - "wer": 0.42857142857142855, - "time_ms": 2908.3175659179688, - "idx": 79, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.0004870723932981491, - "wer": 0.038461538461538464, - "time_ms": 3669.506788253784, - "idx": 81, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.17003053426742554, - "wer": 0.045454545454545456, - "time_ms": 3192.164897918701, - "idx": 82, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.5983250141143799, - "wer": 0.6666666666666666, - "time_ms": 1041.4879322052002, - "idx": 84, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.6702253222465515, - "wer": 0.3, - "time_ms": 1642.9011821746826, - "idx": 86, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.12744781374931335, - "wer": 1.0, - "time_ms": 999.0661144256592, - "idx": 89, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.10663730651140213, - "wer": 0.0967741935483871, - "time_ms": 4758.498668670654, - "idx": 90, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.1803884655237198, - "wer": 0.0625, - "time_ms": 2300.9696006774902, - "idx": 92, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.749545693397522, - "wer": 1.0, - "time_ms": 1019.1571712493896, - "idx": 93, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.08703984320163727, - "wer": 0.05263157894736842, - "time_ms": 2770.5910205841064, - "idx": 94, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.14671510457992554, - "wer": 0.17647058823529413, - "time_ms": 2310.34255027771, - "idx": 96, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.2544420063495636, - "wer": 0.03571428571428571, - "time_ms": 4171.519041061401, - "idx": 97, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.28876355290412903, - "wer": 0.0, - "time_ms": 4423.627138137817, - "idx": 101, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.5084802508354187, - "wer": 0.14285714285714285, - "time_ms": 5331.367969512939, - "idx": 103, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.26397740840911865, - "wer": 0.15, - "time_ms": 2919.948101043701, - "idx": 104, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.4024355113506317, - "wer": 0.10526315789473684, - "time_ms": 2772.6783752441406, - "idx": 108, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.32647010684013367, - "wer": 0.19230769230769232, - "time_ms": 3592.6382541656494, - "idx": 109, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.4224214553833008, - "wer": 0.15, - "time_ms": 3019.343852996826, - "idx": 110, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.39733925461769104, - "wer": 0.3333333333333333, - "time_ms": 1955.40452003479, - "idx": 111, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.7093753814697266, - "wer": 0.0, - "time_ms": 1072.9939937591553, - "idx": 113, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.6385411620140076, - "wer": 0.2, - "time_ms": 1152.4755954742432, - "idx": 114, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.7323590517044067, - "wer": 0.0, - "time_ms": 1597.532033920288, - "idx": 116, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.30537712574005127, - "wer": 0.2727272727272727, - "time_ms": 3334.3567848205566, - "idx": 118, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.5237565040588379, - "wer": 0.2916666666666667, - "time_ms": 3440.000295639038, - "idx": 119, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.1999535858631134, - "wer": 0.20588235294117646, - "time_ms": 5171.050548553467, - "idx": 122, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.6802853941917419, - "wer": 0.17391304347826086, - "time_ms": 3349.6646881103516, - "idx": 123, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.6997215151786804, - "wer": 0.1111111111111111, - "time_ms": 2645.557641983032, - "idx": 124, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.022875254973769188, - "wer": 0.02040816326530612, - "time_ms": 8420.130491256714, - "idx": 125, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.5615862607955933, - "wer": 0.23809523809523808, - "time_ms": 2940.0148391723633, - "idx": 126, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.8017479181289673, - "wer": 0.0, - "time_ms": 1426.316499710083, - "idx": 129, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.1720568686723709, - "wer": 0.038461538461538464, - "time_ms": 3603.764295578003, - "idx": 131, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.5295565724372864, - "wer": 0.14285714285714285, - "time_ms": 2120.6486225128174, - "idx": 132, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.7252486944198608, - "wer": 0.4, - "time_ms": 1163.2027626037598, - "idx": 137, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.7472869157791138, - "wer": 0.3333333333333333, - "time_ms": 1262.4084949493408, - "idx": 139, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.18846166133880615, - "wer": 0.09090909090909091, - "time_ms": 4816.424131393433, - "idx": 140, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.22576309740543365, - "wer": 0.07142857142857142, - "time_ms": 2130.939245223999, - "idx": 141, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.5474753379821777, - "wer": 0.08695652173913043, - "time_ms": 3188.162088394165, - "idx": 143, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.5005326867103577, - "wer": 0.125, - "time_ms": 3403.6245346069336, - "idx": 144, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.11267252266407013, - "wer": 0.2727272727272727, - "time_ms": 1777.1403789520264, - "idx": 145, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.6152558326721191, - "wer": 0.3, - "time_ms": 1640.639066696167, - "idx": 146, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.45703020691871643, - "wer": 0.03571428571428571, - "time_ms": 4134.363174438477, - "idx": 147, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.5782744884490967, - "wer": 0.14285714285714285, - "time_ms": 2919.952392578125, - "idx": 148, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.483723908662796, - "wer": 0.041666666666666664, - "time_ms": 3478.2402515411377, - "idx": 150, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.5905020833015442, - "wer": 0.0, - "time_ms": 2098.22154045105, - "idx": 152, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.17428112030029297, - "wer": 0.09523809523809523, - "time_ms": 2917.447090148926, - "idx": 153, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.8245093822479248, - "wer": 0.2692307692307692, - "time_ms": 3574.190616607666, - "idx": 155, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.22350028157234192, - "wer": 0.1, - "time_ms": 4361.405849456787, - "idx": 157, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.8148385286331177, - "wer": 0.0, - "time_ms": 1162.522792816162, - "idx": 158, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.8588085770606995, - "wer": 1.0, - "time_ms": 1133.1822872161865, - "idx": 163, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6482699513435364, - "wer": 1.0, - "time_ms": 1122.6308345794678, - "idx": 165, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.3253275156021118, - "wer": 0.13157894736842105, - "time_ms": 6104.195833206177, - "idx": 167, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.40134528279304504, - "wer": 0.06521739130434782, - "time_ms": 7413.048505783081, - "idx": 168, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.5548648834228516, - "wer": 1.0, - "time_ms": 1093.008279800415, - "idx": 172, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5598077774047852, - "wer": 1.0, - "time_ms": 1124.5181560516357, - "idx": 173, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.36118191480636597, - "wer": 1.0, - "time_ms": 1104.701280593872, - "idx": 175, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.03309853747487068, - "wer": 0.10714285714285714, - "time_ms": 4165.118932723999, - "idx": 176, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.6281157732009888, - "wer": 1.0, - "time_ms": 1109.7426414489746, - "idx": 177, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.29739734530448914, - "wer": 0.043478260869565216, - "time_ms": 7263.144016265869, - "idx": 180, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.16831502318382263, - "wer": 0.11320754716981132, - "time_ms": 9314.06831741333, - "idx": 181, - "text_len": 53, - "target_length": 1588, - "gen_length_sec": 16.893617021276597 - }, - { - "secs": 0.0961199700832367, - "wer": 0.06896551724137931, - "time_ms": 4363.478183746338, - "idx": 182, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.17891918122768402, - "wer": 0.10869565217391304, - "time_ms": 7348.817825317383, - "idx": 183, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.19920745491981506, - "wer": 0.0625, - "time_ms": 4785.930156707764, - "idx": 185, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.26996910572052, - "wer": 0.5106382978723404, - "time_ms": 8110.407829284668, - "idx": 192, - "text_len": 47, - "target_length": 1419, - "gen_length_sec": 15.095744680851064 - }, - { - "secs": 0.12309533357620239, - "wer": 0.0, - "time_ms": 2300.9748458862305, - "idx": 193, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.10809464007616043, - "wer": 0.2608695652173913, - "time_ms": 3283.1172943115234, - "idx": 194, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.16460908949375153, - "wer": 0.11538461538461539, - "time_ms": 3653.419017791748, - "idx": 195, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.477188378572464, - "wer": 0.1, - "time_ms": 2940.8559799194336, - "idx": 196, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.21880024671554565, - "wer": 0.043478260869565216, - "time_ms": 3267.7693367004395, - "idx": 198, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.15323975682258606, - "wer": 0.21428571428571427, - "time_ms": 4261.01016998291, - "idx": 202, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.513058066368103, - "wer": 0.14285714285714285, - "time_ms": 2214.0963077545166, - "idx": 203, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.4509086012840271, - "wer": 0.4, - "time_ms": 2330.7411670684814, - "idx": 208, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.7849416732788086, - "wer": 0.058823529411764705, - "time_ms": 2302.6559352874756, - "idx": 209, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.7346652150154114, - "wer": 0.08333333333333333, - "time_ms": 1980.1154136657715, - "idx": 210, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.40903371572494507, - "wer": 0.0, - "time_ms": 1614.3710613250732, - "idx": 211, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.7298251390457153, - "wer": 0.3333333333333333, - "time_ms": 1301.1929988861084, - "idx": 218, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7302471995353699, - "wer": 0.0, - "time_ms": 1211.3656997680664, - "idx": 219, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.7141855359077454, - "wer": 0.6818181818181818, - "time_ms": 3275.4967212677, - "idx": 220, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.5800226926803589, - "wer": 0.15, - "time_ms": 2905.1365852355957, - "idx": 222, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.557543158531189, - "wer": 0.0625, - "time_ms": 4885.967493057251, - "idx": 223, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6713714003562927, - "wer": 1.0, - "time_ms": 1168.5523986816406, - "idx": 225, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.5467780232429504, - "wer": 0.1111111111111111, - "time_ms": 4051.10502243042, - "idx": 228, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.3397715091705322, - "wer": 0.06, - "time_ms": 8497.83730506897, - "idx": 229, - "text_len": 50, - "target_length": 1504, - "gen_length_sec": 16.0 - }, - { - "secs": 0.618399977684021, - "wer": 0.07407407407407407, - "time_ms": 4017.265558242798, - "idx": 231, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.521151602268219, - "wer": 0.06976744186046512, - "time_ms": 6910.707235336304, - "idx": 233, - "text_len": 43, - "target_length": 1306, - "gen_length_sec": 13.893617021276595 - }, - { - "secs": 0.5228967666625977, - "wer": 0.08196721311475409, - "time_ms": 11278.16367149353, - "idx": 234, - "text_len": 61, - "target_length": 1814, - "gen_length_sec": 19.29787234042553 - }, - { - "secs": 0.29911577701568604, - "wer": 0.04, - "time_ms": 3620.2311515808105, - "idx": 237, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.8197462558746338, - "wer": 0.0, - "time_ms": 1309.4372749328613, - "idx": 239, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.6127741932868958, - "wer": 0.0, - "time_ms": 1181.6835403442383, - "idx": 244, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.4311879873275757, - "wer": 0.4666666666666667, - "time_ms": 2309.194326400757, - "idx": 246, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.23243895173072815, - "wer": 0.0, - "time_ms": 1125.40864944458, - "idx": 247, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.19541291892528534, - "wer": 0.1111111111111111, - "time_ms": 7581.517219543457, - "idx": 248, - "text_len": 45, - "target_length": 1363, - "gen_length_sec": 14.5 - }, - { - "secs": 0.6894278526306152, - "wer": 0.0, - "time_ms": 1343.1816101074219, - "idx": 249, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.6827991008758545, - "wer": 0.0, - "time_ms": 1461.9052410125732, - "idx": 253, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6054022908210754, - "wer": 0.0, - "time_ms": 1481.4372062683105, - "idx": 255, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2344924360513687, - "wer": 0.15, - "time_ms": 6717.164516448975, - "idx": 258, - "text_len": 40, - "target_length": 1222, - "gen_length_sec": 13.0 - }, - { - "secs": 0.2521670162677765, - "wer": 0.2, - "time_ms": 1169.7168350219727, - "idx": 261, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.2502044141292572, - "wer": 0.08695652173913043, - "time_ms": 3355.5843830108643, - "idx": 265, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.1878688782453537, - "wer": 0.5714285714285714, - "time_ms": 1456.68363571167, - "idx": 266, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.10833500325679779, - "wer": 0.08695652173913043, - "time_ms": 3355.640649795532, - "idx": 268, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.25888901948928833, - "wer": 0.2222222222222222, - "time_ms": 2686.6559982299805, - "idx": 271, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.4106401205062866, - "wer": 0.0, - "time_ms": 3041.4879322052, - "idx": 272, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.5252262353897095, - "wer": 0.4, - "time_ms": 1169.3711280822754, - "idx": 275, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.757026195526123, - "wer": 0.0, - "time_ms": 1176.866054534912, - "idx": 278, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.09323139488697052, - "wer": 0.08, - "time_ms": 3722.3873138427734, - "idx": 281, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.025201117619872093, - "wer": 0.1, - "time_ms": 3027.7915000915527, - "idx": 282, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.10226017981767654, - "wer": 0.375, - "time_ms": 2344.7561264038086, - "idx": 284, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.33576422929763794, - "wer": 0.125, - "time_ms": 1493.6718940734863, - "idx": 285, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.14914821088314056, - "wer": 0.0, - "time_ms": 3617.7830696105957, - "idx": 287, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.677220344543457, - "wer": 0.0, - "time_ms": 1429.1081428527832, - "idx": 289, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2960972487926483, - "wer": 0.16666666666666666, - "time_ms": 1305.9794902801514, - "idx": 291, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.21136930584907532, - "wer": 0.14285714285714285, - "time_ms": 2151.634931564331, - "idx": 292, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.3611273765563965, - "wer": 0.0, - "time_ms": 1462.0575904846191, - "idx": 297, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2566368579864502, - "wer": 0.25, - "time_ms": 1186.9184970855713, - "idx": 298, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.3805943727493286, - "wer": 0.4, - "time_ms": 1189.53537940979, - "idx": 300, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.25719428062438965, - "wer": 1.0, - "time_ms": 1142.8542137145996, - "idx": 302, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.1466832458972931, - "wer": 0.125, - "time_ms": 4978.607654571533, - "idx": 305, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.049519408494234085, - "wer": 0.11290322580645161, - "time_ms": 11692.600965499878, - "idx": 310, - "text_len": 62, - "target_length": 1842, - "gen_length_sec": 19.595744680851062 - }, - { - "secs": 0.06580908596515656, - "wer": 0.14285714285714285, - "time_ms": 5368.703126907349, - "idx": 314, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.7287245392799377, - "wer": 0.0, - "time_ms": 1433.286428451538, - "idx": 316, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.20334145426750183, - "wer": 0.06521739130434782, - "time_ms": 7466.72511100769, - "idx": 317, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.5444697141647339, - "wer": 0.0, - "time_ms": 1451.960802078247, - "idx": 318, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.193912535905838, - "wer": 0.25806451612903225, - "time_ms": 4711.15779876709, - "idx": 321, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.10319754481315613, - "wer": 0.10256410256410256, - "time_ms": 6430.180072784424, - "idx": 324, - "text_len": 39, - "target_length": 1193, - "gen_length_sec": 12.691489361702128 - }, - { - "secs": 0.6905917525291443, - "wer": 0.125, - "time_ms": 1461.2758159637451, - "idx": 325, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.3100742697715759, - "wer": 0.08955223880597014, - "time_ms": 12612.419366836548, - "idx": 326, - "text_len": 67, - "target_length": 1983, - "gen_length_sec": 21.095744680851062 - }, - { - "secs": 0.26392024755477905, - "wer": 0.08695652173913043, - "time_ms": 3338.510274887085, - "idx": 327, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.33354899287223816, - "wer": 0.18181818181818182, - "time_ms": 3266.946792602539, - "idx": 329, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.24512127041816711, - "wer": 0.1111111111111111, - "time_ms": 9725.620985031128, - "idx": 331, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.1014614850282669, - "wer": 0.08695652173913043, - "time_ms": 13714.845657348633, - "idx": 334, - "text_len": 69, - "target_length": 2039, - "gen_length_sec": 21.69148936170213 - }, - { - "secs": 0.4042375981807709, - "wer": 0.14705882352941177, - "time_ms": 5401.417255401611, - "idx": 336, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.005380837246775627, - "wer": 0.23076923076923078, - "time_ms": 2152.632236480713, - "idx": 337, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.3367686867713928, - "wer": 0.08695652173913043, - "time_ms": 3337.3496532440186, - "idx": 338, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.19295482337474823, - "wer": 0.022727272727272728, - "time_ms": 7238.097667694092, - "idx": 339, - "text_len": 44, - "target_length": 1334, - "gen_length_sec": 14.191489361702128 - }, - { - "secs": 0.2470868080854416, - "wer": 0.5, - "time_ms": 1153.7225246429443, - "idx": 340, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.6657766699790955, - "wer": 0.0, - "time_ms": 1164.4165515899658, - "idx": 342, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.38436347246170044, - "wer": 0.4166666666666667, - "time_ms": 2031.6379070281982, - "idx": 347, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.14494484663009644, - "wer": 0.08620689655172414, - "time_ms": 10926.748037338257, - "idx": 350, - "text_len": 58, - "target_length": 1729, - "gen_length_sec": 18.393617021276597 - }, - { - "secs": 0.08728279173374176, - "wer": 0.3, - "time_ms": 3182.9230785369873, - "idx": 356, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.08660467714071274, - "wer": 0.16666666666666666, - "time_ms": 11054.651021957397, - "idx": 357, - "text_len": 60, - "target_length": 1786, - "gen_length_sec": 19.0 - }, - { - "secs": 0.37701427936553955, - "wer": 0.10714285714285714, - "time_ms": 4310.914039611816, - "idx": 358, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.25305095314979553, - "wer": 0.3333333333333333, - "time_ms": 3078.5462856292725, - "idx": 360, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.23845605552196503, - "wer": 0.08333333333333333, - "time_ms": 5713.974237442017, - "idx": 361, - "text_len": 36, - "target_length": 1109, - "gen_length_sec": 11.797872340425531 - }, - { - "secs": 0.16439099609851837, - "wer": 0.125, - "time_ms": 3662.885904312134, - "idx": 362, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.14631453156471252, - "wer": 0.03333333333333333, - "time_ms": 4539.937257766724, - "idx": 363, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.14629700779914856, - "wer": 0.14814814814814814, - "time_ms": 9975.013017654419, - "idx": 364, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.15129898488521576, - "wer": 0.03636363636363636, - "time_ms": 9919.131517410278, - "idx": 365, - "text_len": 55, - "target_length": 1645, - "gen_length_sec": 17.5 - }, - { - "secs": 0.4449402093887329, - "wer": 0.06666666666666667, - "time_ms": 4573.29535484314, - "idx": 367, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.781841516494751, - "wer": 0.0, - "time_ms": 1167.90771484375, - "idx": 368, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.14098967611789703, - "wer": 0.21428571428571427, - "time_ms": 2244.6837425231934, - "idx": 369, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.5645331144332886, - "wer": 0.06666666666666667, - "time_ms": 2390.9294605255127, - "idx": 373, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.3002307116985321, - "wer": 0.03125, - "time_ms": 5047.5568771362305, - "idx": 374, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6792581081390381, - "wer": 0.21052631578947367, - "time_ms": 2781.6123962402344, - "idx": 375, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.47044646739959717, - "wer": 0.0, - "time_ms": 1120.3844547271729, - "idx": 378, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.7749749422073364, - "wer": 0.5, - "time_ms": 1145.4799175262451, - "idx": 381, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6599149703979492, - "wer": 0.05555555555555555, - "time_ms": 2771.425485610962, - "idx": 382, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.5378316640853882, - "wer": 0.09090909090909091, - "time_ms": 4995.489597320557, - "idx": 383, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.7492841482162476, - "wer": 0.14285714285714285, - "time_ms": 1441.7595863342285, - "idx": 385, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - } - ], - "wo_td_cfg": [ - { - "secs": 0.3392982482910156, - "wer": 1.0, - "time_ms": 1179.0587902069092, - "idx": 0, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.18772192299365997, - "wer": 0.1, - "time_ms": 2996.1442947387695, - "idx": 3, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.3318096697330475, - "wer": 0.09523809523809523, - "time_ms": 3134.5016956329346, - "idx": 5, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.3111562430858612, - "wer": 0.13333333333333333, - "time_ms": 2355.360507965088, - "idx": 6, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.07279307395219803, - "wer": 0.17073170731707318, - "time_ms": 6680.054426193237, - "idx": 7, - "text_len": 41, - "target_length": 1250, - "gen_length_sec": 13.297872340425531 - }, - { - "secs": 0.11602123081684113, - "wer": 0.2631578947368421, - "time_ms": 2821.6357231140137, - "idx": 9, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.1613936722278595, - "wer": 0.11764705882352941, - "time_ms": 8929.019212722778, - "idx": 10, - "text_len": 51, - "target_length": 1532, - "gen_length_sec": 16.29787234042553 - }, - { - "secs": 0.27720755338668823, - "wer": 0.2, - "time_ms": 1669.9981689453125, - "idx": 11, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.28912466764450073, - "wer": 0.375, - "time_ms": 1497.3981380462646, - "idx": 15, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.19812701642513275, - "wer": 0.14084507042253522, - "time_ms": 14560.688257217407, - "idx": 16, - "text_len": 71, - "target_length": 2096, - "gen_length_sec": 22.29787234042553 - }, - { - "secs": 0.4999900460243225, - "wer": 0.18181818181818182, - "time_ms": 1825.2954483032227, - "idx": 17, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.48513004183769226, - "wer": 0.6666666666666666, - "time_ms": 1313.7717247009277, - "idx": 18, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.1150314211845398, - "wer": 0.5, - "time_ms": 1103.3604145050049, - "idx": 19, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6791192889213562, - "wer": 0.125, - "time_ms": 1511.9960308074951, - "idx": 22, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.6772955060005188, - "wer": 0.10526315789473684, - "time_ms": 2801.3129234313965, - "idx": 23, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.6014462113380432, - "wer": 0.3, - "time_ms": 1695.8301067352295, - "idx": 24, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.7204212546348572, - "wer": 0.0, - "time_ms": 1271.4509963989258, - "idx": 25, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.503976047039032, - "wer": 0.42857142857142855, - "time_ms": 1473.9198684692383, - "idx": 30, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6161409616470337, - "wer": 0.5, - "time_ms": 1116.2049770355225, - "idx": 31, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.16340216994285583, - "wer": 0.11764705882352941, - "time_ms": 2416.8474674224854, - "idx": 33, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.073515385389328, - "wer": 0.20588235294117646, - "time_ms": 5320.7385540008545, - "idx": 36, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.16806288063526154, - "wer": 0.045454545454545456, - "time_ms": 3342.700719833374, - "idx": 37, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.20445451140403748, - "wer": 0.06896551724137931, - "time_ms": 4480.540990829468, - "idx": 38, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.23613086342811584, - "wer": 0.2857142857142857, - "time_ms": 2303.811550140381, - "idx": 39, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.5749171376228333, - "wer": 0.14705882352941177, - "time_ms": 5189.374208450317, - "idx": 42, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.22971674799919128, - "wer": 0.02631578947368421, - "time_ms": 6284.391641616821, - "idx": 45, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.3374522924423218, - "wer": 0.10204081632653061, - "time_ms": 8630.131483078003, - "idx": 46, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.24101103842258453, - "wer": 0.0, - "time_ms": 4232.214450836182, - "idx": 55, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.19950446486473083, - "wer": 0.2, - "time_ms": 4625.598430633545, - "idx": 56, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.5418001413345337, - "wer": 0.09090909090909091, - "time_ms": 4940.024614334106, - "idx": 57, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.338085800409317, - "wer": 0.1111111111111111, - "time_ms": 2821.0363388061523, - "idx": 59, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.6634930372238159, - "wer": 0.8888888888888888, - "time_ms": 1636.504888534546, - "idx": 60, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.414620965719223, - "wer": 0.041666666666666664, - "time_ms": 3648.550033569336, - "idx": 63, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.31167420744895935, - "wer": 0.08823529411764706, - "time_ms": 5438.395977020264, - "idx": 66, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.28083163499832153, - "wer": 0.09523809523809523, - "time_ms": 3130.0125122070312, - "idx": 67, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.2804258465766907, - "wer": 0.0, - "time_ms": 1510.9527111053467, - "idx": 68, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": -0.12567074596881866, - "wer": 0.4, - "time_ms": 2431.8830966949463, - "idx": 69, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.48059970140457153, - "wer": 0.09523809523809523, - "time_ms": 3018.8865661621094, - "idx": 72, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.45542803406715393, - "wer": 0.0, - "time_ms": 3356.1198711395264, - "idx": 73, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.15243327617645264, - "wer": 0.07692307692307693, - "time_ms": 3838.895320892334, - "idx": 75, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.4052830636501312, - "wer": 0.16666666666666666, - "time_ms": 2020.970344543457, - "idx": 76, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.4327930510044098, - "wer": 0.23076923076923078, - "time_ms": 2178.1935691833496, - "idx": 77, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.30703532695770264, - "wer": 0.1111111111111111, - "time_ms": 1683.840274810791, - "idx": 78, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.3471437096595764, - "wer": 0.14285714285714285, - "time_ms": 3082.231044769287, - "idx": 79, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.0431227833032608, - "wer": 0.038461538461538464, - "time_ms": 3780.9555530548096, - "idx": 81, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.12372363358736038, - "wer": 0.0, - "time_ms": 3371.8185424804688, - "idx": 82, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.626006007194519, - "wer": 1.0, - "time_ms": 1110.0146770477295, - "idx": 84, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5702407956123352, - "wer": 0.4, - "time_ms": 1711.6141319274902, - "idx": 86, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.1045696958899498, - "wer": 1.0, - "time_ms": 1112.9426956176758, - "idx": 89, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.14074595272541046, - "wer": 0.06451612903225806, - "time_ms": 4773.6358642578125, - "idx": 90, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.3077967166900635, - "wer": 0.0625, - "time_ms": 2373.0008602142334, - "idx": 92, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.7511904239654541, - "wer": 1.0, - "time_ms": 1111.3250255584717, - "idx": 93, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.24993012845516205, - "wer": 0.05263157894736842, - "time_ms": 2867.452621459961, - "idx": 94, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.16522519290447235, - "wer": 0.17647058823529413, - "time_ms": 2423.0713844299316, - "idx": 96, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.1974993646144867, - "wer": 0.03571428571428571, - "time_ms": 4323.533535003662, - "idx": 97, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.2291078269481659, - "wer": 0.0, - "time_ms": 4461.703538894653, - "idx": 101, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.38706058263778687, - "wer": 0.08571428571428572, - "time_ms": 5395.045280456543, - "idx": 103, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.16089382767677307, - "wer": 0.1, - "time_ms": 2976.647138595581, - "idx": 104, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.468097984790802, - "wer": 0.10526315789473684, - "time_ms": 2797.8675365448, - "idx": 108, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.30010509490966797, - "wer": 0.19230769230769232, - "time_ms": 3663.4457111358643, - "idx": 109, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.20704597234725952, - "wer": 0.1, - "time_ms": 3006.612539291382, - "idx": 110, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.37324196100234985, - "wer": 0.4166666666666667, - "time_ms": 2083.803415298462, - "idx": 111, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.7150033712387085, - "wer": 0.0, - "time_ms": 1383.0657005310059, - "idx": 113, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.6024324297904968, - "wer": 0.0, - "time_ms": 1259.8583698272705, - "idx": 114, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.5151104927062988, - "wer": 0.1111111111111111, - "time_ms": 1645.2655792236328, - "idx": 116, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.303600549697876, - "wer": 0.3181818181818182, - "time_ms": 3414.064884185791, - "idx": 118, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.5052189230918884, - "wer": 0.20833333333333334, - "time_ms": 5797.114372253418, - "idx": 119, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.24072331190109253, - "wer": 0.11764705882352941, - "time_ms": 5435.757160186768, - "idx": 122, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.5890708565711975, - "wer": 0.21739130434782608, - "time_ms": 3419.7349548339844, - "idx": 123, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.7117500305175781, - "wer": 0.1111111111111111, - "time_ms": 2858.825922012329, - "idx": 124, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.13898874819278717, - "wer": 0.02040816326530612, - "time_ms": 8885.000944137573, - "idx": 125, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.642152726650238, - "wer": 0.19047619047619047, - "time_ms": 3129.9338340759277, - "idx": 126, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.7985363602638245, - "wer": 0.0, - "time_ms": 1474.7843742370605, - "idx": 129, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.08442819863557816, - "wer": 0.19230769230769232, - "time_ms": 3713.101387023926, - "idx": 131, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.2347838282585144, - "wer": 0.07142857142857142, - "time_ms": 2249.265432357788, - "idx": 132, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.7861703634262085, - "wer": 0.0, - "time_ms": 1244.4431781768799, - "idx": 137, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.7310413122177124, - "wer": 0.5, - "time_ms": 1308.5875511169434, - "idx": 139, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.5623592138290405, - "wer": 0.12121212121212122, - "time_ms": 5170.257329940796, - "idx": 140, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.27532118558883667, - "wer": 0.14285714285714285, - "time_ms": 2252.1145343780518, - "idx": 141, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.5335378050804138, - "wer": 0.13043478260869565, - "time_ms": 3415.6014919281006, - "idx": 143, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.5335987210273743, - "wer": 0.125, - "time_ms": 3750.340223312378, - "idx": 144, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.08092678338289261, - "wer": 0.2727272727272727, - "time_ms": 1877.7096271514893, - "idx": 145, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.6247624754905701, - "wer": 0.3, - "time_ms": 1703.385353088379, - "idx": 146, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.5382086038589478, - "wer": 0.0, - "time_ms": 4393.290281295776, - "idx": 147, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.5838649868965149, - "wer": 0.14285714285714285, - "time_ms": 3105.8361530303955, - "idx": 148, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.30911749601364136, - "wer": 0.041666666666666664, - "time_ms": 3753.8540363311768, - "idx": 150, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.7159030437469482, - "wer": 0.0, - "time_ms": 2292.675256729126, - "idx": 152, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.06262186169624329, - "wer": 0.09523809523809523, - "time_ms": 3171.8876361846924, - "idx": 153, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.791455864906311, - "wer": 0.3076923076923077, - "time_ms": 3788.4867191314697, - "idx": 155, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.1469392031431198, - "wer": 0.13333333333333333, - "time_ms": 4524.973154067993, - "idx": 157, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.8676044940948486, - "wer": 0.0, - "time_ms": 1220.0214862823486, - "idx": 158, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.8601194620132446, - "wer": 1.0, - "time_ms": 1138.8673782348633, - "idx": 163, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.73565274477005, - "wer": 1.0, - "time_ms": 1235.3723049163818, - "idx": 165, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.3845851421356201, - "wer": 0.13157894736842105, - "time_ms": 6561.149120330811, - "idx": 167, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.48585712909698486, - "wer": 0.06521739130434782, - "time_ms": 7855.128526687622, - "idx": 168, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.43657150864601135, - "wer": 1.0, - "time_ms": 1115.1790618896484, - "idx": 172, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5516351461410522, - "wer": 1.0, - "time_ms": 1106.8620681762695, - "idx": 173, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.3619770407676697, - "wer": 1.0, - "time_ms": 1107.1488857269287, - "idx": 175, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": -0.017410973086953163, - "wer": 0.03571428571428571, - "time_ms": 4323.535203933716, - "idx": 176, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.4893967807292938, - "wer": 1.0, - "time_ms": 1199.1243362426758, - "idx": 177, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.35251009464263916, - "wer": 0.043478260869565216, - "time_ms": 7633.897542953491, - "idx": 180, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.12812493741512299, - "wer": 0.07547169811320754, - "time_ms": 9422.115087509155, - "idx": 181, - "text_len": 53, - "target_length": 1588, - "gen_length_sec": 16.893617021276597 - }, - { - "secs": 0.12054996192455292, - "wer": 0.034482758620689655, - "time_ms": 4559.674263000488, - "idx": 182, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.22981463372707367, - "wer": 0.08695652173913043, - "time_ms": 7752.050399780273, - "idx": 183, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.17645463347434998, - "wer": 0.0625, - "time_ms": 5008.971691131592, - "idx": 185, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.23806695640087128, - "wer": 0.0851063829787234, - "time_ms": 8646.081447601318, - "idx": 192, - "text_len": 47, - "target_length": 1419, - "gen_length_sec": 15.095744680851064 - }, - { - "secs": 0.13075795769691467, - "wer": 0.058823529411764705, - "time_ms": 2391.981363296509, - "idx": 193, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.06462230533361435, - "wer": 0.2608695652173913, - "time_ms": 3459.7508907318115, - "idx": 194, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.2701970338821411, - "wer": 0.11538461538461539, - "time_ms": 3902.703046798706, - "idx": 195, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.5261938571929932, - "wer": 0.1, - "time_ms": 3143.8169479370117, - "idx": 196, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.320489764213562, - "wer": 0.08695652173913043, - "time_ms": 3424.205541610718, - "idx": 198, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.19778060913085938, - "wer": 0.21428571428571427, - "time_ms": 4339.530944824219, - "idx": 202, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.31665685772895813, - "wer": 0.21428571428571427, - "time_ms": 2243.7636852264404, - "idx": 203, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.41114941239356995, - "wer": 0.4, - "time_ms": 2497.6165294647217, - "idx": 208, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.8162643313407898, - "wer": 0.058823529411764705, - "time_ms": 2432.0170879364014, - "idx": 209, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.7804529666900635, - "wer": 0.08333333333333333, - "time_ms": 2014.7557258605957, - "idx": 210, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.49167463183403015, - "wer": 0.0, - "time_ms": 1606.4529418945312, - "idx": 211, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.6699036955833435, - "wer": 0.3333333333333333, - "time_ms": 1396.5246677398682, - "idx": 218, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7027092576026917, - "wer": 0.0, - "time_ms": 1243.1848049163818, - "idx": 219, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.5178192257881165, - "wer": 0.4090909090909091, - "time_ms": 3353.7778854370117, - "idx": 220, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.4302617609500885, - "wer": 0.15, - "time_ms": 3076.993465423584, - "idx": 222, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.5380795001983643, - "wer": 0.0625, - "time_ms": 5153.215646743774, - "idx": 223, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6863313913345337, - "wer": 1.0, - "time_ms": 1222.1124172210693, - "idx": 225, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.49665382504463196, - "wer": 0.18518518518518517, - "time_ms": 4133.391857147217, - "idx": 228, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.26702195405960083, - "wer": 0.14, - "time_ms": 8821.460962295532, - "idx": 229, - "text_len": 50, - "target_length": 1504, - "gen_length_sec": 16.0 - }, - { - "secs": 0.5601312518119812, - "wer": 0.14814814814814814, - "time_ms": 4172.1343994140625, - "idx": 231, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.5603711605072021, - "wer": 0.06976744186046512, - "time_ms": 7395.04075050354, - "idx": 233, - "text_len": 43, - "target_length": 1306, - "gen_length_sec": 13.893617021276595 - }, - { - "secs": 0.6500290632247925, - "wer": 0.09836065573770492, - "time_ms": 11939.356327056885, - "idx": 234, - "text_len": 61, - "target_length": 1814, - "gen_length_sec": 19.29787234042553 - }, - { - "secs": 0.41631731390953064, - "wer": 0.08, - "time_ms": 3712.1541500091553, - "idx": 237, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.8020921945571899, - "wer": 0.0, - "time_ms": 1556.713581085205, - "idx": 239, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.6701287031173706, - "wer": 0.0, - "time_ms": 1192.8563117980957, - "idx": 244, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.342813104391098, - "wer": 0.3333333333333333, - "time_ms": 2456.5176963806152, - "idx": 246, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.18880131840705872, - "wer": 0.0, - "time_ms": 1130.3951740264893, - "idx": 247, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.27423417568206787, - "wer": 0.08888888888888889, - "time_ms": 7820.793628692627, - "idx": 248, - "text_len": 45, - "target_length": 1363, - "gen_length_sec": 14.5 - }, - { - "secs": 0.6353757381439209, - "wer": 0.0, - "time_ms": 1306.772232055664, - "idx": 249, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.685836911201477, - "wer": 0.0, - "time_ms": 1503.5486221313477, - "idx": 253, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6338347792625427, - "wer": 0.0, - "time_ms": 1454.4618129730225, - "idx": 255, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.206599622964859, - "wer": 0.175, - "time_ms": 6815.863132476807, - "idx": 258, - "text_len": 40, - "target_length": 1222, - "gen_length_sec": 13.0 - }, - { - "secs": 0.21729953587055206, - "wer": 0.2, - "time_ms": 1205.6033611297607, - "idx": 261, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.3687530755996704, - "wer": 0.08695652173913043, - "time_ms": 3403.677463531494, - "idx": 265, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.2154393494129181, - "wer": 0.5714285714285714, - "time_ms": 1527.0767211914062, - "idx": 266, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.09666674584150314, - "wer": 0.08695652173913043, - "time_ms": 3423.478364944458, - "idx": 268, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.2650110721588135, - "wer": 0.16666666666666666, - "time_ms": 2820.2617168426514, - "idx": 271, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.3188486397266388, - "wer": 0.09523809523809523, - "time_ms": 3112.6315593719482, - "idx": 272, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.5809310674667358, - "wer": 0.4, - "time_ms": 1195.5413818359375, - "idx": 275, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6914747357368469, - "wer": 0.0, - "time_ms": 1246.58203125, - "idx": 278, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.25206881761550903, - "wer": 0.08, - "time_ms": 3677.4978637695312, - "idx": 281, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.04028195142745972, - "wer": 0.1, - "time_ms": 3077.812671661377, - "idx": 282, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.18100333213806152, - "wer": 0.375, - "time_ms": 2365.3225898742676, - "idx": 284, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.3306361734867096, - "wer": 0.25, - "time_ms": 1609.0648174285889, - "idx": 285, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.14658592641353607, - "wer": 0.12, - "time_ms": 3769.5302963256836, - "idx": 287, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.650259256362915, - "wer": 0.0, - "time_ms": 1442.1207904815674, - "idx": 289, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.35982707142829895, - "wer": 0.5, - "time_ms": 1338.1741046905518, - "idx": 291, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.3536052703857422, - "wer": 0.14285714285714285, - "time_ms": 2286.095142364502, - "idx": 292, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.21098679304122925, - "wer": 0.0, - "time_ms": 1484.33518409729, - "idx": 297, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2620769143104553, - "wer": 0.75, - "time_ms": 1189.3370151519775, - "idx": 298, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.4100272059440613, - "wer": 0.0, - "time_ms": 1212.2745513916016, - "idx": 300, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.23655663430690765, - "wer": 0.0, - "time_ms": 1201.415777206421, - "idx": 302, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.23959141969680786, - "wer": 0.09375, - "time_ms": 5338.882207870483, - "idx": 305, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.07255019247531891, - "wer": 0.0967741935483871, - "time_ms": 11793.432235717773, - "idx": 310, - "text_len": 62, - "target_length": 1842, - "gen_length_sec": 19.595744680851062 - }, - { - "secs": 0.1957947015762329, - "wer": 0.11428571428571428, - "time_ms": 5419.8737144470215, - "idx": 314, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.7464452981948853, - "wer": 0.0, - "time_ms": 1494.462490081787, - "idx": 316, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.08740080893039703, - "wer": 0.043478260869565216, - "time_ms": 7985.872745513916, - "idx": 317, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.6112457513809204, - "wer": 0.0, - "time_ms": 1505.1567554473877, - "idx": 318, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.27439171075820923, - "wer": 0.1935483870967742, - "time_ms": 4884.276390075684, - "idx": 321, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.22764931619167328, - "wer": 0.10256410256410256, - "time_ms": 6441.506385803223, - "idx": 324, - "text_len": 39, - "target_length": 1193, - "gen_length_sec": 12.691489361702128 - }, - { - "secs": 0.6597126722335815, - "wer": 0.25, - "time_ms": 1534.0819358825684, - "idx": 325, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.32855236530303955, - "wer": 0.05970149253731343, - "time_ms": 12940.213680267334, - "idx": 326, - "text_len": 67, - "target_length": 1983, - "gen_length_sec": 21.095744680851062 - }, - { - "secs": 0.1350022554397583, - "wer": 0.08695652173913043, - "time_ms": 3525.4952907562256, - "idx": 327, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.3857499659061432, - "wer": 0.18181818181818182, - "time_ms": 3423.5079288482666, - "idx": 329, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.3150302767753601, - "wer": 0.3333333333333333, - "time_ms": 10014.909505844116, - "idx": 331, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.22977298498153687, - "wer": 0.057971014492753624, - "time_ms": 14229.495763778687, - "idx": 334, - "text_len": 69, - "target_length": 2039, - "gen_length_sec": 21.69148936170213 - }, - { - "secs": 0.2854556441307068, - "wer": 0.058823529411764705, - "time_ms": 5463.660717010498, - "idx": 336, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.12893174588680267, - "wer": 0.46153846153846156, - "time_ms": 2210.8705043792725, - "idx": 337, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.3849214017391205, - "wer": 0.08695652173913043, - "time_ms": 3453.8466930389404, - "idx": 338, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.17532266676425934, - "wer": 0.0, - "time_ms": 7298.494577407837, - "idx": 339, - "text_len": 44, - "target_length": 1334, - "gen_length_sec": 14.191489361702128 - }, - { - "secs": 0.2225087434053421, - "wer": 0.5, - "time_ms": 1189.6991729736328, - "idx": 340, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.7132148742675781, - "wer": 0.0, - "time_ms": 1168.259620666504, - "idx": 342, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.36362558603286743, - "wer": 0.6666666666666666, - "time_ms": 2043.2884693145752, - "idx": 347, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.23288270831108093, - "wer": 0.10344827586206896, - "time_ms": 11215.120315551758, - "idx": 350, - "text_len": 58, - "target_length": 1729, - "gen_length_sec": 18.393617021276597 - }, - { - "secs": 0.1177520900964737, - "wer": 0.3, - "time_ms": 3044.9862480163574, - "idx": 356, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.06881511211395264, - "wer": 0.15, - "time_ms": 11416.147232055664, - "idx": 357, - "text_len": 60, - "target_length": 1786, - "gen_length_sec": 19.0 - }, - { - "secs": 0.2832352817058563, - "wer": 0.07142857142857142, - "time_ms": 4350.788116455078, - "idx": 358, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.25732487440109253, - "wer": 0.14285714285714285, - "time_ms": 3023.2107639312744, - "idx": 360, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.3117867410182953, - "wer": 0.08333333333333333, - "time_ms": 5674.227237701416, - "idx": 361, - "text_len": 36, - "target_length": 1109, - "gen_length_sec": 11.797872340425531 - }, - { - "secs": 0.19215834140777588, - "wer": 0.20833333333333334, - "time_ms": 3552.2265434265137, - "idx": 362, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.12455634027719498, - "wer": 0.03333333333333333, - "time_ms": 4497.488260269165, - "idx": 363, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.07574845105409622, - "wer": 0.12962962962962962, - "time_ms": 10064.188480377197, - "idx": 364, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.16263198852539062, - "wer": 0.09090909090909091, - "time_ms": 10137.72439956665, - "idx": 365, - "text_len": 55, - "target_length": 1645, - "gen_length_sec": 17.5 - }, - { - "secs": 0.7192111015319824, - "wer": 0.06666666666666667, - "time_ms": 4643.235206604004, - "idx": 367, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.7643480896949768, - "wer": 0.0, - "time_ms": 1160.9585285186768, - "idx": 368, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.23927147686481476, - "wer": 0.14285714285714285, - "time_ms": 2173.7446784973145, - "idx": 369, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.4576646089553833, - "wer": 0.13333333333333333, - "time_ms": 2367.262601852417, - "idx": 373, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.3745838403701782, - "wer": 0.0625, - "time_ms": 4921.855211257935, - "idx": 374, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.60940021276474, - "wer": 0.2631578947368421, - "time_ms": 2908.7634086608887, - "idx": 375, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.47044646739959717, - "wer": 0.0, - "time_ms": 1474.0149974822998, - "idx": 378, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.7348570823669434, - "wer": 0.5, - "time_ms": 1251.3091564178467, - "idx": 381, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6528922319412231, - "wer": 0.05555555555555555, - "time_ms": 2824.329614639282, - "idx": 382, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.6065802574157715, - "wer": 0.09090909090909091, - "time_ms": 4995.379209518433, - "idx": 383, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.7446619868278503, - "wer": 0.14285714285714285, - "time_ms": 1467.6790237426758, - "idx": 385, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - } - ], - "wo_ibop": [ - { - "secs": 0.5699472427368164, - "wer": 1.0, - "time_ms": 1212.6743793487549, - "idx": 0, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.11947351694107056, - "wer": 0.1, - "time_ms": 3089.5442962646484, - "idx": 3, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.20398655533790588, - "wer": 0.14285714285714285, - "time_ms": 3143.0816650390625, - "idx": 5, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.2321963608264923, - "wer": 0.13333333333333333, - "time_ms": 2402.3780822753906, - "idx": 6, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.14648738503456116, - "wer": 0.14634146341463414, - "time_ms": 6843.63055229187, - "idx": 7, - "text_len": 41, - "target_length": 1250, - "gen_length_sec": 13.297872340425531 - }, - { - "secs": 0.1536152958869934, - "wer": 0.21052631578947367, - "time_ms": 2839.138984680176, - "idx": 9, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.1457362025976181, - "wer": 0.0392156862745098, - "time_ms": 8908.461570739746, - "idx": 10, - "text_len": 51, - "target_length": 1532, - "gen_length_sec": 16.29787234042553 - }, - { - "secs": 0.1852186620235443, - "wer": 0.5, - "time_ms": 1731.926441192627, - "idx": 11, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.2515547275543213, - "wer": 0.375, - "time_ms": 1495.6316947937012, - "idx": 15, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.05121901258826256, - "wer": 0.1267605633802817, - "time_ms": 14588.35506439209, - "idx": 16, - "text_len": 71, - "target_length": 2096, - "gen_length_sec": 22.29787234042553 - }, - { - "secs": 0.2798154354095459, - "wer": 0.18181818181818182, - "time_ms": 1877.7034282684326, - "idx": 17, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.40835514664649963, - "wer": 0.8333333333333334, - "time_ms": 1297.3439693450928, - "idx": 18, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.09714044630527496, - "wer": 0.5, - "time_ms": 1126.7671585083008, - "idx": 19, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6924347877502441, - "wer": 0.125, - "time_ms": 1557.7218532562256, - "idx": 22, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.6416478753089905, - "wer": 0.10526315789473684, - "time_ms": 2870.0976371765137, - "idx": 23, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.6515260934829712, - "wer": 0.1, - "time_ms": 1691.8702125549316, - "idx": 24, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.8178004026412964, - "wer": 0.0, - "time_ms": 1330.8804035186768, - "idx": 25, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.450833797454834, - "wer": 0.42857142857142855, - "time_ms": 1419.9926853179932, - "idx": 30, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.5350379943847656, - "wer": 0.5, - "time_ms": 1176.2778759002686, - "idx": 31, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.2831030488014221, - "wer": 0.35294117647058826, - "time_ms": 2445.734977722168, - "idx": 33, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.05660320445895195, - "wer": 0.23529411764705882, - "time_ms": 5332.336187362671, - "idx": 36, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.1904272735118866, - "wer": 0.09090909090909091, - "time_ms": 3404.9644470214844, - "idx": 37, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.12288431078195572, - "wer": 0.06896551724137931, - "time_ms": 4507.817983627319, - "idx": 38, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.20171117782592773, - "wer": 0.14285714285714285, - "time_ms": 2250.004529953003, - "idx": 39, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.6815652251243591, - "wer": 0.11764705882352941, - "time_ms": 5329.8656940460205, - "idx": 42, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.23924550414085388, - "wer": 0.02631578947368421, - "time_ms": 6328.277587890625, - "idx": 45, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.3799828290939331, - "wer": 0.12244897959183673, - "time_ms": 8675.561666488647, - "idx": 46, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.13967178761959076, - "wer": 0.07407407407407407, - "time_ms": 4226.110219955444, - "idx": 55, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.20857858657836914, - "wer": 0.06666666666666667, - "time_ms": 4597.598075866699, - "idx": 56, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.4518612027168274, - "wer": 0.09090909090909091, - "time_ms": 5157.8209400177, - "idx": 57, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.36709126830101013, - "wer": 0.1111111111111111, - "time_ms": 2869.6906566619873, - "idx": 59, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.5052359104156494, - "wer": 1.0, - "time_ms": 1621.109962463379, - "idx": 60, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.400068074464798, - "wer": 0.08333333333333333, - "time_ms": 3692.668676376343, - "idx": 63, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.35548898577690125, - "wer": 0.058823529411764705, - "time_ms": 5294.140338897705, - "idx": 66, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.3931255340576172, - "wer": 0.09523809523809523, - "time_ms": 3153.625011444092, - "idx": 67, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.3771458566188812, - "wer": 0.0, - "time_ms": 1489.100694656372, - "idx": 68, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.01406567171216011, - "wer": 0.13333333333333333, - "time_ms": 2417.908191680908, - "idx": 69, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.49914178252220154, - "wer": 0.047619047619047616, - "time_ms": 3092.1356678009033, - "idx": 72, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.43615806102752686, - "wer": 0.08695652173913043, - "time_ms": 3432.919502258301, - "idx": 73, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.2399103045463562, - "wer": 0.0, - "time_ms": 3683.4371089935303, - "idx": 75, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.6067221164703369, - "wer": 0.16666666666666666, - "time_ms": 2016.7160034179688, - "idx": 76, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.2699476480484009, - "wer": 0.15384615384615385, - "time_ms": 2236.7513179779053, - "idx": 77, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.2955564558506012, - "wer": 0.1111111111111111, - "time_ms": 1664.2677783966064, - "idx": 78, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.14579856395721436, - "wer": 0.2857142857142857, - "time_ms": 3128.265142440796, - "idx": 79, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": -0.0023432080633938313, - "wer": 0.038461538461538464, - "time_ms": 3800.581216812134, - "idx": 81, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.2014978528022766, - "wer": 0.0, - "time_ms": 3370.058298110962, - "idx": 82, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.5970234274864197, - "wer": 0.3333333333333333, - "time_ms": 1089.967966079712, - "idx": 84, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.6211458444595337, - "wer": 0.3, - "time_ms": 1695.9941387176514, - "idx": 86, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.19431012868881226, - "wer": 1.0, - "time_ms": 1114.9260997772217, - "idx": 89, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.16693155467510223, - "wer": 0.06451612903225806, - "time_ms": 4848.606586456299, - "idx": 90, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.2630274295806885, - "wer": 0.0625, - "time_ms": 2313.5130405426025, - "idx": 92, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.7455641627311707, - "wer": 1.0, - "time_ms": 1126.4820098876953, - "idx": 93, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": -0.016995802521705627, - "wer": 0.0, - "time_ms": 2895.658493041992, - "idx": 94, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.24835240840911865, - "wer": 0.11764705882352941, - "time_ms": 2400.1293182373047, - "idx": 96, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.18790863454341888, - "wer": 0.10714285714285714, - "time_ms": 4330.101013183594, - "idx": 97, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.1975749135017395, - "wer": 0.034482758620689655, - "time_ms": 4584.447860717773, - "idx": 101, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.49016430974006653, - "wer": 0.2571428571428571, - "time_ms": 5399.8308181762695, - "idx": 103, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.18949353694915771, - "wer": 0.1, - "time_ms": 3065.9232139587402, - "idx": 104, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.4150078296661377, - "wer": 0.15789473684210525, - "time_ms": 2807.4069023132324, - "idx": 108, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.36343321204185486, - "wer": 0.2692307692307692, - "time_ms": 3776.3261795043945, - "idx": 109, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.081902414560318, - "wer": 0.15, - "time_ms": 3019.516944885254, - "idx": 110, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.3648500442504883, - "wer": 0.3333333333333333, - "time_ms": 2033.8349342346191, - "idx": 111, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.6889201998710632, - "wer": 0.0, - "time_ms": 1106.1456203460693, - "idx": 113, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.541493833065033, - "wer": 0.2, - "time_ms": 1230.52978515625, - "idx": 114, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.7572476863861084, - "wer": 0.1111111111111111, - "time_ms": 1620.119571685791, - "idx": 116, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.2707168459892273, - "wer": 0.18181818181818182, - "time_ms": 3358.600854873657, - "idx": 118, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.541978657245636, - "wer": 0.20833333333333334, - "time_ms": 3692.277193069458, - "idx": 119, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.10617203265428543, - "wer": 0.14705882352941177, - "time_ms": 5295.831680297852, - "idx": 122, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.5688329935073853, - "wer": 0.2608695652173913, - "time_ms": 3398.0250358581543, - "idx": 123, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.6899482011795044, - "wer": 0.1111111111111111, - "time_ms": 2802.7026653289795, - "idx": 124, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.025435125455260277, - "wer": 0.0, - "time_ms": 8739.230155944824, - "idx": 125, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.6622368097305298, - "wer": 0.14285714285714285, - "time_ms": 3079.5674324035645, - "idx": 126, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.7653332352638245, - "wer": 0.14285714285714285, - "time_ms": 1441.2140846252441, - "idx": 129, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.03134370222687721, - "wer": 0.038461538461538464, - "time_ms": 3780.904769897461, - "idx": 131, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.388949453830719, - "wer": 0.07142857142857142, - "time_ms": 2225.01540184021, - "idx": 132, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.7038494944572449, - "wer": 0.4, - "time_ms": 1375.5345344543457, - "idx": 137, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.569023609161377, - "wer": 0.3333333333333333, - "time_ms": 1302.675485610962, - "idx": 139, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.24976330995559692, - "wer": 0.09090909090909091, - "time_ms": 5115.27419090271, - "idx": 140, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.18408241868019104, - "wer": 0.07142857142857142, - "time_ms": 2251.5745162963867, - "idx": 141, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.46378546953201294, - "wer": 0.13043478260869565, - "time_ms": 3401.0696411132812, - "idx": 143, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.5964677333831787, - "wer": 0.125, - "time_ms": 3593.122720718384, - "idx": 144, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.27204862236976624, - "wer": 0.5454545454545454, - "time_ms": 1838.118553161621, - "idx": 145, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.6971082091331482, - "wer": 0.3, - "time_ms": 1734.1458797454834, - "idx": 146, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.5163083076477051, - "wer": 0.03571428571428571, - "time_ms": 4393.140077590942, - "idx": 147, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.3106299042701721, - "wer": 0.14285714285714285, - "time_ms": 3124.0928173065186, - "idx": 148, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.4724201560020447, - "wer": 0.041666666666666664, - "time_ms": 3568.19748878479, - "idx": 150, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.6833779215812683, - "wer": 0.0, - "time_ms": 2209.412097930908, - "idx": 152, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.24286794662475586, - "wer": 0.19047619047619047, - "time_ms": 3044.297456741333, - "idx": 153, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.8169277906417847, - "wer": 0.3076923076923077, - "time_ms": 3744.001865386963, - "idx": 155, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.12108152359724045, - "wer": 0.16666666666666666, - "time_ms": 4530.989170074463, - "idx": 157, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.8075406551361084, - "wer": 0.0, - "time_ms": 1202.1045684814453, - "idx": 158, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.8090764880180359, - "wer": 0.5, - "time_ms": 1107.4883937835693, - "idx": 163, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.7035598754882812, - "wer": 0.6666666666666666, - "time_ms": 1137.3016834259033, - "idx": 165, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.2343989759683609, - "wer": 0.10526315789473684, - "time_ms": 6465.953588485718, - "idx": 167, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.4426153898239136, - "wer": 0.08695652173913043, - "time_ms": 7865.445375442505, - "idx": 168, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.6807124614715576, - "wer": 1.0, - "time_ms": 1112.593412399292, - "idx": 172, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5626481771469116, - "wer": 1.0, - "time_ms": 1099.7111797332764, - "idx": 173, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.35868868231773376, - "wer": 0.5, - "time_ms": 1106.7252159118652, - "idx": 175, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.11080527305603027, - "wer": 0.14285714285714285, - "time_ms": 4243.476629257202, - "idx": 176, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.5978828072547913, - "wer": 0.5, - "time_ms": 1123.5032081604004, - "idx": 177, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.2868909239768982, - "wer": 0.06521739130434782, - "time_ms": 7643.183469772339, - "idx": 180, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.15851393342018127, - "wer": 0.09433962264150944, - "time_ms": 9322.199583053589, - "idx": 181, - "text_len": 53, - "target_length": 1588, - "gen_length_sec": 16.893617021276597 - }, - { - "secs": 0.089958056807518, - "wer": 0.034482758620689655, - "time_ms": 4596.018314361572, - "idx": 182, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.19031250476837158, - "wer": 0.06521739130434782, - "time_ms": 7924.9749183654785, - "idx": 183, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.1912582665681839, - "wer": 0.09375, - "time_ms": 5031.28457069397, - "idx": 185, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.1926622986793518, - "wer": 0.5106382978723404, - "time_ms": 8271.87728881836, - "idx": 192, - "text_len": 47, - "target_length": 1419, - "gen_length_sec": 15.095744680851064 - }, - { - "secs": 0.1453816145658493, - "wer": 0.11764705882352941, - "time_ms": 2414.236307144165, - "idx": 193, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.0816098004579544, - "wer": 0.21739130434782608, - "time_ms": 3340.137004852295, - "idx": 194, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.1324283331632614, - "wer": 0.11538461538461539, - "time_ms": 3863.553524017334, - "idx": 195, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.48070991039276123, - "wer": 0.1, - "time_ms": 3097.33247756958, - "idx": 196, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.19305510818958282, - "wer": 0.043478260869565216, - "time_ms": 3377.171039581299, - "idx": 198, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.33322471380233765, - "wer": 0.10714285714285714, - "time_ms": 4277.3284912109375, - "idx": 202, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.38515332341194153, - "wer": 0.14285714285714285, - "time_ms": 2204.7650814056396, - "idx": 203, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.43323636054992676, - "wer": 0.26666666666666666, - "time_ms": 2427.8042316436768, - "idx": 208, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.8386000394821167, - "wer": 0.058823529411764705, - "time_ms": 2393.512964248657, - "idx": 209, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.7583167552947998, - "wer": 0.08333333333333333, - "time_ms": 2035.9759330749512, - "idx": 210, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.35876116156578064, - "wer": 0.1111111111111111, - "time_ms": 1605.1521301269531, - "idx": 211, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.7021698355674744, - "wer": 0.3333333333333333, - "time_ms": 1278.611183166504, - "idx": 218, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7994676828384399, - "wer": 0.0, - "time_ms": 1211.313009262085, - "idx": 219, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.516089916229248, - "wer": 0.4090909090909091, - "time_ms": 3393.1233882904053, - "idx": 220, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.42784935235977173, - "wer": 0.4, - "time_ms": 3060.7402324676514, - "idx": 222, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.6383591294288635, - "wer": 0.03125, - "time_ms": 4981.189250946045, - "idx": 223, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.644751787185669, - "wer": 1.0, - "time_ms": 1200.64377784729, - "idx": 225, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6183174848556519, - "wer": 0.07407407407407407, - "time_ms": 4318.394184112549, - "idx": 228, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.34814736247062683, - "wer": 0.06, - "time_ms": 8949.586153030396, - "idx": 229, - "text_len": 50, - "target_length": 1504, - "gen_length_sec": 16.0 - }, - { - "secs": 0.6487900614738464, - "wer": 0.07407407407407407, - "time_ms": 5787.627458572388, - "idx": 231, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.6058727502822876, - "wer": 0.06976744186046512, - "time_ms": 11755.56492805481, - "idx": 233, - "text_len": 43, - "target_length": 1306, - "gen_length_sec": 13.893617021276595 - }, - { - "secs": 0.40726977586746216, - "wer": 0.09836065573770492, - "time_ms": 13473.477602005005, - "idx": 234, - "text_len": 61, - "target_length": 1814, - "gen_length_sec": 19.29787234042553 - }, - { - "secs": 0.4817165732383728, - "wer": 0.08, - "time_ms": 5090.685129165649, - "idx": 237, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.8144233822822571, - "wer": 0.0, - "time_ms": 1406.3670635223389, - "idx": 239, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7081912755966187, - "wer": 0.0, - "time_ms": 1219.7153568267822, - "idx": 244, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.438749223947525, - "wer": 0.5333333333333333, - "time_ms": 5609.90834236145, - "idx": 246, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.1756136268377304, - "wer": 0.0, - "time_ms": 4133.368253707886, - "idx": 247, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.44677406549453735, - "wer": 0.06666666666666667, - "time_ms": 12584.280490875244, - "idx": 248, - "text_len": 45, - "target_length": 1363, - "gen_length_sec": 14.5 - }, - { - "secs": 0.6337708830833435, - "wer": 0.0, - "time_ms": 3779.468059539795, - "idx": 249, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.5840575098991394, - "wer": 0.0, - "time_ms": 1667.7496433258057, - "idx": 253, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6256372928619385, - "wer": 0.0, - "time_ms": 2189.166307449341, - "idx": 255, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2236424684524536, - "wer": 0.175, - "time_ms": 7241.826772689819, - "idx": 258, - "text_len": 40, - "target_length": 1222, - "gen_length_sec": 13.0 - }, - { - "secs": 0.14181317389011383, - "wer": 0.2, - "time_ms": 13112.600088119507, - "idx": 261, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.3076290488243103, - "wer": 0.08695652173913043, - "time_ms": 7134.974241256714, - "idx": 265, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.19084709882736206, - "wer": 0.7142857142857143, - "time_ms": 8156.830549240112, - "idx": 266, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.15080305933952332, - "wer": 0.08695652173913043, - "time_ms": 9934.661149978638, - "idx": 268, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.12189222872257233, - "wer": 0.16666666666666666, - "time_ms": 2501.254081726074, - "idx": 271, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.26829734444618225, - "wer": 0.09523809523809523, - "time_ms": 2710.132122039795, - "idx": 272, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.5768176317214966, - "wer": 0.4, - "time_ms": 1371.6716766357422, - "idx": 275, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6661871075630188, - "wer": 0.0, - "time_ms": 1486.999750137329, - "idx": 278, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.07861398160457611, - "wer": 0.08, - "time_ms": 7527.472257614136, - "idx": 281, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.0745561346411705, - "wer": 0.1, - "time_ms": 2775.008201599121, - "idx": 282, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.11896970868110657, - "wer": 0.25, - "time_ms": 4029.522180557251, - "idx": 284, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.3945004343986511, - "wer": 0.125, - "time_ms": 3055.3793907165527, - "idx": 285, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.16538290679454803, - "wer": 0.04, - "time_ms": 3419.9275970458984, - "idx": 287, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.5572595596313477, - "wer": 0.0, - "time_ms": 1548.0875968933105, - "idx": 289, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.27514004707336426, - "wer": 0.3333333333333333, - "time_ms": 1344.5465564727783, - "idx": 291, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.3395036458969116, - "wer": 0.14285714285714285, - "time_ms": 2402.3399353027344, - "idx": 292, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.30002638697624207, - "wer": 0.0, - "time_ms": 1693.3643817901611, - "idx": 297, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.29668182134628296, - "wer": 0.25, - "time_ms": 1270.0910568237305, - "idx": 298, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.45831143856048584, - "wer": 0.0, - "time_ms": 1913.529872894287, - "idx": 300, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.19799931347370148, - "wer": 1.0, - "time_ms": 2186.5715980529785, - "idx": 302, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.11184151470661163, - "wer": 0.125, - "time_ms": 4983.38508605957, - "idx": 305, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.08410923182964325, - "wer": 0.016129032258064516, - "time_ms": 11451.878786087036, - "idx": 310, - "text_len": 62, - "target_length": 1842, - "gen_length_sec": 19.595744680851062 - }, - { - "secs": 0.040705397725105286, - "wer": 1.0, - "time_ms": 5246.816396713257, - "idx": 314, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.7054064869880676, - "wer": 0.0, - "time_ms": 1473.3374118804932, - "idx": 316, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.07137345522642136, - "wer": 0.043478260869565216, - "time_ms": 7689.848184585571, - "idx": 317, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.5928433537483215, - "wer": 0.0, - "time_ms": 1488.098382949829, - "idx": 318, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.20657484233379364, - "wer": 0.16129032258064516, - "time_ms": 4949.763298034668, - "idx": 321, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.24190787971019745, - "wer": 0.10256410256410256, - "time_ms": 6267.183542251587, - "idx": 324, - "text_len": 39, - "target_length": 1193, - "gen_length_sec": 12.691489361702128 - }, - { - "secs": 0.7052621245384216, - "wer": 0.25, - "time_ms": 1543.8072681427002, - "idx": 325, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.24276115000247955, - "wer": 0.07462686567164178, - "time_ms": 13073.855638504028, - "idx": 326, - "text_len": 67, - "target_length": 1983, - "gen_length_sec": 21.095744680851062 - }, - { - "secs": 0.16831505298614502, - "wer": 0.08695652173913043, - "time_ms": 3373.97837638855, - "idx": 327, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.30974283814430237, - "wer": 0.22727272727272727, - "time_ms": 3355.8640480041504, - "idx": 329, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.1633903831243515, - "wer": 0.07407407407407407, - "time_ms": 9967.066764831543, - "idx": 331, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.14297272264957428, - "wer": 0.057971014492753624, - "time_ms": 13759.021282196045, - "idx": 334, - "text_len": 69, - "target_length": 2039, - "gen_length_sec": 21.69148936170213 - }, - { - "secs": 0.320568323135376, - "wer": 0.08823529411764706, - "time_ms": 5475.413799285889, - "idx": 336, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.2022414207458496, - "wer": 0.3076923076923077, - "time_ms": 2250.6721019744873, - "idx": 337, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.47064536809921265, - "wer": 0.13043478260869565, - "time_ms": 4562.304258346558, - "idx": 338, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.21898019313812256, - "wer": 0.06818181818181818, - "time_ms": 7479.485750198364, - "idx": 339, - "text_len": 44, - "target_length": 1334, - "gen_length_sec": 14.191489361702128 - }, - { - "secs": 0.45324450731277466, - "wer": 0.5, - "time_ms": 1218.9719676971436, - "idx": 340, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.7521980404853821, - "wer": 0.0, - "time_ms": 1163.1295680999756, - "idx": 342, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.39866888523101807, - "wer": 0.5833333333333334, - "time_ms": 2102.0727157592773, - "idx": 347, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.1508951485157013, - "wer": 0.06896551724137931, - "time_ms": 11383.659601211548, - "idx": 350, - "text_len": 58, - "target_length": 1729, - "gen_length_sec": 18.393617021276597 - }, - { - "secs": 0.11982938647270203, - "wer": 0.1, - "time_ms": 3101.4060974121094, - "idx": 356, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.16491807997226715, - "wer": 0.2, - "time_ms": 11444.040060043335, - "idx": 357, - "text_len": 60, - "target_length": 1786, - "gen_length_sec": 19.0 - }, - { - "secs": 0.358723908662796, - "wer": 0.10714285714285714, - "time_ms": 4313.297986984253, - "idx": 358, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.16067369282245636, - "wer": 0.19047619047619047, - "time_ms": 3094.2437648773193, - "idx": 360, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.25473377108573914, - "wer": 0.1388888888888889, - "time_ms": 5677.003622055054, - "idx": 361, - "text_len": 36, - "target_length": 1109, - "gen_length_sec": 11.797872340425531 - }, - { - "secs": 0.17641618847846985, - "wer": 0.20833333333333334, - "time_ms": 3823.237419128418, - "idx": 362, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.1287633329629898, - "wer": 0.03333333333333333, - "time_ms": 7401.477098464966, - "idx": 363, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.15639391541481018, - "wer": 0.12962962962962962, - "time_ms": 10040.001153945923, - "idx": 364, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.1890750378370285, - "wer": 0.07272727272727272, - "time_ms": 10393.195867538452, - "idx": 365, - "text_len": 55, - "target_length": 1645, - "gen_length_sec": 17.5 - }, - { - "secs": 0.582460343837738, - "wer": 0.06666666666666667, - "time_ms": 4696.397542953491, - "idx": 367, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.8029996156692505, - "wer": 0.0, - "time_ms": 1210.6099128723145, - "idx": 368, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.23180994391441345, - "wer": 0.14285714285714285, - "time_ms": 2296.692371368408, - "idx": 369, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.5544859766960144, - "wer": 0.13333333333333333, - "time_ms": 2417.54412651062, - "idx": 373, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.21112900972366333, - "wer": 0.0625, - "time_ms": 4985.166072845459, - "idx": 374, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6464191675186157, - "wer": 0.21052631578947367, - "time_ms": 2877.7847290039062, - "idx": 375, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.47044646739959717, - "wer": 0.0, - "time_ms": 1145.0533866882324, - "idx": 378, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.789466142654419, - "wer": 0.5, - "time_ms": 1126.9242763519287, - "idx": 381, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6978387832641602, - "wer": 0.05555555555555555, - "time_ms": 2808.8161945343018, - "idx": 382, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.5870248675346375, - "wer": 0.09090909090909091, - "time_ms": 5140.677452087402, - "idx": 383, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.7264244556427002, - "wer": 0.14285714285714285, - "time_ms": 1647.291660308838, - "idx": 385, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - } - ], - "wo_sbm": [ - { - "secs": 0.3887980878353119, - "wer": 1.0, - "time_ms": 1397.681474685669, - "idx": 0, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.19389058649539948, - "wer": 0.1, - "time_ms": 3065.2308464050293, - "idx": 3, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.27523142099380493, - "wer": 0.09523809523809523, - "time_ms": 3019.404649734497, - "idx": 5, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.31480151414871216, - "wer": 0.2, - "time_ms": 2380.3012371063232, - "idx": 6, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.15999138355255127, - "wer": 0.21951219512195122, - "time_ms": 6633.827924728394, - "idx": 7, - "text_len": 41, - "target_length": 1250, - "gen_length_sec": 13.297872340425531 - }, - { - "secs": 0.10687891393899918, - "wer": 0.15789473684210525, - "time_ms": 2838.925361633301, - "idx": 9, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.09678769111633301, - "wer": 0.058823529411764705, - "time_ms": 8959.716081619263, - "idx": 10, - "text_len": 51, - "target_length": 1532, - "gen_length_sec": 16.29787234042553 - }, - { - "secs": 0.21387198567390442, - "wer": 0.4, - "time_ms": 1746.0463047027588, - "idx": 11, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.49922457337379456, - "wer": 0.5, - "time_ms": 1514.8756504058838, - "idx": 15, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.06555332988500595, - "wer": 0.14084507042253522, - "time_ms": 14045.05705833435, - "idx": 16, - "text_len": 71, - "target_length": 2096, - "gen_length_sec": 22.29787234042553 - }, - { - "secs": 0.43777555227279663, - "wer": 0.2727272727272727, - "time_ms": 1881.2496662139893, - "idx": 17, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.5392704606056213, - "wer": 1.0, - "time_ms": 1311.4159107208252, - "idx": 18, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.14097030460834503, - "wer": 1.0, - "time_ms": 1368.7090873718262, - "idx": 19, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6863713264465332, - "wer": 0.125, - "time_ms": 1561.7029666900635, - "idx": 22, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.6559112668037415, - "wer": 0.10526315789473684, - "time_ms": 2831.3772678375244, - "idx": 23, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.6030592322349548, - "wer": 0.1, - "time_ms": 1668.2257652282715, - "idx": 24, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.7956801652908325, - "wer": 0.0, - "time_ms": 1286.9071960449219, - "idx": 25, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.2429293692111969, - "wer": 0.42857142857142855, - "time_ms": 1525.5134105682373, - "idx": 30, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.5708144307136536, - "wer": 0.5, - "time_ms": 1136.4457607269287, - "idx": 31, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.24382203817367554, - "wer": 0.23529411764705882, - "time_ms": 2465.6875133514404, - "idx": 33, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.08618979901075363, - "wer": 0.23529411764705882, - "time_ms": 5327.136278152466, - "idx": 36, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.17020857334136963, - "wer": 0.13636363636363635, - "time_ms": 3283.8406562805176, - "idx": 37, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.13599056005477905, - "wer": 0.06896551724137931, - "time_ms": 4437.005519866943, - "idx": 38, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.2838330864906311, - "wer": 0.42857142857142855, - "time_ms": 2255.833864212036, - "idx": 39, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.7423495650291443, - "wer": 0.11764705882352941, - "time_ms": 5353.996992111206, - "idx": 42, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.2381402999162674, - "wer": 0.02631578947368421, - "time_ms": 6312.29567527771, - "idx": 45, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.3121061325073242, - "wer": 0.10204081632653061, - "time_ms": 8753.499984741211, - "idx": 46, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.10900220274925232, - "wer": 0.07407407407407407, - "time_ms": 4064.7096633911133, - "idx": 55, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.13127796351909637, - "wer": 0.03333333333333333, - "time_ms": 4556.077241897583, - "idx": 56, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.4198943078517914, - "wer": 0.09090909090909091, - "time_ms": 4895.7390785217285, - "idx": 57, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.3143467307090759, - "wer": 0.1111111111111111, - "time_ms": 2767.561674118042, - "idx": 59, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.5678880214691162, - "wer": 1.0, - "time_ms": 1660.6457233428955, - "idx": 60, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.3880733251571655, - "wer": 0.041666666666666664, - "time_ms": 3582.5343132019043, - "idx": 63, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.38583335280418396, - "wer": 0.08823529411764706, - "time_ms": 5132.673025131226, - "idx": 66, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.2433730959892273, - "wer": 0.19047619047619047, - "time_ms": 3026.4391899108887, - "idx": 67, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.06079470366239548, - "wer": 0.125, - "time_ms": 1489.9024963378906, - "idx": 68, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.018381493166089058, - "wer": 0.06666666666666667, - "time_ms": 2386.8327140808105, - "idx": 69, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.4976930022239685, - "wer": 0.23809523809523808, - "time_ms": 3098.4957218170166, - "idx": 72, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.46057429909706116, - "wer": 0.08695652173913043, - "time_ms": 3431.055784225464, - "idx": 73, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.31051868200302124, - "wer": 0.15384615384615385, - "time_ms": 3756.1280727386475, - "idx": 75, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.49289512634277344, - "wer": 0.16666666666666666, - "time_ms": 2051.781415939331, - "idx": 76, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.37132754921913147, - "wer": 0.0, - "time_ms": 2186.0718727111816, - "idx": 77, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.25785595178604126, - "wer": 0.0, - "time_ms": 1593.89066696167, - "idx": 78, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.31943345069885254, - "wer": 0.38095238095238093, - "time_ms": 3130.115270614624, - "idx": 79, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.0671764463186264, - "wer": 0.038461538461538464, - "time_ms": 3760.204315185547, - "idx": 81, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.15484251081943512, - "wer": 0.0, - "time_ms": 3325.3796100616455, - "idx": 82, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.6290072202682495, - "wer": 1.0, - "time_ms": 1133.8374614715576, - "idx": 84, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5222805142402649, - "wer": 0.2, - "time_ms": 1731.5731048583984, - "idx": 86, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.1398782581090927, - "wer": 1.0, - "time_ms": 1132.4760913848877, - "idx": 89, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.0800573080778122, - "wer": 0.0, - "time_ms": 4945.93358039856, - "idx": 90, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.479640930891037, - "wer": 0.1875, - "time_ms": 2299.6931076049805, - "idx": 92, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.7898225784301758, - "wer": 0.6666666666666666, - "time_ms": 1134.1004371643066, - "idx": 93, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.02475903369486332, - "wer": 0.0, - "time_ms": 2761.0535621643066, - "idx": 94, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.14657175540924072, - "wer": 0.23529411764705882, - "time_ms": 2410.4645252227783, - "idx": 96, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.1836070418357849, - "wer": 0.07142857142857142, - "time_ms": 5688.0223751068115, - "idx": 97, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.21053525805473328, - "wer": 0.0, - "time_ms": 5413.788557052612, - "idx": 101, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.5709930062294006, - "wer": 0.08571428571428572, - "time_ms": 7755.493879318237, - "idx": 103, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.0669063925743103, - "wer": 0.1, - "time_ms": 3861.441135406494, - "idx": 104, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.5045067667961121, - "wer": 0.10526315789473684, - "time_ms": 3006.446361541748, - "idx": 108, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.26613351702690125, - "wer": 0.19230769230769232, - "time_ms": 4132.052421569824, - "idx": 109, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.2726246118545532, - "wer": 0.05, - "time_ms": 3119.4586753845215, - "idx": 110, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.31921619176864624, - "wer": 0.3333333333333333, - "time_ms": 2091.233491897583, - "idx": 111, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.6966614723205566, - "wer": 0.0, - "time_ms": 1475.3022193908691, - "idx": 113, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5893596410751343, - "wer": 0.0, - "time_ms": 1548.6834049224854, - "idx": 114, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.8306715488433838, - "wer": 0.1111111111111111, - "time_ms": 1772.136926651001, - "idx": 116, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.19866874814033508, - "wer": 0.3181818181818182, - "time_ms": 3349.9035835266113, - "idx": 118, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.47074079513549805, - "wer": 0.16666666666666666, - "time_ms": 3683.3760738372803, - "idx": 119, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.11326892673969269, - "wer": 0.11764705882352941, - "time_ms": 5306.7357540130615, - "idx": 122, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.5403794050216675, - "wer": 0.21739130434782608, - "time_ms": 3491.76025390625, - "idx": 123, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.6457275152206421, - "wer": 0.1111111111111111, - "time_ms": 2799.7641563415527, - "idx": 124, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.13639935851097107, - "wer": 0.04081632653061224, - "time_ms": 10052.79016494751, - "idx": 125, - "text_len": 49, - "target_length": 1475, - "gen_length_sec": 15.691489361702128 - }, - { - "secs": 0.634978711605072, - "wer": 0.19047619047619047, - "time_ms": 3966.0067558288574, - "idx": 126, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.7657889127731323, - "wer": 0.0, - "time_ms": 1510.758876800537, - "idx": 129, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.15823693573474884, - "wer": 0.038461538461538464, - "time_ms": 3850.053310394287, - "idx": 131, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.5844376087188721, - "wer": 0.07142857142857142, - "time_ms": 2409.799575805664, - "idx": 132, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.7249207496643066, - "wer": 0.0, - "time_ms": 4015.1658058166504, - "idx": 137, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6391288638114929, - "wer": 0.5, - "time_ms": 1254.3315887451172, - "idx": 139, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.20976485311985016, - "wer": 0.12121212121212122, - "time_ms": 4932.896375656128, - "idx": 140, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.17855201661586761, - "wer": 0.0, - "time_ms": 2173.591375350952, - "idx": 141, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.506332516670227, - "wer": 0.13043478260869565, - "time_ms": 3576.0979652404785, - "idx": 143, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.44213011860847473, - "wer": 0.20833333333333334, - "time_ms": 3691.9913291931152, - "idx": 144, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.22851009666919708, - "wer": 0.2727272727272727, - "time_ms": 1787.0848178863525, - "idx": 145, - "text_len": 11, - "target_length": 404, - "gen_length_sec": 4.297872340425532 - }, - { - "secs": 0.6278233528137207, - "wer": 0.3, - "time_ms": 1623.3813762664795, - "idx": 146, - "text_len": 10, - "target_length": 376, - "gen_length_sec": 4.0 - }, - { - "secs": 0.5769422054290771, - "wer": 0.0, - "time_ms": 4203.324556350708, - "idx": 147, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.45217499136924744, - "wer": 0.09523809523809523, - "time_ms": 6122.4939823150635, - "idx": 148, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.3796223998069763, - "wer": 0.041666666666666664, - "time_ms": 3685.586452484131, - "idx": 150, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.7364220023155212, - "wer": 0.0, - "time_ms": 2146.979808807373, - "idx": 152, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.06588156521320343, - "wer": 0.09523809523809523, - "time_ms": 3098.7911224365234, - "idx": 153, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.7446374297142029, - "wer": 0.3076923076923077, - "time_ms": 3656.8145751953125, - "idx": 155, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.038531962782144547, - "wer": 0.13333333333333333, - "time_ms": 4600.878477096558, - "idx": 157, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.8455790281295776, - "wer": 0.0, - "time_ms": 1348.0885028839111, - "idx": 158, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.8503038883209229, - "wer": 1.0, - "time_ms": 1393.7013149261475, - "idx": 163, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6608384251594543, - "wer": 1.0, - "time_ms": 1403.538703918457, - "idx": 165, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.36719900369644165, - "wer": 0.10526315789473684, - "time_ms": 6208.382844924927, - "idx": 167, - "text_len": 38, - "target_length": 1165, - "gen_length_sec": 12.393617021276595 - }, - { - "secs": 0.4142124652862549, - "wer": 0.08695652173913043, - "time_ms": 7696.364879608154, - "idx": 168, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.6421636343002319, - "wer": 1.0, - "time_ms": 1286.696434020996, - "idx": 172, - "text_len": 3, - "target_length": 178, - "gen_length_sec": 1.8936170212765957 - }, - { - "secs": 0.5695799589157104, - "wer": 1.0, - "time_ms": 1305.4907321929932, - "idx": 173, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.33410558104515076, - "wer": 0.5, - "time_ms": 1275.4921913146973, - "idx": 175, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.08756745606660843, - "wer": 0.10714285714285714, - "time_ms": 4357.569456100464, - "idx": 176, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.5708613395690918, - "wer": 0.5, - "time_ms": 1539.428949356079, - "idx": 177, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.38249656558036804, - "wer": 0.0, - "time_ms": 7559.894800186157, - "idx": 180, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.05422130599617958, - "wer": 0.09433962264150944, - "time_ms": 9531.367063522339, - "idx": 181, - "text_len": 53, - "target_length": 1588, - "gen_length_sec": 16.893617021276597 - }, - { - "secs": 0.12274745106697083, - "wer": 0.06896551724137931, - "time_ms": 4424.314022064209, - "idx": 182, - "text_len": 29, - "target_length": 911, - "gen_length_sec": 9.691489361702128 - }, - { - "secs": 0.22322231531143188, - "wer": 0.06521739130434782, - "time_ms": 7654.062032699585, - "idx": 183, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.19465817511081696, - "wer": 0.0625, - "time_ms": 5002.284049987793, - "idx": 185, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.24790827929973602, - "wer": 0.5106382978723404, - "time_ms": 8463.53268623352, - "idx": 192, - "text_len": 47, - "target_length": 1419, - "gen_length_sec": 15.095744680851064 - }, - { - "secs": 0.09377650171518326, - "wer": 0.0, - "time_ms": 2395.029306411743, - "idx": 193, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.12729203701019287, - "wer": 0.30434782608695654, - "time_ms": 3369.8155879974365, - "idx": 194, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.15840907394886017, - "wer": 0.11538461538461539, - "time_ms": 3754.0040016174316, - "idx": 195, - "text_len": 26, - "target_length": 827, - "gen_length_sec": 8.797872340425531 - }, - { - "secs": 0.4338621497154236, - "wer": 0.1, - "time_ms": 3070.026636123657, - "idx": 196, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.2934127151966095, - "wer": 0.08695652173913043, - "time_ms": 3397.212266921997, - "idx": 198, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.1338375061750412, - "wer": 0.10714285714285714, - "time_ms": 4290.986776351929, - "idx": 202, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.23157081007957458, - "wer": 0.14285714285714285, - "time_ms": 2252.8364658355713, - "idx": 203, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.37132886052131653, - "wer": 0.3333333333333333, - "time_ms": 2399.5165824890137, - "idx": 208, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.764169454574585, - "wer": 0.058823529411764705, - "time_ms": 2438.551425933838, - "idx": 209, - "text_len": 17, - "target_length": 573, - "gen_length_sec": 6.095744680851064 - }, - { - "secs": 0.7376022338867188, - "wer": 0.08333333333333333, - "time_ms": 2052.6139736175537, - "idx": 210, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.4489494264125824, - "wer": 0.1111111111111111, - "time_ms": 1652.20308303833, - "idx": 211, - "text_len": 9, - "target_length": 347, - "gen_length_sec": 3.6914893617021276 - }, - { - "secs": 0.6851053237915039, - "wer": 0.3333333333333333, - "time_ms": 1318.1090354919434, - "idx": 218, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7245178818702698, - "wer": 0.0, - "time_ms": 1306.3654899597168, - "idx": 219, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6577444076538086, - "wer": 0.4090909090909091, - "time_ms": 3423.032760620117, - "idx": 220, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.548701286315918, - "wer": 0.15, - "time_ms": 3063.3955001831055, - "idx": 222, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.6690694093704224, - "wer": 0.0625, - "time_ms": 4983.320951461792, - "idx": 223, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6921694278717041, - "wer": 1.0, - "time_ms": 1424.5154857635498, - "idx": 225, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6066182851791382, - "wer": 0.14814814814814814, - "time_ms": 4145.708084106445, - "idx": 228, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.36229878664016724, - "wer": 0.08, - "time_ms": 8689.23544883728, - "idx": 229, - "text_len": 50, - "target_length": 1504, - "gen_length_sec": 16.0 - }, - { - "secs": 0.4370690882205963, - "wer": 0.07407407407407407, - "time_ms": 4145.111322402954, - "idx": 231, - "text_len": 27, - "target_length": 855, - "gen_length_sec": 9.095744680851064 - }, - { - "secs": 0.4860302805900574, - "wer": 0.09302325581395349, - "time_ms": 7056.772470474243, - "idx": 233, - "text_len": 43, - "target_length": 1306, - "gen_length_sec": 13.893617021276595 - }, - { - "secs": 0.47836318612098694, - "wer": 0.09836065573770492, - "time_ms": 11454.752683639526, - "idx": 234, - "text_len": 61, - "target_length": 1814, - "gen_length_sec": 19.29787234042553 - }, - { - "secs": 0.44300973415374756, - "wer": 0.16, - "time_ms": 3669.7280406951904, - "idx": 237, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.8038159608840942, - "wer": 0.0, - "time_ms": 1331.425428390503, - "idx": 239, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.7148954272270203, - "wer": 0.0, - "time_ms": 1291.3603782653809, - "idx": 244, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.5969453454017639, - "wer": 0.5333333333333333, - "time_ms": 2478.9741039276123, - "idx": 246, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.17885661125183105, - "wer": 0.0, - "time_ms": 1297.7321147918701, - "idx": 247, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.31565073132514954, - "wer": 0.08888888888888889, - "time_ms": 7607.226610183716, - "idx": 248, - "text_len": 45, - "target_length": 1363, - "gen_length_sec": 14.5 - }, - { - "secs": 0.7311010360717773, - "wer": 0.0, - "time_ms": 1351.8123626708984, - "idx": 249, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.613842248916626, - "wer": 0.14285714285714285, - "time_ms": 1485.0823879241943, - "idx": 253, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.6031036972999573, - "wer": 0.0, - "time_ms": 1514.936923980713, - "idx": 255, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.1828315705060959, - "wer": 0.075, - "time_ms": 6450.896263122559, - "idx": 258, - "text_len": 40, - "target_length": 1222, - "gen_length_sec": 13.0 - }, - { - "secs": 0.2637174427509308, - "wer": 0.2, - "time_ms": 1343.9366817474365, - "idx": 261, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.31281134486198425, - "wer": 0.08695652173913043, - "time_ms": 3435.781240463257, - "idx": 265, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.22379423677921295, - "wer": 0.42857142857142855, - "time_ms": 2759.359836578369, - "idx": 266, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.1385502815246582, - "wer": 0.08695652173913043, - "time_ms": 3405.545473098755, - "idx": 268, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.20101889967918396, - "wer": 0.2222222222222222, - "time_ms": 2801.219940185547, - "idx": 271, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.4016900956630707, - "wer": 0.14285714285714285, - "time_ms": 3113.3487224578857, - "idx": 272, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.5871167182922363, - "wer": 0.4, - "time_ms": 1303.5345077514648, - "idx": 275, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.6848756074905396, - "wer": 0.0, - "time_ms": 1337.8572463989258, - "idx": 278, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.16756479442119598, - "wer": 0.08, - "time_ms": 3787.555932998657, - "idx": 281, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.07664526998996735, - "wer": 0.1, - "time_ms": 3261.121988296509, - "idx": 282, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.1147737205028534, - "wer": 0.4375, - "time_ms": 2481.9509983062744, - "idx": 284, - "text_len": 16, - "target_length": 545, - "gen_length_sec": 5.797872340425532 - }, - { - "secs": 0.406973272562027, - "wer": 0.125, - "time_ms": 1793.7898635864258, - "idx": 285, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.18896882236003876, - "wer": 0.0, - "time_ms": 3796.354293823242, - "idx": 287, - "text_len": 25, - "target_length": 799, - "gen_length_sec": 8.5 - }, - { - "secs": 0.6473802328109741, - "wer": 0.0, - "time_ms": 2307.3508739471436, - "idx": 289, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.2931229770183563, - "wer": 0.6666666666666666, - "time_ms": 1744.7121143341064, - "idx": 291, - "text_len": 6, - "target_length": 263, - "gen_length_sec": 2.797872340425532 - }, - { - "secs": 0.4076858162879944, - "wer": 0.14285714285714285, - "time_ms": 2504.640817642212, - "idx": 292, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.44004881381988525, - "wer": 0.2857142857142857, - "time_ms": 2201.396703720093, - "idx": 297, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.366830050945282, - "wer": 0.25, - "time_ms": 2120.577573776245, - "idx": 298, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.43535396456718445, - "wer": 0.2, - "time_ms": 1327.455997467041, - "idx": 300, - "text_len": 5, - "target_length": 235, - "gen_length_sec": 2.5 - }, - { - "secs": 0.20816709101200104, - "wer": 0.0, - "time_ms": 1353.907585144043, - "idx": 302, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.18571829795837402, - "wer": 0.125, - "time_ms": 5106.59646987915, - "idx": 305, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.08387017995119095, - "wer": 0.06451612903225806, - "time_ms": 11917.076587677002, - "idx": 310, - "text_len": 62, - "target_length": 1842, - "gen_length_sec": 19.595744680851062 - }, - { - "secs": 0.0007234811782836914, - "wer": 0.11428571428571428, - "time_ms": 5410.073280334473, - "idx": 314, - "text_len": 35, - "target_length": 1081, - "gen_length_sec": 11.5 - }, - { - "secs": 0.7381525635719299, - "wer": 0.0, - "time_ms": 1541.7752265930176, - "idx": 316, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.12382388114929199, - "wer": 0.043478260869565216, - "time_ms": 7465.759038925171, - "idx": 317, - "text_len": 46, - "target_length": 1391, - "gen_length_sec": 14.797872340425531 - }, - { - "secs": 0.524940013885498, - "wer": 0.0, - "time_ms": 1500.6439685821533, - "idx": 318, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - }, - { - "secs": 0.22775617241859436, - "wer": 0.16129032258064516, - "time_ms": 4748.284101486206, - "idx": 321, - "text_len": 31, - "target_length": 968, - "gen_length_sec": 10.297872340425531 - }, - { - "secs": 0.15064281225204468, - "wer": 0.1282051282051282, - "time_ms": 6211.411952972412, - "idx": 324, - "text_len": 39, - "target_length": 1193, - "gen_length_sec": 12.691489361702128 - }, - { - "secs": 0.6833600401878357, - "wer": 0.25, - "time_ms": 1555.429220199585, - "idx": 325, - "text_len": 8, - "target_length": 319, - "gen_length_sec": 3.393617021276596 - }, - { - "secs": 0.29853391647338867, - "wer": 0.05970149253731343, - "time_ms": 12875.869035720825, - "idx": 326, - "text_len": 67, - "target_length": 1983, - "gen_length_sec": 21.095744680851062 - }, - { - "secs": 0.26092249155044556, - "wer": 0.08695652173913043, - "time_ms": 3479.6180725097656, - "idx": 327, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.3394409716129303, - "wer": 0.18181818181818182, - "time_ms": 3257.7104568481445, - "idx": 329, - "text_len": 22, - "target_length": 714, - "gen_length_sec": 7.595744680851064 - }, - { - "secs": 0.1926424503326416, - "wer": 0.14814814814814814, - "time_ms": 9816.66898727417, - "idx": 331, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.16178825497627258, - "wer": 0.057971014492753624, - "time_ms": 13683.427333831787, - "idx": 334, - "text_len": 69, - "target_length": 2039, - "gen_length_sec": 21.69148936170213 - }, - { - "secs": 0.33846890926361084, - "wer": 0.11764705882352941, - "time_ms": 5164.300203323364, - "idx": 336, - "text_len": 34, - "target_length": 1052, - "gen_length_sec": 11.191489361702128 - }, - { - "secs": 0.01929490454494953, - "wer": 0.3076923076923077, - "time_ms": 2213.824510574341, - "idx": 337, - "text_len": 13, - "target_length": 460, - "gen_length_sec": 4.8936170212765955 - }, - { - "secs": 0.2971690893173218, - "wer": 0.08695652173913043, - "time_ms": 3426.1415004730225, - "idx": 338, - "text_len": 23, - "target_length": 742, - "gen_length_sec": 7.8936170212765955 - }, - { - "secs": 0.19867946207523346, - "wer": 0.045454545454545456, - "time_ms": 7103.1341552734375, - "idx": 339, - "text_len": 44, - "target_length": 1334, - "gen_length_sec": 14.191489361702128 - }, - { - "secs": 0.4274500906467438, - "wer": 0.5, - "time_ms": 1279.6971797943115, - "idx": 340, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.6359113454818726, - "wer": 0.25, - "time_ms": 1298.3441352844238, - "idx": 342, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.4309901297092438, - "wer": 0.5833333333333334, - "time_ms": 2116.772413253784, - "idx": 347, - "text_len": 12, - "target_length": 432, - "gen_length_sec": 4.595744680851064 - }, - { - "secs": 0.09649265557527542, - "wer": 0.06896551724137931, - "time_ms": 11177.406072616577, - "idx": 350, - "text_len": 58, - "target_length": 1729, - "gen_length_sec": 18.393617021276597 - }, - { - "secs": 0.15276426076889038, - "wer": 0.4, - "time_ms": 3091.815948486328, - "idx": 356, - "text_len": 20, - "target_length": 658, - "gen_length_sec": 7.0 - }, - { - "secs": 0.1309194713830948, - "wer": 0.08333333333333333, - "time_ms": 11472.678661346436, - "idx": 357, - "text_len": 60, - "target_length": 1786, - "gen_length_sec": 19.0 - }, - { - "secs": 0.29353561997413635, - "wer": 0.07142857142857142, - "time_ms": 4181.737422943115, - "idx": 358, - "text_len": 28, - "target_length": 883, - "gen_length_sec": 9.393617021276595 - }, - { - "secs": 0.2934585213661194, - "wer": 0.14285714285714285, - "time_ms": 3198.6145973205566, - "idx": 360, - "text_len": 21, - "target_length": 686, - "gen_length_sec": 7.297872340425532 - }, - { - "secs": 0.28012675046920776, - "wer": 0.1388888888888889, - "time_ms": 5698.044538497925, - "idx": 361, - "text_len": 36, - "target_length": 1109, - "gen_length_sec": 11.797872340425531 - }, - { - "secs": 0.2672085464000702, - "wer": 0.4166666666666667, - "time_ms": 3679.0292263031006, - "idx": 362, - "text_len": 24, - "target_length": 770, - "gen_length_sec": 8.191489361702128 - }, - { - "secs": 0.11791912466287613, - "wer": 0.06666666666666667, - "time_ms": 4613.793611526489, - "idx": 363, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.13707734644412994, - "wer": 0.12962962962962962, - "time_ms": 9697.57628440857, - "idx": 364, - "text_len": 54, - "target_length": 1616, - "gen_length_sec": 17.19148936170213 - }, - { - "secs": 0.1518184244632721, - "wer": 0.03636363636363636, - "time_ms": 9779.544115066528, - "idx": 365, - "text_len": 55, - "target_length": 1645, - "gen_length_sec": 17.5 - }, - { - "secs": 0.608623206615448, - "wer": 0.06666666666666667, - "time_ms": 4675.122976303101, - "idx": 367, - "text_len": 30, - "target_length": 940, - "gen_length_sec": 10.0 - }, - { - "secs": 0.7678830623626709, - "wer": 0.0, - "time_ms": 1281.3289165496826, - "idx": 368, - "text_len": 4, - "target_length": 206, - "gen_length_sec": 2.1914893617021276 - }, - { - "secs": 0.16211743652820587, - "wer": 0.14285714285714285, - "time_ms": 2279.064416885376, - "idx": 369, - "text_len": 14, - "target_length": 488, - "gen_length_sec": 5.191489361702128 - }, - { - "secs": 0.688791036605835, - "wer": 0.2, - "time_ms": 2426.374673843384, - "idx": 373, - "text_len": 15, - "target_length": 517, - "gen_length_sec": 5.5 - }, - { - "secs": 0.23875176906585693, - "wer": 0.125, - "time_ms": 5063.087224960327, - "idx": 374, - "text_len": 32, - "target_length": 996, - "gen_length_sec": 10.595744680851064 - }, - { - "secs": 0.6613235473632812, - "wer": 0.21052631578947367, - "time_ms": 2809.5321655273438, - "idx": 375, - "text_len": 19, - "target_length": 629, - "gen_length_sec": 6.691489361702128 - }, - { - "secs": 0.47044646739959717, - "wer": 0.0, - "time_ms": 1284.555196762085, - "idx": 378, - "text_len": 1, - "target_length": 122, - "gen_length_sec": 1.297872340425532 - }, - { - "secs": 0.7719053030014038, - "wer": 0.5, - "time_ms": 1312.8151893615723, - "idx": 381, - "text_len": 2, - "target_length": 150, - "gen_length_sec": 1.5957446808510638 - }, - { - "secs": 0.6740312576293945, - "wer": 0.05555555555555555, - "time_ms": 2837.1095657348633, - "idx": 382, - "text_len": 18, - "target_length": 601, - "gen_length_sec": 6.3936170212765955 - }, - { - "secs": 0.6211265325546265, - "wer": 0.15151515151515152, - "time_ms": 5058.014154434204, - "idx": 383, - "text_len": 33, - "target_length": 1024, - "gen_length_sec": 10.893617021276595 - }, - { - "secs": 0.6311425566673279, - "wer": 0.14285714285714285, - "time_ms": 1639.5142078399658, - "idx": 385, - "text_len": 7, - "target_length": 291, - "gen_length_sec": 3.095744680851064 - } - ] -} \ No newline at end of file diff --git a/results/eval_200_samples.json b/results/eval_200_samples.json deleted file mode 100644 index 3ac9a1b..0000000 --- a/results/eval_200_samples.json +++ /dev/null @@ -1,2236 +0,0 @@ -{ - "metadata": { - "n_samples": 200, - "seed": 42, - "timestamp": "2026-06-25 13:02:55", - "version": "v10.0", - "personavoice_config": { - "steps": 32, - "cfg_strength": 2.5, - "use_oes": true, - "env_weight": 0.0, - "oes_env_scale_init": 0.1, - "use_ceag": false, - "ceag_lambda_max": 0.2, - "ceag_t_start": 0.1, - "ceag_t_end": 0.4, - "ceag_layers": [ - -2, - -1 - ], - "use_duration_predictor": false, - "use_laag": true, - "seconds_per_char": 0.3, - "sway_sampling_coef": -1.0, - "removed_modules": [ - "IBOP", - "AM-ODE", - "TD-CFG", - "SBM", - "GRPO", - "Duration Predictor" - ] - } - }, - "baseline_results": [], - "personavoice_results": [ - { - "secs": 0.27107763290405273, - "wer": 1.0, - "time_ms": 2265.24019241333, - "idx": 0, - "text": "A 'JOLLY' ART CRITIC", - "text_len": 4, - "target_length": 155, - "gen_length_sec": 1.648936170212766, - "speaker_name": "1272" - }, - { - "secs": 0.2732350826263428, - "wer": 0.05, - "time_ms": 4352.713346481323, - "idx": 3, - "text": "He rollicks through art with the recklessness of the tourist and describes its b", - "text_len": 20, - "target_length": 781, - "gen_length_sec": 8.308510638297872, - "speaker_name": "1272" - }, - { - "secs": 0.41273030638694763, - "wer": 0.0, - "time_ms": 6451.563835144043, - "idx": 5, - "text": "After listening so long to the Don Quixote of art, to listen once to Sancho Panz", - "text_len": 21, - "target_length": 1178, - "gen_length_sec": 12.53191489361702, - "speaker_name": "1272" - }, - { - "secs": 0.2974545359611511, - "wer": 0.13333333333333333, - "time_ms": 5326.040029525757, - "idx": 6, - "text": "About artists and their work mr Quilter has, of course, a great deal to say.", - "text_len": 15, - "target_length": 1009, - "gen_length_sec": 10.73404255319149, - "speaker_name": "1272" - }, - { - "secs": 0.43833810091018677, - "wer": 0.14634146341463414, - "time_ms": 9884.914636611938, - "idx": 7, - "text": "Sculpture he regards as 'Painting's poor relation'; so, with the exception of a ", - "text_len": 41, - "target_length": 1846, - "gen_length_sec": 19.638297872340427, - "speaker_name": "1272" - }, - { - "secs": 0.11021759361028671, - "wer": 0.21052631578947367, - "time_ms": 2687.7262592315674, - "idx": 9, - "text": "Painting, he tells us, is 'of a different quality to mathematics,' and finish in", - "text_len": 19, - "target_length": 497, - "gen_length_sec": 5.287234042553192, - "speaker_name": "1272" - }, - { - "secs": 0.39776700735092163, - "wer": 0.0196078431372549, - "time_ms": 8804.027795791626, - "idx": 10, - "text": "Portrait painting is a bad pursuit for an emotional artist as it destroys his pe", - "text_len": 51, - "target_length": 1606, - "gen_length_sec": 17.085106382978722, - "speaker_name": "1272" - }, - { - "secs": 0.32744014263153076, - "wer": 0.3, - "time_ms": 3958.374500274658, - "idx": 11, - "text": "As for etchings, they are of two kinds-British and foreign.", - "text_len": 10, - "target_length": 783, - "gen_length_sec": 8.329787234042554, - "speaker_name": "1272" - }, - { - "secs": 0.2897970974445343, - "wer": 0.0, - "time_ms": 2035.1390838623047, - "idx": 15, - "text": "On the whole, the book will not do.", - "text_len": 8, - "target_length": 361, - "gen_length_sec": 3.8404255319148937, - "speaker_name": "1272" - }, - { - "secs": 0.2673601508140564, - "wer": 0.08450704225352113, - "time_ms": 16322.355031967163, - "idx": 16, - "text": "We fully admit that it is extremely amusing and, no doubt, mr Quilter is quite e", - "text_len": 71, - "target_length": 3165, - "gen_length_sec": 33.670212765957444, - "speaker_name": "1272" - }, - { - "secs": 0.3027217984199524, - "wer": 0.2727272727272727, - "time_ms": 1969.1579341888428, - "idx": 17, - "text": "Sententiae: Artis: First Principles of Art for Painters and Picture Lovers.", - "text_len": 11, - "target_length": 348, - "gen_length_sec": 3.702127659574468, - "speaker_name": "1272" - }, - { - "secs": 0.5485738515853882, - "wer": 0.5, - "time_ms": 1541.9366359710693, - "idx": 18, - "text": "By Harry Quilter, m a (Isbister.)", - "text_len": 6, - "target_length": 180, - "gen_length_sec": 1.9148936170212767, - "speaker_name": "1272" - }, - { - "secs": -0.03095334768295288, - "wer": 1.0, - "time_ms": 1499.0112781524658, - "idx": 19, - "text": "King Kaliko", - "text_len": 2, - "target_length": 85, - "gen_length_sec": 0.9042553191489362, - "speaker_name": "1272" - }, - { - "secs": 0.6019306182861328, - "wer": 0.75, - "time_ms": 1507.4043273925781, - "idx": 22, - "text": "Mainhall liked Alexander because he was an engineer.", - "text_len": 8, - "target_length": 156, - "gen_length_sec": 1.6595744680851063, - "speaker_name": "1462" - }, - { - "secs": 0.6986300945281982, - "wer": 0.5789473684210527, - "time_ms": 9418.711423873901, - "idx": 23, - "text": "He had preconceived ideas about everything, and his idea about Americans was tha", - "text_len": 19, - "target_length": 1594, - "gen_length_sec": 16.95744680851064, - "speaker_name": "1462" - }, - { - "secs": 0.42865651845932007, - "wer": 0.0, - "time_ms": 1546.971082687378, - "idx": 24, - "text": "He hated them when they presumed to be anything else.", - "text_len": 10, - "target_length": 176, - "gen_length_sec": 1.872340425531915, - "speaker_name": "1462" - }, - { - "secs": 0.5925960540771484, - "wer": 0.0, - "time_ms": 1526.1285305023193, - "idx": 25, - "text": "\"It's tremendously well put on, too.", - "text_len": 6, - "target_length": 152, - "gen_length_sec": 1.6170212765957446, - "speaker_name": "1462" - }, - { - "secs": 0.37530001997947693, - "wer": 0.42857142857142855, - "time_ms": 1531.2516689300537, - "idx": 30, - "text": "\"Why, I haven't heard of her for-years.\"", - "text_len": 7, - "target_length": 148, - "gen_length_sec": 1.574468085106383, - "speaker_name": "1462" - }, - { - "secs": 0.5406363010406494, - "wer": 1.0, - "time_ms": 1530.2081108093262, - "idx": 31, - "text": "Mainhall laughed.", - "text_len": 2, - "target_length": 158, - "gen_length_sec": 1.6808510638297873, - "speaker_name": "1462" - }, - { - "secs": 0.6329270005226135, - "wer": 0.29411764705882354, - "time_ms": 2215.72208404541, - "idx": 33, - "text": "It's only lately, since MacConnell and his set have got hold of her, that she's ", - "text_len": 17, - "target_length": 372, - "gen_length_sec": 3.9574468085106385, - "speaker_name": "1462" - }, - { - "secs": 0.0, - "wer": 1.0, - "time_ms": 0.0, - "error": "The size of tensor a (69) must match the size of tensor b (100) at non-singleton dimension 1", - "target_length": 0, - "text_len": 34, - "gen_length_sec": 0.0 - }, - { - "secs": 0.3057604432106018, - "wer": 0.045454545454545456, - "time_ms": 4645.420074462891, - "idx": 37, - "text": "Just then they drove up to the Duke of York's, so Alexander did not commit himse", - "text_len": 22, - "target_length": 817, - "gen_length_sec": 8.691489361702128, - "speaker_name": "1462" - }, - { - "secs": 0.6450440287590027, - "wer": 0.06896551724137931, - "time_ms": 5372.473955154419, - "idx": 38, - "text": "When they entered the stage box on the left the first act was well under way, th", - "text_len": 29, - "target_length": 922, - "gen_length_sec": 9.808510638297872, - "speaker_name": "1462" - }, - { - "secs": 0.17909374833106995, - "wer": 0.0, - "time_ms": 2770.8635330200195, - "idx": 39, - "text": "As they sat down, a burst of applause drew Alexander's attention to the stage.", - "text_len": 14, - "target_length": 518, - "gen_length_sec": 5.51063829787234, - "speaker_name": "1462" - }, - { - "secs": 0.7853341102600098, - "wer": 0.058823529411764705, - "time_ms": 5659.383058547974, - "idx": 42, - "text": "In the age of religious freedom, which was determined by the council of Nice, th", - "text_len": 34, - "target_length": 1096, - "gen_length_sec": 11.659574468085106, - "speaker_name": "1673" - }, - { - "secs": 0.650704026222229, - "wer": 1.0, - "time_ms": 32755.741834640503, - "idx": 45, - "text": "The worthy friend of Athanasius, the worthy antagonist of Julian, he bravely wre", - "text_len": 38, - "target_length": 5923, - "gen_length_sec": 63.01063829787234, - "speaker_name": "1673" - }, - { - "secs": 0.6490217447280884, - "wer": 0.061224489795918366, - "time_ms": 9722.32460975647, - "idx": 46, - "text": "A mystery, which had long floated in the looseness of popular belief, was define", - "text_len": 49, - "target_length": 1818, - "gen_length_sec": 19.340425531914892, - "speaker_name": "1673" - }, - { - "secs": 0.3188856840133667, - "wer": 0.14814814814814814, - "time_ms": 4250.643253326416, - "idx": 55, - "text": "As soon as they beheld the twilight of sense and heresy, they started, measured ", - "text_len": 27, - "target_length": 729, - "gen_length_sec": 7.75531914893617, - "speaker_name": "1673" - }, - { - "secs": 0.6123562455177307, - "wer": 0.0, - "time_ms": 7124.140739440918, - "idx": 56, - "text": "To purge themselves from the guilt or reproach of damnable error, they disavowed", - "text_len": 30, - "target_length": 1420, - "gen_length_sec": 15.106382978723405, - "speaker_name": "1673" - }, - { - "secs": 0.7128543257713318, - "wer": 0.0, - "time_ms": 7156.467437744141, - "idx": 57, - "text": "In the house of his uncle, the archbishop Theophilus, he imbibed the orthodox le", - "text_len": 33, - "target_length": 1380, - "gen_length_sec": 14.680851063829786, - "speaker_name": "1673" - }, - { - "secs": 0.5359652042388916, - "wer": 0.0, - "time_ms": 3385.110378265381, - "idx": 59, - "text": "With the approbation of his uncle, he assumed the office, and acquired the fame,", - "text_len": 18, - "target_length": 684, - "gen_length_sec": 7.276595744680851, - "speaker_name": "1673" - }, - { - "secs": 0.5256399512290955, - "wer": 1.0, - "time_ms": 2909.259080886841, - "idx": 60, - "text": "CHAPTER three-TWO MISFORTUNES MAKE ONE PIECE OF GOOD FORTUNE", - "text_len": 9, - "target_length": 558, - "gen_length_sec": 5.9361702127659575, - "speaker_name": "174" - }, - { - "secs": 0.3750353157520294, - "wer": 0.20833333333333334, - "time_ms": 8557.50560760498, - "idx": 63, - "text": "When he saw Cosette, when he had taken possession of her, carried her off, and d", - "text_len": 24, - "target_length": 1700, - "gen_length_sec": 18.085106382978722, - "speaker_name": "174" - }, - { - "secs": 0.37898218631744385, - "wer": 0.08823529411764706, - "time_ms": 4918.002128601074, - "idx": 66, - "text": "Only, as he was five and fifty, and Cosette eight years of age, all that might h", - "text_len": 34, - "target_length": 874, - "gen_length_sec": 9.297872340425531, - "speaker_name": "174" - }, - { - "secs": 0.5018507242202759, - "wer": 0.0, - "time_ms": 6290.781259536743, - "idx": 67, - "text": "The Bishop had caused the dawn of virtue to rise on his horizon; Cosette caused ", - "text_len": 21, - "target_length": 1220, - "gen_length_sec": 12.97872340425532, - "speaker_name": "174" - }, - { - "secs": 0.6391394138336182, - "wer": 0.5, - "time_ms": 2944.420337677002, - "idx": 68, - "text": "The early days passed in this dazzled state.", - "text_len": 8, - "target_length": 584, - "gen_length_sec": 6.212765957446808, - "speaker_name": "174" - }, - { - "secs": 0.1747751384973526, - "wer": 0.06666666666666667, - "time_ms": 4196.892499923706, - "idx": 69, - "text": "Cosette, on her side, had also, unknown to herself, become another being, poor l", - "text_len": 15, - "target_length": 855, - "gen_length_sec": 9.095744680851064, - "speaker_name": "174" - }, - { - "secs": 0.5763987302780151, - "wer": 0.14285714285714285, - "time_ms": 4726.497650146484, - "idx": 72, - "text": "She had loved the dog, and he had died, after which nothing and nobody would hav", - "text_len": 21, - "target_length": 985, - "gen_length_sec": 10.47872340425532, - "speaker_name": "174" - }, - { - "secs": 0.5659634470939636, - "wer": 0.0, - "time_ms": 3175.6558418273926, - "idx": 73, - "text": "It is a sad thing to say, and we have already intimated it, that, at eight years", - "text_len": 23, - "target_length": 627, - "gen_length_sec": 6.670212765957447, - "speaker_name": "174" - }, - { - "secs": 0.2650850713253021, - "wer": 0.11538461538461539, - "time_ms": 6202.64196395874, - "idx": 75, - "text": "The man no longer produced on her the effect of being old or poor; she thought J", - "text_len": 26, - "target_length": 1151, - "gen_length_sec": 12.24468085106383, - "speaker_name": "174" - }, - { - "secs": 0.3462061882019043, - "wer": 0.0, - "time_ms": 2095.13258934021, - "idx": 76, - "text": "The novelty of the earth and of life counts for something here.", - "text_len": 12, - "target_length": 390, - "gen_length_sec": 4.148936170212766, - "speaker_name": "174" - }, - { - "secs": 0.5651621222496033, - "wer": 0.07692307692307693, - "time_ms": 2765.2158737182617, - "idx": 77, - "text": "Nothing is so charming as the coloring reflection of happiness on a garret.", - "text_len": 13, - "target_length": 536, - "gen_length_sec": 5.702127659574468, - "speaker_name": "174" - }, - { - "secs": 0.3796137869358063, - "wer": 0.0, - "time_ms": 2927.9353618621826, - "idx": 78, - "text": "We all have in our past a delightful garret.", - "text_len": 9, - "target_length": 584, - "gen_length_sec": 6.212765957446808, - "speaker_name": "174" - }, - { - "secs": 0.5936663150787354, - "wer": 0.19047619047619047, - "time_ms": 3424.393653869629, - "idx": 79, - "text": "Nature, a difference of fifty years, had set a profound gulf between Jean Valjea", - "text_len": 21, - "target_length": 714, - "gen_length_sec": 7.595744680851064, - "speaker_name": "174" - }, - { - "secs": 0.2171430140733719, - "wer": 0.07692307692307693, - "time_ms": 4505.856037139893, - "idx": 81, - "text": "The next day, drain them well for six hours, put them into a jar, pour boiling v", - "text_len": 26, - "target_length": 915, - "gen_length_sec": 9.73404255319149, - "speaker_name": "1919" - }, - { - "secs": 0.6449095010757446, - "wer": 0.09090909090909091, - "time_ms": 8797.14822769165, - "idx": 82, - "text": "In a short time, boil up the vinegar again, add pepper and ginger in the above p", - "text_len": 22, - "target_length": 1581, - "gen_length_sec": 16.819148936170212, - "speaker_name": "1919" - }, - { - "secs": 0.6905227899551392, - "wer": 0.3333333333333333, - "time_ms": 1646.3649272918701, - "idx": 84, - "text": "[Illustration: LONG PEPPER.]", - "text_len": 3, - "target_length": 173, - "gen_length_sec": 1.8404255319148937, - "speaker_name": "1919" - }, - { - "secs": 0.6383596062660217, - "wer": 0.1, - "time_ms": 2007.967472076416, - "idx": 86, - "text": "The romans indulged in them to a most extravagant degree.", - "text_len": 10, - "target_length": 311, - "gen_length_sec": 3.3085106382978724, - "speaker_name": "1919" - }, - { - "secs": 0.11642695218324661, - "wer": 1.0, - "time_ms": 1656.4273834228516, - "idx": 89, - "text": "four hundred.", - "text_len": 2, - "target_length": 201, - "gen_length_sec": 2.1382978723404253, - "speaker_name": "1919" - }, - { - "secs": 0.7020225524902344, - "wer": 0.0, - "time_ms": 10593.189239501953, - "idx": 90, - "text": "Let it stand four or five days, boil it all up, and when cold, strain the liquor", - "text_len": 31, - "target_length": 2031, - "gen_length_sec": 21.606382978723403, - "speaker_name": "1919" - }, - { - "secs": 0.44212764501571655, - "wer": 0.0625, - "time_ms": 2882.709264755249, - "idx": 92, - "text": "Now wash them well in fresh water, and dress as usual with pepper, vinegar, and ", - "text_len": 16, - "target_length": 558, - "gen_length_sec": 5.9361702127659575, - "speaker_name": "1919" - }, - { - "secs": 0.6264216303825378, - "wer": 1.0, - "time_ms": 1638.0164623260498, - "idx": 93, - "text": "[Illustration: THE CUCUMBER.]", - "text_len": 3, - "target_length": 179, - "gen_length_sec": 1.9042553191489362, - "speaker_name": "1919" - }, - { - "secs": 0.16016967594623566, - "wer": 0.0, - "time_ms": 7840.317964553833, - "idx": 94, - "text": "If they are not a fine green, change the water again, cover them as before, and ", - "text_len": 19, - "target_length": 1457, - "gen_length_sec": 15.5, - "speaker_name": "1919" - }, - { - "secs": 0.6174471974372864, - "wer": 0.0, - "time_ms": 2123.7735748291016, - "idx": 96, - "text": "Let them remain for two days, changing the water twice each day, to draw out the", - "text_len": 17, - "target_length": 399, - "gen_length_sec": 4.24468085106383, - "speaker_name": "1919" - }, - { - "secs": 0.5926114320755005, - "wer": 0.0, - "time_ms": 9582.210540771484, - "idx": 97, - "text": "Boil the syrup once in two or three days for three weeks; strengthen it if requi", - "text_len": 28, - "target_length": 1859, - "gen_length_sec": 19.77659574468085, - "speaker_name": "1919" - }, - { - "secs": 0.43996015191078186, - "wer": 0.3448275862068966, - "time_ms": 10990.026235580444, - "idx": 101, - "text": "We were taking them some provisions, as they had come to live on a wild place wh", - "text_len": 29, - "target_length": 1965, - "gen_length_sec": 20.904255319148938, - "speaker_name": "1988" - }, - { - "secs": 0.6575297117233276, - "wer": 0.0, - "time_ms": 7482.638120651245, - "idx": 103, - "text": "I could hardly wait to see what lay beyond that cornfield; but there was only re", - "text_len": 35, - "target_length": 1490, - "gen_length_sec": 15.851063829787234, - "speaker_name": "1988" - }, - { - "secs": 0.5792771577835083, - "wer": 0.0, - "time_ms": 4519.529104232788, - "idx": 104, - "text": "The road ran about like a wild thing, avoiding the deep draws, crossing them whe", - "text_len": 20, - "target_length": 930, - "gen_length_sec": 9.893617021276595, - "speaker_name": "1988" - }, - { - "secs": 0.38278064131736755, - "wer": 0.0, - "time_ms": 2275.0556468963623, - "idx": 108, - "text": "They could not speak enough English to ask for advice, or even to make their mos", - "text_len": 19, - "target_length": 478, - "gen_length_sec": 5.085106382978723, - "speaker_name": "1988" - }, - { - "secs": 0.6384974718093872, - "wer": 0.07692307692307693, - "time_ms": 4156.432628631592, - "idx": 109, - "text": "One son, Fuchs said, was well grown, and strong enough to work the land; but the", - "text_len": 26, - "target_length": 729, - "gen_length_sec": 7.75531914893617, - "speaker_name": "1988" - }, - { - "secs": 0.6919047236442566, - "wer": 0.05, - "time_ms": 5570.333480834961, - "idx": 110, - "text": "\"If they're nice people, I hate to think of them spending the winter in that cav", - "text_len": 20, - "target_length": 1060, - "gen_length_sec": 11.27659574468085, - "speaker_name": "1988" - }, - { - "secs": 0.33602800965309143, - "wer": 0.16666666666666666, - "time_ms": 1744.89426612854, - "idx": 111, - "text": "\"It's no better than a badger hole; no proper dugout at all.", - "text_len": 12, - "target_length": 253, - "gen_length_sec": 2.6914893617021276, - "speaker_name": "1988" - }, - { - "secs": 0.5260140299797058, - "wer": 0.3333333333333333, - "time_ms": 2126.4517307281494, - "idx": 113, - "text": "Grandmother looked interested.", - "text_len": 3, - "target_length": 398, - "gen_length_sec": 4.23404255319149, - "speaker_name": "1988" - }, - { - "secs": 0.6505724191665649, - "wer": 0.2, - "time_ms": 2280.6968688964844, - "idx": 114, - "text": "\"Now, why is that, Otto?\"", - "text_len": 5, - "target_length": 465, - "gen_length_sec": 4.946808510638298, - "speaker_name": "1988" - }, - { - "secs": 0.6435026526451111, - "wer": 0.0, - "time_ms": 1729.661226272583, - "idx": 116, - "text": "It would take me a long while to explain.\"", - "text_len": 9, - "target_length": 244, - "gen_length_sec": 2.595744680851064, - "speaker_name": "1988" - }, - { - "secs": 0.5484553575515747, - "wer": 0.09090909090909091, - "time_ms": 6496.44660949707, - "idx": 118, - "text": "Presently, against one of those banks, I saw a sort of shed, thatched with the s", - "text_len": 22, - "target_length": 1281, - "gen_length_sec": 13.627659574468085, - "speaker_name": "1988" - }, - { - "secs": 0.7567923069000244, - "wer": 0.0, - "time_ms": 3224.262237548828, - "idx": 119, - "text": "We drove up to this skeleton to tie our horses, and then I saw a door and window", - "text_len": 24, - "target_length": 669, - "gen_length_sec": 7.117021276595745, - "speaker_name": "1988" - }, - { - "secs": 0.6442071795463562, - "wer": 0.058823529411764705, - "time_ms": 4976.388454437256, - "idx": 122, - "text": "So food for four mouths was all she was now required to find; only for three she", - "text_len": 34, - "target_length": 948, - "gen_length_sec": 10.085106382978724, - "speaker_name": "1993" - }, - { - "secs": 0.6256280541419983, - "wer": 0.0, - "time_ms": 3220.858335494995, - "idx": 123, - "text": "Her plan of living was so far arranged, when she heard, with keen sorrow, that W", - "text_len": 23, - "target_length": 656, - "gen_length_sec": 6.9787234042553195, - "speaker_name": "1993" - }, - { - "secs": 0.728340208530426, - "wer": 0.0, - "time_ms": 2946.8820095062256, - "idx": 124, - "text": "They were like many a pair of twins, and seemed to have but one life divided bet", - "text_len": 18, - "target_length": 597, - "gen_length_sec": 6.351063829787234, - "speaker_name": "1993" - }, - { - "secs": 0.46935364603996277, - "wer": 0.10204081632653061, - "time_ms": 19119.640827178955, - "idx": 125, - "text": "But when the twins, after ailing many days, and caring little for their meat, fe", - "text_len": 49, - "target_length": 3597, - "gen_length_sec": 38.265957446808514, - "speaker_name": "1993" - }, - { - "secs": 0.642058253288269, - "wer": 0.047619047619047616, - "time_ms": 3237.8156185150146, - "idx": 126, - "text": "Over the child, which yet breathed, the father bent, watching anxiously for some", - "text_len": 21, - "target_length": 672, - "gen_length_sec": 7.148936170212766, - "speaker_name": "1993" - }, - { - "secs": 0.7479057312011719, - "wer": 0.0, - "time_ms": 1656.7089557647705, - "idx": 129, - "text": "He cannot die while she's wishing him.\"", - "text_len": 7, - "target_length": 226, - "gen_length_sec": 2.404255319148936, - "speaker_name": "1993" - }, - { - "secs": 0.4603855013847351, - "wer": 0.0, - "time_ms": 8503.385782241821, - "idx": 131, - "text": "She bent down, and fondly, oh! with what passionate fondness, kissed her child, ", - "text_len": 26, - "target_length": 1612, - "gen_length_sec": 17.148936170212767, - "speaker_name": "1993" - }, - { - "secs": 0.7353982925415039, - "wer": 0.0, - "time_ms": 3232.5961589813232, - "idx": 132, - "text": "Nature's struggles were soon exhausted, and he breathed his little life away in ", - "text_len": 14, - "target_length": 666, - "gen_length_sec": 7.085106382978723, - "speaker_name": "1993" - }, - { - "secs": 0.638843834400177, - "wer": 0.0, - "time_ms": 1620.018482208252, - "idx": 137, - "text": "\"They're both gone,\" said she.", - "text_len": 5, - "target_length": 139, - "gen_length_sec": 1.4787234042553192, - "speaker_name": "1993" - }, - { - "secs": 0.5468279719352722, - "wer": 0.6666666666666666, - "time_ms": 1690.166711807251, - "idx": 139, - "text": "They took worse about two o'clock.", - "text_len": 6, - "target_length": 112, - "gen_length_sec": 1.1914893617021276, - "speaker_name": "1993" - }, - { - "secs": 0.72887122631073, - "wer": 0.0, - "time_ms": 5719.5563316345215, - "idx": 140, - "text": "She was four years older than I, to be sure, and had seen more of the world; but", - "text_len": 33, - "target_length": 1051, - "gen_length_sec": 11.180851063829786, - "speaker_name": "2035" - }, - { - "secs": 0.2292698323726654, - "wer": 0.0, - "time_ms": 2114.2749786376953, - "idx": 141, - "text": "I offered to take her on the pony, and she got up behind me.", - "text_len": 14, - "target_length": 398, - "gen_length_sec": 4.23404255319149, - "speaker_name": "2035" - }, - { - "secs": 0.6581439971923828, - "wer": 0.08695652173913043, - "time_ms": 6135.331630706787, - "idx": 143, - "text": "Within a week all the blooming roads had been despoiled-hundreds of miles of yel", - "text_len": 23, - "target_length": 1058, - "gen_length_sec": 11.25531914893617, - "speaker_name": "2035" - }, - { - "secs": 0.38071417808532715, - "wer": 0.08333333333333333, - "time_ms": 2246.94561958313, - "idx": 144, - "text": "As we rode away with the spade, Antonia suggested that we stop at the prairie do", - "text_len": 24, - "target_length": 453, - "gen_length_sec": 4.819148936170213, - "speaker_name": "2035" - }, - { - "secs": 0.46251440048217773, - "wer": 0.45454545454545453, - "time_ms": 4180.472850799561, - "idx": 145, - "text": "We might get some puppies, or owl eggs, or snake skins.", - "text_len": 11, - "target_length": 852, - "gen_length_sec": 9.063829787234043, - "speaker_name": "2035" - }, - { - "secs": 0.6779918074607849, - "wer": 0.1, - "time_ms": 2014.2149925231934, - "idx": 146, - "text": "The dog town was spread out over perhaps ten acres.", - "text_len": 10, - "target_length": 316, - "gen_length_sec": 3.3617021276595747, - "speaker_name": "2035" - }, - { - "secs": 0.515079915523529, - "wer": 0.0, - "time_ms": 5046.637296676636, - "idx": 147, - "text": "The holes were several yards apart, and were disposed with a good deal of regula", - "text_len": 28, - "target_length": 968, - "gen_length_sec": 10.297872340425531, - "speaker_name": "2035" - }, - { - "secs": 0.7569690942764282, - "wer": 0.047619047619047616, - "time_ms": 6286.0212326049805, - "idx": 148, - "text": "The dogs were out, as usual, dozens of them, sitting up on their hind legs over ", - "text_len": 21, - "target_length": 1232, - "gen_length_sec": 13.106382978723405, - "speaker_name": "2035" - }, - { - "secs": 0.5576000213623047, - "wer": 0.041666666666666664, - "time_ms": 5628.365755081177, - "idx": 150, - "text": "Before the mouths of the holes were little patches of sand and gravel, scratched", - "text_len": 24, - "target_length": 1116, - "gen_length_sec": 11.872340425531915, - "speaker_name": "2035" - }, - { - "secs": 0.6705622673034668, - "wer": 0.0, - "time_ms": 2052.1652698516846, - "idx": 152, - "text": "I was walking backward, in a crouching position, when I heard Antonia scream.", - "text_len": 13, - "target_length": 341, - "gen_length_sec": 3.627659574468085, - "speaker_name": "2035" - }, - { - "secs": 0.5325301289558411, - "wer": 0.0, - "time_ms": 3934.3838691711426, - "idx": 153, - "text": "When I turned he was lying in long loose waves, like a letter \"W.\" He twitched a", - "text_len": 21, - "target_length": 744, - "gen_length_sec": 7.914893617021277, - "speaker_name": "2035" - }, - { - "secs": 0.7742722630500793, - "wer": 1.0, - "time_ms": 10734.864950180054, - "idx": 155, - "text": "I did n't run because I did n't think of it-if my back had been against a stone ", - "text_len": 26, - "target_length": 1875, - "gen_length_sec": 19.9468085106383, - "speaker_name": "2035" - }, - { - "secs": 0.7050472497940063, - "wer": 0.0, - "time_ms": 4822.720289230347, - "idx": 157, - "text": "I ran up and drove at his head with my spade, struck him fairly across the neck,", - "text_len": 30, - "target_length": 882, - "gen_length_sec": 9.382978723404255, - "speaker_name": "2035" - }, - { - "secs": 0.6658248901367188, - "wer": 0.0, - "time_ms": 1605.2513122558594, - "idx": 158, - "text": "I struck now from hate.", - "text_len": 5, - "target_length": 142, - "gen_length_sec": 1.5106382978723405, - "speaker_name": "2035" - }, - { - "secs": 0.8217276334762573, - "wer": 1.0, - "time_ms": 2078.468084335327, - "idx": 163, - "text": "seventeen seventeen.", - "text_len": 2, - "target_length": 372, - "gen_length_sec": 3.9574468085106385, - "speaker_name": "2078" - }, - { - "secs": 0.4581278860569, - "wer": 0.6666666666666666, - "time_ms": 1649.794578552246, - "idx": 165, - "text": "[Illustration: TIN BREAD.]", - "text_len": 3, - "target_length": 186, - "gen_length_sec": 1.9787234042553192, - "speaker_name": "2078" - }, - { - "secs": 0.39522784948349, - "wer": 0.0, - "time_ms": 6583.547592163086, - "idx": 167, - "text": "Strew plenty of flour on the top; throw a thick clean cloth over, and set it whe", - "text_len": 38, - "target_length": 1309, - "gen_length_sec": 13.925531914893616, - "speaker_name": "2078" - }, - { - "secs": 0.6023036241531372, - "wer": 0.021739130434782608, - "time_ms": 5072.61061668396, - "idx": 168, - "text": "Look at it from time to time: when it has been laid for nearly an hour, and when", - "text_len": 46, - "target_length": 989, - "gen_length_sec": 10.52127659574468, - "speaker_name": "2078" - }, - { - "secs": 0.6614422798156738, - "wer": 1.0, - "time_ms": 1668.9190864562988, - "idx": 172, - "text": "[Illustration: ITALIAN MILLET.]", - "text_len": 3, - "target_length": 205, - "gen_length_sec": 2.1808510638297873, - "speaker_name": "2078" - }, - { - "secs": 0.14065054059028625, - "wer": 1.0, - "time_ms": 1673.0871200561523, - "idx": 173, - "text": "RICE BREAD.", - "text_len": 2, - "target_length": 78, - "gen_length_sec": 0.8297872340425532, - "speaker_name": "2078" - }, - { - "secs": 0.13627545535564423, - "wer": 1.0, - "time_ms": 1616.136074066162, - "idx": 175, - "text": "SODA BREAD.", - "text_len": 2, - "target_length": 78, - "gen_length_sec": 0.8297872340425532, - "speaker_name": "2078" - }, - { - "secs": 0.32471543550491333, - "wer": 0.14285714285714285, - "time_ms": 5482.749223709106, - "idx": 176, - "text": "INGREDIENTS.--To every two pounds. of flour allow one teaspoonful of tartaric ac", - "text_len": 28, - "target_length": 1107, - "gen_length_sec": 11.77659574468085, - "speaker_name": "2078" - }, - { - "secs": 0.5688883662223816, - "wer": 1.0, - "time_ms": 1597.7058410644531, - "idx": 177, - "text": "[Illustration: ROLLS.]", - "text_len": 2, - "target_length": 136, - "gen_length_sec": 1.446808510638298, - "speaker_name": "2078" - }, - { - "secs": 0.6801303029060364, - "wer": 0.10869565217391304, - "time_ms": 4411.1268520355225, - "idx": 180, - "text": "WHEN a writer calls his work a Romance, it need hardly be observed that he wishe", - "text_len": 46, - "target_length": 818, - "gen_length_sec": 8.702127659574469, - "speaker_name": "2086" - }, - { - "secs": 0.31896743178367615, - "wer": 0.16981132075471697, - "time_ms": 18171.47707939148, - "idx": 181, - "text": "The former-while, as a work of art, it must rigidly subject itself to laws, and ", - "text_len": 53, - "target_length": 3353, - "gen_length_sec": 35.670212765957444, - "speaker_name": "2086" - }, - { - "secs": 0.5652904510498047, - "wer": 0.06896551724137931, - "time_ms": 8774.428129196167, - "idx": 182, - "text": "If he think fit, also, he may so manage his atmospherical medium as to bring out", - "text_len": 29, - "target_length": 1535, - "gen_length_sec": 16.329787234042552, - "speaker_name": "2086" - }, - { - "secs": 0.5049606561660767, - "wer": 0.021739130434782608, - "time_ms": 7446.396112442017, - "idx": 183, - "text": "He will be wise, no doubt, to make a very moderate use of the privileges here st", - "text_len": 46, - "target_length": 1458, - "gen_length_sec": 15.51063829787234, - "speaker_name": "2086" - }, - { - "secs": 0.6142274141311646, - "wer": 0.0, - "time_ms": 3911.141872406006, - "idx": 185, - "text": "The point of view in which this tale comes under the Romantic definition lies in", - "text_len": 32, - "target_length": 752, - "gen_length_sec": 8.0, - "speaker_name": "2086" - }, - { - "secs": 0.5706803798675537, - "wer": 0.10638297872340426, - "time_ms": 8251.282453536987, - "idx": 192, - "text": "A high truth, indeed, fairly, finely, and skilfully wrought out, brightening at ", - "text_len": 47, - "target_length": 1595, - "gen_length_sec": 16.96808510638298, - "speaker_name": "2086" - }, - { - "secs": 0.5642995834350586, - "wer": 0.0, - "time_ms": 3942.4471855163574, - "idx": 193, - "text": "The reader may perhaps choose to assign an actual locality to the imaginary even", - "text_len": 17, - "target_length": 782, - "gen_length_sec": 8.319148936170214, - "speaker_name": "2086" - }, - { - "secs": 0.42356282472610474, - "wer": 0.21739130434782608, - "time_ms": 5076.808214187622, - "idx": 194, - "text": "If permitted by the historical connection,--which, though slight, was essential ", - "text_len": 23, - "target_length": 990, - "gen_length_sec": 10.53191489361702, - "speaker_name": "2086" - }, - { - "secs": 0.4859374761581421, - "wer": 0.0, - "time_ms": 7298.36893081665, - "idx": 195, - "text": "A hop vine, springing from last year's root, was beginning to clamber over it, b", - "text_len": 26, - "target_length": 1422, - "gen_length_sec": 15.127659574468085, - "speaker_name": "2086" - }, - { - "secs": 0.6268647909164429, - "wer": 0.05, - "time_ms": 3416.6054725646973, - "idx": 196, - "text": "Three of the seven gables either fronted or looked sideways, with a dark solemni", - "text_len": 20, - "target_length": 713, - "gen_length_sec": 7.585106382978723, - "speaker_name": "2086" - }, - { - "secs": 0.4656131863594055, - "wer": 0.043478260869565216, - "time_ms": 4965.883731842041, - "idx": 198, - "text": "Phoebe saw, however, that their growth must have been checked by a degree of car", - "text_len": 23, - "target_length": 908, - "gen_length_sec": 9.659574468085106, - "speaker_name": "2086" - }, - { - "secs": 0.5942751169204712, - "wer": 0.03571428571428571, - "time_ms": 7364.241600036621, - "idx": 202, - "text": "It was on the third floor, the front windows looking down into the street, where", - "text_len": 28, - "target_length": 1464, - "gen_length_sec": 15.574468085106384, - "speaker_name": "2277" - }, - { - "secs": 0.5552817583084106, - "wer": 0.0, - "time_ms": 2012.7196311950684, - "idx": 203, - "text": "Her husband asked a few questions and sat down to read the evening paper.", - "text_len": 14, - "target_length": 308, - "gen_length_sec": 3.276595744680851, - "speaker_name": "2277" - }, - { - "secs": 0.5744425058364868, - "wer": 0.2, - "time_ms": 2880.3811073303223, - "idx": 208, - "text": "She had some slight gift of observation and that sense, so rich in every woman-i", - "text_len": 15, - "target_length": 551, - "gen_length_sec": 5.861702127659575, - "speaker_name": "2277" - }, - { - "secs": 0.7904633283615112, - "wer": 0.11764705882352941, - "time_ms": 2119.9324131011963, - "idx": 209, - "text": "Then she walked and sang to it, until Hanson, disturbed in his reading, came and", - "text_len": 17, - "target_length": 413, - "gen_length_sec": 4.3936170212765955, - "speaker_name": "2277" - }, - { - "secs": 0.6885713338851929, - "wer": 0.0, - "time_ms": 1768.0554389953613, - "idx": 210, - "text": "A pleasant side to his nature came out here. He was patient.", - "text_len": 12, - "target_length": 265, - "gen_length_sec": 2.8191489361702127, - "speaker_name": "2277" - }, - { - "secs": 0.439483106136322, - "wer": 0.1111111111111111, - "time_ms": 1756.3328742980957, - "idx": 211, - "text": "\"Well, we'll go out Sunday and see Lincoln Park.\"", - "text_len": 9, - "target_length": 268, - "gen_length_sec": 2.851063829787234, - "speaker_name": "2277" - }, - { - "secs": 0.6225277781486511, - "wer": 0.3333333333333333, - "time_ms": 1671.2677478790283, - "idx": 218, - "text": "\"At about twenty minutes of five.\"", - "text_len": 6, - "target_length": 210, - "gen_length_sec": 2.234042553191489, - "speaker_name": "2277" - }, - { - "secs": 0.6585513949394226, - "wer": 0.0, - "time_ms": 1601.7789840698242, - "idx": 219, - "text": "He could not come here.", - "text_len": 5, - "target_length": 106, - "gen_length_sec": 1.127659574468085, - "speaker_name": "2277" - }, - { - "secs": 0.7009989619255066, - "wer": 0.22727272727272727, - "time_ms": 4185.856103897095, - "idx": 220, - "text": "The Author wishes it to be understood that Erewhon is pronounced as a word of th", - "text_len": 22, - "target_length": 865, - "gen_length_sec": 9.202127659574469, - "speaker_name": "2412" - }, - { - "secs": 0.6603825092315674, - "wer": 0.05, - "time_ms": 4433.022737503052, - "idx": 222, - "text": "On my return, I purposely avoided looking into it until I had sent back my last ", - "text_len": 20, - "target_length": 870, - "gen_length_sec": 9.25531914893617, - "speaker_name": "2412" - }, - { - "secs": 0.7663170695304871, - "wer": 0.03125, - "time_ms": 7830.128908157349, - "idx": 223, - "text": "Then I had much pleasure in reading it, but was indeed surprised at the many lit", - "text_len": 32, - "target_length": 1507, - "gen_length_sec": 16.03191489361702, - "speaker_name": "2412" - }, - { - "secs": 0.7407243251800537, - "wer": 0.8, - "time_ms": 3427.6392459869385, - "idx": 225, - "text": "june ninth eighteen seventy two", - "text_len": 5, - "target_length": 720, - "gen_length_sec": 7.659574468085107, - "speaker_name": "2412" - }, - { - "secs": 0.6604575514793396, - "wer": 0.037037037037037035, - "time_ms": 7320.247411727905, - "idx": 228, - "text": "Having now, I fear, at too great length done what I was asked to do, I should li", - "text_len": 27, - "target_length": 1395, - "gen_length_sec": 14.840425531914894, - "speaker_name": "2412" - }, - { - "secs": 0.6273906826972961, - "wer": 0.02, - "time_ms": 8723.622560501099, - "idx": 229, - "text": "Suffice it, that when I left home it was with the intention of going to some new", - "text_len": 50, - "target_length": 1682, - "gen_length_sec": 17.893617021276597, - "speaker_name": "2412" - }, - { - "secs": 0.7665560245513916, - "wer": 0.037037037037037035, - "time_ms": 6797.813415527344, - "idx": 231, - "text": "The climate was temperate, and very healthy; there were no wild animals, nor wer", - "text_len": 27, - "target_length": 1288, - "gen_length_sec": 13.702127659574469, - "speaker_name": "2412" - }, - { - "secs": 0.6538539528846741, - "wer": 0.0, - "time_ms": 11232.752561569214, - "idx": 233, - "text": "I had a telescope and a dog, and would take bread and meat and tobacco with me. ", - "text_len": 43, - "target_length": 2166, - "gen_length_sec": 23.04255319148936, - "speaker_name": "2412" - }, - { - "secs": 0.8070616722106934, - "wer": 0.03278688524590164, - "time_ms": 19843.915939331055, - "idx": 234, - "text": "I knew that there was a range still farther back; but except from one place near", - "text_len": 61, - "target_length": 3143, - "gen_length_sec": 33.43617021276596, - "speaker_name": "2412" - }, - { - "secs": 0.6259363889694214, - "wer": 0.24, - "time_ms": 14918.198347091675, - "idx": 237, - "text": "Alas! they are both mistaken; the ewe is not the lamb's ewe, they are neither ki", - "text_len": 25, - "target_length": 2929, - "gen_length_sec": 31.159574468085108, - "speaker_name": "2412" - }, - { - "secs": 0.7333503365516663, - "wer": 0.0, - "time_ms": 1611.6912364959717, - "idx": 239, - "text": "Could I hope to cross it?", - "text_len": 6, - "target_length": 116, - "gen_length_sec": 1.2340425531914894, - "speaker_name": "2412" - }, - { - "secs": 0.45363330841064453, - "wer": 0.25, - "time_ms": 1598.876714706421, - "idx": 244, - "text": "That was the advertisement.", - "text_len": 4, - "target_length": 100, - "gen_length_sec": 1.0638297872340425, - "speaker_name": "2428" - }, - { - "secs": 0.44281598925590515, - "wer": 0.06666666666666667, - "time_ms": 2212.6898765563965, - "idx": 246, - "text": "I was not sure what, in this case, an old-fashioned Christmas might happen to me", - "text_len": 15, - "target_length": 428, - "gen_length_sec": 4.553191489361702, - "speaker_name": "2428" - }, - { - "secs": 0.21574045717716217, - "wer": 0.0, - "time_ms": 1635.1325511932373, - "idx": 247, - "text": "Yes!", - "text_len": 1, - "target_length": 206, - "gen_length_sec": 2.1914893617021276, - "speaker_name": "2428" - }, - { - "secs": 0.5777386426925659, - "wer": 0.0, - "time_ms": 12083.4321975708, - "idx": 248, - "text": "She and her husband, who suffers from melancholia, and all the other complaints ", - "text_len": 45, - "target_length": 2259, - "gen_length_sec": 24.03191489361702, - "speaker_name": "2428" - }, - { - "secs": 0.4410702586174011, - "wer": 0.0, - "time_ms": 1611.0150814056396, - "idx": 249, - "text": "So I wrote to the advertiser.", - "text_len": 6, - "target_length": 134, - "gen_length_sec": 1.425531914893617, - "speaker_name": "2428" - }, - { - "secs": 0.4247892200946808, - "wer": 0.0, - "time_ms": 1619.7290420532227, - "idx": 253, - "text": "The whole thing was a trifle odd.", - "text_len": 7, - "target_length": 153, - "gen_length_sec": 1.627659574468085, - "speaker_name": "2428" - }, - { - "secs": 0.5395867228507996, - "wer": 0.14285714285714285, - "time_ms": 1734.8277568817139, - "idx": 255, - "text": "No references were offered or asked for.", - "text_len": 7, - "target_length": 248, - "gen_length_sec": 2.6382978723404253, - "speaker_name": "2428" - }, - { - "secs": 0.46704575419425964, - "wer": 0.075, - "time_ms": 5154.598951339722, - "idx": 258, - "text": "She was apparently the careworn mother of a \"lively family.\" Under such circumst", - "text_len": 40, - "target_length": 1027, - "gen_length_sec": 10.925531914893616, - "speaker_name": "2428" - }, - { - "secs": 0.07226021587848663, - "wer": 0.4, - "time_ms": 1629.7380924224854, - "idx": 261, - "text": "one Death Strikes a King", - "text_len": 5, - "target_length": 186, - "gen_length_sec": 1.9787234042553192, - "speaker_name": "251" - }, - { - "secs": 0.3163744807243347, - "wer": 0.0, - "time_ms": 3926.4776706695557, - "idx": 265, - "text": "She threw up her head in a gusty gesture of wrath and despair as the thunder of ", - "text_len": 23, - "target_length": 763, - "gen_length_sec": 8.117021276595745, - "speaker_name": "251" - }, - { - "secs": 0.1493312418460846, - "wer": 0.0, - "time_ms": 2258.894920349121, - "idx": 266, - "text": "'The priests and their clamor!' she exclaimed.", - "text_len": 7, - "target_length": 475, - "gen_length_sec": 5.053191489361702, - "speaker_name": "251" - }, - { - "secs": 0.18090854585170746, - "wer": 0.08695652173913043, - "time_ms": 4804.728269577026, - "idx": 268, - "text": "He is dying now-and I stand here helpless, who would burn the whole city and spi", - "text_len": 23, - "target_length": 1006, - "gen_length_sec": 10.702127659574469, - "speaker_name": "251" - }, - { - "secs": 0.1926124542951584, - "wer": 0.16666666666666666, - "time_ms": 6273.835897445679, - "idx": 271, - "text": "Five skulls bleaching on the Tower of the Kites can testify to attempts which we", - "text_len": 18, - "target_length": 1220, - "gen_length_sec": 12.97872340425532, - "speaker_name": "251" - }, - { - "secs": 0.4029429256916046, - "wer": 0.047619047619047616, - "time_ms": 2684.649705886841, - "idx": 272, - "text": "She ceased as the king spoke; his livid lips did not move, and there was no reco", - "text_len": 21, - "target_length": 497, - "gen_length_sec": 5.287234042553192, - "speaker_name": "251" - }, - { - "secs": 0.38715463876724243, - "wer": 0.6, - "time_ms": 1625.4701614379883, - "idx": 275, - "text": "I can not find you.", - "text_len": 5, - "target_length": 93, - "gen_length_sec": 0.9893617021276596, - "speaker_name": "251" - }, - { - "secs": 0.4091877043247223, - "wer": 0.2, - "time_ms": 1617.9633140563965, - "idx": 278, - "text": "Do you not know me-'", - "text_len": 5, - "target_length": 84, - "gen_length_sec": 0.8936170212765957, - "speaker_name": "251" - }, - { - "secs": 0.19973985850811005, - "wer": 0.0, - "time_ms": 9295.806646347046, - "idx": 281, - "text": "The wind was fair, nevertheless, and blew steadily from the southwest; but the c", - "text_len": 25, - "target_length": 1717, - "gen_length_sec": 18.26595744680851, - "speaker_name": "2803" - }, - { - "secs": 0.21929308772087097, - "wer": 0.0, - "time_ms": 5598.790407180786, - "idx": 282, - "text": "The heavy, lumpy sea strained her cordage, her timbers creaked, and she labored ", - "text_len": 20, - "target_length": 1069, - "gen_length_sec": 11.372340425531915, - "speaker_name": "2803" - }, - { - "secs": 0.28681638836860657, - "wer": 0.0, - "time_ms": 4665.126323699951, - "idx": 284, - "text": "Then they returned to the narrow space, fitter for stowing cargo than accommodat", - "text_len": 16, - "target_length": 963, - "gen_length_sec": 10.24468085106383, - "speaker_name": "2803" - }, - { - "secs": 0.2908633351325989, - "wer": 0.0, - "time_ms": 2013.6959552764893, - "idx": 285, - "text": "Their friends did their best to amuse them.", - "text_len": 8, - "target_length": 333, - "gen_length_sec": 3.5425531914893615, - "speaker_name": "2803" - }, - { - "secs": 0.40402984619140625, - "wer": 0.04, - "time_ms": 4873.603105545044, - "idx": 287, - "text": "Much as they had been interested in his dissertation on the Pampas, or Australia", - "text_len": 25, - "target_length": 935, - "gen_length_sec": 9.946808510638299, - "speaker_name": "2803" - }, - { - "secs": 0.48355570435523987, - "wer": 0.0, - "time_ms": 1623.1496334075928, - "idx": 289, - "text": "He could not stay in one place.", - "text_len": 7, - "target_length": 180, - "gen_length_sec": 1.9148936170212767, - "speaker_name": "2803" - }, - { - "secs": 0.2460787147283554, - "wer": 0.0, - "time_ms": 1995.0671195983887, - "idx": 291, - "text": "Glenarvan shook his head in dissent.", - "text_len": 6, - "target_length": 304, - "gen_length_sec": 3.234042553191489, - "speaker_name": "2803" - }, - { - "secs": 0.429821252822876, - "wer": 0.0, - "time_ms": 4807.914733886719, - "idx": 292, - "text": "\"And yet,\" said the young captain, \"you must be longing to quit this vessel.", - "text_len": 14, - "target_length": 1009, - "gen_length_sec": 10.73404255319149, - "speaker_name": "2803" - }, - { - "secs": 0.48847222328186035, - "wer": 0.0, - "time_ms": 1728.062629699707, - "idx": 297, - "text": "\"I am not looking for the land.\"", - "text_len": 7, - "target_length": 248, - "gen_length_sec": 2.6382978723404253, - "speaker_name": "2803" - }, - { - "secs": 0.22531476616859436, - "wer": 0.0, - "time_ms": 1639.9335861206055, - "idx": 298, - "text": "\"What then, my Lord?\"", - "text_len": 4, - "target_length": 177, - "gen_length_sec": 1.8829787234042554, - "speaker_name": "2803" - }, - { - "secs": 0.30257347226142883, - "wer": 0.0, - "time_ms": 1776.3986587524414, - "idx": 300, - "text": "This is a LibriVox recording.", - "text_len": 5, - "target_length": 245, - "gen_length_sec": 2.606382978723404, - "speaker_name": "2902" - }, - { - "secs": 0.2546756863594055, - "wer": 1.0, - "time_ms": 1737.569808959961, - "idx": 302, - "text": "HYPATIA", - "text_len": 1, - "target_length": 217, - "gen_length_sec": 2.3085106382978724, - "speaker_name": "2902" - }, - { - "secs": 0.5939136743545532, - "wer": 0.03125, - "time_ms": 5158.262491226196, - "idx": 305, - "text": "A picture of life in the fifth century must needs contain much which will be pai", - "text_len": 32, - "target_length": 985, - "gen_length_sec": 10.47872340425532, - "speaker_name": "2902" - }, - { - "secs": 0.5356156826019287, - "wer": 0.016129032258064516, - "time_ms": 10593.171119689941, - "idx": 310, - "text": "And He who excited the craving, was also furnishing that which would satisfy it;", - "text_len": 62, - "target_length": 2117, - "gen_length_sec": 22.52127659574468, - "speaker_name": "2902" - }, - { - "secs": 0.3705868124961853, - "wer": 0.08571428571428572, - "time_ms": 8935.197353363037, - "idx": 314, - "text": "They brought before the minds of churchmen a thousand new questions which must b", - "text_len": 35, - "target_length": 1826, - "gen_length_sec": 19.425531914893618, - "speaker_name": "2902" - }, - { - "secs": 0.49749213457107544, - "wer": 0.14285714285714285, - "time_ms": 1651.902437210083, - "idx": 316, - "text": "And their strength was felt at once.", - "text_len": 7, - "target_length": 159, - "gen_length_sec": 1.6914893617021276, - "speaker_name": "2902" - }, - { - "secs": 0.581058919429779, - "wer": 0.043478260869565216, - "time_ms": 11519.500494003296, - "idx": 317, - "text": "Their vanguard, confined with difficulty for three centuries beyond the Eastern ", - "text_len": 46, - "target_length": 2364, - "gen_length_sec": 25.148936170212767, - "speaker_name": "2902" - }, - { - "secs": 0.3486821949481964, - "wer": 0.0, - "time_ms": 1633.2333087921143, - "idx": 318, - "text": "But now the main body had arrived.", - "text_len": 7, - "target_length": 186, - "gen_length_sec": 1.9787234042553192, - "speaker_name": "2902" - }, - { - "secs": 0.2582034468650818, - "wer": 0.22580645161290322, - "time_ms": 9650.505542755127, - "idx": 321, - "text": "The distance is about a hundred miles, and will take some of the time we hear so", - "text_len": 31, - "target_length": 1868, - "gen_length_sec": 19.872340425531913, - "speaker_name": "3000" - }, - { - "secs": 0.664265513420105, - "wer": 0.07692307692307693, - "time_ms": 12768.735408782959, - "idx": 324, - "text": "Some have strange, morbid fears as soon as they find themselves with Nature, eve", - "text_len": 39, - "target_length": 2507, - "gen_length_sec": 26.670212765957448, - "speaker_name": "3000" - }, - { - "secs": 0.43077030777931213, - "wer": 0.875, - "time_ms": 1608.046531677246, - "idx": 325, - "text": "But it is far better to go afoot.", - "text_len": 8, - "target_length": 133, - "gen_length_sec": 1.4148936170212767, - "speaker_name": "3000" - }, - { - "secs": 0.5778879523277283, - "wer": 0.0, - "time_ms": 10454.42509651184, - "idx": 326, - "text": "Then you are free to make wide waverings and zigzags away from the roads to visi", - "text_len": 67, - "target_length": 1997, - "gen_length_sec": 21.24468085106383, - "speaker_name": "3000" - }, - { - "secs": 0.2660723328590393, - "wer": 0.0, - "time_ms": 7163.1410121917725, - "idx": 327, - "text": "One blanket will be enough to carry, or you may forego the pleasure and burden a", - "text_len": 23, - "target_length": 1364, - "gen_length_sec": 14.51063829787234, - "speaker_name": "3000" - }, - { - "secs": 0.384576678276062, - "wer": 0.13636363636363635, - "time_ms": 6239.370107650757, - "idx": 329, - "text": "Berries and plums abound in season, and quail and grouse and deer-the magnificen", - "text_len": 22, - "target_length": 1199, - "gen_length_sec": 12.75531914893617, - "speaker_name": "3000" - }, - { - "secs": 0.32151612639427185, - "wer": 0.09259259259259259, - "time_ms": 9331.047773361206, - "idx": 331, - "text": "One glacier after another comes into view, and the outlines of the mountain are ", - "text_len": 54, - "target_length": 1825, - "gen_length_sec": 19.414893617021278, - "speaker_name": "3000" - }, - { - "secs": 0.6166120171546936, - "wer": 0.014492753623188406, - "time_ms": 16635.32018661499, - "idx": 334, - "text": "And the wandering winds, how busy they are, and what a breadth of sound and moti", - "text_len": 69, - "target_length": 3304, - "gen_length_sec": 35.148936170212764, - "speaker_name": "3000" - }, - { - "secs": 0.4440804123878479, - "wer": 0.0, - "time_ms": 7389.375448226929, - "idx": 336, - "text": "Then the shining of the wet leaves is delightful, and the steamy fragrance, and ", - "text_len": 34, - "target_length": 1450, - "gen_length_sec": 15.425531914893616, - "speaker_name": "3000" - }, - { - "secs": 0.2967967689037323, - "wer": 0.15384615384615385, - "time_ms": 2936.182975769043, - "idx": 337, - "text": "The nights, too, are delightful, watching with Shasta beneath the great starry d", - "text_len": 13, - "target_length": 600, - "gen_length_sec": 6.382978723404255, - "speaker_name": "3000" - }, - { - "secs": 0.5126329660415649, - "wer": 0.0, - "time_ms": 5798.789262771606, - "idx": 338, - "text": "A thousand thousand voices are heard, but so finely blended they seem a part of ", - "text_len": 23, - "target_length": 1153, - "gen_length_sec": 12.26595744680851, - "speaker_name": "3000" - }, - { - "secs": 0.43992310762405396, - "wer": 0.022727272727272728, - "time_ms": 15734.193325042725, - "idx": 339, - "text": "And how grandly do the great logs and branches of your campfire give forth the h", - "text_len": 44, - "target_length": 2752, - "gen_length_sec": 29.27659574468085, - "speaker_name": "3000" - }, - { - "secs": 0.2956528067588806, - "wer": 0.0, - "time_ms": 1776.820421218872, - "idx": 340, - "text": "How quickly he disappeared!\"", - "text_len": 4, - "target_length": 289, - "gen_length_sec": 3.074468085106383, - "speaker_name": "3081" - }, - { - "secs": 0.6578259468078613, - "wer": 0.0, - "time_ms": 1633.7132453918457, - "idx": 342, - "text": "\"So they tell me.\"", - "text_len": 4, - "target_length": 111, - "gen_length_sec": 1.1808510638297873, - "speaker_name": "3081" - }, - { - "secs": 0.47821420431137085, - "wer": 0.3333333333333333, - "time_ms": 2945.4174041748047, - "idx": 347, - "text": "I Turn Out A Worthless Fellow-My Good Fortune-I Become A Rich Nobleman", - "text_len": 12, - "target_length": 591, - "gen_length_sec": 6.287234042553192, - "speaker_name": "3170" - }, - { - "secs": 0.3941524028778076, - "wer": 0.017241379310344827, - "time_ms": 7127.444267272949, - "idx": 350, - "text": "I felt that in my first profession, as I was not blessed with the vocation neces", - "text_len": 58, - "target_length": 1371, - "gen_length_sec": 14.585106382978724, - "speaker_name": "3170" - }, - { - "secs": 0.3930961787700653, - "wer": 0.15, - "time_ms": 2898.05269241333, - "idx": 356, - "text": "We did the same with physicians, whom we often sent half dressed to some noblema", - "text_len": 20, - "target_length": 567, - "gen_length_sec": 6.031914893617022, - "speaker_name": "3170" - }, - { - "secs": 0.4751744270324707, - "wer": 0.11666666666666667, - "time_ms": 11486.397504806519, - "idx": 357, - "text": "Whenever we could contrive to get into a church tower we thought it great fun to", - "text_len": 60, - "target_length": 2358, - "gen_length_sec": 25.085106382978722, - "speaker_name": "3170" - }, - { - "secs": 0.4658663272857666, - "wer": 0.0, - "time_ms": 5100.255489349365, - "idx": 358, - "text": "The city was alive with complaints, and we laughed at the useless search made by", - "text_len": 28, - "target_length": 928, - "gen_length_sec": 9.872340425531915, - "speaker_name": "3170" - }, - { - "secs": 0.4644694924354553, - "wer": 0.09523809523809523, - "time_ms": 4656.24213218689, - "idx": 360, - "text": "In every one of the seventy two parishes of the city of Venice, there is a large", - "text_len": 21, - "target_length": 955, - "gen_length_sec": 10.159574468085106, - "speaker_name": "3170" - }, - { - "secs": 0.5484161972999573, - "wer": 0.08333333333333333, - "time_ms": 10487.039566040039, - "idx": 361, - "text": "People can likewise eat in the 'magazzino', but they must obtain what they want ", - "text_len": 36, - "target_length": 2093, - "gen_length_sec": 22.26595744680851, - "speaker_name": "3170" - }, - { - "secs": 0.33635613322257996, - "wer": 0.08333333333333333, - "time_ms": 4708.503246307373, - "idx": 362, - "text": "The nobility, the merchants, even workmen in good circumstances, are never seen ", - "text_len": 24, - "target_length": 791, - "gen_length_sec": 8.414893617021276, - "speaker_name": "3170" - }, - { - "secs": 0.4734167158603668, - "wer": 0.9666666666666667, - "time_ms": 11225.918531417847, - "idx": 363, - "text": "Yet there are a few private rooms which contain a table surrounded with benches,", - "text_len": 30, - "target_length": 2164, - "gen_length_sec": 23.02127659574468, - "speaker_name": "3170" - }, - { - "secs": 0.4828583002090454, - "wer": 0.7407407407407407, - "time_ms": 27425.631523132324, - "idx": 364, - "text": "It was during the Carnival of seventeen forty five, after midnight; we were, all", - "text_len": 54, - "target_length": 5123, - "gen_length_sec": 54.5, - "speaker_name": "3170" - }, - { - "secs": 0.5536553859710693, - "wer": 0.03636363636363636, - "time_ms": 18187.044858932495, - "idx": 365, - "text": "Our chief, a noble Venetian belonging to the Balbi family, said to us, \"It would", - "text_len": 55, - "target_length": 3354, - "gen_length_sec": 35.680851063829785, - "speaker_name": "3170" - }, - { - "secs": 0.7762069702148438, - "wer": 0.0, - "time_ms": 4468.628168106079, - "idx": 367, - "text": "She, however, flattered herself that what she had said might be excused as a mis", - "text_len": 30, - "target_length": 757, - "gen_length_sec": 8.053191489361701, - "speaker_name": "3536" - }, - { - "secs": 0.5873372554779053, - "wer": 0.0, - "time_ms": 1628.2918453216553, - "idx": 368, - "text": "Her guardian looked astonished.", - "text_len": 4, - "target_length": 125, - "gen_length_sec": 1.3297872340425532, - "speaker_name": "3536" - }, - { - "secs": 0.21976079046726227, - "wer": 0.2857142857142857, - "time_ms": 2897.127389907837, - "idx": 369, - "text": "\"I mistook-for I had before given my word that I should pass it abroad.\"", - "text_len": 14, - "target_length": 558, - "gen_length_sec": 5.9361702127659575, - "speaker_name": "3536" - }, - { - "secs": 0.6873931884765625, - "wer": 0.0, - "time_ms": 3437.9959106445312, - "idx": 373, - "text": "\"What could be worse, Madam?\" cried Miss Milner; \"am not I disappointed of the b", - "text_len": 15, - "target_length": 718, - "gen_length_sec": 7.638297872340425, - "speaker_name": "3536" - }, - { - "secs": 0.6291590929031372, - "wer": 0.03125, - "time_ms": 9525.448560714722, - "idx": 374, - "text": "\"It is not the first time, I believe, you have acted contrary to that, Miss Miln", - "text_len": 32, - "target_length": 1814, - "gen_length_sec": 19.29787234042553, - "speaker_name": "3536" - }, - { - "secs": 0.6860816478729248, - "wer": 0.10526315789473684, - "time_ms": 2266.7465209960938, - "idx": 375, - "text": "Dorriforth read on, and seemed afraid of looking up, lest he should see what he ", - "text_len": 19, - "target_length": 483, - "gen_length_sec": 5.138297872340425, - "speaker_name": "3536" - }, - { - "secs": 0.5194693207740784, - "wer": 1.0, - "time_ms": 1612.7452850341797, - "idx": 378, - "text": "mr", - "text_len": 1, - "target_length": 124, - "gen_length_sec": 1.3191489361702127, - "speaker_name": "3536" - }, - { - "secs": 0.6828760504722595, - "wer": 1.0, - "time_ms": 1617.1801090240479, - "idx": 381, - "text": "\"Gammon, Bozzle.\"", - "text_len": 2, - "target_length": 112, - "gen_length_sec": 1.1914893617021276, - "speaker_name": "3536" - }, - { - "secs": 0.6598377227783203, - "wer": 0.05555555555555555, - "time_ms": 2091.919422149658, - "idx": 382, - "text": "We'd all be shewed up in the papers as that black, that they'd hoot us along the", - "text_len": 18, - "target_length": 376, - "gen_length_sec": 4.0, - "speaker_name": "3536" - }, - { - "secs": 0.508304238319397, - "wer": 1.0, - "time_ms": 12175.996541976929, - "idx": 383, - "text": "It ain't the regular line of business, Bozzle; and there ain't no good to be got", - "text_len": 33, - "target_length": 2403, - "gen_length_sec": 25.56382978723404, - "speaker_name": "3536" - }, - { - "secs": 0.7741633653640747, - "wer": 0.42857142857142855, - "time_ms": 1578.7169933319092, - "idx": 385, - "text": "\"Of course it ain't,\" said mrs Bozzle.", - "text_len": 7, - "target_length": 168, - "gen_length_sec": 1.7872340425531914, - "speaker_name": "3536" - } - ] -} \ No newline at end of file diff --git a/results/figures/figure4_pareto_frontier.png b/results/figures/figure4_pareto_frontier.png deleted file mode 100644 index e5cdf8c..0000000 Binary files a/results/figures/figure4_pareto_frontier.png and /dev/null differ diff --git a/results/figures/figure5_waveform_spectrogram.png b/results/figures/figure5_waveform_spectrogram.png deleted file mode 100644 index 324bcce..0000000 Binary files a/results/figures/figure5_waveform_spectrogram.png and /dev/null differ diff --git a/results/figures/figure6_attention_heatmap.png b/results/figures/figure6_attention_heatmap.png deleted file mode 100644 index e3bde92..0000000 Binary files a/results/figures/figure6_attention_heatmap.png and /dev/null differ diff --git a/scripts/download_models.py b/scripts/download_models.py deleted file mode 100644 index 8e57896..0000000 --- a/scripts/download_models.py +++ /dev/null @@ -1,184 +0,0 @@ -"""PersonaVoice v10.4.8 — 预训练模型一键下载脚本. - -架构位置: 开源基础设施层, 在首次运行 PersonaVoice 之前执行一次, -将所有 HuggingFace 模型预下载到本地缓存, 之后可完全离线运行. - -包含的模型: - 1. SWivid/F5-TTS (~1.5GB, TTS 骨干) - 2. speechbrain/spkrec-ecapa-voxceleb (~80MB, 说话人编码器) - 3. charactr/vocos-mel-24khz (~50MB, 神经声码器) - 4. bert-base-chinese (~400MB, 人格文本编码器) - 5. openai/whisper-large-v3-turbo (~1.5GB, WER 评估用 ASR) - 6. Qwen/Qwen2.5-0.5B (~1GB, 可选, LLM 人格对话) - 7. sentence-transformers/all-MiniLM-L6-v2 (~90MB, 可选, 文本嵌入) - -使用方法: - python scripts/download_models.py # 下载所有必需模型 - python scripts/download_models.py --optional # 同时下载可选模型 - python scripts/download_models.py --check # 仅检查缓存状态 - -注意: - - 默认下载到 HuggingFace 默认缓存目录 (~/.cache/huggingface/hub) - - 可通过环境变量 HF_HOME 自定义缓存目录 - - 国内用户可设置 HF_ENDPOINT=https://hf-mirror.com 使用镜像加速 -""" -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path -from typing import Dict, List, Tuple - -# 项目根目录 -PROJECT_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(PROJECT_ROOT)) - - -# 必需模型 (运行 PersonaVoice 主流程必需) -REQUIRED_MODELS: Dict[str, str] = { - "f5_tts": "SWivid/F5-TTS", - "ecapa_voxceleb": "speechbrain/spkrec-ecapa-voxceleb", - "vocos_mel_24khz": "charactr/vocos-mel-24khz", - "bert_chinese": "bert-base-chinese", - "whisper_large_v3_turbo": "openai/whisper-large-v3-turbo", -} - -# 可选模型 (实验脚本或扩展功能需要) -OPTIONAL_MODELS: Dict[str, str] = { - "qwen2.5_0.5b": "Qwen/Qwen2.5-0.5B", - "minilm_l6": "sentence-transformers/all-MiniLM-L6-v2", -} - - -def check_cache_status(models: Dict[str, str]) -> Tuple[List[str], List[str]]: - """检查模型缓存状态. - - Returns: - (cached_keys, missing_keys) - """ - try: - from personavoice.common.local_models import is_model_cached - except ImportError: - # 项目未安装时, 简化检查 - from huggingface_hub import constants - cache_dir = Path(constants.HF_HUB_CACHE) - - def is_model_cached(model_id: str) -> bool: - cache_name = "models--" + model_id.replace("/", "--") - snapshots = cache_dir / cache_name / "snapshots" - return snapshots.exists() and any(snapshots.iterdir()) - - cached, missing = [], [] - for key, model_id in models.items(): - if is_model_cached(model_id): - cached.append(key) - else: - missing.append(key) - return cached, missing - - -def download_model(model_id: str, key: str) -> bool: - """下载单个模型. - - Returns: - True if success, False otherwise. - """ - print(f"\n[{key}] 下载 {model_id} ...") - try: - from huggingface_hub import snapshot_download - snapshot_download( - repo_id=model_id, - resume_download=True, - max_workers=4, - ) - print(f" ✓ {model_id} 下载完成") - return True - except Exception as e: - print(f" ✗ {model_id} 下载失败: {e}") - return False - - -def main(): - parser = argparse.ArgumentParser( - description="PersonaVoice 预训练模型下载脚本", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - python scripts/download_models.py # 下载所有必需模型 - python scripts/download_models.py --optional # 同时下载可选模型 - python scripts/download_models.py --check # 仅检查缓存状态 - python scripts/download_models.py --only f5_tts ecapa_voxceleb # 仅下载指定模型 - """, - ) - parser.add_argument( - "--optional", action="store_true", - help="同时下载可选模型 (Qwen2.5, MiniLM)", - ) - parser.add_argument( - "--check", action="store_true", - help="仅检查缓存状态, 不下载", - ) - parser.add_argument( - "--only", nargs="+", default=None, - help="仅下载指定的模型别名 (空格分隔)", - ) - args = parser.parse_args() - - print("=" * 60) - print(" PersonaVoice v10.4.8 预训练模型下载脚本") - print("=" * 60) - - # 选择要处理的模型 - if args.only: - all_models = {**REQUIRED_MODELS, **OPTIONAL_MODELS} - models = {k: all_models[k] for k in args.only if k in all_models} - if not models: - print(f" [ERROR] 未找到指定的模型别名: {args.only}") - print(f" 可用别名: {list(all_models.keys())}") - return 1 - elif args.optional: - models = {**REQUIRED_MODELS, **OPTIONAL_MODELS} - else: - models = REQUIRED_MODELS - - # 检查缓存状态 - cached, missing = check_cache_status(models) - print(f"\n 缓存目录: {os.environ.get('HF_HOME', '~/.cache/huggingface')}") - print(f" 已缓存: {len(cached)}/{len(models)}") - print(f" 缺失: {len(missing)}/{len(models)}") - if missing: - print(f" 缺失模型: {missing}") - - if args.check: - print("\n[CHECK] 仅检查模式, 不执行下载") - return 0 - - if not missing: - print("\n ✓ 所有模型已缓存, 无需下载") - return 0 - - # 下载缺失模型 - print(f"\n 开始下载 {len(missing)} 个缺失模型...") - success, failed = 0, 0 - for key in missing: - model_id = models[key] - if download_model(model_id, key): - success += 1 - else: - failed += 1 - - # 汇总 - print("\n" + "=" * 60) - print(f" 下载完成: {success} 成功, {failed} 失败") - if failed: - print(f" 失败模型, 请稍后重试或手动下载:") - for key in missing: - model_id = models[key] - print(f" huggingface-cli download {model_id}") - print("=" * 60) - return 0 if failed == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/prepare_data.py b/scripts/prepare_data.py deleted file mode 100644 index 44532cd..0000000 --- a/scripts/prepare_data.py +++ /dev/null @@ -1,243 +0,0 @@ -"""PersonaVoice v10.4.8 — LibriTTS 评估数据集准备脚本. - -架构位置: 开源基础设施层, 在首次复现 200 样本评估之前执行一次, -下载并预处理 LibriTTS dev-clean 子集, 生成项目评估所需的 -libritts_devclean_processed.pt 文件. - -数据来源: - - LibriTTS dev-clean (https://openslr.org/60/) - - 6 个说话人, ~5.4 小时, 16kHz - -输出: - data/libritts_processed/libritts_devclean_processed.pt - - tensors.mel: (N, 100, T) mel 频谱 - - tensors.ecapa_emb: (N, 192) ECAPA 说话人嵌入 - - tensors.wav: (N, T_samples) 原始波形 (24kHz) - - metadata.texts: List[str] 文本内容 - - metadata.speaker_ids: List[int] 说话人 ID - -使用方法: - python scripts/prepare_data.py # 完整下载+预处理 - python scripts/prepare_data.py --skip-download # 跳过下载, 仅预处理 - python scripts/prepare_data.py --n_samples 200 # 限制样本数 (测试用) -""" -from __future__ import annotations - -import argparse -import os -import sys -import tarfile -import urllib.request -from pathlib import Path -from typing import List, Dict, Tuple - -import numpy as np -import torch -import torchaudio - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(PROJECT_ROOT)) - -# LibriTTS dev-clean 下载地址 -LIBRITTS_DEVCLEAN_URL = "https://openslr.org/resources/60/dev-clean.tar.gz" - -# 输出路径 -DATA_DIR = PROJECT_ROOT / "data" -RAW_DIR = DATA_DIR / "LibriTTS" -PROCESSED_DIR = DATA_DIR / "libritts_processed" -PROCESSED_FILE = PROCESSED_DIR / "libritts_devclean_processed.pt" - - -def download_libritts(): - """下载 LibriTTS dev-clean 数据集 (~1.2GB).""" - if (RAW_DIR / "dev-clean").exists(): - print(f" ✓ LibriTTS dev-clean 已存在: {RAW_DIR / 'dev-clean'}") - return - - archive_path = DATA_DIR / "dev-clean.tar.gz" - if not archive_path.exists(): - print(f" 下载 LibriTTS dev-clean (~1.2GB) ...") - print(f" URL: {LIBRITTS_DEVCLEAN_URL}") - DATA_DIR.mkdir(parents=True, exist_ok=True) - try: - urllib.request.urlretrieve( - LIBRITTS_DEVCLEAN_URL, - str(archive_path), - ) - print(f" ✓ 下载完成: {archive_path}") - except Exception as e: - print(f" ✗ 下载失败: {e}") - print(f" 请手动下载: {LIBRITTS_DEVCLEAN_URL}") - print(f" 并解压到: {DATA_DIR}") - raise - - print(f" 解压 {archive_path} → {DATA_DIR}") - with tarfile.open(archive_path, "r:gz") as tar: - tar.extractall(str(DATA_DIR)) - print(f" ✓ 解压完成: {RAW_DIR / 'dev-clean'}") - - -def collect_audio_files() -> List[Tuple[Path, str, int]]: - """收集所有 (wav_path, text, speaker_id) 三元组. - - Returns: - List of (wav_path, normalized_text, speaker_id) - """ - samples = [] - dev_clean = RAW_DIR / "dev-clean" - if not dev_clean.exists(): - raise FileNotFoundError( - f"LibriTTS dev-clean 不存在: {dev_clean}\n" - f"请先运行: python scripts/prepare_data.py" - ) - - for speaker_dir in sorted(dev_clean.iterdir()): - if not speaker_dir.is_dir(): - continue - speaker_id = int(speaker_dir.name) - for chapter_dir in sorted(speaker_dir.iterdir()): - if not chapter_dir.is_dir(): - continue - # 找到 .wav 和对应的 .normalized.txt - wav_files = sorted(chapter_dir.glob("*.wav")) - for wav_file in wav_files: - base = wav_file.stem - text_file = chapter_dir / f"{base}.normalized.txt" - if not text_file.exists(): - continue - text = text_file.read_text(encoding="utf-8").strip() - if not text: - continue - samples.append((wav_file, text, speaker_id)) - - return samples - - -def extract_mel(wav_path: Path, sample_rate: int = 24000, n_mels: int = 100) -> torch.Tensor: - """提取 mel 频谱 (100-bin, 24kHz, 与 Vocos 兼容).""" - wav, sr = torchaudio.load(str(wav_path)) - if wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - if sr != sample_rate: - wav = torchaudio.functional.resample(wav, sr, sample_rate) - - mel_transform = torchaudio.transforms.MelSpectrogram( - sample_rate=sample_rate, - n_fft=1024, - hop_length=256, - win_length=1024, - n_mels=n_mels, - ) - mel = mel_transform(wav) # (1, n_mels, T) - mel = torch.log(torch.clamp(mel, min=1e-5)) - return mel.squeeze(0) # (n_mels, T) - - -def extract_ecapa_embedding( - wav_path: Path, - ecapa_encoder, - sample_rate: int = 16000, -) -> torch.Tensor: - """提取 ECAPA 说话人嵌入 (192-d).""" - wav, sr = torchaudio.load(str(wav_path)) - if wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - if sr != sample_rate: - wav = torchaudio.functional.resample(wav, sr, sample_rate) - with torch.no_grad(): - emb = ecapa_encoder.encode_batch(wav).squeeze(0) # (192,) - return emb - - -def preprocess_data(n_samples: int = 0): - """预处理 LibriTTS dev-clean → libritts_devclean_processed.pt.""" - samples = collect_audio_files() - if n_samples > 0: - samples = samples[:n_samples] - print(f" 共 {len(samples)} 个样本待处理") - - # 加载 ECAPA 编码器 - print(" 加载 ECAPA-TDNN 编码器 ...") - from speechbrain.inference.speaker import EncoderClassifier - ecapa_encoder = EncoderClassifier.from_hparams( - source="speechbrain/spkrec-ecapa-voxceleb", - savedir=str(PROJECT_ROOT / "models" / "ecapa_tdnn"), - run_opts={"device": "cpu"}, - ) - - mels, embeddings, wavs, texts, speaker_ids = [], [], [], [], [] - for i, (wav_path, text, sid) in enumerate(samples): - if (i + 1) % 50 == 0: - print(f" 处理中: {i+1}/{len(samples)}") - try: - mel = extract_mel(wav_path) - emb = extract_ecapa_embedding(wav_path, ecapa_encoder) - wav, sr = torchaudio.load(str(wav_path)) - if wav.shape[0] > 1: - wav = wav.mean(dim=0, keepdim=True) - if sr != 24000: - wav = torchaudio.functional.resample(wav, sr, 24000) - mels.append(mel) - embeddings.append(emb) - wavs.append(wav.squeeze(0)) - texts.append(text) - speaker_ids.append(sid) - except Exception as e: - print(f" [WARN] 跳过 {wav_path.name}: {e}") - - # 保存 - PROCESSED_DIR.mkdir(parents=True, exist_ok=True) - data = { - "tensors": { - "mel": torch.nn.utils.rnn.pad_sequence(mels, batch_first=True).permute(0, 2, 1), - "ecapa_emb": torch.stack(embeddings), - "wav": wavs, - }, - "metadata": { - "texts": texts, - "speaker_ids": speaker_ids, - }, - } - torch.save(data, str(PROCESSED_FILE)) - print(f" ✓ 保存到: {PROCESSED_FILE}") - print(f" mel: {data['tensors']['mel'].shape}") - print(f" ecapa_emb: {data['tensors']['ecapa_emb'].shape}") - print(f" wav: {len(data['tensors']['wav'])} 条") - print(f" texts: {len(data['metadata']['texts'])} 条") - - -def main(): - parser = argparse.ArgumentParser( - description="PersonaVoice LibriTTS 评估数据集准备脚本", - ) - parser.add_argument( - "--skip-download", action="store_true", - help="跳过下载步骤 (假设 LibriTTS 已存在)", - ) - parser.add_argument( - "--n_samples", type=int, default=0, - help="限制样本数 (测试用, 0=全部)", - ) - args = parser.parse_args() - - print("=" * 60) - print(" PersonaVoice v10.4.8 — LibriTTS 数据集准备") - print("=" * 60) - - if not args.skip_download: - download_libritts() - else: - print(" [--skip-download] 跳过下载步骤") - - if not PROCESSED_FILE.exists(): - preprocess_data(n_samples=args.n_samples) - else: - print(f" ✓ 已存在预处理文件: {PROCESSED_FILE}") - print(f" 如需重新生成, 请删除后重跑") - - print("\n ✓ 数据集准备完成") - print("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/scripts/setup_env.ps1 b/scripts/setup_env.ps1 deleted file mode 100644 index c879f65..0000000 --- a/scripts/setup_env.ps1 +++ /dev/null @@ -1,104 +0,0 @@ -# ============================================================ -# PersonaVoice v10.4.8 — 一键环境搭建脚本 (Windows PowerShell) -# ============================================================ -# 用法: -# powershell -ExecutionPolicy Bypass -File scripts\setup_env.ps1 -# -# 功能: -# 1. 创建 .venv 虚拟环境 -# 2. 升级 pip -# 3. 安装 PyTorch (CUDA 11.8 / CPU) -# 4. 安装 PersonaVoice 依赖 -# 5. 安装 silero-vad, f5-tts -# 6. 验证安装 -# ============================================================ - -param( - [switch]$UseCPU, - [switch]$SkipModels -) - -$ErrorActionPreference = "Stop" -$ProjectRoot = Split-Path -Parent $PSScriptRoot -Set-Location $ProjectRoot - -Write-Host "" -Write-Host "============================================================" -ForegroundColor Cyan -Write-Host " PersonaVoice v10.4.8 环境搭建 (Windows PowerShell)" -ForegroundColor Cyan -Write-Host "============================================================" -ForegroundColor Cyan -Write-Host "" - -# ── 1. 检查 Python ── -Write-Host "[1/6] 检查 Python ..." -ForegroundColor Yellow -try { - $pythonVersion = python --version 2>&1 - Write-Host " ✓ $pythonVersion" -} catch { - Write-Host " [ERROR] 未找到 Python, 请先安装 Python 3.10+: https://www.python.org/downloads/" -ForegroundColor Red - exit 1 -} - -# ── 2. 创建虚拟环境 ── -Write-Host "" -Write-Host "[2/6] 创建虚拟环境 .venv ..." -ForegroundColor Yellow -if (Test-Path ".venv") { - Write-Host " .venv 已存在, 跳过创建" -} else { - # 继承系统 site-packages (复用已安装的 torch/speechbrain/vocos) - python -m venv --system-site-packages .venv - Write-Host " ✓ 虚拟环境创建完成" -} - -# ── 3. 升级 pip ── -Write-Host "" -Write-Host "[3/6] 升级 pip ..." -ForegroundColor Yellow -& .\.venv\Scripts\python.exe -m pip install --upgrade pip -Write-Host " ✓ pip 升级完成" - -# ── 4. 安装 PyTorch ── -Write-Host "" -Write-Host "[4/6] 安装 PyTorch ..." -ForegroundColor Yellow -if ($UseCPU) { - Write-Host " 安装 CPU 版本 (--UseCPU)" - & .\.venv\Scripts\pip.exe install torch torchaudio --index-url https://download.pytorch.org/whl/cpu -} else { - Write-Host " 安装 CUDA 11.8 版本 (默认)" - Write-Host " 如需 CPU 版本, 请加 -UseCPU 参数" - & .\.venv\Scripts\pip.exe install torch torchaudio --index-url https://download.pytorch.org/whl/cu118 -} -Write-Host " ✓ PyTorch 安装完成" - -# ── 5. 安装 PersonaVoice 依赖 ── -Write-Host "" -Write-Host "[5/6] 安装 PersonaVoice 依赖 ..." -ForegroundColor Yellow -& .\.venv\Scripts\pip.exe install -r requirements.txt -& .\.venv\Scripts\pip.exe install silero-vad f5-tts imageio-ffmpeg -& .\.venv\Scripts\pip.exe install -e . -Write-Host " ✓ 依赖安装完成" - -# ── 6. 验证安装 ── -Write-Host "" -Write-Host "[6/6] 验证安装 ..." -ForegroundColor Yellow -& .\.venv\Scripts\python.exe -c "import torch; print(f' ✓ PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}')" -& .\.venv\Scripts\python.exe -c "import personavoice; print(f' ✓ PersonaVoice {personavoice.__version__}')" -& .\.venv\Scripts\python.exe -c "from f5_tts.api import F5TTS; print(' ✓ F5-TTS')" -& .\.venv\Scripts\python.exe -c "import speechbrain; print(f' ✓ SpeechBrain {speechbrain.__version__}')" -& .\.venv\Scripts\python.exe -c "import silero_vad; print(' ✓ Silero VAD')" - -# ── 下载模型 (可选) ── -if (-not $SkipModels) { - Write-Host "" - Write-Host "下载预训练模型 (首次运行需要, ~3GB)..." -ForegroundColor Yellow - & .\.venv\Scripts\python.exe scripts\download_models.py -} - -Write-Host "" -Write-Host "============================================================" -ForegroundColor Green -Write-Host " ✓ PersonaVoice 环境搭建完成!" -ForegroundColor Green -Write-Host "============================================================" -ForegroundColor Green -Write-Host "" -Write-Host " 下一步:" -Write-Host " 1. 下载评估数据集: .\.venv\Scripts\python.exe scripts\prepare_data.py" -Write-Host " 2. 运行 Web Demo: .\.venv\Scripts\python.exe -m personavoice.demo.api_server" -Write-Host " 3. 运行 1111 测试: .\.venv\Scripts\python.exe test_1111_clone.py" -Write-Host "" diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh deleted file mode 100644 index 2605afe..0000000 --- a/scripts/setup_env.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# ============================================================ -# PersonaVoice v10.4.8 — 一键环境搭建脚本 (Linux/macOS) -# ============================================================ -# 用法: -# bash scripts/setup_env.sh # 默认 CUDA 11.8 -# bash scripts/setup_env.sh --cpu # CPU 版本 -# bash scripts/setup_env.sh --skip-models # 跳过模型下载 -# ============================================================ - -set -e - -PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$PROJECT_ROOT" - -USE_CPU=0 -SKIP_MODELS=0 - -for arg in "$@"; do - case "$arg" in - --cpu) USE_CPU=1 ;; - --skip-models) SKIP_MODELS=1 ;; - esac -done - -echo "" -echo "============================================================" -echo " PersonaVoice v10.4.8 环境搭建 (Linux/macOS)" -echo "============================================================" -echo "" - -# ── 1. 检查 Python ── -echo "[1/6] 检查 Python ..." -if ! command -v python3 &> /dev/null; then - echo " [ERROR] 未找到 python3, 请先安装 Python 3.10+" - exit 1 -fi -PYTHON_VERSION=$(python3 --version) -echo " ✓ $PYTHON_VERSION" - -# ── 2. 创建虚拟环境 ── -echo "" -echo "[2/6] 创建虚拟环境 .venv ..." -if [ -d ".venv" ]; then - echo " .venv 已存在, 跳过创建" -else - python3 -m venv --system-site-packages .venv - echo " ✓ 虚拟环境创建完成" -fi - -# ── 3. 升级 pip ── -echo "" -echo "[3/6] 升级 pip ..." -./.venv/bin/python -m pip install --upgrade pip -echo " ✓ pip 升级完成" - -# ── 4. 安装 PyTorch ── -echo "" -echo "[4/6] 安装 PyTorch ..." -if [ "$USE_CPU" = "1" ]; then - echo " 安装 CPU 版本 (--cpu)" - ./.venv/bin/pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu -else - echo " 安装 CUDA 11.8 版本 (默认)" - echo " 如需 CPU 版本, 请加 --cpu 参数" - ./.venv/bin/pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu118 -fi -echo " ✓ PyTorch 安装完成" - -# ── 5. 安装 PersonaVoice 依赖 ── -echo "" -echo "[5/6] 安装 PersonaVoice 依赖 ..." -./.venv/bin/pip install -r requirements.txt -./.venv/bin/pip install silero-vad f5-tts imageio-ffmpeg -./.venv/bin/pip install -e . -echo " ✓ 依赖安装完成" - -# ── 6. 验证安装 ── -echo "" -echo "[6/6] 验证安装 ..." -./.venv/bin/python -c "import torch; print(f' ✓ PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}')" -./.venv/bin/python -c "import personavoice; print(f' ✓ PersonaVoice {personavoice.__version__}')" -./.venv/bin/python -c "from f5_tts.api import F5TTS; print(' ✓ F5-TTS')" -./.venv/bin/python -c "import speechbrain; print(f' ✓ SpeechBrain {speechbrain.__version__}')" -./.venv/bin/python -c "import silero_vad; print(' ✓ Silero VAD')" - -# ── 下载模型 (可选) ── -if [ "$SKIP_MODELS" = "0" ]; then - echo "" - echo "下载预训练模型 (首次运行需要, ~3GB)..." - ./.venv/bin/python scripts/download_models.py -fi - -echo "" -echo "============================================================" -echo " ✓ PersonaVoice 环境搭建完成!" -echo "============================================================" -echo "" -echo " 下一步:" -echo " 1. 下载评估数据集: ./.venv/bin/python scripts/prepare_data.py" -echo " 2. 运行 Web Demo: ./.venv/bin/python -m personavoice.demo.api_server" -echo " 3. 运行 1111 测试: ./.venv/bin/python test_1111_clone.py" -echo "" diff --git a/setup.py b/setup.py deleted file mode 100644 index 8a47c01..0000000 --- a/setup.py +++ /dev/null @@ -1,52 +0,0 @@ -from setuptools import setup, find_packages - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setup( - name="personavoice", - version="10.4.8", - description="PersonaVoice: Plug-in Adapter Architecture for 1-Second Extreme Voice Cloning", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/personavoice/personavoice", - packages=find_packages(), - python_requires=">=3.10", - install_requires=[ - "torch>=2.0.0", - "torchaudio>=2.0.0", - "speechbrain>=0.8.0", - "transformers>=4.35.0", - "accelerate>=0.24.0", - "numpy>=1.24.0", - "librosa>=0.10.0", - "soundfile>=0.12.0", - "scipy>=1.11.0", - "scikit-learn>=1.3.0", - "pyyaml>=6.0", - "omegaconf>=2.3.0", - "einops>=0.7.0", - "tqdm>=4.66.0", - "silero-vad>=5.0", - "f5-tts>=1.0", - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - "imageio-ffmpeg>=0.4.9", - "Pillow>=10.0.0", - ], - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Multimedia :: Sound/Audio :: Speech", - ], - keywords=( - "voice-cloning, zero-shot-tts, few-shot-voice-cloning, flow-matching, " - "F5-TTS, speech-synthesis, persona-driven-tts, emotional-tts, " - "plug-in-adapter, 1-second-cloning, speaker-embedding, ECAPA-TDNN, " - "orthogonal-decomposition, length-adaptive-generation, FiLM-modulation" - ), -) diff --git a/supplement/README.md b/supplement/README.md new file mode 100644 index 0000000..d626fec --- /dev/null +++ b/supplement/README.md @@ -0,0 +1,14 @@ +# Standalone supplements + +The two directories are independent analysis packages. Each reads only the +public snapshots stored in its own `artifacts/` directory, writes derived JSON +and publication figures, and can be tested without the removed PersonaVoice +runtime: + +```bash +python -m pytest -q supplement +``` + +`language_provenance` covers provenance failure auditing, observed evidence +budgets, and a fixed no-provenance replay. `speech_shortref` covers aggregate +metrics, the SECS/WER trade-off, length-stratified CFR, and ablation summaries. diff --git a/supplement/language_provenance/README.md b/supplement/language_provenance/README.md new file mode 100644 index 0000000..899b147 --- /dev/null +++ b/supplement/language_provenance/README.md @@ -0,0 +1,28 @@ +# Language provenance supplement + +This package is a standalone, CPU-only replay layer for *Provenance-Constrained Selective Language Style Planning from Sparse Evidence*. It deliberately contains no import from the historical `personavoice` implementation. The `artifacts/` directory contains ID-only public test evidence, three frozen prediction snapshots, the measured error audit source, and a manifest. + +Run from this directory: + +```bash +python audit_failures.py +python failure_by_budget.py +python cross_dataset_audit.py +python budget_curve.py +python make_figures.py +python make_architecture_figure.py +``` + +The budget curve uses the recorded constrained decisions and a provenance-free score (`support_probability * style_probability`). The requested 1/2/3/5/10 grid is recorded in the artifact; only 5 and 10 are genuine cells in the frozen predictions. The 1/2/3 cells are intentionally marked unavailable, because truncating a full-budget row would not be a new planner inference. If validation snapshots are available, pass `--validation ../path/snli_semantic_mlp_seed{seed}.validation.json` to fit the no-provenance threshold on validation only. No threshold is tuned on test. + +The benchmark is a programmatic SNLI--PersonaChat proxy. It is not consent evidence, natural memorial speech, or a deployment safety guarantee. The taxonomy intentionally reports that conflict failures are absent in this construction rather than manufacturing a conflict result. + +`cross_dataset_audit.py` downloads the official DailyDialog test archive from +Hugging Face, verifies the archive through the emitted SHA256, and computes a +deterministic lexical-evidence availability audit at budgets 1/2/3/5. Its +labels are token-overlap proxies, not entailment annotations, and its numbers +must not be pooled with the planner endpoint. + +`make_architecture_figure.py` emits the mechanism schematic as both PDF and +editable SVG. It is drawn on a wide paper-scale canvas with a fixed minimum +label size, so the figure remains legible when embedded as a two-column figure. diff --git a/supplement/language_provenance/artifacts/SHA256SUMS.txt b/supplement/language_provenance/artifacts/SHA256SUMS.txt new file mode 100644 index 0000000..ae6d3e2 --- /dev/null +++ b/supplement/language_provenance/artifacts/SHA256SUMS.txt @@ -0,0 +1,10 @@ +d717a09b263b65f9d98635733291c917110f9554702fa50d60ec421b75169d50 budget_curve.json +4d941eb49150eb5e4abfabf004cdf0d67dfbecccc1c32e6b392b7e772cce3b79 dailydialog_cross_dataset.json +8bf89ccddf75cd062b689a4426ac5d789e19d0c00e2c3871de7bc911ead58841 failure_audit.json +93f552ab788ac0a73d585979655c93a7198367eaac526a61318928e1449e919d failure_by_budget.json +77a2a09e2125e11dcd3337c696b96c8ea207228304f5d10531186b7ca1c9be97 manifest.json +0119d83cf53fc3ed9d955cfd4e3163c58d0fab11f46e2f30477a8198e062ea27 snli_error_analysis.json +84bbe8256b28cd4844718177e18a79ee35b7deac40049dd8537d5fa3dde95f60 snli_semantic_mlp_seed0.json +c5b968a2595ca152ceacf85ca67a2354493d43dca2b3aa69f8d97ff8d353a04c snli_semantic_mlp_seed1.json +5ea39fa98db65cba20c08017cf39d47483afbc488d210d77b7987755cb7152b2 snli_semantic_mlp_seed2.json +7dd31e83826f151001ee48cecfea41181a0b189cbb33fe0f384042c8252e84e8 snli_compositional_test.jsonl diff --git a/supplement/language_provenance/artifacts/budget_curve.json b/supplement/language_provenance/artifacts/budget_curve.json new file mode 100644 index 0000000..7a35600 --- /dev/null +++ b/supplement/language_provenance/artifacts/budget_curve.json @@ -0,0 +1,161 @@ +{ + "availability_note": "The frozen prediction snapshots contain no genuine planner re-evaluation at budgets 1, 2, or 3. These cells are intentionally absent rather than reconstructed by truncating full-budget predictions.", + "budgets": { + "10": { + "constrained": { + "accepted": 4, + "accepted_citation_precision": 1.0, + "citation_precision": 0.7214199759326113, + "citation_recall": 0.762936221419976, + "false_acceptance": 0.5, + "n": 831, + "positive_coverage": 0.020202020202020204 + }, + "unconstrained": { + "accepted": 86, + "accepted_citation_precision": 0.8837209302325582, + "citation_precision": 0.7214199759326113, + "citation_recall": 0.762936221419976, + "false_acceptance": 0.46511627906976744, + "n": 831, + "positive_coverage": 0.46464646464646464 + } + }, + "11": { + "constrained": { + "accepted": 12, + "accepted_citation_precision": 1.0, + "citation_precision": 0.6922246220302376, + "citation_recall": 0.7134629229661628, + "false_acceptance": 0.25, + "n": 1389, + "positive_coverage": 0.031578947368421054 + }, + "unconstrained": { + "accepted": 231, + "accepted_citation_precision": 0.8787878787878788, + "citation_precision": 0.6922246220302376, + "citation_recall": 0.7134629229661628, + "false_acceptance": 0.33766233766233766, + "n": 1389, + "positive_coverage": 0.5368421052631579 + } + }, + "12": { + "constrained": { + "accepted": 0, + "accepted_citation_precision": 0.0, + "citation_precision": 0.20833333333333334, + "citation_recall": 0.25, + "false_acceptance": 0.0, + "n": 12, + "positive_coverage": 0.0 + }, + "unconstrained": { + "accepted": 0, + "accepted_citation_precision": 0.0, + "citation_precision": 0.20833333333333334, + "citation_recall": 0.25, + "false_acceptance": 0.0, + "n": 12, + "positive_coverage": 0.0 + } + }, + "14": { + "constrained": { + "accepted": 0, + "accepted_citation_precision": 0.0, + "citation_precision": 0.5, + "citation_recall": 0.5, + "false_acceptance": 0.0, + "n": 6, + "positive_coverage": 0.0 + }, + "unconstrained": { + "accepted": 0, + "accepted_citation_precision": 0.0, + "citation_precision": 0.5, + "citation_recall": 0.5, + "false_acceptance": 0.0, + "n": 6, + "positive_coverage": 0.0 + } + }, + "5": { + "constrained": { + "accepted": 17, + "accepted_citation_precision": 0.9411764705882353, + "citation_precision": 0.7302543507362784, + "citation_recall": 0.7590361445783133, + "false_acceptance": 0.23529411764705882, + "n": 2241, + "positive_coverage": 0.036414565826330535 + }, + "unconstrained": { + "accepted": 347, + "accepted_citation_precision": 0.8976945244956772, + "citation_precision": 0.7302543507362784, + "citation_recall": 0.7590361445783133, + "false_acceptance": 0.484149855907781, + "n": 2241, + "positive_coverage": 0.5014005602240896 + } + }, + "6": { + "constrained": { + "accepted": 19, + "accepted_citation_precision": 0.9210526315789473, + "citation_precision": 0.7084967320261438, + "citation_recall": 0.7403050108932462, + "false_acceptance": 0.3684210526315789, + "n": 2295, + "positive_coverage": 0.031496062992125984 + }, + "unconstrained": { + "accepted": 333, + "accepted_citation_precision": 0.8858858858858859, + "citation_precision": 0.7084967320261438, + "citation_recall": 0.7403050108932462, + "false_acceptance": 0.45345345345345345, + "n": 2295, + "positive_coverage": 0.4776902887139108 + } + }, + "9": { + "constrained": { + "accepted": 23, + "accepted_citation_precision": 0.9347826086956522, + "citation_precision": 0.7120395327942498, + "citation_recall": 0.7331536388140162, + "false_acceptance": 0.13043478260869565, + "n": 2226, + "positive_coverage": 0.05291005291005291 + }, + "unconstrained": { + "accepted": 299, + "accepted_citation_precision": 0.9080267558528428, + "citation_precision": 0.7120395327942498, + "citation_recall": 0.7331536388140162, + "false_acceptance": 0.39464882943143814, + "n": 2226, + "positive_coverage": 0.47883597883597884 + } + } + }, + "requested_budgets": [ + 1, + 2, + 3, + 5, + 10 + ], + "schema_version": "personavoice-language-budget-curve-v1", + "status": "measured_replay", + "unavailable_requested_budgets": [ + 1, + 2, + 3 + ], + "unconstrained_score": "support_probability * style_probability; no citation or conflict gate", + "unconstrained_threshold": 0.5 +} diff --git a/supplement/language_provenance/artifacts/dailydialog_cross_dataset.json b/supplement/language_provenance/artifacts/dailydialog_cross_dataset.json new file mode 100644 index 0000000..dc26817 --- /dev/null +++ b/supplement/language_provenance/artifacts/dailydialog_cross_dataset.json @@ -0,0 +1,38 @@ +{ + "budgets": { + "1": { + "n_supported_episodes": 276, + "support_coverage_over_eligible": 0.2881002087682672, + "top1_citation_precision": 1.0, + "top1_citation_recall": 1.0 + }, + "2": { + "n_supported_episodes": 418, + "support_coverage_over_eligible": 0.4363256784968685, + "top1_citation_precision": 0.8732057416267942, + "top1_citation_recall": 1.0 + }, + "3": { + "n_supported_episodes": 468, + "support_coverage_over_eligible": 0.48851774530271397, + "top1_citation_precision": 0.8002136752136753, + "top1_citation_recall": 1.0 + }, + "5": { + "n_supported_episodes": 534, + "support_coverage_over_eligible": 0.55741127348643, + "top1_citation_precision": 0.7083645443196005, + "top1_citation_recall": 1.0 + } + }, + "claim_boundary": "This is not a planner rerun, entailment annotation, or natural-person authorization result; it is a deterministic cross-domain input-structure audit.", + "dataset": "DailyDialog official test archive", + "episodes_considered": 958, + "label_definition": "lexical evidence = prior turn sharing at least one non-stopword with the target", + "schema_version": "personavoice-language-dailydialog-audit-v1", + "source_sha256": "cfffc0bab93a3697c2562a9ee35c4533f0de2a93e86a4e4d3ea45cbde4bea00f", + "source_url": "https://huggingface.co/datasets/roskoN/dailydialog/resolve/main/test.zip", + "split": "test", + "status": "measured_auxiliary_audit", + "target_turn": "sixth turn when available; otherwise final turn" +} diff --git a/supplement/language_provenance/artifacts/failure_audit.json b/supplement/language_provenance/artifacts/failure_audit.json new file mode 100644 index 0000000..f0b5607 --- /dev/null +++ b/supplement/language_provenance/artifacts/failure_audit.json @@ -0,0 +1,354 @@ +{ + "by_evidence_budget": { + "10": { + "false_abstention:attribution_underconfidence": 2, + "false_abstention:selection_threshold": 95, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 1 + }, + "11": { + "false_abstention:attribution_underconfidence": 25, + "false_abstention:selection_threshold": 251, + "false_acceptance:selection_calibration": 3 + }, + "5": { + "false_abstention:attribution_underconfidence": 18, + "false_abstention:selection_threshold": 326, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 3 + }, + "6": { + "false_abstention:attribution_underconfidence": 9, + "false_abstention:selection_threshold": 360, + "false_acceptance:selection_calibration": 7 + }, + "9": { + "false_abstention:attribution_underconfidence": 33, + "false_abstention:selection_threshold": 325, + "false_acceptance:selection_calibration": 3 + } + }, + "by_selected_source_type": { + "factual_relation": { + "false_abstention:selection_threshold": 1357, + "false_acceptance:missing_support": 2, + "false_acceptance:style_overfit": 17 + }, + "none_selected": { + "false_abstention:attribution_underconfidence": 11, + "false_abstention:selection_threshold": 76 + } + }, + "counts": { + "false_abstention:attribution_underconfidence": 87, + "false_abstention:selection_threshold": 1357, + "false_acceptance:missing_support": 2, + "false_acceptance:selection_calibration": 17 + }, + "derived_false_acceptance": { + "false_acceptance:missing_support": 2, + "false_acceptance:style_overfit": 17 + }, + "evaluation_rows": 9000, + "examples": { + "false_abstention:attribution_underconfidence": [ + { + "episode_id": "snli-compositional:test:598:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:3041:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed1.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed1.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:4240:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed1.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:4824:positive", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed1.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed2.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:4240:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed2.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:5433:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed2.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:7265:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed2.json", + "selected_source_types": [] + }, + { + "episode_id": "snli-compositional:test:8681:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed2.json", + "selected_source_types": [] + } + ], + "false_abstention:selection_threshold": [ + { + "episode_id": "snli-compositional:test:6:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:13:positive", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:26:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:32:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:48:positive", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:56:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:76:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:89:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:121:positive", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:173:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:182:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:198:positive", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + } + ], + "false_acceptance:missing_support": [ + { + "episode_id": "snli-compositional:test:6260:positive", + "evidence_budget": 10, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:9399:positive", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + } + ], + "false_acceptance:style_overfit": [ + { + "episode_id": "snli-compositional:test:6:negative", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:48:negative", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:856:negative", + "evidence_budget": 5, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:1009:negative", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:1195:negative", + "evidence_budget": 11, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:2017:negative", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:3566:negative", + "evidence_budget": 10, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:4186:negative", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:5553:negative", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:6430:negative", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:7276:negative", + "evidence_budget": 9, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + }, + { + "episode_id": "snli-compositional:test:8994:negative", + "evidence_budget": 6, + "seed_file": "snli_semantic_mlp_seed0.json", + "selected_source_types": [ + "factual_relation" + ] + } + ] + }, + "interpretation": "Selection/calibration and style-overfit dominate this proxy; conflict failures are zero because the released test construction contains no accepted contradictory cases. This is a benchmark limitation, not evidence that conflict handling is solved.", + "rates_among_tagged_failures": { + "false_abstention:attribution_underconfidence": 0.059466848940533154, + "false_abstention:selection_threshold": 0.9275461380724539, + "false_acceptance:missing_support": 0.001367053998632946, + "false_acceptance:selection_calibration": 0.01161995898838004 + }, + "schema_version": "personavoice-language-failure-audit-v1", + "status": "measured_replay", + "taxonomy": { + "false_abstention:attribution_underconfidence": "positive episode rejected with no attribution score at gate", + "false_abstention:selection_threshold": "positive episode rejected by the joint threshold", + "false_acceptance:missing_support": "accepted positive-style episode without gold support", + "false_acceptance:selection_calibration": "accepted negative episode not explained by the typed labels", + "false_acceptance:style_overfit": "accepted negative style pair; semantic support exists but style gate is wrong", + "false_acceptance:unresolved_conflict": "accepted an episode labelled contradictory" + } +} diff --git a/supplement/language_provenance/artifacts/failure_by_budget.json b/supplement/language_provenance/artifacts/failure_by_budget.json new file mode 100644 index 0000000..2f1f46d --- /dev/null +++ b/supplement/language_provenance/artifacts/failure_by_budget.json @@ -0,0 +1,85 @@ +{ + "budgets": { + "10": { + "counts": { + "false_abstention:attribution_underconfidence": 2, + "false_abstention:selection_threshold": 95, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 1 + }, + "false_abstention_attribution_rate": 0.0024067388688327317, + "false_abstention_selection_rate": 0.11432009626955475, + "false_acceptance_missing_support_rate": 0.0012033694344163659, + "false_acceptance_selection_calibration_rate": 0.0012033694344163659, + "n": 831 + }, + "11": { + "counts": { + "false_abstention:attribution_underconfidence": 25, + "false_abstention:selection_threshold": 251, + "false_acceptance:selection_calibration": 3 + }, + "false_abstention_attribution_rate": 0.017998560115190784, + "false_abstention_selection_rate": 0.18070554355651547, + "false_acceptance_missing_support_rate": 0.0, + "false_acceptance_selection_calibration_rate": 0.0021598272138228943, + "n": 1389 + }, + "12": { + "counts": {}, + "false_abstention_attribution_rate": 0.0, + "false_abstention_selection_rate": 0.0, + "false_acceptance_missing_support_rate": 0.0, + "false_acceptance_selection_calibration_rate": 0.0, + "n": 12 + }, + "14": { + "counts": {}, + "false_abstention_attribution_rate": 0.0, + "false_abstention_selection_rate": 0.0, + "false_acceptance_missing_support_rate": 0.0, + "false_acceptance_selection_calibration_rate": 0.0, + "n": 6 + }, + "5": { + "counts": { + "false_abstention:attribution_underconfidence": 18, + "false_abstention:selection_threshold": 326, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 3 + }, + "false_abstention_attribution_rate": 0.008032128514056224, + "false_abstention_selection_rate": 0.14547077197679606, + "false_acceptance_missing_support_rate": 0.0004462293618920125, + "false_acceptance_selection_calibration_rate": 0.0013386880856760374, + "n": 2241 + }, + "6": { + "counts": { + "false_abstention:attribution_underconfidence": 9, + "false_abstention:selection_threshold": 360, + "false_acceptance:selection_calibration": 7 + }, + "false_abstention_attribution_rate": 0.00392156862745098, + "false_abstention_selection_rate": 0.1568627450980392, + "false_acceptance_missing_support_rate": 0.0, + "false_acceptance_selection_calibration_rate": 0.0030501089324618735, + "n": 2295 + }, + "9": { + "counts": { + "false_abstention:attribution_underconfidence": 33, + "false_abstention:selection_threshold": 325, + "false_acceptance:selection_calibration": 3 + }, + "false_abstention_attribution_rate": 0.014824797843665768, + "false_abstention_selection_rate": 0.14600179694519316, + "false_acceptance_missing_support_rate": 0.0, + "false_acceptance_selection_calibration_rate": 0.0013477088948787063, + "n": 2226 + } + }, + "interpretation": "Rates use pooled frozen rows from three registered seeds; cells are the observed budgets only.", + "schema_version": "personavoice-language-failure-by-budget-v1", + "status": "measured_replay" +} diff --git a/supplement/language_provenance/artifacts/manifest.json b/supplement/language_provenance/artifacts/manifest.json new file mode 100644 index 0000000..5ff1f6e --- /dev/null +++ b/supplement/language_provenance/artifacts/manifest.json @@ -0,0 +1,31 @@ +{ + "citation_semantics": "the annotated SNLI premise is the provenance source for all three NLI labels", + "counts": { + "test": 3000, + "train": 12000, + "validation": 3000 + }, + "distractors_per_example": 3, + "fingerprint": "sha256:2a97e9948608eb12ce6f33ca312c020611efaca7ab8bfd785d91cd6ecf7d0bd5", + "gold_position": "source-index modulo evidence count", + "label_mapping": { + "0": "entailment", + "1": "neutral", + "2": "contradiction" + }, + "living_speaker_authorization": false, + "natural_generation_claim": false, + "sampling": "fixed-seed label-balanced rows; official splits preserved", + "schema_version": "personavoice-snli-compositional-v1", + "source_file_sha256": { + "test": "4696deda851c4d2385f26b58f2e13f9ed9f08ea7b42a3f4c2b97a9d08448878c", + "train": "ef9a7b25d97390a62aeda7abe26aec8640600f50b818eaeb9107097d60ac6620", + "validation": "00f5ed8deaed007fef3022f0215b287efbab815b1bb31ac3f3ff4f4129d41ffe" + }, + "source_uri": "https://huggingface.co/datasets/stanfordnlp/snli", + "sources": { + "factual": "SNLI 1.0 human NLI labels (CC BY-SA 4.0)", + "style": "PersonaChat role-instance evidence (CC BY 4.0)", + "auxiliary_cross_dataset": "DailyDialog official test archive (CC BY-NC-SA 4.0; lexical audit only)" + } +} diff --git a/supplement/language_provenance/artifacts/snli_compositional_test.jsonl b/supplement/language_provenance/artifacts/snli_compositional_test.jsonl new file mode 100644 index 0000000..cb65f44 --- /dev/null +++ b/supplement/language_provenance/artifacts/snli_compositional_test.jsonl @@ -0,0 +1,3000 @@ +{"candidate_speaker_id": "snli:test:6", "candidate_text": "I wonder whether a man poses in front of an ad?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9854:premise", "snli:test:173:premise", "snli:test:6:premise", "snli:test:9814:premise", "style:personachat:test:0:self:1:positive:0"], "evidence_texts": ["Kids play in the water in the middle of the street.", "A man wearing a red uniform and helmet stands on his motorbike.", "An old man with a package poses in front of an advertisement.", "A child being watched while getting something out of a tank.", "so ? lol i want to meet my family at home in bora"], "speaker_id": "snli:test:6", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6", "candidate_text": "well, a man poses in front of an ad", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9854:premise", "snli:test:173:premise", "snli:test:6:premise", "snli:test:9814:premise", "style:personachat:test:1:self:1:positive:0"], "evidence_texts": ["Kids play in the water in the middle of the street.", "A man wearing a red uniform and helmet stands on his motorbike.", "An old man with a package poses in front of an advertisement.", "A child being watched while getting something out of a tank.", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:6", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:13", "candidate_text": "I wonder whether a vehicle is crossing a river?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:13:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9950:premise", "snli:test:13:premise", "snli:test:253:premise", "snli:test:9871:premise", "style:personachat:test:0:self:2:positive:0", "style:personachat:test:0:self:2:positive:1"], "evidence_texts": ["Two men in karate gear using fighting sticks.", "A land rover is being driven across a river.", "Four adults eat while sitting on a tile floor.", "A bearded man sits near a white van.", "why does that matter any ? i went outdoors to play frisbee", "so ? lol i want to meet my family at home in bora"], "speaker_id": "snli:test:13", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:13", "candidate_text": "Honestly, A vehicle is crossing a river!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:13:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9950:premise", "snli:test:13:premise", "snli:test:253:premise", "snli:test:9871:premise", "style:personachat:test:21:self:2:positive:0", "style:personachat:test:21:self:2:positive:1"], "evidence_texts": ["Two men in karate gear using fighting sticks.", "A land rover is being driven across a river.", "Four adults eat while sitting on a tile floor.", "A bearded man sits near a white van.", "yes i thought they were real too , i got duped by my supplier", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:13", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:26", "candidate_text": "I wonder whether a family is at the beach?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:26:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9902:premise", "snli:test:34:premise", "snli:test:26:premise", "snli:test:9889:premise", "style:personachat:test:0:self:5:positive:0", "style:personachat:test:0:self:5:positive:1", "style:personachat:test:0:self:5:positive:2", "style:personachat:test:0:self:5:positive:3", "style:personachat:test:0:self:5:positive:4"], "evidence_texts": ["The older woman smiles as she holds up a hammer.", "A man reads the paper in a bar with green lighting.", "A young family enjoys feeling ocean waves lap at their feet.", "Girl wearing white shirt sings on stage while playing guitar", "what club did you go to ? me an timothy watched tv", "lol oh okay kind of random", "we watched a show about animals like him", "why does that matter any ? i went outdoors to play frisbee", "so ? lol i want to meet my family at home in bora"], "speaker_id": "snli:test:26", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:26", "candidate_text": "A family is at the beach.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:26:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9902:premise", "snli:test:34:premise", "snli:test:26:premise", "snli:test:9889:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The older woman smiles as she holds up a hammer.", "A man reads the paper in a bar with green lighting.", "A young family enjoys feeling ocean waves lap at their feet.", "Girl wearing white shirt sings on stage while playing guitar", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:26", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:30", "candidate_text": "I wonder whether three people sit by a busy street bareheaded?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:30:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9967:premise", "snli:test:9649:premise", "snli:test:30:premise", "snli:test:238:premise", "style:personachat:test:0:self:7:positive:0", "style:personachat:test:0:self:7:positive:1", "style:personachat:test:0:self:7:positive:2", "style:personachat:test:0:self:7:positive:3", "style:personachat:test:0:self:7:positive:4", "style:personachat:test:0:self:7:positive:5", "style:personachat:test:0:self:7:positive:6"], "evidence_texts": ["Man on bike with female standing on rear of back with arms around his neck.", "The man in black is flying off the full in front of a red fence.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "A group of adults stands in the bathroom looking down a small child in the bath.", "i am well an loving this interaction how are you ?", "this is my favorite time of the year season wise", "what club did you go to ? me an timothy watched tv", "lol oh okay kind of random", "we watched a show about animals like him", "why does that matter any ? i went outdoors to play frisbee", "so ? lol i want to meet my family at home in bora"], "speaker_id": "snli:test:30", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:30", "candidate_text": "well, three people sit by a busy street bareheaded", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:30:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9967:premise", "snli:test:9649:premise", "snli:test:30:premise", "snli:test:238:premise", "style:personachat:test:1:self:7:positive:0", "style:personachat:test:1:self:7:positive:1", "style:personachat:test:1:self:7:positive:2", "style:personachat:test:1:self:7:positive:3", "style:personachat:test:1:self:7:positive:4", "style:personachat:test:1:self:7:positive:5", "style:personachat:test:1:self:7:positive:6"], "evidence_texts": ["Man on bike with female standing on rear of back with arms around his neck.", "The man in black is flying off the full in front of a red fence.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "A group of adults stands in the bathroom looking down a small child in the bath.", "hi , my dad is old as well , they live close to me and i see them often", "sure , i pick him up for church every sunday with my ford pickup", "i guess that means you do not go hunting often ? i love hunting , i own 3 guns", "yeah me too , i am conservative so i love church and hunting", "fishing is good . i love fishing as well", "yes that must be hard , i hope things get better for you", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:30", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:32", "candidate_text": "well, three hood wearing people stand in a street", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:32:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:32:premise", "snli:test:238:premise", "snli:test:9967:premise", "snli:test:321:premise", "style:personachat:test:1:self:1:positive:0"], "evidence_texts": ["3 young man in hoods standing in the middle of a quiet street facing the camera.", "A group of adults stands in the bathroom looking down a small child in the bath.", "Man on bike with female standing on rear of back with arms around his neck.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:32", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:32", "candidate_text": "Three hood wearing people stand in a street.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:32:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:32:premise", "snli:test:238:premise", "snli:test:9967:premise", "snli:test:321:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["3 young man in hoods standing in the middle of a quiet street facing the camera.", "A group of adults stands in the bathroom looking down a small child in the bath.", "Man on bike with female standing on rear of back with arms around his neck.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:32", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:34", "candidate_text": "well, the man is reading the sportspage", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:34:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:26:premise", "snli:test:212:premise", "snli:test:34:premise", "snli:test:9902:premise", "style:personachat:test:1:self:2:positive:0", "style:personachat:test:1:self:2:positive:1"], "evidence_texts": ["A young family enjoys feeling ocean waves lap at their feet.", "A person in a red shirt and black pants hunched over.", "A man reads the paper in a bar with green lighting.", "The older woman smiles as she holds up a hammer.", "yes that must be hard , i hope things get better for you", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:34", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:34", "candidate_text": "I wonder whether the man is reading the sportspage?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:34:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:26:premise", "snli:test:212:premise", "snli:test:34:premise", "snli:test:9902:premise", "style:personachat:test:2:self:2:positive:0", "style:personachat:test:2:self:2:positive:1"], "evidence_texts": ["A young family enjoys feeling ocean waves lap at their feet.", "A person in a red shirt and black pants hunched over.", "A man reads the paper in a bar with green lighting.", "The older woman smiles as she holds up a hammer.", "yes i am . i have always been a christian .", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:34", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:36", "candidate_text": "well, three firefighters putting out a fire inside of a subway station", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:36:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:36:premise", "snli:test:9996:premise", "snli:test:9620:premise", "snli:test:121:premise", "style:personachat:test:1:self:5:positive:0", "style:personachat:test:1:self:5:positive:1", "style:personachat:test:1:self:5:positive:2", "style:personachat:test:1:self:5:positive:3", "style:personachat:test:1:self:5:positive:4"], "evidence_texts": ["Three firefighter come out of subway station.", "Two women are observing something together.", "A woman holding a newborn baby.", "Two teenage girls conversing next to lockers.", "i guess that means you do not go hunting often ? i love hunting , i own 3 guns", "yeah me too , i am conservative so i love church and hunting", "fishing is good . i love fishing as well", "yes that must be hard , i hope things get better for you", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:36", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:36", "candidate_text": "Honestly, Three firefighters putting out a fire inside of a subway station!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:36:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:36:premise", "snli:test:9996:premise", "snli:test:9620:premise", "snli:test:121:premise", "style:personachat:test:29:self:5:positive:0", "style:personachat:test:29:self:5:positive:1", "style:personachat:test:29:self:5:positive:2", "style:personachat:test:29:self:5:positive:3", "style:personachat:test:29:self:5:positive:4"], "evidence_texts": ["Three firefighter come out of subway station.", "Two women are observing something together.", "A woman holding a newborn baby.", "Two teenage girls conversing next to lockers.", "that is a great industry . what kind of music do you like ?", "yes . i love to cook and i am looking for someone to build me some cooling racks .", "me too what is your favorite ? chevelle ? cadillac ? oldsmobile ?", "i prefer to cook vegan food . what do you like to eat ?", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:36", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:38", "candidate_text": "well, three firefighters playing cards inside a fire station", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:38:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:121:premise", "snli:test:9996:premise", "snli:test:38:premise", "snli:test:198:premise", "style:personachat:test:1:self:7:positive:0", "style:personachat:test:1:self:7:positive:1", "style:personachat:test:1:self:7:positive:2", "style:personachat:test:1:self:7:positive:3", "style:personachat:test:1:self:7:positive:4", "style:personachat:test:1:self:7:positive:5", "style:personachat:test:1:self:7:positive:6"], "evidence_texts": ["Two teenage girls conversing next to lockers.", "Two women are observing something together.", "Three firefighter come out of subway station.", "Two men climbing on a wooden scaffold.", "hi , my dad is old as well , they live close to me and i see them often", "sure , i pick him up for church every sunday with my ford pickup", "i guess that means you do not go hunting often ? i love hunting , i own 3 guns", "yeah me too , i am conservative so i love church and hunting", "fishing is good . i love fishing as well", "yes that must be hard , i hope things get better for you", "yes i agree . i try to enjoy life too , whenever i am not working"], "speaker_id": "snli:test:38", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:38", "candidate_text": "Three firefighters playing cards inside a fire station.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:38:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:121:premise", "snli:test:9996:premise", "snli:test:38:premise", "snli:test:198:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two teenage girls conversing next to lockers.", "Two women are observing something together.", "Three firefighter come out of subway station.", "Two men climbing on a wooden scaffold.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:38", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:40", "candidate_text": "I wonder whether a person is selling coconuts?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:40:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:40:premise", "snli:test:9681:premise", "snli:test:314:premise", "snli:test:9475:premise", "style:personachat:test:2:self:1:positive:0"], "evidence_texts": ["A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:40", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:40", "candidate_text": "A person is selling coconuts.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:40:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:40:premise", "snli:test:9681:premise", "snli:test:314:premise", "snli:test:9475:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:40", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:48", "candidate_text": "I wonder whether a boy is on a playground?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:48:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:48:premise", "snli:test:9843:premise", "snli:test:282:premise", "snli:test:9785:premise", "style:personachat:test:2:self:2:positive:0", "style:personachat:test:2:self:2:positive:1"], "evidence_texts": ["A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "yes i am . i have always been a christian .", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:48", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:48", "candidate_text": "well, a boy is on a playground", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:48:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:48:premise", "snli:test:9843:premise", "snli:test:282:premise", "snli:test:9785:premise", "style:personachat:test:7:self:2:positive:0", "style:personachat:test:7:self:2:positive:1"], "evidence_texts": ["A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "do not you have to be older than 15 to work at mcdonalds ? that sounds illegal", "that is horrible and verry illegal"], "speaker_id": "snli:test:48", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:55", "candidate_text": "I wonder whether two children are playing catch at a park?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:55:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9998:premise", "snli:test:9858:premise", "snli:test:114:premise", "snli:test:55:premise", "style:personachat:test:2:self:5:positive:0", "style:personachat:test:2:self:5:positive:1", "style:personachat:test:2:self:5:positive:2", "style:personachat:test:2:self:5:positive:3", "style:personachat:test:2:self:5:positive:4"], "evidence_texts": ["A man in a black leather jacket and a book in his hand speaks in a classroom.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "very cool . do you have pets ?", "yes . two dogs and a cat . they are my babies .", "i live in texas . i love riding my bike here .", "yes i am . i have always been a christian .", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:55", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:55", "candidate_text": "Honestly, Two children are playing catch at a park!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:55:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9998:premise", "snli:test:9858:premise", "snli:test:114:premise", "snli:test:55:premise", "style:personachat:test:32:self:5:positive:0", "style:personachat:test:32:self:5:positive:1", "style:personachat:test:32:self:5:positive:2", "style:personachat:test:32:self:5:positive:3", "style:personachat:test:32:self:5:positive:4"], "evidence_texts": ["A man in a black leather jacket and a book in his hand speaks in a classroom.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "amazon is my favorite store , you cannot go wrong .", "no i pick pretty well trust me", "no i do not an have a good 12", "i do but not now", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:55", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:56", "candidate_text": "I wonder whether two children are on a rug?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:56:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:56:premise", "snli:test:114:premise", "snli:test:9998:premise", "snli:test:332:premise", "style:personachat:test:2:self:7:positive:0", "style:personachat:test:2:self:7:positive:1", "style:personachat:test:2:self:7:positive:2", "style:personachat:test:2:self:7:positive:3", "style:personachat:test:2:self:7:positive:4", "style:personachat:test:2:self:7:positive:5", "style:personachat:test:2:self:7:positive:6"], "evidence_texts": ["Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "no , i work full time at a nursing home . i am a nurses aide .", "nice . yes i do . caring for people is the joy of my life .", "very cool . do you have pets ?", "yes . two dogs and a cat . they are my babies .", "i live in texas . i love riding my bike here .", "yes i am . i have always been a christian .", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:56", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:56", "candidate_text": "Two children are on a rug.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:56:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:56:premise", "snli:test:114:premise", "snli:test:9998:premise", "snli:test:332:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:56", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:62", "candidate_text": "I wonder whether the two girls play in the autumn?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:62:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9919:premise", "snli:test:538:premise", "snli:test:62:premise", "snli:test:9763:premise", "style:personachat:test:3:self:1:positive:0"], "evidence_texts": ["Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A black and brown dog is jumping on blocks of concrete in the water.", "anytime are you okay now ?"], "speaker_id": "snli:test:62", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:62", "candidate_text": "well, the two girls play in the autumn", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:62:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9919:premise", "snli:test:538:premise", "snli:test:62:premise", "snli:test:9763:premise", "style:personachat:test:9:self:1:positive:0"], "evidence_texts": ["Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A black and brown dog is jumping on blocks of concrete in the water.", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:62", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:68", "candidate_text": "I wonder whether a snowmobile in a blizzard?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:68:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:68:premise", "snli:test:9944:premise", "snli:test:240:premise", "snli:test:9864:premise", "style:personachat:test:3:self:2:positive:0", "style:personachat:test:3:self:2:positive:1"], "evidence_texts": ["A snowboarder on a wide plain of snow", "A woman is petting a dog outside.", "A goofy looking woman is singing on stage.", "a man walks on the icy sidewalk.", "yes get on it now", "anytime are you okay now ?"], "speaker_id": "snli:test:68", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:68", "candidate_text": "Honestly, A snowmobile in a blizzard!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:68:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:68:premise", "snli:test:9944:premise", "snli:test:240:premise", "snli:test:9864:premise", "style:personachat:test:36:self:2:positive:0", "style:personachat:test:36:self:2:positive:1"], "evidence_texts": ["A snowboarder on a wide plain of snow", "A woman is petting a dog outside.", "A goofy looking woman is singing on stage.", "a man walks on the icy sidewalk.", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:68", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:76", "candidate_text": "I wonder whether a man is in a black shirt?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:76:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:76:premise", "snli:test:9910:premise", "snli:test:157:premise", "snli:test:9640:premise", "style:personachat:test:3:self:5:positive:0", "style:personachat:test:3:self:5:positive:1", "style:personachat:test:3:self:5:positive:2", "style:personachat:test:3:self:5:positive:3", "style:personachat:test:3:self:5:positive:4"], "evidence_texts": ["A man in a black shirt is looking at a bike in a workshop.", "A woman in a black coat eats dinner while her dog looks on.", "A couple is eating outside at a table and he is pointing at something.", "A group of young men in a gym take turns scoring in basketball.", "yes i do i work on the baby floor an i want no kids lol", "no it is easy especially being single an i like it", "i do get it i do", "yes get on it now", "anytime are you okay now ?"], "speaker_id": "snli:test:76", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:76", "candidate_text": "A man is in a black shirt.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:76:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:76:premise", "snli:test:9910:premise", "snli:test:157:premise", "snli:test:9640:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a black shirt is looking at a bike in a workshop.", "A woman in a black coat eats dinner while her dog looks on.", "A couple is eating outside at a table and he is pointing at something.", "A group of young men in a gym take turns scoring in basketball.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:76", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:86", "candidate_text": "I wonder whether a child wearing a red top is standing behind a pretty blond headed child?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:86:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9782:premise", "snli:test:295:premise", "snli:test:86:premise", "snli:test:9703:premise", "style:personachat:test:3:self:7:positive:0", "style:personachat:test:3:self:7:positive:1", "style:personachat:test:3:self:7:positive:2", "style:personachat:test:3:self:7:positive:3", "style:personachat:test:3:self:7:positive:4", "style:personachat:test:3:self:7:positive:5", "style:personachat:test:3:self:7:positive:6"], "evidence_texts": ["The young girl is swinging on a swing, and her hair is flying out behind her.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "yes we got to act quickly", "i never have juice , just water .", "yes i do i work on the baby floor an i want no kids lol", "no it is easy especially being single an i like it", "i do get it i do", "yes get on it now", "anytime are you okay now ?"], "speaker_id": "snli:test:86", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:86", "candidate_text": "well, a child wearing a red top is standing behind a pretty blond headed child", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:86:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9782:premise", "snli:test:295:premise", "snli:test:86:premise", "snli:test:9703:premise", "style:personachat:test:9:self:6:positive:0", "style:personachat:test:9:self:6:positive:1", "style:personachat:test:9:self:6:positive:2", "style:personachat:test:9:self:6:positive:3", "style:personachat:test:9:self:6:positive:4", "style:personachat:test:9:self:6:positive:5"], "evidence_texts": ["The young girl is swinging on a swing, and her hair is flying out behind her.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "i am great ! that is cool , i just got home from hunting .", "i do like the outdoors . . . as well as restoring classic cars . i am in missouri .", "that is good . i work as a handyman for a company that rents properties .", "i am a single parent as well . just me and my baby girl .", "it is fun to go shopping for her", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:86", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:89", "candidate_text": "I wonder whether a person outside pushing a stroller?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:89:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9990:premise", "snli:test:89:premise", "snli:test:546:premise", "snli:test:9954:premise", "style:personachat:test:4:self:1:positive:0"], "evidence_texts": ["A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:89", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:89", "candidate_text": "Honestly, A person outside pushing a stroller!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:89:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9990:premise", "snli:test:89:premise", "snli:test:546:premise", "snli:test:9954:premise", "style:personachat:test:45:self:1:positive:0"], "evidence_texts": ["A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:89", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:95", "candidate_text": "I wonder whether people are screaming at a boxing match?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:95:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9982:premise", "snli:test:167:premise", "snli:test:9845:premise", "snli:test:95:premise", "style:personachat:test:4:self:2:positive:0", "style:personachat:test:4:self:2:positive:1"], "evidence_texts": ["Two gondola operators sit along a road in Italy.", "The flight attendant dressed in yellow demonstrates life vest usage.", "People landscaping and gardening the areas around the walkway.", "People are conversing at a dining table under a canopy.", "i certainly am , i am part native american , i live here in oklahoma", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:95", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:95", "candidate_text": "People are screaming at a boxing match.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:95:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9982:premise", "snli:test:167:premise", "snli:test:9845:premise", "snli:test:95:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two gondola operators sit along a road in Italy.", "The flight attendant dressed in yellow demonstrates life vest usage.", "People landscaping and gardening the areas around the walkway.", "People are conversing at a dining table under a canopy.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:95", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:114", "candidate_text": "I wonder whether the crowd is calling for help for the people on the roof?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:114:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:56:premise", "snli:test:332:premise", "snli:test:114:premise", "snli:test:55:premise", "style:personachat:test:4:self:5:positive:0", "style:personachat:test:4:self:5:positive:1", "style:personachat:test:4:self:5:positive:2", "style:personachat:test:4:self:5:positive:3", "style:personachat:test:4:self:5:positive:4"], "evidence_texts": ["Two children re laying on a rug with some wooden bricks laid out in a square between them.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "are you afraid of heights ? i certainly am", "sure i like tv , what do you watch ?", "i do not have time to watch tv , i am an attorney so i work alot", "i certainly am , i am part native american , i live here in oklahoma", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:114", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:114", "candidate_text": "well, the crowd is calling for help for the people on the roof", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:114:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:56:premise", "snli:test:332:premise", "snli:test:114:premise", "snli:test:55:premise", "style:personachat:test:10:self:5:positive:0", "style:personachat:test:10:self:5:positive:1", "style:personachat:test:10:self:5:positive:2", "style:personachat:test:10:self:5:positive:3", "style:personachat:test:10:self:5:positive:4"], "evidence_texts": ["Two children re laying on a rug with some wooden bricks laid out in a square between them.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "nice , my job is being a truck driver , i have seven kids to feed .", "i like running , but i have no time .", "it does , i like to wear my favorite color blue when i do run .", "there is a local trail near my house in the mountains that i like .", "yes , the peach and the quiet , it is heaven ."], "speaker_id": "snli:test:114", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:121", "candidate_text": "I wonder whether people talking next to lockers?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:121:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:38:premise", "snli:test:121:premise", "snli:test:198:premise", "snli:test:36:premise", "style:personachat:test:4:self:7:positive:0", "style:personachat:test:4:self:7:positive:1", "style:personachat:test:4:self:7:positive:2", "style:personachat:test:4:self:7:positive:3", "style:personachat:test:4:self:7:positive:4", "style:personachat:test:4:self:7:positive:5", "style:personachat:test:4:self:7:positive:6"], "evidence_texts": ["Three firefighter come out of subway station.", "Two teenage girls conversing next to lockers.", "Two men climbing on a wooden scaffold.", "Three firefighter come out of subway station.", "good , do you like animals ?", "i have 2 dogs , they are great , where do you work ?", "are you afraid of heights ? i certainly am", "sure i like tv , what do you watch ?", "i do not have time to watch tv , i am an attorney so i work alot", "i certainly am , i am part native american , i live here in oklahoma", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:121", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:121", "candidate_text": "Honestly, People talking next to lockers!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:121:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:38:premise", "snli:test:121:premise", "snli:test:198:premise", "snli:test:36:premise", "style:personachat:test:45:self:7:positive:0", "style:personachat:test:45:self:7:positive:1", "style:personachat:test:45:self:7:positive:2", "style:personachat:test:45:self:7:positive:3", "style:personachat:test:45:self:7:positive:4", "style:personachat:test:45:self:7:positive:5", "style:personachat:test:45:self:7:positive:6"], "evidence_texts": ["Three firefighter come out of subway station.", "Two teenage girls conversing next to lockers.", "Two men climbing on a wooden scaffold.", "Three firefighter come out of subway station.", "hi , i am lucy . just getting ready for another day at the office", "i am a secretary . that is neat , what subject do you want to teach", "kindergarten would be fun ! do you play any sports", "i played soccer in college ! we won a cup one season", "that sounds like fun ! i just almost choked on a piece of candy", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:121", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:157", "candidate_text": "I wonder whether the man is pointing at a purse snatcher?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:157:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:76:premise", "snli:test:157:premise", "snli:test:160:premise", "snli:test:9910:premise", "style:personachat:test:5:self:1:positive:0"], "evidence_texts": ["A man in a black shirt is looking at a bike in a workshop.", "A couple is eating outside at a table and he is pointing at something.", "Two women walk down a sidewalk along a busy street in a downtown area.", "A woman in a black coat eats dinner while her dog looks on.", "lol . i am scared of heights too . do you like to listen to music ?"], "speaker_id": "snli:test:157", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:157", "candidate_text": "The man is pointing at a purse snatcher.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:157:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:76:premise", "snli:test:157:premise", "snli:test:160:premise", "snli:test:9910:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a black shirt is looking at a bike in a workshop.", "A couple is eating outside at a table and he is pointing at something.", "Two women walk down a sidewalk along a busy street in a downtown area.", "A woman in a black coat eats dinner while her dog looks on.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:157", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:160", "candidate_text": "I wonder whether the women were picking berries in the countryside?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:160:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:160:premise", "snli:test:157:premise", "snli:test:248:premise", "snli:test:76:premise", "style:personachat:test:5:self:2:positive:0", "style:personachat:test:5:self:2:positive:1"], "evidence_texts": ["Two women walk down a sidewalk along a busy street in a downtown area.", "A couple is eating outside at a table and he is pointing at something.", "A man, woman, and child get their picture taken in front of the mountains.", "A man in a black shirt is looking at a bike in a workshop.", "california . where do you live ?", "lol . i am scared of heights too . do you like to listen to music ?"], "speaker_id": "snli:test:160", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:160", "candidate_text": "well, the women were picking berries in the countryside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:160:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:160:premise", "snli:test:157:premise", "snli:test:248:premise", "snli:test:76:premise", "style:personachat:test:11:self:2:positive:0", "style:personachat:test:11:self:2:positive:1"], "evidence_texts": ["Two women walk down a sidewalk along a busy street in a downtown area.", "A couple is eating outside at a table and he is pointing at something.", "A man, woman, and child get their picture taken in front of the mountains.", "A man in a black shirt is looking at a bike in a workshop.", "i live close to where daniel boone roamed .", "i have been to small town there called mansfield ."], "speaker_id": "snli:test:160", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:167", "candidate_text": "I wonder whether a woman is preparing drinks on an airplane?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:167:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:95:premise", "snli:test:263:premise", "snli:test:9982:premise", "snli:test:167:premise", "style:personachat:test:5:self:5:positive:0", "style:personachat:test:5:self:5:positive:1", "style:personachat:test:5:self:5:positive:2", "style:personachat:test:5:self:5:positive:3", "style:personachat:test:5:self:5:positive:4"], "evidence_texts": ["People are conversing at a dining table under a canopy.", "A person in orange clothing rests above a metro entrance.", "Two gondola operators sit along a road in Italy.", "The flight attendant dressed in yellow demonstrates life vest usage.", "i get that ! i am a doctor . pediatrics to be exact . do anything for fun ?", "me too ! well , i do not have any dogs , but i run everyday !", "yes , it helps me clear my mind . i also love to read to escape reality sometimes too", "california . where do you live ?", "lol . i am scared of heights too . do you like to listen to music ?"], "speaker_id": "snli:test:167", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:167", "candidate_text": "Honestly, A woman is preparing drinks on an airplane!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:167:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:95:premise", "snli:test:263:premise", "snli:test:9982:premise", "snli:test:167:premise", "style:personachat:test:50:self:5:positive:0", "style:personachat:test:50:self:5:positive:1", "style:personachat:test:50:self:5:positive:2", "style:personachat:test:50:self:5:positive:3", "style:personachat:test:50:self:5:positive:4"], "evidence_texts": ["People are conversing at a dining table under a canopy.", "A person in orange clothing rests above a metro entrance.", "Two gondola operators sit along a road in Italy.", "The flight attendant dressed in yellow demonstrates life vest usage.", "really ? i went to school at usc . i have a few contacts out there .", "dad tending the farm and mom a strict teacher , anything artsy was always frowned upon !", "where are you from ? i am proudly born and raised in south carolina !", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:167", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:170", "candidate_text": "I wonder whether girls were going to the park?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:170:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9256:premise", "snli:test:186:premise", "snli:test:170:premise", "snli:test:8213:premise", "style:personachat:test:5:self:7:positive:0", "style:personachat:test:5:self:7:positive:1", "style:personachat:test:5:self:7:positive:2", "style:personachat:test:5:self:7:positive:3", "style:personachat:test:5:self:7:positive:4", "style:personachat:test:5:self:7:positive:5", "style:personachat:test:5:self:7:positive:6"], "evidence_texts": ["A woman happy about something.", "Six dogs swimming in a river.", "The girls walk down the street.", "An SUV traveling through mud.", "i am well . how about you ?", "me too . what do you do for a living ?", "i get that ! i am a doctor . pediatrics to be exact . do anything for fun ?", "me too ! well , i do not have any dogs , but i run everyday !", "yes , it helps me clear my mind . i also love to read to escape reality sometimes too", "california . where do you live ?", "lol . i am scared of heights too . do you like to listen to music ?"], "speaker_id": "snli:test:170", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:170", "candidate_text": "Girls were going to the park.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:170:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9256:premise", "snli:test:186:premise", "snli:test:170:premise", "snli:test:8213:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman happy about something.", "Six dogs swimming in a river.", "The girls walk down the street.", "An SUV traveling through mud.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:170", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:173", "candidate_text": "well, a man standing on his motorbike", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:173:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6:premise", "snli:test:173:premise", "snli:test:319:premise", "snli:test:9854:premise", "style:personachat:test:6:self:1:positive:0"], "evidence_texts": ["An old man with a package poses in front of an advertisement.", "A man wearing a red uniform and helmet stands on his motorbike.", "A large group of people walking in a busy city at night.", "Kids play in the water in the middle of the street.", "no i do not . only fruit and coconut juice . also , water and supplements"], "speaker_id": "snli:test:173", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:173", "candidate_text": "Honestly, a man standing on his motorbike!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:173:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6:premise", "snli:test:173:premise", "snli:test:319:premise", "snli:test:9854:premise", "style:personachat:test:69:self:1:positive:0"], "evidence_texts": ["An old man with a package poses in front of an advertisement.", "A man wearing a red uniform and helmet stands on his motorbike.", "A large group of people walking in a busy city at night.", "Kids play in the water in the middle of the street.", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:173", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:180", "candidate_text": "well, a boy holding onto the wall of an old brick house's raised foundation as construction occurs", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:180:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:180:premise", "snli:test:9958:premise", "snli:test:9865:premise", "snli:test:393:premise", "style:personachat:test:6:self:2:positive:0", "style:personachat:test:6:self:2:positive:1"], "evidence_texts": ["a boy holding onto the wall of an old brick house's raised foundation.", "A white duck is spreading its wings while sitting on the water.", "A group of people wait to cross the street in new york", "People are riding bicycles in the street, and they are all wearing helmets.", "am i how much younger ? how old are you ?", "no i do not . only fruit and coconut juice . also , water and supplements"], "speaker_id": "snli:test:180", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:180", "candidate_text": "a boy holding onto the wall of an old brick house's raised foundation as construction occurs.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:180:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:180:premise", "snli:test:9958:premise", "snli:test:9865:premise", "snli:test:393:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["a boy holding onto the wall of an old brick house's raised foundation.", "A white duck is spreading its wings while sitting on the water.", "A group of people wait to cross the street in new york", "People are riding bicycles in the street, and they are all wearing helmets.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:180", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:182", "candidate_text": "well, a boy is against the wall of a raised foundation", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:182:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:393:premise", "snli:test:9958:premise", "snli:test:182:premise", "snli:test:502:premise", "style:personachat:test:6:self:5:positive:0", "style:personachat:test:6:self:5:positive:1", "style:personachat:test:6:self:5:positive:2", "style:personachat:test:6:self:5:positive:3", "style:personachat:test:6:self:5:positive:4"], "evidence_texts": ["People are riding bicycles in the street, and they are all wearing helmets.", "A white duck is spreading its wings while sitting on the water.", "a boy holding onto the wall of an old brick house's raised foundation.", "A group of kids is splashing in deep water nearby a rock formation.", "my family is nice . we spend time at the weight room a lot", "we work out to hip hop artists and i love the 1990 era", "yes it is . i cherish those moments .", "am i how much younger ? how old are you ?", "no i do not . only fruit and coconut juice . also , water and supplements"], "speaker_id": "snli:test:182", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:182", "candidate_text": "I wonder whether a boy is against the wall of a raised foundation?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:182:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:393:premise", "snli:test:9958:premise", "snli:test:182:premise", "snli:test:502:premise", "style:personachat:test:12:self:5:positive:0", "style:personachat:test:12:self:5:positive:1", "style:personachat:test:12:self:5:positive:2", "style:personachat:test:12:self:5:positive:3", "style:personachat:test:12:self:5:positive:4"], "evidence_texts": ["People are riding bicycles in the street, and they are all wearing helmets.", "A white duck is spreading its wings while sitting on the water.", "a boy holding onto the wall of an old brick house's raised foundation.", "A group of kids is splashing in deep water nearby a rock formation.", "is it the video where they are wearing purple ? best color ever", "the one at the hospital ? i am a nurse at a pediatric one", "i do not believe in sheltering animals . i am a catholic", "you could make them into a pizza . i love to eat pizza so i would buy", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:182", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:186", "candidate_text": "well, six dogs swim across the river", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:186:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:170:premise", "snli:test:9256:premise", "snli:test:186:premise", "snli:test:193:premise", "style:personachat:test:6:self:7:positive:0", "style:personachat:test:6:self:7:positive:1", "style:personachat:test:6:self:7:positive:2", "style:personachat:test:6:self:7:positive:3", "style:personachat:test:6:self:7:positive:4", "style:personachat:test:6:self:7:positive:5", "style:personachat:test:6:self:7:positive:6"], "evidence_texts": ["The girls walk down the street.", "A woman happy about something.", "Six dogs swimming in a river.", "a group of people dancing together.", "a younger brother and no children is all . besides dad", "really . fish raw is my meal . what kind of fish ?", "my family is nice . we spend time at the weight room a lot", "we work out to hip hop artists and i love the 1990 era", "yes it is . i cherish those moments .", "am i how much younger ? how old are you ?", "no i do not . only fruit and coconut juice . also , water and supplements"], "speaker_id": "snli:test:186", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:186", "candidate_text": "Honestly, Six dogs swim across the river!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:186:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:170:premise", "snli:test:9256:premise", "snli:test:186:premise", "snli:test:193:premise", "style:personachat:test:69:self:7:positive:0", "style:personachat:test:69:self:7:positive:1", "style:personachat:test:69:self:7:positive:2", "style:personachat:test:69:self:7:positive:3", "style:personachat:test:69:self:7:positive:4", "style:personachat:test:69:self:7:positive:5", "style:personachat:test:69:self:7:positive:6"], "evidence_texts": ["The girls walk down the street.", "A woman happy about something.", "Six dogs swimming in a river.", "a group of people dancing together.", "hello . i am well . how are you doing ?", "interesting . where are you from ?", "i want to visit italy one day .", "do you cook italian food ?", "i understand , i am from the middle east .", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:186", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:188", "candidate_text": "well, the puppies are setting on the couch", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:188:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:188:premise", "snli:test:193:premise", "snli:test:170:premise", "snli:test:801:premise", "style:personachat:test:7:self:1:positive:0"], "evidence_texts": ["Six dogs swimming in a river.", "a group of people dancing together.", "The girls walk down the street.", "Two people talking on a dock.", "that is horrible and verry illegal"], "speaker_id": "snli:test:188", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:188", "candidate_text": "The puppies are setting on the couch.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:188:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:188:premise", "snli:test:193:premise", "snli:test:170:premise", "snli:test:801:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Six dogs swimming in a river.", "a group of people dancing together.", "The girls walk down the street.", "Two people talking on a dock.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:188", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:193", "candidate_text": "well, they are doing the tango", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:193:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:188:premise", "snli:test:193:premise", "snli:test:801:premise", "snli:test:186:premise", "style:personachat:test:7:self:2:positive:0", "style:personachat:test:7:self:2:positive:1"], "evidence_texts": ["Six dogs swimming in a river.", "a group of people dancing together.", "Two people talking on a dock.", "Six dogs swimming in a river.", "do not you have to be older than 15 to work at mcdonalds ? that sounds illegal", "that is horrible and verry illegal"], "speaker_id": "snli:test:193", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:193", "candidate_text": "I wonder whether they are doing the tango?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:193:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:188:premise", "snli:test:193:premise", "snli:test:801:premise", "snli:test:186:premise", "style:personachat:test:15:self:2:positive:0", "style:personachat:test:15:self:2:positive:1"], "evidence_texts": ["Six dogs swimming in a river.", "a group of people dancing together.", "Two people talking on a dock.", "Six dogs swimming in a river.", "i do ! i play ice hockey sometimes", "can you do lots of tricks ?"], "speaker_id": "snli:test:193", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:198", "candidate_text": "well, two people climbing on a wooden scaffold", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:198:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:121:premise", "snli:test:215:premise", "snli:test:198:premise", "snli:test:38:premise", "style:personachat:test:7:self:5:positive:0", "style:personachat:test:7:self:5:positive:1", "style:personachat:test:7:self:5:positive:2", "style:personachat:test:7:self:5:positive:3", "style:personachat:test:7:self:5:positive:4"], "evidence_texts": ["Two teenage girls conversing next to lockers.", "This child is on the library steps.", "Two men climbing on a wooden scaffold.", "Three firefighter come out of subway station.", "i am almost done i only have two years left in law school .", "that sounds important . maybe one day you will go back to school", "i do not know if we would be able to do anything because is has been so long", "do not you have to be older than 15 to work at mcdonalds ? that sounds illegal", "that is horrible and verry illegal"], "speaker_id": "snli:test:198", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:198", "candidate_text": "Honestly, Two people climbing on a wooden scaffold!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:198:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:121:premise", "snli:test:215:premise", "snli:test:198:premise", "snli:test:38:premise", "style:personachat:test:70:self:5:positive:0", "style:personachat:test:70:self:5:positive:1", "style:personachat:test:70:self:5:positive:2", "style:personachat:test:70:self:5:positive:3", "style:personachat:test:70:self:5:positive:4"], "evidence_texts": ["Two teenage girls conversing next to lockers.", "This child is on the library steps.", "Two men climbing on a wooden scaffold.", "Three firefighter come out of subway station.", "i like to cook to much to worry about my figure ! do you like food ?", "hmm . . you sound like a liberal .", "i am proud to say i voted for trump . do you like him ?", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:198", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:205", "candidate_text": "well, a young boy is playing in the field because his mother kicked him out of the house", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:205:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9861:premise", "snli:test:205:premise", "snli:test:225:premise", "snli:test:9860:premise", "style:personachat:test:7:self:7:positive:0", "style:personachat:test:7:self:7:positive:1", "style:personachat:test:7:self:7:positive:2", "style:personachat:test:7:self:7:positive:3", "style:personachat:test:7:self:7:positive:4", "style:personachat:test:7:self:7:positive:5", "style:personachat:test:7:self:7:positive:6"], "evidence_texts": ["A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "that is really sad . do you have any siblings", "that was nice of him . i am almost done with college", "i am almost done i only have two years left in law school .", "that sounds important . maybe one day you will go back to school", "i do not know if we would be able to do anything because is has been so long", "do not you have to be older than 15 to work at mcdonalds ? that sounds illegal", "that is horrible and verry illegal"], "speaker_id": "snli:test:205", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:205", "candidate_text": "A young boy is playing in the field because his mother kicked him out of the house.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:205:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9861:premise", "snli:test:205:premise", "snli:test:225:premise", "snli:test:9860:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:205", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:212", "candidate_text": "I wonder whether the person is wearing a green shirt?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:212:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:212:premise", "snli:test:34:premise", "snli:test:564:premise", "snli:test:26:premise", "style:personachat:test:8:self:1:positive:0"], "evidence_texts": ["A person in a red shirt and black pants hunched over.", "A man reads the paper in a bar with green lighting.", "Four boys are about to be hit by an approaching wave.", "A young family enjoys feeling ocean waves lap at their feet.", "i am a secretary like my mom hard living and you ?"], "speaker_id": "snli:test:212", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:212", "candidate_text": "The person is wearing a green shirt.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:212:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:212:premise", "snli:test:34:premise", "snli:test:564:premise", "snli:test:26:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person in a red shirt and black pants hunched over.", "A man reads the paper in a bar with green lighting.", "Four boys are about to be hit by an approaching wave.", "A young family enjoys feeling ocean waves lap at their feet.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:212", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:215", "candidate_text": "I wonder whether the child is on the steps inside the library?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:215:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:198:premise", "snli:test:367:premise", "snli:test:121:premise", "snli:test:215:premise", "style:personachat:test:8:self:2:positive:0", "style:personachat:test:8:self:2:positive:1"], "evidence_texts": ["Two men climbing on a wooden scaffold.", "Two men are standing in a boat.", "Two teenage girls conversing next to lockers.", "This child is on the library steps.", "i have a cat and a dog you would not be scared of my dog", "i am a secretary like my mom hard living and you ?"], "speaker_id": "snli:test:215", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:215", "candidate_text": "well, the child is on the steps inside the library", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:215:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:198:premise", "snli:test:367:premise", "snli:test:121:premise", "snli:test:215:premise", "style:personachat:test:17:self:2:positive:0", "style:personachat:test:17:self:2:positive:1"], "evidence_texts": ["Two men climbing on a wooden scaffold.", "Two men are standing in a boat.", "Two teenage girls conversing next to lockers.", "This child is on the library steps.", "love is hard and hurtful . alone is best , i have found .", "my best friend betrayed me jerry springer style in high school and it still stings ."], "speaker_id": "snli:test:215", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:225", "candidate_text": "I wonder whether an old woman has a light jean jacket?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:225:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:205:premise", "snli:test:225:premise", "snli:test:1291:premise", "snli:test:9861:premise", "style:personachat:test:8:self:5:positive:0", "style:personachat:test:8:self:5:positive:1", "style:personachat:test:8:self:5:positive:2", "style:personachat:test:8:self:5:positive:3", "style:personachat:test:8:self:5:positive:4"], "evidence_texts": ["A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "scotch goes good with meat", "i have a brother who likes scotch", "i have just the one brother and 3 sisters", "i have a cat and a dog you would not be scared of my dog", "i am a secretary like my mom hard living and you ?"], "speaker_id": "snli:test:225", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:225", "candidate_text": "Honestly, An old woman has a light jean jacket!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:225:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:205:premise", "snli:test:225:premise", "snli:test:1291:premise", "snli:test:9861:premise", "style:personachat:test:72:self:5:positive:0", "style:personachat:test:72:self:5:positive:1", "style:personachat:test:72:self:5:positive:2", "style:personachat:test:72:self:5:positive:3", "style:personachat:test:72:self:5:positive:4"], "evidence_texts": ["A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "3 kids and a great husband too . we live beside the sea .", "that is a lot ! they would not fit into my mustang convertible .", "do you like to work out ? i am trying to lose some weight .", "i work out about 2 hours a day but it is hard to fit it all in .", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:225", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:238", "candidate_text": "I wonder whether people are in the house?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:238:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:32:premise", "snli:test:321:premise", "snli:test:238:premise", "snli:test:30:premise", "style:personachat:test:8:self:7:positive:0", "style:personachat:test:8:self:7:positive:1", "style:personachat:test:8:self:7:positive:2", "style:personachat:test:8:self:7:positive:3", "style:personachat:test:8:self:7:positive:4", "style:personachat:test:8:self:7:positive:5", "style:personachat:test:8:self:7:positive:6"], "evidence_texts": ["3 young man in hoods standing in the middle of a quiet street facing the camera.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A group of adults stands in the bathroom looking down a small child in the bath.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "great , i just finished a huge steak how are you ?", "i am major meat eater to build muscles", "scotch goes good with meat", "i have a brother who likes scotch", "i have just the one brother and 3 sisters", "i have a cat and a dog you would not be scared of my dog", "i am a secretary like my mom hard living and you ?"], "speaker_id": "snli:test:238", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:238", "candidate_text": "People are in the house.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:238:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:32:premise", "snli:test:321:premise", "snli:test:238:premise", "snli:test:30:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["3 young man in hoods standing in the middle of a quiet street facing the camera.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A group of adults stands in the bathroom looking down a small child in the bath.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:238", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:240", "candidate_text": "well, a woman is singing", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:240:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:240:premise", "snli:test:68:premise", "snli:test:9944:premise", "snli:test:309:premise", "style:personachat:test:9:self:1:positive:0"], "evidence_texts": ["A goofy looking woman is singing on stage.", "A snowboarder on a wide plain of snow", "A woman is petting a dog outside.", "Two men prepare a fish at a dock.", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:240", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:240", "candidate_text": "Honestly, A woman is singing!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:240:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:240:premise", "snli:test:68:premise", "snli:test:9944:premise", "snli:test:309:premise", "style:personachat:test:74:self:1:positive:0"], "evidence_texts": ["A goofy looking woman is singing on stage.", "A snowboarder on a wide plain of snow", "A woman is petting a dog outside.", "Two men prepare a fish at a dock.", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:240", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:242", "candidate_text": "well, a blonde woman is singing and dancing on the stage", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:242:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:309:premise", "snli:test:68:premise", "snli:test:242:premise", "snli:test:310:premise", "style:personachat:test:9:self:2:positive:0", "style:personachat:test:9:self:2:positive:1"], "evidence_texts": ["Two men prepare a fish at a dock.", "A snowboarder on a wide plain of snow", "A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "it is fun to go shopping for her", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:242", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:242", "candidate_text": "A blonde woman is singing and dancing on the stage.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:242:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:309:premise", "snli:test:68:premise", "snli:test:242:premise", "snli:test:310:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two men prepare a fish at a dock.", "A snowboarder on a wide plain of snow", "A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:242", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:248", "candidate_text": "well, a family is in their car", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:248:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:248:premise", "snli:test:160:premise", "snli:test:348:premise", "snli:test:157:premise", "style:personachat:test:9:self:5:positive:0", "style:personachat:test:9:self:5:positive:1", "style:personachat:test:9:self:5:positive:2", "style:personachat:test:9:self:5:positive:3", "style:personachat:test:9:self:5:positive:4"], "evidence_texts": ["A man, woman, and child get their picture taken in front of the mountains.", "Two women walk down a sidewalk along a busy street in a downtown area.", "A golden dog passes a blue ball to another golden dog on the beach.", "A couple is eating outside at a table and he is pointing at something.", "i do like the outdoors . . . as well as restoring classic cars . i am in missouri .", "that is good . i work as a handyman for a company that rents properties .", "i am a single parent as well . just me and my baby girl .", "it is fun to go shopping for her", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:248", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:248", "candidate_text": "I wonder whether a family is in their car?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:248:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:248:premise", "snli:test:160:premise", "snli:test:348:premise", "snli:test:157:premise", "style:personachat:test:22:self:5:positive:0", "style:personachat:test:22:self:5:positive:1", "style:personachat:test:22:self:5:positive:2", "style:personachat:test:22:self:5:positive:3", "style:personachat:test:22:self:5:positive:4"], "evidence_texts": ["A man, woman, and child get their picture taken in front of the mountains.", "Two women walk down a sidewalk along a busy street in a downtown area.", "A golden dog passes a blue ball to another golden dog on the beach.", "A couple is eating outside at a table and he is pointing at something.", "awe how old are they ?", "i went here in ohio like pizza ?", "what is your favorite holiday ?", "my fav is christmas i love that day", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:248", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:253", "candidate_text": "well, two couples play pool in a bar", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:253:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:13:premise", "snli:test:253:premise", "snli:test:281:premise", "snli:test:9950:premise", "style:personachat:test:9:self:6:positive:0", "style:personachat:test:9:self:6:positive:1", "style:personachat:test:9:self:6:positive:2", "style:personachat:test:9:self:6:positive:3", "style:personachat:test:9:self:6:positive:4", "style:personachat:test:9:self:6:positive:5"], "evidence_texts": ["A land rover is being driven across a river.", "Four adults eat while sitting on a tile floor.", "A skateboarding youth does a trick on a rail.", "Two men in karate gear using fighting sticks.", "i am great ! that is cool , i just got home from hunting .", "i do like the outdoors . . . as well as restoring classic cars . i am in missouri .", "that is good . i work as a handyman for a company that rents properties .", "i am a single parent as well . just me and my baby girl .", "it is fun to go shopping for her", "she happens to like pink a lot . . ."], "speaker_id": "snli:test:253", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:253", "candidate_text": "Honestly, Two couples play pool in a bar!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:253:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:13:premise", "snli:test:253:premise", "snli:test:281:premise", "snli:test:9950:premise", "style:personachat:test:74:self:6:positive:0", "style:personachat:test:74:self:6:positive:1", "style:personachat:test:74:self:6:positive:2", "style:personachat:test:74:self:6:positive:3", "style:personachat:test:74:self:6:positive:4", "style:personachat:test:74:self:6:positive:5"], "evidence_texts": ["A land rover is being driven across a river.", "Four adults eat while sitting on a tile floor.", "A skateboarding youth does a trick on a rail.", "Two men in karate gear using fighting sticks.", "me too ! how are you today ?", "great ! what is your favorite animal ? i love riding horses on my dads farm", "awe , i love turtles too . any hobbies ?", "cool . my favorite hobby is watching horror films", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:253", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:263", "candidate_text": "well, a person is waiting for a train", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:263:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:167:premise", "snli:test:289:premise", "snli:test:95:premise", "snli:test:263:premise", "style:personachat:test:10:self:1:positive:0"], "evidence_texts": ["The flight attendant dressed in yellow demonstrates life vest usage.", "A group of people gathered at night watching an event.", "People are conversing at a dining table under a canopy.", "A person in orange clothing rests above a metro entrance.", "yes , the peach and the quiet , it is heaven ."], "speaker_id": "snli:test:263", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:263", "candidate_text": "A person is waiting for a train.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:263:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:167:premise", "snli:test:289:premise", "snli:test:95:premise", "snli:test:263:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The flight attendant dressed in yellow demonstrates life vest usage.", "A group of people gathered at night watching an event.", "People are conversing at a dining table under a canopy.", "A person in orange clothing rests above a metro entrance.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:263", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:270", "candidate_text": "well, a man is making a sandwich while a crowd gasps in awe", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:270:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9977:premise", "snli:test:300:premise", "snli:test:270:premise", "snli:test:9882:premise", "style:personachat:test:10:self:2:positive:0", "style:personachat:test:10:self:2:positive:1"], "evidence_texts": ["A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "there is a local trail near my house in the mountains that i like .", "yes , the peach and the quiet , it is heaven ."], "speaker_id": "snli:test:270", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:270", "candidate_text": "I wonder whether a man is making a sandwich while a crowd gasps in awe?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:270:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9977:premise", "snli:test:300:premise", "snli:test:270:premise", "snli:test:9882:premise", "style:personachat:test:26:self:2:positive:0", "style:personachat:test:26:self:2:positive:1"], "evidence_texts": ["A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "do you work ? or your parents work ?", "college ? studying anything in particular ?"], "speaker_id": "snli:test:270", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:281", "candidate_text": "well, a young boy showing his new skateboard tricks to his friends", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:281:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:253:premise", "snli:test:281:premise", "snli:test:299:premise", "snli:test:13:premise", "style:personachat:test:10:self:5:positive:0", "style:personachat:test:10:self:5:positive:1", "style:personachat:test:10:self:5:positive:2", "style:personachat:test:10:self:5:positive:3", "style:personachat:test:10:self:5:positive:4"], "evidence_texts": ["Four adults eat while sitting on a tile floor.", "A skateboarding youth does a trick on a rail.", "A girl holding a beverage points at a painting.", "A land rover is being driven across a river.", "nice , my job is being a truck driver , i have seven kids to feed .", "i like running , but i have no time .", "it does , i like to wear my favorite color blue when i do run .", "there is a local trail near my house in the mountains that i like .", "yes , the peach and the quiet , it is heaven ."], "speaker_id": "snli:test:281", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:281", "candidate_text": "Honestly, A young boy showing his new skateboard tricks to his friends!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:281:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:253:premise", "snli:test:281:premise", "snli:test:299:premise", "snli:test:13:premise", "style:personachat:test:84:self:5:positive:0", "style:personachat:test:84:self:5:positive:1", "style:personachat:test:84:self:5:positive:2", "style:personachat:test:84:self:5:positive:3", "style:personachat:test:84:self:5:positive:4"], "evidence_texts": ["Four adults eat while sitting on a tile floor.", "A skateboarding youth does a trick on a rail.", "A girl holding a beverage points at a painting.", "A land rover is being driven across a river.", "the sea is my home . the aquatic creatures are my friends .", "sure you could call it that . do have any hobbies ?", "how old are you ? i spend all day at school ! i would rather be playing .", "do you sell cars too", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:281", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:282", "candidate_text": "well, he is wearing a button-up shirt", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:282:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:48:premise", "snli:test:9843:premise", "snli:test:282:premise", "snli:test:641:premise", "style:personachat:test:10:self:7:positive:0", "style:personachat:test:10:self:7:positive:1", "style:personachat:test:10:self:7:positive:2", "style:personachat:test:10:self:7:positive:3", "style:personachat:test:10:self:7:positive:4", "style:personachat:test:10:self:7:positive:5", "style:personachat:test:10:self:7:positive:6"], "evidence_texts": ["A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "sorry , i do not like music , i like reading mystery books .", "growing up on a farm , you learn to read a lot more .", "nice , my job is being a truck driver , i have seven kids to feed .", "i like running , but i have no time .", "it does , i like to wear my favorite color blue when i do run .", "there is a local trail near my house in the mountains that i like .", "yes , the peach and the quiet , it is heaven ."], "speaker_id": "snli:test:282", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:282", "candidate_text": "He is wearing a button-up shirt.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:282:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:48:premise", "snli:test:9843:premise", "snli:test:282:premise", "snli:test:641:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:282", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:284", "candidate_text": "well, a man is crouching on a handrail", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:284:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:284:premise", "snli:test:641:premise", "snli:test:48:premise", "snli:test:937:premise", "style:personachat:test:11:self:1:positive:0"], "evidence_texts": ["A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "i have been to small town there called mansfield ."], "speaker_id": "snli:test:284", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:284", "candidate_text": "I wonder whether a man is crouching on a handrail?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:284:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:284:premise", "snli:test:641:premise", "snli:test:48:premise", "snli:test:937:premise", "style:personachat:test:28:self:1:positive:0"], "evidence_texts": ["A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:284", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:289", "candidate_text": "well, a group of humans are looking at the same direction", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:289:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:263:premise", "snli:test:289:premise", "snli:test:354:premise", "snli:test:167:premise", "style:personachat:test:11:self:2:positive:0", "style:personachat:test:11:self:2:positive:1"], "evidence_texts": ["A person in orange clothing rests above a metro entrance.", "A group of people gathered at night watching an event.", "A woman is weaving with a comb in her hand.", "The flight attendant dressed in yellow demonstrates life vest usage.", "i live close to where daniel boone roamed .", "i have been to small town there called mansfield ."], "speaker_id": "snli:test:289", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:289", "candidate_text": "Honestly, A group of humans are looking at the same direction!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:289:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:263:premise", "snli:test:289:premise", "snli:test:354:premise", "snli:test:167:premise", "style:personachat:test:88:self:2:positive:0", "style:personachat:test:88:self:2:positive:1"], "evidence_texts": ["A person in orange clothing rests above a metro entrance.", "A group of people gathered at night watching an event.", "A woman is weaving with a comb in her hand.", "The flight attendant dressed in yellow demonstrates life vest usage.", "oh that is neat ! before my parents died a couple years back , we went to shows .", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:289", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:295", "candidate_text": "well, a person is dancing and singing", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:295:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:86:premise", "snli:test:378:premise", "snli:test:9782:premise", "snli:test:295:premise", "style:personachat:test:11:self:5:positive:0", "style:personachat:test:11:self:5:positive:1", "style:personachat:test:11:self:5:positive:2", "style:personachat:test:11:self:5:positive:3", "style:personachat:test:11:self:5:positive:4"], "evidence_texts": ["A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "agree . i live in a small town in kentucky .", "so we are semi close neighbors .", "no . i enjoy crafts and this new game that i am trying to learn .", "i live close to where daniel boone roamed .", "i have been to small town there called mansfield ."], "speaker_id": "snli:test:295", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:295", "candidate_text": "A person is dancing and singing.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:295:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:86:premise", "snli:test:378:premise", "snli:test:9782:premise", "snli:test:295:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:295", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:299", "candidate_text": "well, the girl has a beverage", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:299:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:281:premise", "snli:test:326:premise", "snli:test:253:premise", "snli:test:299:premise", "style:personachat:test:11:self:7:positive:0", "style:personachat:test:11:self:7:positive:1", "style:personachat:test:11:self:7:positive:2", "style:personachat:test:11:self:7:positive:3", "style:personachat:test:11:self:7:positive:4", "style:personachat:test:11:self:7:positive:5", "style:personachat:test:11:self:7:positive:6"], "evidence_texts": ["A skateboarding youth does a trick on a rail.", "A softball player throws the ball to her teammate.", "Four adults eat while sitting on a tile floor.", "A girl holding a beverage points at a painting.", "hello , doing good thanks ! no i am a wildcats fan lol", "no i ma not . go figure i am cats fan and allergic to cats lol .", "agree . i live in a small town in kentucky .", "so we are semi close neighbors .", "no . i enjoy crafts and this new game that i am trying to learn .", "i live close to where daniel boone roamed .", "i have been to small town there called mansfield ."], "speaker_id": "snli:test:299", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:299", "candidate_text": "I wonder whether the girl has a beverage?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:299:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:281:premise", "snli:test:326:premise", "snli:test:253:premise", "snli:test:299:premise", "style:personachat:test:28:self:7:positive:0", "style:personachat:test:28:self:7:positive:1", "style:personachat:test:28:self:7:positive:2", "style:personachat:test:28:self:7:positive:3", "style:personachat:test:28:self:7:positive:4", "style:personachat:test:28:self:7:positive:5", "style:personachat:test:28:self:7:positive:6"], "evidence_texts": ["A skateboarding youth does a trick on a rail.", "A softball player throws the ball to her teammate.", "Four adults eat while sitting on a tile floor.", "A girl holding a beverage points at a painting.", "hello i am doing well . hanging out with my dog pedro", "he is a yorkie . perfect size for me since i am only five feet tall .", "he is five . he likes to eat muffins like me .", "yes , there is one at my job at the healthcare industry .", "yes . i also like film do you ?", "so do i . i also like documentaries , do you ?", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:299", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:300", "candidate_text": "I wonder whether a man in a black shirt, in a commercial kitchen, holding up the old meat he took out of a bag?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:300:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:300:premise", "snli:test:270:premise", "snli:test:9977:premise", "snli:test:667:premise", "style:personachat:test:12:self:1:positive:0"], "evidence_texts": ["A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:300", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:300", "candidate_text": "well, a man in a black shirt, in a commercial kitchen, holding up the old meat he took out of a bag", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:300:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:300:premise", "snli:test:270:premise", "snli:test:9977:premise", "snli:test:667:premise", "style:personachat:test:23:self:1:positive:0"], "evidence_texts": ["A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:300", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:301", "candidate_text": "I wonder whether a woman in a black shirt, in a commercial kitchen, holding up meat he took out of a bag?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:301:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:667:premise", "snli:test:301:premise", "snli:test:270:premise", "snli:test:1156:premise", "style:personachat:test:12:self:2:positive:0", "style:personachat:test:12:self:2:positive:1"], "evidence_texts": ["A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "you could make them into a pizza . i love to eat pizza so i would buy", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:301", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:301", "candidate_text": "Honestly, A woman in a black shirt, in a commercial kitchen, holding up meat he took out of a bag!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:301:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:667:premise", "snli:test:301:premise", "snli:test:270:premise", "snli:test:1156:premise", "style:personachat:test:105:self:2:positive:0", "style:personachat:test:105:self:2:positive:1"], "evidence_texts": ["A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:301", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:309", "candidate_text": "I wonder whether two men have just come in from fishing all day?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:309:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:242:premise", "snli:test:309:premise", "snli:test:240:premise", "snli:test:68:premise", "style:personachat:test:12:self:5:positive:0", "style:personachat:test:12:self:5:positive:1", "style:personachat:test:12:self:5:positive:2", "style:personachat:test:12:self:5:positive:3", "style:personachat:test:12:self:5:positive:4"], "evidence_texts": ["A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "A goofy looking woman is singing on stage.", "A snowboarder on a wide plain of snow", "is it the video where they are wearing purple ? best color ever", "the one at the hospital ? i am a nurse at a pediatric one", "i do not believe in sheltering animals . i am a catholic", "you could make them into a pizza . i love to eat pizza so i would buy", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:309", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:309", "candidate_text": "Two men have just come in from fishing all day.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:309:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:242:premise", "snli:test:309:premise", "snli:test:240:premise", "snli:test:68:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "A goofy looking woman is singing on stage.", "A snowboarder on a wide plain of snow", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:309", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:310", "candidate_text": "I wonder whether two men are cleaning their fish?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:310:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:242:premise", "snli:test:469:premise", "snli:test:310:premise", "snli:test:240:premise", "style:personachat:test:12:self:6:positive:0", "style:personachat:test:12:self:6:positive:1", "style:personachat:test:12:self:6:positive:2", "style:personachat:test:12:self:6:positive:3", "style:personachat:test:12:self:6:positive:4", "style:personachat:test:12:self:6:positive:5"], "evidence_texts": ["A goofy looking woman is singing on stage.", "Children in yellow sports uniforms climbing a tower.", "Two men prepare a fish at a dock.", "A goofy looking woman is singing on stage.", "i am great . i am watching one tree hill . that is my favorite show", "is it the video where they are wearing purple ? best color ever", "the one at the hospital ? i am a nurse at a pediatric one", "i do not believe in sheltering animals . i am a catholic", "you could make them into a pizza . i love to eat pizza so i would buy", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:310", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:310", "candidate_text": "well, two men are cleaning their fish", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:310:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:242:premise", "snli:test:469:premise", "snli:test:310:premise", "snli:test:240:premise", "style:personachat:test:23:self:6:positive:0", "style:personachat:test:23:self:6:positive:1", "style:personachat:test:23:self:6:positive:2", "style:personachat:test:23:self:6:positive:3", "style:personachat:test:23:self:6:positive:4", "style:personachat:test:23:self:6:positive:5"], "evidence_texts": ["A goofy looking woman is singing on stage.", "Children in yellow sports uniforms climbing a tower.", "Two men prepare a fish at a dock.", "A goofy looking woman is singing on stage.", "i am doing pretty good , how are you ?", "green . what do you do for fun ?", "kool ! i like to play fantasy video games .", "just like that , i also build model spaceships in my free time .", "yes , we do . do you like animals ?", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:310", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:311", "candidate_text": "well, two men are sitting in a canoe with their fishing poles", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:311:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:469:premise", "snli:test:475:premise", "snli:test:242:premise", "snli:test:311:premise", "style:personachat:test:13:self:1:positive:0"], "evidence_texts": ["Children in yellow sports uniforms climbing a tower.", "A crowded city street with lots of pedestrians.", "A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "too busy with the boys for much besides hiking ."], "speaker_id": "snli:test:311", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:311", "candidate_text": "Two men are sitting in a canoe with their fishing poles.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:311:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:469:premise", "snli:test:475:premise", "snli:test:242:premise", "snli:test:311:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Children in yellow sports uniforms climbing a tower.", "A crowded city street with lots of pedestrians.", "A goofy looking woman is singing on stage.", "Two men prepare a fish at a dock.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:311", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:314", "candidate_text": "well, the woman is blind", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:314:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:40:premise", "snli:test:342:premise", "snli:test:314:premise", "snli:test:9681:premise", "style:personachat:test:13:self:2:positive:0", "style:personachat:test:13:self:2:positive:1"], "evidence_texts": ["A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "it is a great time of year for sure . do you have any hobbies ?", "too busy with the boys for much besides hiking ."], "speaker_id": "snli:test:314", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:314", "candidate_text": "I wonder whether the woman is blind?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:314:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:40:premise", "snli:test:342:premise", "snli:test:314:premise", "snli:test:9681:premise", "style:personachat:test:37:self:2:positive:0", "style:personachat:test:37:self:2:positive:1"], "evidence_texts": ["A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "i did not even know gucci was getting married . who to ?", "do they have any kids together ?"], "speaker_id": "snli:test:314", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:319", "candidate_text": "well, people are walking a city", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:319:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:173:premise", "snli:test:333:premise", "snli:test:6:premise", "snli:test:319:premise", "style:personachat:test:13:self:5:positive:0", "style:personachat:test:13:self:5:positive:1", "style:personachat:test:13:self:5:positive:2", "style:personachat:test:13:self:5:positive:3", "style:personachat:test:13:self:5:positive:4"], "evidence_texts": ["A man wearing a red uniform and helmet stands on his motorbike.", "Island native fishermen reeling in their nets after a long day's work.", "An old man with a package poses in front of an advertisement.", "A large group of people walking in a busy city at night.", "i am just an ohio mom with two amazing sons . not married though .", "great pet name . i do not have any pets unless you count my car , sally .", "yep . the leaves change here and it is beautiful . i like to hike in fall .", "it is a great time of year for sure . do you have any hobbies ?", "too busy with the boys for much besides hiking ."], "speaker_id": "snli:test:319", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:319", "candidate_text": "Honestly, People are walking a city!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:319:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:173:premise", "snli:test:333:premise", "snli:test:6:premise", "snli:test:319:premise", "style:personachat:test:119:self:5:positive:0", "style:personachat:test:119:self:5:positive:1", "style:personachat:test:119:self:5:positive:2", "style:personachat:test:119:self:5:positive:3", "style:personachat:test:119:self:5:positive:4"], "evidence_texts": ["A man wearing a red uniform and helmet stands on his motorbike.", "Island native fishermen reeling in their nets after a long day's work.", "An old man with a package poses in front of an advertisement.", "A large group of people walking in a busy city at night.", "unless you are under 18 , lol .", "what is your tattoo of ?", "is it on your lower back ? my daughter wants one , ugh !", "that was my next guess ! my ex wife has a similar one . you been married before ?", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:319", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:321", "candidate_text": "well, a woman is sitting in a room with other people", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:321:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:238:premise", "snli:test:321:premise", "snli:test:521:premise", "snli:test:32:premise", "style:personachat:test:13:self:6:positive:0", "style:personachat:test:13:self:6:positive:1", "style:personachat:test:13:self:6:positive:2", "style:personachat:test:13:self:6:positive:3", "style:personachat:test:13:self:6:positive:4", "style:personachat:test:13:self:6:positive:5"], "evidence_texts": ["A group of adults stands in the bathroom looking down a small child in the bath.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "hi . i am more the mousy type . numbers are my world at my day job . you ?", "i am just an ohio mom with two amazing sons . not married though .", "great pet name . i do not have any pets unless you count my car , sally .", "yep . the leaves change here and it is beautiful . i like to hike in fall .", "it is a great time of year for sure . do you have any hobbies ?", "too busy with the boys for much besides hiking ."], "speaker_id": "snli:test:321", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:321", "candidate_text": "A woman is sitting in a room with other people.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:321:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:238:premise", "snli:test:321:premise", "snli:test:521:premise", "snli:test:32:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A group of adults stands in the bathroom looking down a small child in the bath.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:321", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:326", "candidate_text": "well, two softball players are sitting on a bench", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:326:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:299:premise", "snli:test:417:premise", "snli:test:326:premise", "snli:test:281:premise", "style:personachat:test:14:self:1:positive:0"], "evidence_texts": ["A girl holding a beverage points at a painting.", "A boy hits a tennis ball on a court.", "A softball player throws the ball to her teammate.", "A skateboarding youth does a trick on a rail.", "no mother in laws i hear are far worse ."], "speaker_id": "snli:test:326", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:326", "candidate_text": "I wonder whether two softball players are sitting on a bench?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:326:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:299:premise", "snli:test:417:premise", "snli:test:326:premise", "snli:test:281:premise", "style:personachat:test:39:self:1:positive:0"], "evidence_texts": ["A girl holding a beverage points at a painting.", "A boy hits a tennis ball on a court.", "A softball player throws the ball to her teammate.", "A skateboarding youth does a trick on a rail.", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:326", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:332", "candidate_text": "well, the boys are siblings playing with each other", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:332:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:332:premise", "snli:test:114:premise", "snli:test:647:premise", "snli:test:56:premise", "style:personachat:test:14:self:2:positive:0", "style:personachat:test:14:self:2:positive:1"], "evidence_texts": ["Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "well if you can find a sea hag to turn her into one .", "no mother in laws i hear are far worse ."], "speaker_id": "snli:test:332", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:332", "candidate_text": "Honestly, The boys are siblings playing with each other!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:332:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:332:premise", "snli:test:114:premise", "snli:test:647:premise", "snli:test:56:premise", "style:personachat:test:127:self:2:positive:0", "style:personachat:test:127:self:2:positive:1"], "evidence_texts": ["Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "i was homeschooled , that is pretty boring too", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:332", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:333", "candidate_text": "well, the men caught many fish", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:333:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:319:premise", "snli:test:333:premise", "snli:test:173:premise", "snli:test:567:premise", "style:personachat:test:14:self:5:positive:0", "style:personachat:test:14:self:5:positive:1", "style:personachat:test:14:self:5:positive:2", "style:personachat:test:14:self:5:positive:3", "style:personachat:test:14:self:5:positive:4"], "evidence_texts": ["A large group of people walking in a busy city at night.", "Island native fishermen reeling in their nets after a long day's work.", "A man wearing a red uniform and helmet stands on his motorbike.", "A woman, standing behind a girl, helping the girl with an experiment.", "oh i spend most of my time at school , when i am not in the water .", "how sweet ! do you live on land ? i live way under the sea !", "oh so you have legs and feet ?", "well if you can find a sea hag to turn her into one .", "no mother in laws i hear are far worse ."], "speaker_id": "snli:test:333", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:333", "candidate_text": "The men caught many fish.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:333:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:319:premise", "snli:test:333:premise", "snli:test:173:premise", "snli:test:567:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A large group of people walking in a busy city at night.", "Island native fishermen reeling in their nets after a long day's work.", "A man wearing a red uniform and helmet stands on his motorbike.", "A woman, standing behind a girl, helping the girl with an experiment.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:333", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:334", "candidate_text": "well, the men did not go to work today but instead played bridge", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:334:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:567:premise", "snli:test:319:premise", "snli:test:334:premise", "snli:test:569:premise", "style:personachat:test:14:self:6:positive:0", "style:personachat:test:14:self:6:positive:1", "style:personachat:test:14:self:6:positive:2", "style:personachat:test:14:self:6:positive:3", "style:personachat:test:14:self:6:positive:4", "style:personachat:test:14:self:6:positive:5"], "evidence_texts": ["A woman, standing behind a girl, helping the girl with an experiment.", "A large group of people walking in a busy city at night.", "Island native fishermen reeling in their nets after a long day's work.", "A woman, standing behind a girl, helping the girl with an experiment.", "oh i am so well ! how are you my land friend ?", "oh i spend most of my time at school , when i am not in the water .", "how sweet ! do you live on land ? i live way under the sea !", "oh so you have legs and feet ?", "well if you can find a sea hag to turn her into one .", "no mother in laws i hear are far worse ."], "speaker_id": "snli:test:334", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:334", "candidate_text": "I wonder whether the men did not go to work today but instead played bridge?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:334:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:567:premise", "snli:test:319:premise", "snli:test:334:premise", "snli:test:569:premise", "style:personachat:test:39:self:7:positive:0", "style:personachat:test:39:self:7:positive:1", "style:personachat:test:39:self:7:positive:2", "style:personachat:test:39:self:7:positive:3", "style:personachat:test:39:self:7:positive:4", "style:personachat:test:39:self:7:positive:5", "style:personachat:test:39:self:7:positive:6"], "evidence_texts": ["A woman, standing behind a girl, helping the girl with an experiment.", "A large group of people walking in a busy city at night.", "Island native fishermen reeling in their nets after a long day's work.", "A woman, standing behind a girl, helping the girl with an experiment.", "i am doing great . just watching a movie .", "i worked at dairy queen . i am in school now .", "what kind of skating do you do ?", "i used to skateboard . now i just study and play video games .", "i love chips so much . what food do you like ?", "i like pizza too . the sauce is the best part .", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:334", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:342", "candidate_text": "I wonder whether a lineman is trying to tackle the quarterback before he makes the game winning touchdown?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:342:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:314:premise", "snli:test:424:premise", "snli:test:342:premise", "snli:test:40:premise", "style:personachat:test:15:self:1:positive:0"], "evidence_texts": ["A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "can you do lots of tricks ?"], "speaker_id": "snli:test:342", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:342", "candidate_text": "well, a lineman is trying to tackle the quarterback before he makes the game winning touchdown", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:342:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:314:premise", "snli:test:424:premise", "snli:test:342:premise", "snli:test:40:premise", "style:personachat:test:27:self:1:positive:0"], "evidence_texts": ["A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:342", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:348", "candidate_text": "I wonder whether the dogs are friends?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:348:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:348:premise", "snli:test:248:premise", "snli:test:383:premise", "snli:test:160:premise", "style:personachat:test:15:self:2:positive:0", "style:personachat:test:15:self:2:positive:1"], "evidence_texts": ["A golden dog passes a blue ball to another golden dog on the beach.", "A man, woman, and child get their picture taken in front of the mountains.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "Two women walk down a sidewalk along a busy street in a downtown area.", "i do ! i play ice hockey sometimes", "can you do lots of tricks ?"], "speaker_id": "snli:test:348", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:348", "candidate_text": "Honestly, The dogs are friends!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:348:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:348:premise", "snli:test:248:premise", "snli:test:383:premise", "snli:test:160:premise", "style:personachat:test:130:self:2:positive:0", "style:personachat:test:130:self:2:positive:1"], "evidence_texts": ["A golden dog passes a blue ball to another golden dog on the beach.", "A man, woman, and child get their picture taken in front of the mountains.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "Two women walk down a sidewalk along a busy street in a downtown area.", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:348", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:354", "candidate_text": "I wonder whether a woman is at work?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:354:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:289:premise", "snli:test:412:premise", "snli:test:354:premise", "snli:test:263:premise", "style:personachat:test:15:self:5:positive:0", "style:personachat:test:15:self:5:positive:1", "style:personachat:test:15:self:5:positive:2", "style:personachat:test:15:self:5:positive:3", "style:personachat:test:15:self:5:positive:4"], "evidence_texts": ["A group of people gathered at night watching an event.", "A guy and a girl look down a mountain range.", "A woman is weaving with a comb in her hand.", "A person in orange clothing rests above a metro entrance.", "nice . i am 16 and i would like to be in the army", "i like making websites and i drink a lot of lipton", "what type of job should i look at ?", "i do ! i play ice hockey sometimes", "can you do lots of tricks ?"], "speaker_id": "snli:test:354", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:354", "candidate_text": "A woman is at work.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:354:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:289:premise", "snli:test:412:premise", "snli:test:354:premise", "snli:test:263:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of people gathered at night watching an event.", "A guy and a girl look down a mountain range.", "A woman is weaving with a comb in her hand.", "A person in orange clothing rests above a metro entrance.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:354", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:367", "candidate_text": "I wonder whether a few men are fishing on a boat?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:367:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:215:premise", "snli:test:749:premise", "snli:test:198:premise", "snli:test:367:premise", "style:personachat:test:15:self:7:positive:0", "style:personachat:test:15:self:7:positive:1", "style:personachat:test:15:self:7:positive:2", "style:personachat:test:15:self:7:positive:3", "style:personachat:test:15:self:7:positive:4", "style:personachat:test:15:self:7:positive:5", "style:personachat:test:15:self:7:positive:6"], "evidence_texts": ["This child is on the library steps.", "Two dogs run together near the leaves.", "Two men climbing on a wooden scaffold.", "Two men are standing in a boat.", "i live in florida but my family is from toronto", "cool . how old are you ?", "nice . i am 16 and i would like to be in the army", "i like making websites and i drink a lot of lipton", "what type of job should i look at ?", "i do ! i play ice hockey sometimes", "can you do lots of tricks ?"], "speaker_id": "snli:test:367", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:367", "candidate_text": "well, a few men are fishing on a boat", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:367:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:215:premise", "snli:test:749:premise", "snli:test:198:premise", "snli:test:367:premise", "style:personachat:test:27:self:7:positive:0", "style:personachat:test:27:self:7:positive:1", "style:personachat:test:27:self:7:positive:2", "style:personachat:test:27:self:7:positive:3", "style:personachat:test:27:self:7:positive:4", "style:personachat:test:27:self:7:positive:5", "style:personachat:test:27:self:7:positive:6"], "evidence_texts": ["This child is on the library steps.", "Two dogs run together near the leaves.", "Two men climbing on a wooden scaffold.", "Two men are standing in a boat.", "hello , i am ok just hanging out here . do you have pets ?", "i work as a carpenter , stay at home mostly . kinda a loner", "i can not see my self as a salesman not good at socializing", "i use to drink a lot and since i quit so did my friends", "i would agree that is it good . my friends were not true friends", "well that is human nature min . you are no fun people run .", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:367", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:378", "candidate_text": "I wonder whether the man is cooking food?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:378:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:295:premise", "snli:test:700:premise", "snli:test:378:premise", "snli:test:86:premise", "style:personachat:test:16:self:1:positive:0"], "evidence_texts": ["A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "well , i ride show horses , but i am hoping to retire soon , you ?"], "speaker_id": "snli:test:378", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:378", "candidate_text": "Honestly, The man is cooking food!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:378:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:295:premise", "snli:test:700:premise", "snli:test:378:premise", "snli:test:86:premise", "style:personachat:test:133:self:1:positive:0"], "evidence_texts": ["A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:378", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:383", "candidate_text": "I wonder whether pete sampras plays tennis?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:383:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:348:premise", "snli:test:432:premise", "snli:test:248:premise", "snli:test:383:premise", "style:personachat:test:16:self:2:positive:0", "style:personachat:test:16:self:2:positive:1"], "evidence_texts": ["A golden dog passes a blue ball to another golden dog on the beach.", "A man wearing a red vest is walking past a black and green fence.", "A man, woman, and child get their picture taken in front of the mountains.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "well that is nice that you all get along .", "well , i ride show horses , but i am hoping to retire soon , you ?"], "speaker_id": "snli:test:383", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:383", "candidate_text": "Pete Sampras plays tennis.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:383:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:348:premise", "snli:test:432:premise", "snli:test:248:premise", "snli:test:383:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A golden dog passes a blue ball to another golden dog on the beach.", "A man wearing a red vest is walking past a black and green fence.", "A man, woman, and child get their picture taken in front of the mountains.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:383", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:386", "candidate_text": "I wonder whether a bunch of women are playing volleyball?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:386:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9479:premise", "snli:test:581:premise", "snli:test:386:premise", "snli:test:6790:premise", "style:personachat:test:16:self:5:positive:0", "style:personachat:test:16:self:5:positive:1", "style:personachat:test:16:self:5:positive:2", "style:personachat:test:16:self:5:positive:3", "style:personachat:test:16:self:5:positive:4"], "evidence_texts": ["A boy plays basketball.", "A hockey player in helmet.", "Several women are playing volleyball.", "Woman adjusting her eyeglasses.", "i am just about to cook dinner ; what will you order ?", "i like pizza . do you let your son make the decisions ?", "that is lucky for him .", "well that is nice that you all get along .", "well , i ride show horses , but i am hoping to retire soon , you ?"], "speaker_id": "snli:test:386", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:386", "candidate_text": "well, a bunch of women are playing volleyball", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:386:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9479:premise", "snli:test:581:premise", "snli:test:386:premise", "snli:test:6790:premise", "style:personachat:test:30:self:5:positive:0", "style:personachat:test:30:self:5:positive:1", "style:personachat:test:30:self:5:positive:2", "style:personachat:test:30:self:5:positive:3", "style:personachat:test:30:self:5:positive:4"], "evidence_texts": ["A boy plays basketball.", "A hockey player in helmet.", "Several women are playing volleyball.", "Woman adjusting her eyeglasses.", "i know how that is . i love to craft but being colorblind makes it hard", "i love to sing . sometimes i go sing karaoke .", "you should , it is great fun !", "that must be a lot of work but very rewarding i bet", "not yet . i have not found the right man to settle down with ."], "speaker_id": "snli:test:386", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:391", "candidate_text": "I wonder whether a group of people are changing a bike tire?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:391:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9385:premise", "snli:test:1245:premise", "snli:test:8225:premise", "snli:test:391:premise", "style:personachat:test:16:self:7:positive:0", "style:personachat:test:16:self:7:positive:1", "style:personachat:test:16:self:7:positive:2", "style:personachat:test:16:self:7:positive:3", "style:personachat:test:16:self:7:positive:4", "style:personachat:test:16:self:7:positive:5", "style:personachat:test:16:self:7:positive:6"], "evidence_texts": ["A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "hello , my leg is really giving me pain today .", "thank you . what are you doing ?", "i am just about to cook dinner ; what will you order ?", "i like pizza . do you let your son make the decisions ?", "that is lucky for him .", "well that is nice that you all get along .", "well , i ride show horses , but i am hoping to retire soon , you ?"], "speaker_id": "snli:test:391", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:391", "candidate_text": "Honestly, A group of people are changing a bike tire!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:391:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9385:premise", "snli:test:1245:premise", "snli:test:8225:premise", "snli:test:391:premise", "style:personachat:test:133:self:7:positive:0", "style:personachat:test:133:self:7:positive:1", "style:personachat:test:133:self:7:positive:2", "style:personachat:test:133:self:7:positive:3", "style:personachat:test:133:self:7:positive:4", "style:personachat:test:133:self:7:positive:5", "style:personachat:test:133:self:7:positive:6"], "evidence_texts": ["A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "no , not anymore . i barely have time for that , with the family !", "i have 3 children , and 2 kitties , that think they are my children as well !", "do you like to cook ? i make homemade gourmet cat food , and people food , too !", "i love cooking ! and , i am pretty good at it , too ! that explains the extra pounds !", "if i was taller , i might feel better about my body . what do you do ?", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:391", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:393", "candidate_text": "well, a group of friends are biking through a street, all wearing their helmets", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:393:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:182:premise", "snli:test:393:premise", "snli:test:502:premise", "snli:test:180:premise", "style:personachat:test:17:self:1:positive:0"], "evidence_texts": ["a boy holding onto the wall of an old brick house's raised foundation.", "People are riding bicycles in the street, and they are all wearing helmets.", "A group of kids is splashing in deep water nearby a rock formation.", "a boy holding onto the wall of an old brick house's raised foundation.", "my best friend betrayed me jerry springer style in high school and it still stings ."], "speaker_id": "snli:test:393", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:393", "candidate_text": "I wonder whether a group of friends are biking through a street, all wearing their helmets?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:393:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:182:premise", "snli:test:393:premise", "snli:test:502:premise", "snli:test:180:premise", "style:personachat:test:43:self:1:positive:0"], "evidence_texts": ["a boy holding onto the wall of an old brick house's raised foundation.", "People are riding bicycles in the street, and they are all wearing helmets.", "A group of kids is splashing in deep water nearby a rock formation.", "a boy holding onto the wall of an old brick house's raised foundation.", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:393", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:397", "candidate_text": "well, an american theater", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:397:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3834:premise", "snli:test:397:premise", "snli:test:1500:premise", "snli:test:1010:premise", "style:personachat:test:17:self:2:positive:0", "style:personachat:test:17:self:2:positive:1"], "evidence_texts": ["Men play hockey.", "A middle eastern marketplace.", "Military personnel are shopping", "A dog runs.", "love is hard and hurtful . alone is best , i have found .", "my best friend betrayed me jerry springer style in high school and it still stings ."], "speaker_id": "snli:test:397", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:397", "candidate_text": "Honestly, An American theater!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:397:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3834:premise", "snli:test:397:premise", "snli:test:1500:premise", "snli:test:1010:premise", "style:personachat:test:137:self:2:positive:0", "style:personachat:test:137:self:2:positive:1"], "evidence_texts": ["Men play hockey.", "A middle eastern marketplace.", "Military personnel are shopping", "A dog runs.", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:397", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:412", "candidate_text": "well, the couple is going to climb the mountains", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:412:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:412:premise", "snli:test:354:premise", "snli:test:606:premise", "snli:test:289:premise", "style:personachat:test:17:self:5:positive:0", "style:personachat:test:17:self:5:positive:1", "style:personachat:test:17:self:5:positive:2", "style:personachat:test:17:self:5:positive:3", "style:personachat:test:17:self:5:positive:4"], "evidence_texts": ["A guy and a girl look down a mountain range.", "A woman is weaving with a comb in her hand.", "The little boy gets ready to kick the soccer ball.", "A group of people gathered at night watching an event.", "you skateboard professionally ? wow , how exciting ! lots of travel ?", "what do you do for a living ?", "oh my , and i thought my life was dramatic ! do you have a family ?", "love is hard and hurtful . alone is best , i have found .", "my best friend betrayed me jerry springer style in high school and it still stings ."], "speaker_id": "snli:test:412", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:412", "candidate_text": "The couple is going to climb the mountains.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:412:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:412:premise", "snli:test:354:premise", "snli:test:606:premise", "snli:test:289:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A guy and a girl look down a mountain range.", "A woman is weaving with a comb in her hand.", "The little boy gets ready to kick the soccer ball.", "A group of people gathered at night watching an event.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:412", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:417", "candidate_text": "well, a girl is hitting the ball", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:417:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:326:premise", "snli:test:417:premise", "snli:test:598:premise", "snli:test:299:premise", "style:personachat:test:17:self:6:positive:0", "style:personachat:test:17:self:6:positive:1", "style:personachat:test:17:self:6:positive:2", "style:personachat:test:17:self:6:positive:3", "style:personachat:test:17:self:6:positive:4", "style:personachat:test:17:self:6:positive:5"], "evidence_texts": ["A softball player throws the ball to her teammate.", "A boy hits a tennis ball on a court.", "A woman wearing jeans is walking down the street.", "A girl holding a beverage points at a painting.", "i prefer pop music . i try to stay up with the trends .", "you skateboard professionally ? wow , how exciting ! lots of travel ?", "what do you do for a living ?", "oh my , and i thought my life was dramatic ! do you have a family ?", "love is hard and hurtful . alone is best , i have found .", "my best friend betrayed me jerry springer style in high school and it still stings ."], "speaker_id": "snli:test:417", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:417", "candidate_text": "I wonder whether a girl is hitting the ball?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:417:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:326:premise", "snli:test:417:premise", "snli:test:598:premise", "snli:test:299:premise", "style:personachat:test:43:self:7:positive:0", "style:personachat:test:43:self:7:positive:1", "style:personachat:test:43:self:7:positive:2", "style:personachat:test:43:self:7:positive:3", "style:personachat:test:43:self:7:positive:4", "style:personachat:test:43:self:7:positive:5", "style:personachat:test:43:self:7:positive:6"], "evidence_texts": ["A softball player throws the ball to her teammate.", "A boy hits a tennis ball on a court.", "A woman wearing jeans is walking down the street.", "A girl holding a beverage points at a painting.", "i am well . how are you ?", "that is great . i live far from the beach .", "i keep busy with my seven children .", "do you reach mysteries to your children ? they are my favorite type of novel .", "do you teach them about animals , as i grew up on a farm .", "great , how about colors ? my favorite is blue", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:417", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:424", "candidate_text": "well, the railing is in front of a frozen lake", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:424:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:424:premise", "snli:test:342:premise", "snli:test:452:premise", "snli:test:314:premise", "style:personachat:test:18:self:1:positive:0"], "evidence_texts": ["A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "we like our schedule . that way we do not get tired of each other"], "speaker_id": "snli:test:424", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:424", "candidate_text": "Honestly, The railing is in front of a frozen lake!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:424:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:424:premise", "snli:test:342:premise", "snli:test:452:premise", "snli:test:314:premise", "style:personachat:test:140:self:1:positive:0"], "evidence_texts": ["A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A woman with her face partially covered in silver face paint sits on a plastic tote looking in a mirror.", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:424", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:432", "candidate_text": "well, the man wearing the vest is walking down the street", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:432:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:432:premise", "snli:test:383:premise", "snli:test:348:premise", "snli:test:248:premise", "style:personachat:test:18:self:2:positive:0", "style:personachat:test:18:self:2:positive:1"], "evidence_texts": ["A man wearing a red vest is walking past a black and green fence.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "A golden dog passes a blue ball to another golden dog on the beach.", "A man, woman, and child get their picture taken in front of the mountains.", "i am a chef during the day at work and again at home for the kids", "we like our schedule . that way we do not get tired of each other"], "speaker_id": "snli:test:432", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:432", "candidate_text": "The man wearing the vest is walking down the street.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:432:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:432:premise", "snli:test:383:premise", "snli:test:348:premise", "snli:test:248:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man wearing a red vest is walking past a black and green fence.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "A golden dog passes a blue ball to another golden dog on the beach.", "A man, woman, and child get their picture taken in front of the mountains.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:432", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:433", "candidate_text": "well, a man is wearing a vest", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:433:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:383:premise", "snli:test:433:premise", "snli:test:575:premise", "snli:test:348:premise", "style:personachat:test:18:self:5:positive:0", "style:personachat:test:18:self:5:positive:1", "style:personachat:test:18:self:5:positive:2", "style:personachat:test:18:self:5:positive:3", "style:personachat:test:18:self:5:positive:4"], "evidence_texts": ["A woman in tennis clothes running to hit a ball with her tennis racket.", "A man wearing a red vest is walking past a black and green fence.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A golden dog passes a blue ball to another golden dog on the beach.", "short hair is ok for guys and dogs but my husband hates it on women .", "i can make 5 different kinds depending what cheese and toppings", "i have 3 hungry kids and a husband to cook for . luckily he works at night .", "i am a chef during the day at work and again at home for the kids", "we like our schedule . that way we do not get tired of each other"], "speaker_id": "snli:test:433", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:433", "candidate_text": "I wonder whether a man is wearing a vest?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:433:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:383:premise", "snli:test:433:premise", "snli:test:575:premise", "snli:test:348:premise", "style:personachat:test:47:self:5:positive:0", "style:personachat:test:47:self:5:positive:1", "style:personachat:test:47:self:5:positive:2", "style:personachat:test:47:self:5:positive:3", "style:personachat:test:47:self:5:positive:4"], "evidence_texts": ["A woman in tennis clothes running to hit a ball with her tennis racket.", "A man wearing a red vest is walking past a black and green fence.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A golden dog passes a blue ball to another golden dog on the beach.", "i will try that , any hobbies , i want to try writing fantasy books .", "i like that show to , greys anatomy is a favorite of mine , any kids or other person ?", "i think a fantasy world something like avatar , i always wanted a part 2 .", "the blue people , any hobbies for you ?", "sounds interesting , what is your favorite of your collection ?"], "speaker_id": "snli:test:433", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:434", "candidate_text": "well, the man wearing the vest is sitting on the sofa", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:434:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:575:premise", "snli:test:681:premise", "snli:test:434:premise", "snli:test:383:premise", "style:personachat:test:18:self:7:positive:0", "style:personachat:test:18:self:7:positive:1", "style:personachat:test:18:self:7:positive:2", "style:personachat:test:18:self:7:positive:3", "style:personachat:test:18:self:7:positive:4", "style:personachat:test:18:self:7:positive:5", "style:personachat:test:18:self:7:positive:6"], "evidence_texts": ["One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "A man wearing a red vest is walking past a black and green fence.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "hi , sure if you stop cutting womens hair so short", "i am a chef and i can cook good but i am not happy about my weight .", "short hair is ok for guys and dogs but my husband hates it on women .", "i can make 5 different kinds depending what cheese and toppings", "i have 3 hungry kids and a husband to cook for . luckily he works at night .", "i am a chef during the day at work and again at home for the kids", "we like our schedule . that way we do not get tired of each other"], "speaker_id": "snli:test:434", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:434", "candidate_text": "Honestly, The man wearing the vest is sitting on the sofa!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:434:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:575:premise", "snli:test:681:premise", "snli:test:434:premise", "snli:test:383:premise", "style:personachat:test:140:self:7:positive:0", "style:personachat:test:140:self:7:positive:1", "style:personachat:test:140:self:7:positive:2", "style:personachat:test:140:self:7:positive:3", "style:personachat:test:140:self:7:positive:4", "style:personachat:test:140:self:7:positive:5", "style:personachat:test:140:self:7:positive:6"], "evidence_texts": ["One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "A man wearing a red vest is walking past a black and green fence.", "A woman in tennis clothes running to hit a ball with her tennis racket.", "great ! just grading some papers and throwing some things in the crock pot .", "very cool ! my daughter wants a fish tank !", "i am a literature teacher ! nothing like a good book ! nice ! !", "i wish my husband was ! my daughter just loves cooking like me .", "it is fun ! i love granola . what else are you into ?", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:434", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:452", "candidate_text": "well, the women are having a conversation at the office,", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:452:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:452:premise", "snli:test:424:premise", "snli:test:994:premise", "snli:test:342:premise", "style:personachat:test:19:self:1:positive:0"], "evidence_texts": ["Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "well , got to go now . . . . see you ."], "speaker_id": "snli:test:452", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:452", "candidate_text": "The women are having a conversation at the office,.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:452:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:452:premise", "snli:test:424:premise", "snli:test:994:premise", "snli:test:342:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "Two people are playing American Football and one of them is trying to stop the other from throwing the ball.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:452", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:469", "candidate_text": "well, the kids crawl in sand", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:469:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:311:premise", "snli:test:469:premise", "snli:test:475:premise", "snli:test:310:premise", "style:personachat:test:19:self:2:positive:0", "style:personachat:test:19:self:2:positive:1"], "evidence_texts": ["Two men prepare a fish at a dock.", "Children in yellow sports uniforms climbing a tower.", "A crowded city street with lots of pedestrians.", "Two men prepare a fish at a dock.", "have your very own test subjects .", "well , got to go now . . . . see you ."], "speaker_id": "snli:test:469", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:469", "candidate_text": "I wonder whether the kids crawl in sand?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:469:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:311:premise", "snli:test:469:premise", "snli:test:475:premise", "snli:test:310:premise", "style:personachat:test:48:self:2:positive:0", "style:personachat:test:48:self:2:positive:1"], "evidence_texts": ["Two men prepare a fish at a dock.", "Children in yellow sports uniforms climbing a tower.", "A crowded city street with lots of pedestrians.", "Two men prepare a fish at a dock.", "i love to go running , its my favorite passion . i wish i ran for a living", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:469", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:475", "candidate_text": "well, there are many pedestrians on the city street", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:475:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:469:premise", "snli:test:563:premise", "snli:test:311:premise", "snli:test:475:premise", "style:personachat:test:19:self:5:positive:0", "style:personachat:test:19:self:5:positive:1", "style:personachat:test:19:self:5:positive:2", "style:personachat:test:19:self:5:positive:3", "style:personachat:test:19:self:5:positive:4"], "evidence_texts": ["Children in yellow sports uniforms climbing a tower.", "Boys with their backs against an incoming wave.", "Two men prepare a fish at a dock.", "A crowded city street with lots of pedestrians.", "oh , i no longer work at the bank but for exercise walk a mile a day .", "better than walking alone . i also make quilts and give them away to family .", "what is your major in school ?", "have your very own test subjects .", "well , got to go now . . . . see you ."], "speaker_id": "snli:test:475", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:475", "candidate_text": "Honestly, There are many pedestrians on the city street!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:475:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:469:premise", "snli:test:563:premise", "snli:test:311:premise", "snli:test:475:premise", "style:personachat:test:144:self:5:positive:0", "style:personachat:test:144:self:5:positive:1", "style:personachat:test:144:self:5:positive:2", "style:personachat:test:144:self:5:positive:3", "style:personachat:test:144:self:5:positive:4"], "evidence_texts": ["Children in yellow sports uniforms climbing a tower.", "Boys with their backs against an incoming wave.", "Two men prepare a fish at a dock.", "A crowded city street with lots of pedestrians.", "i teach kindergarten . barely covers the bills ?", "no problem . 7 kids must fry your brain ! lol", "purple ! i live in seattle . how about you ?", "i went to college in san diego , but moved out here 2 years ago after i graduated .", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:475", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:502", "candidate_text": "well, the kids are in deep water", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:502:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:393:premise", "snli:test:895:premise", "snli:test:502:premise", "snli:test:182:premise", "style:personachat:test:19:self:6:positive:0", "style:personachat:test:19:self:6:positive:1", "style:personachat:test:19:self:6:positive:2", "style:personachat:test:19:self:6:positive:3", "style:personachat:test:19:self:6:positive:4", "style:personachat:test:19:self:6:positive:5"], "evidence_texts": ["People are riding bicycles in the street, and they are all wearing helmets.", "A brown dog carries an object in its mouth on a snowy hillside.", "A group of kids is splashing in deep water nearby a rock formation.", "a boy holding onto the wall of an old brick house's raised foundation.", "just two right now and a siamese cat .", "oh , i no longer work at the bank but for exercise walk a mile a day .", "better than walking alone . i also make quilts and give them away to family .", "what is your major in school ?", "have your very own test subjects .", "well , got to go now . . . . see you ."], "speaker_id": "snli:test:502", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:502", "candidate_text": "The kids are in deep water.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:502:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:393:premise", "snli:test:895:premise", "snli:test:502:premise", "snli:test:182:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["People are riding bicycles in the street, and they are all wearing helmets.", "A brown dog carries an object in its mouth on a snowy hillside.", "A group of kids is splashing in deep water nearby a rock formation.", "a boy holding onto the wall of an old brick house's raised foundation.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:502", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:521", "candidate_text": "well, a man hits on a woman", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:521:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:321:premise", "snli:test:521:premise", "snli:test:1035:premise", "snli:test:238:premise", "style:personachat:test:20:self:1:positive:0"], "evidence_texts": ["A woman, in red and white, and wearing glasses, sits in a room with other people.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "A group of adults stands in the bathroom looking down a small child in the bath.", "i want a ranch full of animals . i am getting hungry . i need to start making dinner ."], "speaker_id": "snli:test:521", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:521", "candidate_text": "I wonder whether a man hits on a woman?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:521:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:321:premise", "snli:test:521:premise", "snli:test:1035:premise", "snli:test:238:premise", "style:personachat:test:53:self:1:positive:0"], "evidence_texts": ["A woman, in red and white, and wearing glasses, sits in a room with other people.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "A group of adults stands in the bathroom looking down a small child in the bath.", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:521", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:538", "candidate_text": "well, a girl with a ponytail is riding a bike at a park", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:538:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:62:premise", "snli:test:543:premise", "snli:test:538:premise", "snli:test:9919:premise", "style:personachat:test:20:self:2:positive:0", "style:personachat:test:20:self:2:positive:1"], "evidence_texts": ["The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "i like to run , create art , and take naps ! how about you ?", "i want a ranch full of animals . i am getting hungry . i need to start making dinner ."], "speaker_id": "snli:test:538", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:538", "candidate_text": "Honestly, A girl with a ponytail is riding a bike at a park!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:538:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:62:premise", "snli:test:543:premise", "snli:test:538:premise", "snli:test:9919:premise", "style:personachat:test:146:self:2:positive:0", "style:personachat:test:146:self:2:positive:1"], "evidence_texts": ["The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "so you like the hot weather ? i prefer cold and snow", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:538", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:543", "candidate_text": "well, he is wearing the newspaper on his head", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:543:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:538:premise", "snli:test:808:premise", "snli:test:62:premise", "snli:test:543:premise", "style:personachat:test:20:self:5:positive:0", "style:personachat:test:20:self:5:positive:1", "style:personachat:test:20:self:5:positive:2", "style:personachat:test:20:self:5:positive:3", "style:personachat:test:20:self:5:positive:4"], "evidence_texts": ["A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "A man has flung himself over a pole with people and canopies in the background.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "my grandfather , all my uncles and my dad were police officers .", "i have always wanted to be a cop . i work with animals .", "that is pretty sweet . i love making everyday sunday i cant get him off my mind .", "i like to run , create art , and take naps ! how about you ?", "i want a ranch full of animals . i am getting hungry . i need to start making dinner ."], "speaker_id": "snli:test:543", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:543", "candidate_text": "He is wearing the newspaper on his head.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:543:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:538:premise", "snli:test:808:premise", "snli:test:62:premise", "snli:test:543:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "A man has flung himself over a pole with people and canopies in the background.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:543", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:546", "candidate_text": "well, a human jumping", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:546:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:89:premise", "snli:test:9990:premise", "snli:test:546:premise", "snli:test:1263:premise", "style:personachat:test:20:self:7:positive:0", "style:personachat:test:20:self:7:positive:1", "style:personachat:test:20:self:7:positive:2", "style:personachat:test:20:self:7:positive:3", "style:personachat:test:20:self:7:positive:4", "style:personachat:test:20:self:7:positive:5", "style:personachat:test:20:self:7:positive:6"], "evidence_texts": ["A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "i drive a toyota ! what is your favorite band ?", "i am still rocking to nirvana . i love animals !", "my grandfather , all my uncles and my dad were police officers .", "i have always wanted to be a cop . i work with animals .", "that is pretty sweet . i love making everyday sunday i cant get him off my mind .", "i like to run , create art , and take naps ! how about you ?", "i want a ranch full of animals . i am getting hungry . i need to start making dinner ."], "speaker_id": "snli:test:546", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:546", "candidate_text": "I wonder whether a human jumping?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:546:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:89:premise", "snli:test:9990:premise", "snli:test:546:premise", "snli:test:1263:premise", "style:personachat:test:53:self:7:positive:0", "style:personachat:test:53:self:7:positive:1", "style:personachat:test:53:self:7:positive:2", "style:personachat:test:53:self:7:positive:3", "style:personachat:test:53:self:7:positive:4", "style:personachat:test:53:self:7:positive:5", "style:personachat:test:53:self:7:positive:6"], "evidence_texts": ["A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "i am doing well , how about you ?", "that is cool , i am currently building a tree house in my spare time", "i do , i am divorced but have some great kids", "i used to as well but now i am 40 so it is too late for me", "yeah i really need to . i just hate sitting around and being still", "helps me stay healthy and avoid thinking about my ex wife", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:546", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:548", "candidate_text": "Honestly, A tall human jumping!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:548:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:548:premise", "snli:test:1263:premise", "snli:test:89:premise", "snli:test:1265:premise", "style:personachat:test:21:self:1:positive:0"], "evidence_texts": ["A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:548", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:548", "candidate_text": "well, a tall human jumping", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:548:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:548:premise", "snli:test:1263:premise", "snli:test:89:premise", "snli:test:1265:premise", "style:personachat:test:41:self:1:positive:0"], "evidence_texts": ["A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "i m having a hard time typing sorry"], "speaker_id": "snli:test:548", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:563", "candidate_text": "Honestly, A group of people sit on the beach!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:563:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:475:premise", "snli:test:685:premise", "snli:test:469:premise", "snli:test:563:premise", "style:personachat:test:21:self:2:positive:0", "style:personachat:test:21:self:2:positive:1"], "evidence_texts": ["A crowded city street with lots of pedestrians.", "Five children playing soccer chase after a ball.", "Children in yellow sports uniforms climbing a tower.", "Boys with their backs against an incoming wave.", "yes i thought they were real too , i got duped by my supplier", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:563", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:563", "candidate_text": "A group of people sit on the beach.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:563:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:475:premise", "snli:test:685:premise", "snli:test:469:premise", "snli:test:563:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A crowded city street with lots of pedestrians.", "Five children playing soccer chase after a ball.", "Children in yellow sports uniforms climbing a tower.", "Boys with their backs against an incoming wave.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:563", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:564", "candidate_text": "Honestly, A wave is about hit some boys!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:564:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:564:premise", "snli:test:212:premise", "snli:test:753:premise", "snli:test:34:premise", "style:personachat:test:21:self:5:positive:0", "style:personachat:test:21:self:5:positive:1", "style:personachat:test:21:self:5:positive:2", "style:personachat:test:21:self:5:positive:3", "style:personachat:test:21:self:5:positive:4"], "evidence_texts": ["Four boys are about to be hit by an approaching wave.", "A person in a red shirt and black pants hunched over.", "A crowd wearing orange cheering for their team in a stadium.", "A man reads the paper in a bar with green lighting.", "cool i am in jail right now so can i ask you some questions ?", "so i am here for selling sea monkeys like you see in comic books", "what do you mean ? i got convicted for mail fraud cuz sea monkeys do not exist", "yes i thought they were real too , i got duped by my supplier", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:564", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:564", "candidate_text": "I wonder whether a wave is about hit some boys?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:564:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:564:premise", "snli:test:212:premise", "snli:test:753:premise", "snli:test:34:premise", "style:personachat:test:60:self:5:positive:0", "style:personachat:test:60:self:5:positive:1", "style:personachat:test:60:self:5:positive:2", "style:personachat:test:60:self:5:positive:3", "style:personachat:test:60:self:5:positive:4"], "evidence_texts": ["Four boys are about to be hit by an approaching wave.", "A person in a red shirt and black pants hunched over.", "A crowd wearing orange cheering for their team in a stadium.", "A man reads the paper in a bar with green lighting.", "i like the atmosphere that it offers . what led you to nursing ?", "good a reason as any . does she work at a hospital ?", "do you also want to work with babies ?", "that is sweet of you . do you like working out ?", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:564", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:567", "candidate_text": "Honestly, A mother is helping her child complete the experiment!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:567:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:334:premise", "snli:test:333:premise", "snli:test:591:premise", "snli:test:567:premise", "style:personachat:test:21:self:7:positive:0", "style:personachat:test:21:self:7:positive:1", "style:personachat:test:21:self:7:positive:2", "style:personachat:test:21:self:7:positive:3", "style:personachat:test:21:self:7:positive:4", "style:personachat:test:21:self:7:positive:5", "style:personachat:test:21:self:7:positive:6"], "evidence_texts": ["Island native fishermen reeling in their nets after a long day's work.", "Island native fishermen reeling in their nets after a long day's work.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "just as fantastic as i possibly can be and yourself ?", "what are you studying , may i ask ?", "cool i am in jail right now so can i ask you some questions ?", "so i am here for selling sea monkeys like you see in comic books", "what do you mean ? i got convicted for mail fraud cuz sea monkeys do not exist", "yes i thought they were real too , i got duped by my supplier", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:567", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:567", "candidate_text": "well, a mother is helping her child complete the experiment", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:567:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:334:premise", "snli:test:333:premise", "snli:test:591:premise", "snli:test:567:premise", "style:personachat:test:41:self:7:positive:0", "style:personachat:test:41:self:7:positive:1", "style:personachat:test:41:self:7:positive:2", "style:personachat:test:41:self:7:positive:3", "style:personachat:test:41:self:7:positive:4", "style:personachat:test:41:self:7:positive:5", "style:personachat:test:41:self:7:positive:6"], "evidence_texts": ["Island native fishermen reeling in their nets after a long day's work.", "Island native fishermen reeling in their nets after a long day's work.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "registered nurse it s not raining fine", "watching one tree hill is relaxing for me", "you guys like pizza it s my favorite", "hey i am a cat catholic devout", "i m not remain silent but purple is my favorite", "sorry for the typo what i mean is", "i m having a hard time typing sorry"], "speaker_id": "snli:test:567", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:569", "candidate_text": "I wonder whether a woman is helping a girl?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:569:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:591:premise", "snli:test:569:premise", "snli:test:334:premise", "snli:test:624:premise", "style:personachat:test:22:self:1:positive:0"], "evidence_texts": ["couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "Island native fishermen reeling in their nets after a long day's work.", "A carpenter is working on a roof at a house being built.", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:569", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:569", "candidate_text": "Honestly, A woman is helping a girl!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:569:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:591:premise", "snli:test:569:premise", "snli:test:334:premise", "snli:test:624:premise", "style:personachat:test:157:self:1:positive:0"], "evidence_texts": ["couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "Island native fishermen reeling in their nets after a long day's work.", "A carpenter is working on a roof at a house being built.", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:569", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:575", "candidate_text": "I wonder whether one guy kicks another?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:575:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:434:premise", "snli:test:681:premise", "snli:test:433:premise", "snli:test:575:premise", "style:personachat:test:22:self:2:positive:0", "style:personachat:test:22:self:2:positive:1"], "evidence_texts": ["A man wearing a red vest is walking past a black and green fence.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "A man wearing a red vest is walking past a black and green fence.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "my fav is christmas i love that day", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:575", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:575", "candidate_text": "One guy kicks another.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:575:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:434:premise", "snli:test:681:premise", "snli:test:433:premise", "snli:test:575:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man wearing a red vest is walking past a black and green fence.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "A man wearing a red vest is walking past a black and green fence.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:575", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:577", "candidate_text": "I wonder whether mob gathers as flutes fight to the death?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:577:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9741:premise", "snli:test:577:premise", "snli:test:783:premise", "snli:test:9448:premise", "style:personachat:test:22:self:5:positive:0", "style:personachat:test:22:self:5:positive:1", "style:personachat:test:22:self:5:positive:2", "style:personachat:test:22:self:5:positive:3", "style:personachat:test:22:self:5:positive:4"], "evidence_texts": ["A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "awe how old are they ?", "i went here in ohio like pizza ?", "what is your favorite holiday ?", "my fav is christmas i love that day", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:577", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:577", "candidate_text": "well, mob gathers as flutes fight to the death", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:577:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9741:premise", "snli:test:577:premise", "snli:test:783:premise", "snli:test:9448:premise", "style:personachat:test:44:self:5:positive:0", "style:personachat:test:44:self:5:positive:1", "style:personachat:test:44:self:5:positive:2", "style:personachat:test:44:self:5:positive:3", "style:personachat:test:44:self:5:positive:4"], "evidence_texts": ["A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "well i am only 17 , would rather do something with fashion though", "cause my pops wants it . one of those things .", "lol ! try racing a beetle , that was something dad said no to right away", "i do not want that day to come . i want to hold onto my youth !", "oh i miss the days of crayons being my hardest decision already"], "speaker_id": "snli:test:577", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:581", "candidate_text": "I wonder whether they are playing hockey?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:581:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:386:premise", "snli:test:581:premise", "snli:test:673:premise", "snli:test:9479:premise", "style:personachat:test:22:self:7:positive:0", "style:personachat:test:22:self:7:positive:1", "style:personachat:test:22:self:7:positive:2", "style:personachat:test:22:self:7:positive:3", "style:personachat:test:22:self:7:positive:4", "style:personachat:test:22:self:7:positive:5", "style:personachat:test:22:self:7:positive:6"], "evidence_texts": ["Several women are playing volleyball.", "A hockey player in helmet.", "Man holding torch of fire.", "A boy plays basketball.", "i am well an you ? like dogs .", "names ? i have 3 dogs", "awe how old are they ?", "i went here in ohio like pizza ?", "what is your favorite holiday ?", "my fav is christmas i love that day", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:581", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:581", "candidate_text": "Honestly, They are playing hockey!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:581:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:386:premise", "snli:test:581:premise", "snli:test:673:premise", "snli:test:9479:premise", "style:personachat:test:157:self:7:positive:0", "style:personachat:test:157:self:7:positive:1", "style:personachat:test:157:self:7:positive:2", "style:personachat:test:157:self:7:positive:3", "style:personachat:test:157:self:7:positive:4", "style:personachat:test:157:self:7:positive:5", "style:personachat:test:157:self:7:positive:6"], "evidence_texts": ["Several women are playing volleyball.", "A hockey player in helmet.", "Man holding torch of fire.", "A boy plays basketball.", "i am doing great ! i am just currently exploring with my sea creature friends !", "i like rush as well , i have to see them whenever i can above the sea .", "well starfish , jellyfish , squids , fish , they are all my friends !", "that sounds awesome , i have a tail as feet , so i can not work .", "no , i am not , i do have long wavy hair though !", "i wish i could let you see me , i have to eat dinner with my family though .", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:581", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:588", "candidate_text": "well, two men are recording a cooking tutorial", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:588:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:588:premise", "snli:test:9898:premise", "snli:test:605:premise", "snli:test:9820:premise", "style:personachat:test:23:self:1:positive:0"], "evidence_texts": ["A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:588", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:588", "candidate_text": "I wonder whether two men are recording a cooking tutorial?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:588:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:588:premise", "snli:test:9898:premise", "snli:test:605:premise", "snli:test:9820:premise", "style:personachat:test:64:self:1:positive:0"], "evidence_texts": ["A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:588", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:591", "candidate_text": "well, the couple are eating steak", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:591:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:569:premise", "snli:test:624:premise", "snli:test:567:premise", "snli:test:591:premise", "style:personachat:test:23:self:2:positive:0", "style:personachat:test:23:self:2:positive:1"], "evidence_texts": ["A woman, standing behind a girl, helping the girl with an experiment.", "A carpenter is working on a roof at a house being built.", "A woman, standing behind a girl, helping the girl with an experiment.", "couples enjoying a meal, while a bunch of dogs take a nap.", "yes , we do . do you like animals ?", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:591", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:591", "candidate_text": "Honestly, THE COUPLE ARE EATING STEAK!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:591:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:569:premise", "snli:test:624:premise", "snli:test:567:premise", "snli:test:591:premise", "style:personachat:test:159:self:2:positive:0", "style:personachat:test:159:self:2:positive:1"], "evidence_texts": ["A woman, standing behind a girl, helping the girl with an experiment.", "A carpenter is working on a roof at a house being built.", "A woman, standing behind a girl, helping the girl with an experiment.", "couples enjoying a meal, while a bunch of dogs take a nap.", "ah . have any pets ? i have 3 . my dogs are my babies .", "i bet your singing is not that bad !"], "speaker_id": "snli:test:591", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:598", "candidate_text": "well, a person is outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:598:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:417:premise", "snli:test:865:premise", "snli:test:598:premise", "snli:test:326:premise", "style:personachat:test:23:self:5:positive:0", "style:personachat:test:23:self:5:positive:1", "style:personachat:test:23:self:5:positive:2", "style:personachat:test:23:self:5:positive:3", "style:personachat:test:23:self:5:positive:4"], "evidence_texts": ["A boy hits a tennis ball on a court.", "A car is loaded with items on the top.", "A woman wearing jeans is walking down the street.", "A softball player throws the ball to her teammate.", "green . what do you do for fun ?", "kool ! i like to play fantasy video games .", "just like that , i also build model spaceships in my free time .", "yes , we do . do you like animals ?", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:598", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:598", "candidate_text": "A person is outside.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:598:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:417:premise", "snli:test:865:premise", "snli:test:598:premise", "snli:test:326:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A boy hits a tennis ball on a court.", "A car is loaded with items on the top.", "A woman wearing jeans is walking down the street.", "A softball player throws the ball to her teammate.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:598", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:605", "candidate_text": "well, a group of people dressed as clowns stroll into the bigtop circus holding signs", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:605:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:588:premise", "snli:test:605:premise", "snli:test:1248:premise", "snli:test:9898:premise", "style:personachat:test:23:self:6:positive:0", "style:personachat:test:23:self:6:positive:1", "style:personachat:test:23:self:6:positive:2", "style:personachat:test:23:self:6:positive:3", "style:personachat:test:23:self:6:positive:4", "style:personachat:test:23:self:6:positive:5"], "evidence_texts": ["A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "i am doing pretty good , how are you ?", "green . what do you do for fun ?", "kool ! i like to play fantasy video games .", "just like that , i also build model spaceships in my free time .", "yes , we do . do you like animals ?", "not really , i am pretty shy . i like cats ."], "speaker_id": "snli:test:605", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:605", "candidate_text": "I wonder whether a group of people dressed as clowns stroll into the bigtop circus holding signs?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:605:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:588:premise", "snli:test:605:premise", "snli:test:1248:premise", "snli:test:9898:premise", "style:personachat:test:64:self:6:positive:0", "style:personachat:test:64:self:6:positive:1", "style:personachat:test:64:self:6:positive:2", "style:personachat:test:64:self:6:positive:3", "style:personachat:test:64:self:6:positive:4", "style:personachat:test:64:self:6:positive:5"], "evidence_texts": ["A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "hello , i am a web designer .", "no , i do not . i like it hot .", "i am a web designer , i do coding for several country musicians .", "yes , i am . i only listen to country .", "i hate snow , i am so glad i made the move .", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:605", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:606", "candidate_text": "well, the boy gets ready", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:606:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:412:premise", "snli:test:831:premise", "snli:test:606:premise", "snli:test:354:premise", "style:personachat:test:24:self:1:positive:0"], "evidence_texts": ["A guy and a girl look down a mountain range.", "Four women are taking a walk down an icy road.", "The little boy gets ready to kick the soccer ball.", "A woman is weaving with a comb in her hand.", "gosh , i thought all the children would need to be asleep by then"], "speaker_id": "snli:test:606", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:606", "candidate_text": "Honestly, the boy gets ready!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:606:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:412:premise", "snli:test:831:premise", "snli:test:606:premise", "snli:test:354:premise", "style:personachat:test:174:self:1:positive:0"], "evidence_texts": ["A guy and a girl look down a mountain range.", "Four women are taking a walk down an icy road.", "The little boy gets ready to kick the soccer ball.", "A woman is weaving with a comb in her hand.", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:606", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:616", "candidate_text": "well, a person is talking to a child while another person watches", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:616:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:616:premise", "snli:test:8990:premise", "snli:test:2059:premise", "snli:test:8838:premise", "style:personachat:test:24:self:2:positive:0", "style:personachat:test:24:self:2:positive:1"], "evidence_texts": ["A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "tricky what about in the evening ? the office is my fave at the moment", "gosh , i thought all the children would need to be asleep by then"], "speaker_id": "snli:test:616", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:616", "candidate_text": "A person is talking to a child while another person watches.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:616:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:616:premise", "snli:test:8990:premise", "snli:test:2059:premise", "snli:test:8838:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:616", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:624", "candidate_text": "well, a man is hammering the final shingle on a roof", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:624:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:624:premise", "snli:test:591:premise", "snli:test:569:premise", "snli:test:660:premise", "style:personachat:test:24:self:5:positive:0", "style:personachat:test:24:self:5:positive:1", "style:personachat:test:24:self:5:positive:2", "style:personachat:test:24:self:5:positive:3", "style:personachat:test:24:self:5:positive:4"], "evidence_texts": ["A carpenter is working on a roof at a house being built.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "A blond woman in a black shirt is standing behind a counter.", "yeah , that is fine . i prefer savoury foods to sweet anyway . do you like music ?", "i would , but i live in new york so do not drive , which is difficult", "could hike there , but camping might be tricky ? do you watch much tv", "tricky what about in the evening ? the office is my fave at the moment", "gosh , i thought all the children would need to be asleep by then"], "speaker_id": "snli:test:624", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:624", "candidate_text": "I wonder whether a man is hammering the final shingle on a roof?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:624:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:624:premise", "snli:test:591:premise", "snli:test:569:premise", "snli:test:660:premise", "style:personachat:test:66:self:5:positive:0", "style:personachat:test:66:self:5:positive:1", "style:personachat:test:66:self:5:positive:2", "style:personachat:test:66:self:5:positive:3", "style:personachat:test:66:self:5:positive:4"], "evidence_texts": ["A carpenter is working on a roof at a house being built.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A woman, standing behind a girl, helping the girl with an experiment.", "A blond woman in a black shirt is standing behind a counter.", "do you teach dogs obedience ? that is good . i am afraid of them .", "there were sometimes animals around when i worked at the carnival .", "when i want to relax i watch gritty movies and drink scotch .", "when did you buy it ? do you live alone ?", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:624", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:626", "candidate_text": "well, a boy is jumping up and down", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:626:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:660:premise", "snli:test:591:premise", "snli:test:626:premise", "snli:test:693:premise", "style:personachat:test:24:self:6:positive:0", "style:personachat:test:24:self:6:positive:1", "style:personachat:test:24:self:6:positive:2", "style:personachat:test:24:self:6:positive:3", "style:personachat:test:24:self:6:positive:4", "style:personachat:test:24:self:6:positive:5"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A carpenter is working on a roof at a house being built.", "Here is a picture of people getting drunk at a house party.", "no ! i am a jew ! i assume you like bacon ?", "yeah , that is fine . i prefer savoury foods to sweet anyway . do you like music ?", "i would , but i live in new york so do not drive , which is difficult", "could hike there , but camping might be tricky ? do you watch much tv", "tricky what about in the evening ? the office is my fave at the moment", "gosh , i thought all the children would need to be asleep by then"], "speaker_id": "snli:test:626", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:626", "candidate_text": "Honestly, A boy is jumping up and down!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:626:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:660:premise", "snli:test:591:premise", "snli:test:626:premise", "snli:test:693:premise", "style:personachat:test:174:self:7:positive:0", "style:personachat:test:174:self:7:positive:1", "style:personachat:test:174:self:7:positive:2", "style:personachat:test:174:self:7:positive:3", "style:personachat:test:174:self:7:positive:4", "style:personachat:test:174:self:7:positive:5", "style:personachat:test:174:self:7:positive:6"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "couples enjoying a meal, while a bunch of dogs take a nap.", "A carpenter is working on a roof at a house being built.", "Here is a picture of people getting drunk at a house party.", "wonderful . just finished work . how are you ?", "right now i am modeling . it is why i moved to this city . what do you do ?", "that is why i do not ever want kids . where are you from ?", "new york . a great place to be single ! married yourself ?", "came here to be a singer . why did you leave ?", "it is fun . but modeling pays more ! you have hobbies ?", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:626", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:641", "candidate_text": "well, two research scientists conduct research in disguise", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:641:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:284:premise", "snli:test:641:premise", "snli:test:937:premise", "snli:test:282:premise", "style:personachat:test:25:self:1:positive:0"], "evidence_texts": ["A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "mostly country . like the way it makes me feel and relates to life"], "speaker_id": "snli:test:641", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:641", "candidate_text": "Two research scientists conduct research in disguise.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:641:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:284:premise", "snli:test:641:premise", "snli:test:937:premise", "snli:test:282:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:641", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:647", "candidate_text": "well, two friends sit against a wall with their pet dog", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:647:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:332:premise", "snli:test:671:premise", "snli:test:114:premise", "snli:test:647:premise", "style:personachat:test:25:self:2:positive:0", "style:personachat:test:25:self:2:positive:1"], "evidence_texts": ["Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "big city now . helps with my career in singing", "mostly country . like the way it makes me feel and relates to life"], "speaker_id": "snli:test:647", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:647", "candidate_text": "I wonder whether two friends sit against a wall with their pet dog?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:647:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:332:premise", "snli:test:671:premise", "snli:test:114:premise", "snli:test:647:premise", "style:personachat:test:67:self:2:positive:0", "style:personachat:test:67:self:2:positive:1"], "evidence_texts": ["Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "A crowd of people looking up at 3 people on the edge of the roof of a building.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "ooh , i love italian food , especially lasagna . do you have a favorite ?", "odd . you need a singer in your metal band ? the church choir bores me ."], "speaker_id": "snli:test:647", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:660", "candidate_text": "well, the woman is standing", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:660:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:660:premise", "snli:test:626:premise", "snli:test:693:premise", "snli:test:624:premise", "style:personachat:test:25:self:5:positive:0", "style:personachat:test:25:self:5:positive:1", "style:personachat:test:25:self:5:positive:2", "style:personachat:test:25:self:5:positive:3", "style:personachat:test:25:self:5:positive:4"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "A carpenter is working on a roof at a house being built.", "Here is a picture of people getting drunk at a house party.", "A carpenter is working on a roof at a house being built.", "my name is joanna and i like horror films", "interesting , what do you do for work ?", "i can understand that .", "big city now . helps with my career in singing", "mostly country . like the way it makes me feel and relates to life"], "speaker_id": "snli:test:660", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:660", "candidate_text": "Honestly, The woman is standing!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:660:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:660:premise", "snli:test:626:premise", "snli:test:693:premise", "snli:test:624:premise", "style:personachat:test:182:self:5:positive:0", "style:personachat:test:182:self:5:positive:1", "style:personachat:test:182:self:5:positive:2", "style:personachat:test:182:self:5:positive:3", "style:personachat:test:182:self:5:positive:4"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "A carpenter is working on a roof at a house being built.", "Here is a picture of people getting drunk at a house party.", "A carpenter is working on a roof at a house being built.", "you should ! i moved to the states to become a stock broker but i miss it .", "one of my sisters is in fashion . i have a brother too but he is a doctor .", "are you close to your siblings ?", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:660", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:667", "candidate_text": "well, a couple is eating in a restaurant", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:667:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:301:premise", "snli:test:1156:premise", "snli:test:300:premise", "snli:test:667:premise", "style:personachat:test:25:self:7:positive:0", "style:personachat:test:25:self:7:positive:1", "style:personachat:test:25:self:7:positive:2", "style:personachat:test:25:self:7:positive:3", "style:personachat:test:25:self:7:positive:4", "style:personachat:test:25:self:7:positive:5", "style:personachat:test:25:self:7:positive:6"], "evidence_texts": ["A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "ok lets do it . movies ?", "i was born in south carolina you ? i am a singer", "my name is joanna and i like horror films", "interesting , what do you do for work ?", "i can understand that .", "big city now . helps with my career in singing", "mostly country . like the way it makes me feel and relates to life"], "speaker_id": "snli:test:667", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:667", "candidate_text": "A couple is eating in a restaurant.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:667:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:301:premise", "snli:test:1156:premise", "snli:test:300:premise", "snli:test:667:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:667", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:671", "candidate_text": "I wonder whether a person drives away in a winnebago?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:671:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:647:premise", "snli:test:687:premise", "snli:test:332:premise", "snli:test:671:premise", "style:personachat:test:26:self:1:positive:0"], "evidence_texts": ["Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "college ? studying anything in particular ?"], "speaker_id": "snli:test:671", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:671", "candidate_text": "A person drives away in a winnebago.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:671:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:647:premise", "snli:test:687:premise", "snli:test:332:premise", "snli:test:671:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "Two little boys are smiling and laughing while one is standing and one is in a bouncy seat.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:671", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:673", "candidate_text": "I wonder whether there is a man?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:673:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:581:premise", "snli:test:673:premise", "snli:test:723:premise", "snli:test:386:premise", "style:personachat:test:26:self:2:positive:0", "style:personachat:test:26:self:2:positive:1"], "evidence_texts": ["A hockey player in helmet.", "Man holding torch of fire.", "Boy in midair on bicycle", "Several women are playing volleyball.", "do you work ? or your parents work ?", "college ? studying anything in particular ?"], "speaker_id": "snli:test:673", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:673", "candidate_text": "well, there is a man", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:673:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:581:premise", "snli:test:673:premise", "snli:test:723:premise", "snli:test:386:premise", "style:personachat:test:52:self:2:positive:0", "style:personachat:test:52:self:2:positive:1"], "evidence_texts": ["A hockey player in helmet.", "Man holding torch of fire.", "Boy in midair on bicycle", "Several women are playing volleyball.", "i try to be i will not drink soda or even tea , just water", "it was hard to get use to at first . i use to love soda"], "speaker_id": "snli:test:673", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:681", "candidate_text": "I wonder whether a child is screaming at the person that almost hit her taxi cab?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:681:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:575:premise", "snli:test:681:premise", "snli:test:1280:premise", "snli:test:434:premise", "style:personachat:test:26:self:5:positive:0", "style:personachat:test:26:self:5:positive:1", "style:personachat:test:26:self:5:positive:2", "style:personachat:test:26:self:5:positive:3", "style:personachat:test:26:self:5:positive:4"], "evidence_texts": ["One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "Three army soldiers are holding guns and peering around a corner of a building.", "A man wearing a red vest is walking past a black and green fence.", "folk music , its kind of like storytelling . people dig it , you listen to music ?", "i am average height , i am the shortest of my 3 sisters . you ?", "do you have any siblings ?", "do you work ? or your parents work ?", "college ? studying anything in particular ?"], "speaker_id": "snli:test:681", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:681", "candidate_text": "Honestly, A child is screaming at the person that almost hit her taxi cab!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:681:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:575:premise", "snli:test:681:premise", "snli:test:1280:premise", "snli:test:434:premise", "style:personachat:test:185:self:5:positive:0", "style:personachat:test:185:self:5:positive:1", "style:personachat:test:185:self:5:positive:2", "style:personachat:test:185:self:5:positive:3", "style:personachat:test:185:self:5:positive:4"], "evidence_texts": ["One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "Three army soldiers are holding guns and peering around a corner of a building.", "A man wearing a red vest is walking past a black and green fence.", "that is sick ! i want to act when i am an adult", "sick as in cool ! i am too shy to start now . i just came out as trans", "good luck to you too ! do you like animals ?", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:681", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:685", "candidate_text": "I wonder whether five children are playing?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:685:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:563:premise", "snli:test:685:premise", "snli:test:1067:premise", "snli:test:475:premise", "style:personachat:test:26:self:7:positive:0", "style:personachat:test:26:self:7:positive:1", "style:personachat:test:26:self:7:positive:2", "style:personachat:test:26:self:7:positive:3", "style:personachat:test:26:self:7:positive:4", "style:personachat:test:26:self:7:positive:5", "style:personachat:test:26:self:7:positive:6"], "evidence_texts": ["Boys with their backs against an incoming wave.", "Five children playing soccer chase after a ball.", "A schoolgirl is jumping over a low hurdle.", "A crowded city street with lots of pedestrians.", "hey there ! i am wonderful how are you doing today ?", "i am a singer actually which can be work but i love it , how about you ?", "folk music , its kind of like storytelling . people dig it , you listen to music ?", "i am average height , i am the shortest of my 3 sisters . you ?", "do you have any siblings ?", "do you work ? or your parents work ?", "college ? studying anything in particular ?"], "speaker_id": "snli:test:685", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:685", "candidate_text": "Five children are playing.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:685:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:563:premise", "snli:test:685:premise", "snli:test:1067:premise", "snli:test:475:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Boys with their backs against an incoming wave.", "Five children playing soccer chase after a ball.", "A schoolgirl is jumping over a low hurdle.", "A crowded city street with lots of pedestrians.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:685", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:687", "candidate_text": "well, a woman stands with her child", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:687:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:671:premise", "snli:test:690:premise", "snli:test:647:premise", "snli:test:687:premise", "style:personachat:test:27:self:1:positive:0"], "evidence_texts": ["Three men are carrying a red bag into a boat with another person and boat in the background.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:687", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:687", "candidate_text": "Honestly, a woman stands with her child!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:687:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:671:premise", "snli:test:690:premise", "snli:test:647:premise", "snli:test:687:premise", "style:personachat:test:188:self:1:positive:0"], "evidence_texts": ["Three men are carrying a red bag into a boat with another person and boat in the background.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "Two homeless men dressed in gray sit against a wall with a sleeping dog on a purple leash.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:687", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:690", "candidate_text": "well, a boy swims while a seagull shits on the sand", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:690:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:687:premise", "snli:test:712:premise", "snli:test:690:premise", "snli:test:671:premise", "style:personachat:test:27:self:2:positive:0", "style:personachat:test:27:self:2:positive:1"], "evidence_texts": ["A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "well that is human nature min . you are no fun people run .", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:690", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:690", "candidate_text": "A boy swims while a seagull shits on the sand.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:690:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:687:premise", "snli:test:712:premise", "snli:test:690:premise", "snli:test:671:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "Three men are carrying a red bag into a boat with another person and boat in the background.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:690", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:693", "candidate_text": "well, some people are by the side of a swimming pool partying", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:693:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:660:premise", "snli:test:693:premise", "snli:test:626:premise", "snli:test:698:premise", "style:personachat:test:27:self:5:positive:0", "style:personachat:test:27:self:5:positive:1", "style:personachat:test:27:self:5:positive:2", "style:personachat:test:27:self:5:positive:3", "style:personachat:test:27:self:5:positive:4"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "Here is a picture of people getting drunk at a house party.", "A carpenter is working on a roof at a house being built.", "A busy street with numerous people interacting and going about their lives.", "i can not see my self as a salesman not good at socializing", "i use to drink a lot and since i quit so did my friends", "i would agree that is it good . my friends were not true friends", "well that is human nature min . you are no fun people run .", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:693", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:693", "candidate_text": "I wonder whether some people are by the side of a swimming pool partying?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:693:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:660:premise", "snli:test:693:premise", "snli:test:626:premise", "snli:test:698:premise", "style:personachat:test:78:self:5:positive:0", "style:personachat:test:78:self:5:positive:1", "style:personachat:test:78:self:5:positive:2", "style:personachat:test:78:self:5:positive:3", "style:personachat:test:78:self:5:positive:4"], "evidence_texts": ["A blond woman in a black shirt is standing behind a counter.", "Here is a picture of people getting drunk at a house party.", "A carpenter is working on a roof at a house being built.", "A busy street with numerous people interacting and going about their lives.", "that is awesome ! ! i worked at a carnival once upon a time .", "no , i worked at the ferris wheel .", "agreed . they do not have dogs , which i am terrified of .", "i had a rough experience with one . a poodle .", "they are . you like scotch ?"], "speaker_id": "snli:test:693", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:695", "candidate_text": "well, people were celebrating at a christmas themed party", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:695:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:698:premise", "snli:test:660:premise", "snli:test:705:premise", "snli:test:695:premise", "style:personachat:test:27:self:7:positive:0", "style:personachat:test:27:self:7:positive:1", "style:personachat:test:27:self:7:positive:2", "style:personachat:test:27:self:7:positive:3", "style:personachat:test:27:self:7:positive:4", "style:personachat:test:27:self:7:positive:5", "style:personachat:test:27:self:7:positive:6"], "evidence_texts": ["A busy street with numerous people interacting and going about their lives.", "A blond woman in a black shirt is standing behind a counter.", "A variety of people are sitting around a tale doing craft projects.", "Here is a picture of people getting drunk at a house party.", "hello , i am ok just hanging out here . do you have pets ?", "i work as a carpenter , stay at home mostly . kinda a loner", "i can not see my self as a salesman not good at socializing", "i use to drink a lot and since i quit so did my friends", "i would agree that is it good . my friends were not true friends", "well that is human nature min . you are no fun people run .", "lol that can be just has fun or bad just depends if you are responsible"], "speaker_id": "snli:test:695", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:695", "candidate_text": "Honestly, People were celebrating at a Christmas themed party!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:695:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:698:premise", "snli:test:660:premise", "snli:test:705:premise", "snli:test:695:premise", "style:personachat:test:188:self:7:positive:0", "style:personachat:test:188:self:7:positive:1", "style:personachat:test:188:self:7:positive:2", "style:personachat:test:188:self:7:positive:3", "style:personachat:test:188:self:7:positive:4", "style:personachat:test:188:self:7:positive:5", "style:personachat:test:188:self:7:positive:6"], "evidence_texts": ["A busy street with numerous people interacting and going about their lives.", "A blond woman in a black shirt is standing behind a counter.", "A variety of people are sitting around a tale doing craft projects.", "Here is a picture of people getting drunk at a house party.", "no , i can barely pay my monthly bills .", "i teach kindergarten , but i hate children .", "where do you get the tattoos ?", "i live in seattle with two derelicts . they drink all the time .", "do you have a southern accent ?", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:695", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:698", "candidate_text": "I wonder whether the morning rush hour fills the streets with busy people?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:698:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:695:premise", "snli:test:705:premise", "snli:test:698:premise", "snli:test:693:premise", "style:personachat:test:28:self:1:positive:0"], "evidence_texts": ["Here is a picture of people getting drunk at a house party.", "A variety of people are sitting around a tale doing craft projects.", "A busy street with numerous people interacting and going about their lives.", "Here is a picture of people getting drunk at a house party.", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:698", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:698", "candidate_text": "Honestly, The morning rush hour fills the streets with busy people!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:698:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:695:premise", "snli:test:705:premise", "snli:test:698:premise", "snli:test:693:premise", "style:personachat:test:200:self:1:positive:0"], "evidence_texts": ["Here is a picture of people getting drunk at a house party.", "A variety of people are sitting around a tale doing craft projects.", "A busy street with numerous people interacting and going about their lives.", "Here is a picture of people getting drunk at a house party.", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:698", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:700", "candidate_text": "I wonder whether the player is flying planes?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:700:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:700:premise", "snli:test:378:premise", "snli:test:778:premise", "snli:test:295:premise", "style:personachat:test:28:self:2:positive:0", "style:personachat:test:28:self:2:positive:1"], "evidence_texts": ["A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "so do i . i also like documentaries , do you ?", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:700", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:700", "candidate_text": "the player is flying planes.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:700:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:700:premise", "snli:test:378:premise", "snli:test:778:premise", "snli:test:295:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A person with blue hair, a baseball cap, and a hood on stands outdoors in a crowd.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:700", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:705", "candidate_text": "I wonder whether the people are unique?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:705:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:698:premise", "snli:test:705:premise", "snli:test:708:premise", "snli:test:695:premise", "style:personachat:test:28:self:5:positive:0", "style:personachat:test:28:self:5:positive:1", "style:personachat:test:28:self:5:positive:2", "style:personachat:test:28:self:5:positive:3", "style:personachat:test:28:self:5:positive:4"], "evidence_texts": ["A busy street with numerous people interacting and going about their lives.", "A variety of people are sitting around a tale doing craft projects.", "A woman in black reviews a message as she walks to work.", "Here is a picture of people getting drunk at a house party.", "he is five . he likes to eat muffins like me .", "yes , there is one at my job at the healthcare industry .", "yes . i also like film do you ?", "so do i . i also like documentaries , do you ?", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:705", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:705", "candidate_text": "well, the people are unique", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:705:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:698:premise", "snli:test:705:premise", "snli:test:708:premise", "snli:test:695:premise", "style:personachat:test:55:self:5:positive:0", "style:personachat:test:55:self:5:positive:1", "style:personachat:test:55:self:5:positive:2", "style:personachat:test:55:self:5:positive:3", "style:personachat:test:55:self:5:positive:4"], "evidence_texts": ["A busy street with numerous people interacting and going about their lives.", "A variety of people are sitting around a tale doing craft projects.", "A woman in black reviews a message as she walks to work.", "Here is a picture of people getting drunk at a house party.", "they are a main staple of my diet , i work in the arctic .", "that is horrible , i love dogs . do you fish ?", "mine taught me too , he was a geologist .", "i only study ice and explore , but i wish i would have paid attention more .", "i never did enlist , thought about it ."], "speaker_id": "snli:test:705", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:708", "candidate_text": "I wonder whether the woman in black is being fired via text message?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:708:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:708:premise", "snli:test:705:premise", "snli:test:902:premise", "snli:test:698:premise", "style:personachat:test:28:self:7:positive:0", "style:personachat:test:28:self:7:positive:1", "style:personachat:test:28:self:7:positive:2", "style:personachat:test:28:self:7:positive:3", "style:personachat:test:28:self:7:positive:4", "style:personachat:test:28:self:7:positive:5", "style:personachat:test:28:self:7:positive:6"], "evidence_texts": ["A woman in black reviews a message as she walks to work.", "A variety of people are sitting around a tale doing craft projects.", "Two men in neon yellow shirts busily sawing a log in half.", "A busy street with numerous people interacting and going about their lives.", "hello i am doing well . hanging out with my dog pedro", "he is a yorkie . perfect size for me since i am only five feet tall .", "he is five . he likes to eat muffins like me .", "yes , there is one at my job at the healthcare industry .", "yes . i also like film do you ?", "so do i . i also like documentaries , do you ?", "same here . anything else you want to share with me ?"], "speaker_id": "snli:test:708", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:708", "candidate_text": "Honestly, The woman in black is being fired via text message!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:708:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:708:premise", "snli:test:705:premise", "snli:test:902:premise", "snli:test:698:premise", "style:personachat:test:200:self:7:positive:0", "style:personachat:test:200:self:7:positive:1", "style:personachat:test:200:self:7:positive:2", "style:personachat:test:200:self:7:positive:3", "style:personachat:test:200:self:7:positive:4", "style:personachat:test:200:self:7:positive:5", "style:personachat:test:200:self:7:positive:6"], "evidence_texts": ["A woman in black reviews a message as she walks to work.", "A variety of people are sitting around a tale doing craft projects.", "Two men in neon yellow shirts busily sawing a log in half.", "A busy street with numerous people interacting and going about their lives.", "judy . what do you like about sundays ? football watching is what i like !", "what is your favorite food ? cheese pizza is mine .", "do you have any pets ? i have three . all dogs . i love them !", "wow expensive . i have a yorkie , maltese and a mix breed .", "christmas , joy and jingle . i love christmas ! my favorite holiday . how about you ?", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:708", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:712", "candidate_text": "Honestly, A pitcher is playing baseball!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:712:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:712:premise", "snli:test:690:premise", "snli:test:738:premise", "snli:test:687:premise", "style:personachat:test:29:self:1:positive:0"], "evidence_texts": ["A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:712", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:712", "candidate_text": "I wonder whether a pitcher is playing baseball?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:712:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:712:premise", "snli:test:690:premise", "snli:test:738:premise", "snli:test:687:premise", "style:personachat:test:87:self:1:positive:0"], "evidence_texts": ["A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A woman in red blouse is standing with small blond child in front of a small folding chalkboard.", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:712", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:723", "candidate_text": "Honestly, The boy is falling off of his bicycle!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:723:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:673:premise", "snli:test:798:premise", "snli:test:581:premise", "snli:test:723:premise", "style:personachat:test:29:self:2:positive:0", "style:personachat:test:29:self:2:positive:1"], "evidence_texts": ["Man holding torch of fire.", "Workers standing on a lift.", "A hockey player in helmet.", "Boy in midair on bicycle", "i prefer to cook vegan food . what do you like to eat ?", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:723", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:723", "candidate_text": "well, the boy is falling off of his bicycle", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:723:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:673:premise", "snli:test:798:premise", "snli:test:581:premise", "snli:test:723:premise", "style:personachat:test:56:self:2:positive:0", "style:personachat:test:56:self:2:positive:1"], "evidence_texts": ["Man holding torch of fire.", "Workers standing on a lift.", "A hockey player in helmet.", "Boy in midair on bicycle", "the only thing i cannot eat is shellfish . i am allergic .", "i do not have any pets , but i do not mind dogs ."], "speaker_id": "snli:test:723", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:734", "candidate_text": "Honestly, two men play with a snowman!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:734:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5780:premise", "snli:test:6938:premise", "snli:test:734:premise", "snli:test:5779:premise", "style:personachat:test:29:self:5:positive:0", "style:personachat:test:29:self:5:positive:1", "style:personachat:test:29:self:5:positive:2", "style:personachat:test:29:self:5:positive:3", "style:personachat:test:29:self:5:positive:4"], "evidence_texts": ["Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "that is a great industry . what kind of music do you like ?", "yes . i love to cook and i am looking for someone to build me some cooling racks .", "me too what is your favorite ? chevelle ? cadillac ? oldsmobile ?", "i prefer to cook vegan food . what do you like to eat ?", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:734", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:734", "candidate_text": "two men play with a snowman.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:734:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5780:premise", "snli:test:6938:premise", "snli:test:734:premise", "snli:test:5779:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:734", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:738", "candidate_text": "Honestly, There are children in the streets!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:738:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:712:premise", "snli:test:876:premise", "snli:test:738:premise", "snli:test:690:premise", "style:personachat:test:29:self:7:positive:0", "style:personachat:test:29:self:7:positive:1", "style:personachat:test:29:self:7:positive:2", "style:personachat:test:29:self:7:positive:3", "style:personachat:test:29:self:7:positive:4", "style:personachat:test:29:self:7:positive:5", "style:personachat:test:29:self:7:positive:6"], "evidence_texts": ["A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "i am great . just relaxing and knitting some hats for the babies at my old job .", "childrens hospital until i became disabled . what do you do ?", "that is a great industry . what kind of music do you like ?", "yes . i love to cook and i am looking for someone to build me some cooling racks .", "me too what is your favorite ? chevelle ? cadillac ? oldsmobile ?", "i prefer to cook vegan food . what do you like to eat ?", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:738", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:738", "candidate_text": "I wonder whether there are children in the streets?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:738:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:712:premise", "snli:test:876:premise", "snli:test:738:premise", "snli:test:690:premise", "style:personachat:test:87:self:7:positive:0", "style:personachat:test:87:self:7:positive:1", "style:personachat:test:87:self:7:positive:2", "style:personachat:test:87:self:7:positive:3", "style:personachat:test:87:self:7:positive:4", "style:personachat:test:87:self:7:positive:5", "style:personachat:test:87:self:7:positive:6"], "evidence_texts": ["A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A boy swims shirtless with a black and red bathing suit while a seagull sits on the sand.", "hi . i am doing great thanks for asking . i do play poker sometimes .", "sorry to hear . i walk a lot since i hike in the nature . do you hike ?", "do you know how to draw ? i do not . i want to learn how .", "that is cool . do you like reading ? i do . mainly a lot of poetry .", "how about collections do you have one ? mine is weird stickers .", "nice ! do you like to try different foods ? i will try at least once .", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:738", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:749", "candidate_text": "well, the two dogs are running to play in the leaves", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:749:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:367:premise", "snli:test:749:premise", "snli:test:856:premise", "snli:test:215:premise", "style:personachat:test:30:self:1:positive:0"], "evidence_texts": ["Two men are standing in a boat.", "Two dogs run together near the leaves.", "Four women competitively rollerskating around an area.", "This child is on the library steps.", "not yet . i have not found the right man to settle down with ."], "speaker_id": "snli:test:749", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:749", "candidate_text": "Honestly, The two dogs are running to play in the leaves!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:749:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:367:premise", "snli:test:749:premise", "snli:test:856:premise", "snli:test:215:premise", "style:personachat:test:212:self:1:positive:0"], "evidence_texts": ["Two men are standing in a boat.", "Two dogs run together near the leaves.", "Four women competitively rollerskating around an area.", "This child is on the library steps.", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:749", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:753", "candidate_text": "well, people wearing orange", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:753:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:564:premise", "snli:test:753:premise", "snli:test:212:premise", "snli:test:820:premise", "style:personachat:test:30:self:2:positive:0", "style:personachat:test:30:self:2:positive:1"], "evidence_texts": ["Four boys are about to be hit by an approaching wave.", "A crowd wearing orange cheering for their team in a stadium.", "A person in a red shirt and black pants hunched over.", "A person is sitting in front of a graffiti covered wall.", "that must be a lot of work but very rewarding i bet", "not yet . i have not found the right man to settle down with ."], "speaker_id": "snli:test:753", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:753", "candidate_text": "People wearing orange.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:753:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:564:premise", "snli:test:753:premise", "snli:test:212:premise", "snli:test:820:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Four boys are about to be hit by an approaching wave.", "A crowd wearing orange cheering for their team in a stadium.", "A person in a red shirt and black pants hunched over.", "A person is sitting in front of a graffiti covered wall.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:753", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:754", "candidate_text": "well, a group of drag queens walking down the street", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:754:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:820:premise", "snli:test:564:premise", "snli:test:754:premise", "snli:test:841:premise", "style:personachat:test:30:self:5:positive:0", "style:personachat:test:30:self:5:positive:1", "style:personachat:test:30:self:5:positive:2", "style:personachat:test:30:self:5:positive:3", "style:personachat:test:30:self:5:positive:4"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "Four boys are about to be hit by an approaching wave.", "A crowd wearing orange cheering for their team in a stadium.", "A group of footballs players is standing behind a coaching official.", "i know how that is . i love to craft but being colorblind makes it hard", "i love to sing . sometimes i go sing karaoke .", "you should , it is great fun !", "that must be a lot of work but very rewarding i bet", "not yet . i have not found the right man to settle down with ."], "speaker_id": "snli:test:754", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:754", "candidate_text": "I wonder whether a group of drag queens walking down the street?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:754:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:820:premise", "snli:test:564:premise", "snli:test:754:premise", "snli:test:841:premise", "style:personachat:test:91:self:5:positive:0", "style:personachat:test:91:self:5:positive:1", "style:personachat:test:91:self:5:positive:2", "style:personachat:test:91:self:5:positive:3", "style:personachat:test:91:self:5:positive:4"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "Four boys are about to be hit by an approaching wave.", "A crowd wearing orange cheering for their team in a stadium.", "A group of footballs players is standing behind a coaching official.", "eh ! iced tea with home grown canada syrup delicious", "red or blue ? coding is so much fun ! how old are you ? i am 22", "i freelance mostly website coding and such .", "cool ! ! any property in canada ? my family is looking to upgrade .", "what do you like to do ?"], "speaker_id": "snli:test:754", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:778", "candidate_text": "well, a blond little boy is cutting out little stars for his little sister", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:778:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:700:premise", "snli:test:860:premise", "snli:test:778:premise", "snli:test:378:premise", "style:personachat:test:30:self:7:positive:0", "style:personachat:test:30:self:7:positive:1", "style:personachat:test:30:self:7:positive:2", "style:personachat:test:30:self:7:positive:3", "style:personachat:test:30:self:7:positive:4", "style:personachat:test:30:self:7:positive:5", "style:personachat:test:30:self:7:positive:6"], "evidence_texts": ["A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "i am great . what are you doing up so late ?", "i am definitely a night owl . you know chatting will nott help you sleep .", "i know how that is . i love to craft but being colorblind makes it hard", "i love to sing . sometimes i go sing karaoke .", "you should , it is great fun !", "that must be a lot of work but very rewarding i bet", "not yet . i have not found the right man to settle down with ."], "speaker_id": "snli:test:778", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:778", "candidate_text": "Honestly, A blond little boy is cutting out little stars for his little sister!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:778:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:700:premise", "snli:test:860:premise", "snli:test:778:premise", "snli:test:378:premise", "style:personachat:test:212:self:6:positive:0", "style:personachat:test:212:self:6:positive:1", "style:personachat:test:212:self:6:positive:2", "style:personachat:test:212:self:6:positive:3", "style:personachat:test:212:self:6:positive:4", "style:personachat:test:212:self:6:positive:5"], "evidence_texts": ["A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A man in an apron smiles as he pokes a frying donut with a little metal stick.", "omg ! i love one tree hill", "what is your favorite color ? mine is purple .", "i do see things differently ! what do you do for work ?", "i help people everyday in a hospital .", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:778", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:781", "candidate_text": "well, the female awaiting the man leaping is his sister", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:781:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6722:premise", "snli:test:781:premise", "snli:test:1307:premise", "snli:test:6721:premise", "style:personachat:test:31:self:1:positive:0"], "evidence_texts": ["A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "good for him . i love my stereo"], "speaker_id": "snli:test:781", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:781", "candidate_text": "The female awaiting the man leaping is his sister.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:781:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6722:premise", "snli:test:781:premise", "snli:test:1307:premise", "snli:test:6721:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:781", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:783", "candidate_text": "well, the woman balancing a basket on her head is heading to her neighbors house", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:783:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:577:premise", "snli:test:1044:premise", "snli:test:9741:premise", "snli:test:783:premise", "style:personachat:test:31:self:2:positive:0", "style:personachat:test:31:self:2:positive:1"], "evidence_texts": ["Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "no , i never got into it", "good for him . i love my stereo"], "speaker_id": "snli:test:783", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:783", "candidate_text": "I wonder whether the woman balancing a basket on her head is heading to her neighbors house?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:783:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:577:premise", "snli:test:1044:premise", "snli:test:9741:premise", "snli:test:783:premise", "style:personachat:test:102:self:2:positive:0", "style:personachat:test:102:self:2:positive:1"], "evidence_texts": ["Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "you are a vet ? i think my cat is getting sick .", "oh you are a veteran ?"], "speaker_id": "snli:test:783", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:798", "candidate_text": "well, workers walk off a lift", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:798:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:723:premise", "snli:test:1177:premise", "snli:test:798:premise", "snli:test:673:premise", "style:personachat:test:31:self:5:positive:0", "style:personachat:test:31:self:5:positive:1", "style:personachat:test:31:self:5:positive:2", "style:personachat:test:31:self:5:positive:3", "style:personachat:test:31:self:5:positive:4"], "evidence_texts": ["Boy in midair on bicycle", "skateboarder sliding down a ramp", "Workers standing on a lift.", "Man holding torch of fire.", "i do not have any siblings . i go alone or with friends", "she is a nurse in a hospital", "there is not much to tell . i drive her to work sometimes in my truck", "no , i never got into it", "good for him . i love my stereo"], "speaker_id": "snli:test:798", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:798", "candidate_text": "Honestly, Workers walk off a lift!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:798:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:723:premise", "snli:test:1177:premise", "snli:test:798:premise", "snli:test:673:premise", "style:personachat:test:216:self:5:positive:0", "style:personachat:test:216:self:5:positive:1", "style:personachat:test:216:self:5:positive:2", "style:personachat:test:216:self:5:positive:3", "style:personachat:test:216:self:5:positive:4"], "evidence_texts": ["Boy in midair on bicycle", "skateboarder sliding down a ramp", "Workers standing on a lift.", "Man holding torch of fire.", "sorry to hear that ! must be hard", "i am hoping retire soon maybe i can write then", "construction , my left leg hurts every day", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:798", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:801", "candidate_text": "well, fishermen at the dock", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:801:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:193:premise", "snli:test:801:premise", "snli:test:1697:premise", "snli:test:188:premise", "style:personachat:test:31:self:7:positive:0", "style:personachat:test:31:self:7:positive:1", "style:personachat:test:31:self:7:positive:2", "style:personachat:test:31:self:7:positive:3", "style:personachat:test:31:self:7:positive:4", "style:personachat:test:31:self:7:positive:5", "style:personachat:test:31:self:7:positive:6"], "evidence_texts": ["a group of people dancing together.", "Two people talking on a dock.", "A woman holding bags behind plantains.", "Six dogs swimming in a river.", "i like to be outdoors . hiking , camping , you know", "sometimes i will fish on my trips , but that is it .", "i do not have any siblings . i go alone or with friends", "she is a nurse in a hospital", "there is not much to tell . i drive her to work sometimes in my truck", "no , i never got into it", "good for him . i love my stereo"], "speaker_id": "snli:test:801", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:801", "candidate_text": "fishermen at the dock.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:801:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:193:premise", "snli:test:801:premise", "snli:test:1697:premise", "snli:test:188:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["a group of people dancing together.", "Two people talking on a dock.", "A woman holding bags behind plantains.", "Six dogs swimming in a river.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:801", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:808", "candidate_text": "Honestly, The man is running after a robber!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:808:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:808:premise", "snli:test:543:premise", "snli:test:928:premise", "snli:test:538:premise", "style:personachat:test:32:self:1:positive:0"], "evidence_texts": ["A man has flung himself over a pole with people and canopies in the background.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:808", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:808", "candidate_text": "I wonder whether the man is running after a robber?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:808:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:808:premise", "snli:test:543:premise", "snli:test:928:premise", "snli:test:538:premise", "style:personachat:test:108:self:1:positive:0"], "evidence_texts": ["A man has flung himself over a pole with people and canopies in the background.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A young man waring black and carrying a skateboard looks at an outdoor skateboard park.", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:808", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:820", "candidate_text": "Honestly, A person is sitting outside!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:820:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:820:premise", "snli:test:754:premise", "snli:test:841:premise", "snli:test:753:premise", "style:personachat:test:32:self:2:positive:0", "style:personachat:test:32:self:2:positive:1"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "A crowd wearing orange cheering for their team in a stadium.", "A group of footballs players is standing behind a coaching official.", "A crowd wearing orange cheering for their team in a stadium.", "i do but not now", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:820", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:820", "candidate_text": "well, a person is sitting outside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:820:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:820:premise", "snli:test:754:premise", "snli:test:841:premise", "snli:test:753:premise", "style:personachat:test:59:self:2:positive:0", "style:personachat:test:59:self:2:positive:1"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "A crowd wearing orange cheering for their team in a stadium.", "A group of footballs players is standing behind a coaching official.", "A crowd wearing orange cheering for their team in a stadium.", "i mostly be at school , the little free time i spend at home", "yeah a few different artist but mainly old music"], "speaker_id": "snli:test:820", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:831", "candidate_text": "Honestly, The road is dangerous for the four women to try to walk on because it is covered in ice!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:831:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:606:premise", "snli:test:983:premise", "snli:test:412:premise", "snli:test:831:premise", "style:personachat:test:32:self:5:positive:0", "style:personachat:test:32:self:5:positive:1", "style:personachat:test:32:self:5:positive:2", "style:personachat:test:32:self:5:positive:3", "style:personachat:test:32:self:5:positive:4"], "evidence_texts": ["The little boy gets ready to kick the soccer ball.", "A man stands by two face structures on Easter Island.", "A guy and a girl look down a mountain range.", "Four women are taking a walk down an icy road.", "amazon is my favorite store , you cannot go wrong .", "no i pick pretty well trust me", "no i do not an have a good 12", "i do but not now", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:831", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:831", "candidate_text": "The road is dangerous for the four women to try to walk on because it is covered in ice.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:831:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:606:premise", "snli:test:983:premise", "snli:test:412:premise", "snli:test:831:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The little boy gets ready to kick the soccer ball.", "A man stands by two face structures on Easter Island.", "A guy and a girl look down a mountain range.", "Four women are taking a walk down an icy road.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:831", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:841", "candidate_text": "Honestly, They are college football players!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:841:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:820:premise", "snli:test:841:premise", "snli:test:1142:premise", "snli:test:754:premise", "style:personachat:test:32:self:7:positive:0", "style:personachat:test:32:self:7:positive:1", "style:personachat:test:32:self:7:positive:2", "style:personachat:test:32:self:7:positive:3", "style:personachat:test:32:self:7:positive:4", "style:personachat:test:32:self:7:positive:5", "style:personachat:test:32:self:7:positive:6"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "A group of footballs players is standing behind a coaching official.", "A woman is in the middle of hitting a tennis ball.", "A crowd wearing orange cheering for their team in a stadium.", "i am fine shopping for clothes wyd ?", "i like jeans an boots", "amazon is my favorite store , you cannot go wrong .", "no i pick pretty well trust me", "no i do not an have a good 12", "i do but not now", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:841", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:841", "candidate_text": "I wonder whether they are college football players?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:841:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:820:premise", "snli:test:841:premise", "snli:test:1142:premise", "snli:test:754:premise", "style:personachat:test:108:self:7:positive:0", "style:personachat:test:108:self:7:positive:1", "style:personachat:test:108:self:7:positive:2", "style:personachat:test:108:self:7:positive:3", "style:personachat:test:108:self:7:positive:4", "style:personachat:test:108:self:7:positive:5", "style:personachat:test:108:self:7:positive:6"], "evidence_texts": ["A person is sitting in front of a graffiti covered wall.", "A group of footballs players is standing behind a coaching official.", "A woman is in the middle of hitting a tennis ball.", "A crowd wearing orange cheering for their team in a stadium.", "hi ! ! ! ! ! ! fall is my favorite i love pumpkin spice lattes", "i love candy and everything sweet . what do you love about fall", "where do your parents live", "new york . i jut bought a house !", "i love to dance at the clubs .", "all of my pictures of me", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:841", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:856", "candidate_text": "I wonder whether women rollerskating around an are?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:856:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:856:premise", "snli:test:749:premise", "snli:test:892:premise", "snli:test:367:premise", "style:personachat:test:33:self:1:positive:0"], "evidence_texts": ["Four women competitively rollerskating around an area.", "Two dogs run together near the leaves.", "A brown dog jumps over an obstacle.", "Two men are standing in a boat.", "cool do you like robots ? i have one that is really cool"], "speaker_id": "snli:test:856", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:856", "candidate_text": "well, women rollerskating around an are", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:856:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:856:premise", "snli:test:749:premise", "snli:test:892:premise", "snli:test:367:premise", "style:personachat:test:61:self:1:positive:0"], "evidence_texts": ["Four women competitively rollerskating around an area.", "Two dogs run together near the leaves.", "A brown dog jumps over an obstacle.", "Two men are standing in a boat.", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:856", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:860", "candidate_text": "I wonder whether a group of cleaners after a parade?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:860:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:860:premise", "snli:test:778:premise", "snli:test:1132:premise", "snli:test:700:premise", "style:personachat:test:33:self:2:positive:0", "style:personachat:test:33:self:2:positive:1"], "evidence_texts": ["A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "what color is your hair ? i dyed mine blue and not sure why", "cool do you like robots ? i have one that is really cool"], "speaker_id": "snli:test:860", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:860", "candidate_text": "Honestly, A group of cleaners after a parade!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:860:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:860:premise", "snli:test:778:premise", "snli:test:1132:premise", "snli:test:700:premise", "style:personachat:test:226:self:2:positive:0", "style:personachat:test:226:self:2:positive:1"], "evidence_texts": ["A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A female softball player wearing blue and red crouches in the infield, waiting for the next play.", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:860", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:865", "candidate_text": "I wonder whether the car is going on a trip?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:865:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:598:premise", "snli:test:865:premise", "snli:test:942:premise", "snli:test:417:premise", "style:personachat:test:33:self:5:positive:0", "style:personachat:test:33:self:5:positive:1", "style:personachat:test:33:self:5:positive:2", "style:personachat:test:33:self:5:positive:3", "style:personachat:test:33:self:5:positive:4"], "evidence_texts": ["A woman wearing jeans is walking down the street.", "A car is loaded with items on the top.", "A wet child stands in chest deep ocean water.", "A boy hits a tennis ball on a court.", "haha do you ever cosplay i got into it lately at school", "nice i go to homeschool but they let me do events at public school", "its ok parents pulled me from public saying its too dangerous", "what color is your hair ? i dyed mine blue and not sure why", "cool do you like robots ? i have one that is really cool"], "speaker_id": "snli:test:865", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:865", "candidate_text": "The car is going on a trip.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:865:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:598:premise", "snli:test:865:premise", "snli:test:942:premise", "snli:test:417:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman wearing jeans is walking down the street.", "A car is loaded with items on the top.", "A wet child stands in chest deep ocean water.", "A boy hits a tennis ball on a court.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:865", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:876", "candidate_text": "I wonder whether someone person was near a bunch of birds?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:876:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:876:premise", "snli:test:738:premise", "snli:test:1149:premise", "snli:test:712:premise", "style:personachat:test:33:self:7:positive:0", "style:personachat:test:33:self:7:positive:1", "style:personachat:test:33:self:7:positive:2", "style:personachat:test:33:self:7:positive:3", "style:personachat:test:33:self:7:positive:4", "style:personachat:test:33:self:7:positive:5", "style:personachat:test:33:self:7:positive:6"], "evidence_texts": ["The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "not bad its a beautiful night in california and you ?", "yes they are pretty cool", "haha do you ever cosplay i got into it lately at school", "nice i go to homeschool but they let me do events at public school", "its ok parents pulled me from public saying its too dangerous", "what color is your hair ? i dyed mine blue and not sure why", "cool do you like robots ? i have one that is really cool"], "speaker_id": "snli:test:876", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:876", "candidate_text": "well, someone person was near a bunch of birds", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:876:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:876:premise", "snli:test:738:premise", "snli:test:1149:premise", "snli:test:712:premise", "style:personachat:test:61:self:7:positive:0", "style:personachat:test:61:self:7:positive:1", "style:personachat:test:61:self:7:positive:2", "style:personachat:test:61:self:7:positive:3", "style:personachat:test:61:self:7:positive:4", "style:personachat:test:61:self:7:positive:5", "style:personachat:test:61:self:7:positive:6"], "evidence_texts": ["The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A pro-baseball pitcher sends a curve ball to his opponent while many fans relax and enjoy the game.", "hello , i am good . i was drinking my favorite drink ice tea .", "i have been listening to country music all day .", "where are you from ? i originate from canada .", "i like to dance in my spare time .", "that is a great gift to have .", "i work for a large online retail . i like it .", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:876", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:892", "candidate_text": "well, the dog is in a pool", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:892:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:892:premise", "snli:test:856:premise", "snli:test:1128:premise", "snli:test:749:premise", "style:personachat:test:34:self:1:positive:0"], "evidence_texts": ["A brown dog jumps over an obstacle.", "Four women competitively rollerskating around an area.", "Two guys walking, one carrying a skateboard", "Two dogs run together near the leaves.", "its so much fun . i am single and alone so i use it to catch women"], "speaker_id": "snli:test:892", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:892", "candidate_text": "The dog is in a pool.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:892:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:892:premise", "snli:test:856:premise", "snli:test:1128:premise", "snli:test:749:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A brown dog jumps over an obstacle.", "Four women competitively rollerskating around an area.", "Two guys walking, one carrying a skateboard", "Two dogs run together near the leaves.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:892", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:895", "candidate_text": "well, a dog is carrying something", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:895:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:502:premise", "snli:test:1110:premise", "snli:test:393:premise", "snli:test:895:premise", "style:personachat:test:34:self:2:positive:0", "style:personachat:test:34:self:2:positive:1"], "evidence_texts": ["A group of kids is splashing in deep water nearby a rock formation.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "People are riding bicycles in the street, and they are all wearing helmets.", "A brown dog carries an object in its mouth on a snowy hillside.", "plus i work out in the gym 7days a week no time", "its so much fun . i am single and alone so i use it to catch women"], "speaker_id": "snli:test:895", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:895", "candidate_text": "I wonder whether a dog is carrying something?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:895:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:502:premise", "snli:test:1110:premise", "snli:test:393:premise", "snli:test:895:premise", "style:personachat:test:111:self:2:positive:0", "style:personachat:test:111:self:2:positive:1"], "evidence_texts": ["A group of kids is splashing in deep water nearby a rock formation.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "People are riding bicycles in the street, and they are all wearing helmets.", "A brown dog carries an object in its mouth on a snowy hillside.", "three . do you have a cat or dog ?", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:895", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:902", "candidate_text": "well, two men are using a saw", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:902:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:708:premise", "snli:test:1042:premise", "snli:test:902:premise", "snli:test:705:premise", "style:personachat:test:34:self:5:positive:0", "style:personachat:test:34:self:5:positive:1", "style:personachat:test:34:self:5:positive:2", "style:personachat:test:34:self:5:positive:3", "style:personachat:test:34:self:5:positive:4"], "evidence_texts": ["A woman in black reviews a message as she walks to work.", "A man is walking past a large sign that says E.S.E. Electronics.", "Two men in neon yellow shirts busily sawing a log in half.", "A variety of people are sitting around a tale doing craft projects.", "awesome . tell me more about yourself", "i work at the hospital in labor and delivery", "i work around them so much i do not want none", "plus i work out in the gym 7days a week no time", "its so much fun . i am single and alone so i use it to catch women"], "speaker_id": "snli:test:902", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:902", "candidate_text": "Honestly, Two men are using a saw!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:902:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:708:premise", "snli:test:1042:premise", "snli:test:902:premise", "snli:test:705:premise", "style:personachat:test:231:self:5:positive:0", "style:personachat:test:231:self:5:positive:1", "style:personachat:test:231:self:5:positive:2", "style:personachat:test:231:self:5:positive:3", "style:personachat:test:231:self:5:positive:4"], "evidence_texts": ["A woman in black reviews a message as she walks to work.", "A man is walking past a large sign that says E.S.E. Electronics.", "Two men in neon yellow shirts busily sawing a log in half.", "A variety of people are sitting around a tale doing craft projects.", "i grew up in the last was . not sure what that means but it is true", "brr ! what do your parents do ? my dad was a firefighter and mom was a teacher", "wow ! i am not much into teaching . too torn up over my divorce .", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:902", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:908", "candidate_text": "well, a man is wearing a red belt", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:908:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:908:premise", "snli:test:7110:premise", "snli:test:1692:premise", "snli:test:5732:premise", "style:personachat:test:34:self:7:positive:0", "style:personachat:test:34:self:7:positive:1", "style:personachat:test:34:self:7:positive:2", "style:personachat:test:34:self:7:positive:3", "style:personachat:test:34:self:7:positive:4", "style:personachat:test:34:self:7:positive:5", "style:personachat:test:34:self:7:positive:6"], "evidence_texts": ["A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "i am fine and you", "amazing fresh foods and veggies sounds yummy", "awesome . tell me more about yourself", "i work at the hospital in labor and delivery", "i work around them so much i do not want none", "plus i work out in the gym 7days a week no time", "its so much fun . i am single and alone so i use it to catch women"], "speaker_id": "snli:test:908", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:908", "candidate_text": "A man is wearing a red belt.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:908:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:908:premise", "snli:test:7110:premise", "snli:test:1692:premise", "snli:test:5732:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:908", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:928", "candidate_text": "well, an asian man is pushing a cart up the street", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:928:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:928:premise", "snli:test:808:premise", "snli:test:1061:premise", "snli:test:543:premise", "style:personachat:test:35:self:1:positive:0"], "evidence_texts": ["An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A man has flung himself over a pole with people and canopies in the background.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "i have been so confused lately . my parents were never there ."], "speaker_id": "snli:test:928", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:928", "candidate_text": "I wonder whether an asian man is pushing a cart up the street?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:928:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:928:premise", "snli:test:808:premise", "snli:test:1061:premise", "snli:test:543:premise", "style:personachat:test:113:self:1:positive:0"], "evidence_texts": ["An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A man has flung himself over a pole with people and canopies in the background.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A man in a blue collared t-shirt posing at someone while holding a hebrew newspaper.", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:928", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:937", "candidate_text": "well, the dress was a gift", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:937:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:641:premise", "snli:test:937:premise", "snli:test:1467:premise", "snli:test:284:premise", "style:personachat:test:35:self:2:positive:0", "style:personachat:test:35:self:2:positive:1"], "evidence_texts": ["Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "i have been doing community service for stealing .", "i have been so confused lately . my parents were never there ."], "speaker_id": "snli:test:937", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:937", "candidate_text": "Honestly, The dress was a gift!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:937:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:641:premise", "snli:test:937:premise", "snli:test:1467:premise", "snli:test:284:premise", "style:personachat:test:232:self:2:positive:0", "style:personachat:test:232:self:2:positive:1"], "evidence_texts": ["Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "A guy wearing jeans and a hat is crouching on a handrail with a concrete embankment located in front of him.", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:937", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:941", "candidate_text": "well, two males are involved in martial arts", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:941:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9688:premise", "snli:test:941:premise", "snli:test:2499:premise", "snli:test:5427:premise", "style:personachat:test:35:self:5:positive:0", "style:personachat:test:35:self:5:positive:1", "style:personachat:test:35:self:5:positive:2", "style:personachat:test:35:self:5:positive:3", "style:personachat:test:35:self:5:positive:4"], "evidence_texts": ["A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "i am in fashion design school . i want to have my own fashion line one day .", "what dish do you like to cook the most ?", "yum . that sounds absolutely delicious . i would love to try it .", "i have been doing community service for stealing .", "i have been so confused lately . my parents were never there ."], "speaker_id": "snli:test:941", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:941", "candidate_text": "Two males are involved in martial arts.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:941:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9688:premise", "snli:test:941:premise", "snli:test:2499:premise", "snli:test:5427:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:941", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:942", "candidate_text": "well, a child is in the ocean", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:942:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:865:premise", "snli:test:598:premise", "snli:test:942:premise", "snli:test:1019:premise", "style:personachat:test:35:self:7:positive:0", "style:personachat:test:35:self:7:positive:1", "style:personachat:test:35:self:7:positive:2", "style:personachat:test:35:self:7:positive:3", "style:personachat:test:35:self:7:positive:4", "style:personachat:test:35:self:7:positive:5", "style:personachat:test:35:self:7:positive:6"], "evidence_texts": ["A car is loaded with items on the top.", "A woman wearing jeans is walking down the street.", "A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "great and how was your day ?", "i can imagine . i thought about having children but i am not ready yet .", "i am in fashion design school . i want to have my own fashion line one day .", "what dish do you like to cook the most ?", "yum . that sounds absolutely delicious . i would love to try it .", "i have been doing community service for stealing .", "i have been so confused lately . my parents were never there ."], "speaker_id": "snli:test:942", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:942", "candidate_text": "I wonder whether a child is in the ocean?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:942:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:865:premise", "snli:test:598:premise", "snli:test:942:premise", "snli:test:1019:premise", "style:personachat:test:113:self:6:positive:0", "style:personachat:test:113:self:6:positive:1", "style:personachat:test:113:self:6:positive:2", "style:personachat:test:113:self:6:positive:3", "style:personachat:test:113:self:6:positive:4", "style:personachat:test:113:self:6:positive:5"], "evidence_texts": ["A car is loaded with items on the top.", "A woman wearing jeans is walking down the street.", "A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "i do enjoy classical music !", "ah , i get it . i just finished making dinner .", "i made pesto chicken and a salad .", "i do ! i would like a big family .", "what kind of engineer are you ?", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:942", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:944", "candidate_text": "Honestly, The child is a boy!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:944:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:944:premise", "snli:test:1019:premise", "snli:test:865:premise", "snli:test:1050:premise", "style:personachat:test:36:self:1:positive:0"], "evidence_texts": ["A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "A car is loaded with items on the top.", "A girl wearing a scarf is brushing a horse.", "that sounds like hard work !"], "speaker_id": "snli:test:944", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:944", "candidate_text": "well, the child is a boy", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:944:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:944:premise", "snli:test:1019:premise", "snli:test:865:premise", "snli:test:1050:premise", "style:personachat:test:68:self:1:positive:0"], "evidence_texts": ["A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "A car is loaded with items on the top.", "A girl wearing a scarf is brushing a horse.", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:944", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:983", "candidate_text": "Honestly, A tourist has his picture taken on Easter Island!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:983:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:831:premise", "snli:test:1160:premise", "snli:test:606:premise", "snli:test:983:premise", "style:personachat:test:36:self:2:positive:0", "style:personachat:test:36:self:2:positive:1"], "evidence_texts": ["Four women are taking a walk down an icy road.", "A group of men and boys look upwards at something.", "The little boy gets ready to kick the soccer ball.", "A man stands by two face structures on Easter Island.", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:983", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:983", "candidate_text": "A tourist has his picture taken on Easter Island.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:983:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:831:premise", "snli:test:1160:premise", "snli:test:606:premise", "snli:test:983:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Four women are taking a walk down an icy road.", "A group of men and boys look upwards at something.", "The little boy gets ready to kick the soccer ball.", "A man stands by two face structures on Easter Island.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:983", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:994", "candidate_text": "Honestly, Men in overalls work to fix a tractor!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:994:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:452:premise", "snli:test:1054:premise", "snli:test:994:premise", "snli:test:424:premise", "style:personachat:test:36:self:5:positive:0", "style:personachat:test:36:self:5:positive:1", "style:personachat:test:36:self:5:positive:2", "style:personachat:test:36:self:5:positive:3", "style:personachat:test:36:self:5:positive:4"], "evidence_texts": ["Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "i hope you feel better soon . what do you like to do for fun ?", "i love to dance . and bake i love sweets !", "how fun ! what a great place to take pictures !", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:994", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:994", "candidate_text": "I wonder whether men in overalls work to fix a tractor?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:994:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:452:premise", "snli:test:1054:premise", "snli:test:994:premise", "snli:test:424:premise", "style:personachat:test:115:self:5:positive:0", "style:personachat:test:115:self:5:positive:1", "style:personachat:test:115:self:5:positive:2", "style:personachat:test:115:self:5:positive:3", "style:personachat:test:115:self:5:positive:4"], "evidence_texts": ["Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "A young man in a heavy brown winter coat stands in front of a blue railing with his arms spread.", "that is good . my cousin loves to play soccer .", "i am single , my ex bf getting married . congratulations for you", "i hope so . break up is bad , specially when you love someone .", "yes i am working but still he is in my mind .", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:994", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1009", "candidate_text": "Honestly, The dog runs!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1009:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5418:premise", "snli:test:1009:premise", "snli:test:3834:premise", "snli:test:397:premise", "style:personachat:test:36:self:7:positive:0", "style:personachat:test:36:self:7:positive:1", "style:personachat:test:36:self:7:positive:2", "style:personachat:test:36:self:7:positive:3", "style:personachat:test:36:self:7:positive:4", "style:personachat:test:36:self:7:positive:5", "style:personachat:test:36:self:7:positive:6"], "evidence_texts": ["A wakeboarder.", "A dog runs.", "Men play hockey.", "A middle eastern marketplace.", "i am great ! just got home from working with dogs all day !", "i love dogs ! i have dogs and i get to train them at work , too !", "i hope you feel better soon . what do you like to do for fun ?", "i love to dance . and bake i love sweets !", "how fun ! what a great place to take pictures !", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:1009", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1009", "candidate_text": "well, the dog runs", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1009:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5418:premise", "snli:test:1009:premise", "snli:test:3834:premise", "snli:test:397:premise", "style:personachat:test:68:self:7:positive:0", "style:personachat:test:68:self:7:positive:1", "style:personachat:test:68:self:7:positive:2", "style:personachat:test:68:self:7:positive:3", "style:personachat:test:68:self:7:positive:4", "style:personachat:test:68:self:7:positive:5", "style:personachat:test:68:self:7:positive:6"], "evidence_texts": ["A wakeboarder.", "A dog runs.", "Men play hockey.", "A middle eastern marketplace.", "how are you doing this late night let us chat .", "oh i do not think i could do that was never to good in the study .", "wonderful cool but i did get my diploma from school to college .", "so so i used to travel a lot from new jersey to atlantic city .", "it was when i was going but now it probably is not so nice now .", "my best kind of hobby i like is entertaining board games and gambling .", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:1009", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1010", "candidate_text": "I wonder whether the dog is walking?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1010:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3834:premise", "snli:test:5418:premise", "snli:test:1010:premise", "snli:test:397:premise", "style:personachat:test:37:self:1:positive:0"], "evidence_texts": ["Men play hockey.", "A wakeboarder.", "A dog runs.", "A middle eastern marketplace.", "do they have any kids together ?"], "speaker_id": "snli:test:1010", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1010", "candidate_text": "Honestly, The dog is walking!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1010:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3834:premise", "snli:test:5418:premise", "snli:test:1010:premise", "snli:test:397:premise", "style:personachat:test:249:self:1:positive:0"], "evidence_texts": ["Men play hockey.", "A wakeboarder.", "A dog runs.", "A middle eastern marketplace.", "i met my wife at a wedding !"], "speaker_id": "snli:test:1010", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1019", "candidate_text": "I wonder whether a group of kids are playing in the city park?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1019:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:944:premise", "snli:test:1050:premise", "snli:test:942:premise", "snli:test:1019:premise", "style:personachat:test:37:self:2:positive:0", "style:personachat:test:37:self:2:positive:1"], "evidence_texts": ["A wet child stands in chest deep ocean water.", "A girl wearing a scarf is brushing a horse.", "A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "i did not even know gucci was getting married . who to ?", "do they have any kids together ?"], "speaker_id": "snli:test:1019", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1019", "candidate_text": "A group of kids are playing in the city park.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1019:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:944:premise", "snli:test:1050:premise", "snli:test:942:premise", "snli:test:1019:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A wet child stands in chest deep ocean water.", "A girl wearing a scarf is brushing a horse.", "A wet child stands in chest deep ocean water.", "A group of kids play on a colorful structure.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1019", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1035", "candidate_text": "I wonder whether a group of friends are at the table?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1035:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:521:premise", "snli:test:1203:premise", "snli:test:321:premise", "snli:test:1035:premise", "style:personachat:test:37:self:5:positive:0", "style:personachat:test:37:self:5:positive:1", "style:personachat:test:37:self:5:positive:2", "style:personachat:test:37:self:5:positive:3", "style:personachat:test:37:self:5:positive:4"], "evidence_texts": ["A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "that is interesting . i am also a vegan and a vet . i love animals .", "a sister in california , where we grew up . i am in florida now . you ?", "yes ! i rap on the weekends myself too . my friends like country , boo .", "i did not even know gucci was getting married . who to ?", "do they have any kids together ?"], "speaker_id": "snli:test:1035", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1035", "candidate_text": "well, a group of friends are at the table", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1035:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:521:premise", "snli:test:1203:premise", "snli:test:321:premise", "snli:test:1035:premise", "style:personachat:test:71:self:5:positive:0", "style:personachat:test:71:self:5:positive:1", "style:personachat:test:71:self:5:positive:2", "style:personachat:test:71:self:5:positive:3", "style:personachat:test:71:self:5:positive:4"], "evidence_texts": ["A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "A woman, in red and white, and wearing glasses, sits in a room with other people.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "i love to sing all the time but will never have a kid", "oh really ? are you single ? i model here in nyc", "really ? i thought you were an artist ,", "really ? let me see your work", "send it to me online"], "speaker_id": "snli:test:1035", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1042", "candidate_text": "I wonder whether the man works at e.s.e electronics?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1042:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:902:premise", "snli:test:1189:premise", "snli:test:1042:premise", "snli:test:708:premise", "style:personachat:test:37:self:7:positive:0", "style:personachat:test:37:self:7:positive:1", "style:personachat:test:37:self:7:positive:2", "style:personachat:test:37:self:7:positive:3", "style:personachat:test:37:self:7:positive:4", "style:personachat:test:37:self:7:positive:5", "style:personachat:test:37:self:7:positive:6"], "evidence_texts": ["Two men in neon yellow shirts busily sawing a log in half.", "There is a group of children getting their picture taken with presents.", "A man is walking past a large sign that says E.S.E. Electronics.", "A woman in black reviews a message as she walks to work.", "not really . i enjoy video games and reading mostly . do you workout ?", "are you a bodybuilder ? what do you do for a living ?", "that is interesting . i am also a vegan and a vet . i love animals .", "a sister in california , where we grew up . i am in florida now . you ?", "yes ! i rap on the weekends myself too . my friends like country , boo .", "i did not even know gucci was getting married . who to ?", "do they have any kids together ?"], "speaker_id": "snli:test:1042", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1042", "candidate_text": "Honestly, The man works at E.S.E Electronics!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1042:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:902:premise", "snli:test:1189:premise", "snli:test:1042:premise", "snli:test:708:premise", "style:personachat:test:249:self:7:positive:0", "style:personachat:test:249:self:7:positive:1", "style:personachat:test:249:self:7:positive:2", "style:personachat:test:249:self:7:positive:3", "style:personachat:test:249:self:7:positive:4", "style:personachat:test:249:self:7:positive:5", "style:personachat:test:249:self:7:positive:6"], "evidence_texts": ["Two men in neon yellow shirts busily sawing a log in half.", "There is a group of children getting their picture taken with presents.", "A man is walking past a large sign that says E.S.E. Electronics.", "A woman in black reviews a message as she walks to work.", "i just got home from a nice nature hike . do you like the outdoors ?", "hiking gives me inspiration for my drawing class . do you like art ?", "yes ! what kind of work do you do indoors ?", "i would put wierd stickers all over the walls ! do you have any hobbies ?", "i collect stickers , i try all kinds of strange food . can you recommend any strange food ?", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:1042", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1044", "candidate_text": "well, black man poses in front of e.s.e. sign after making a purchase at a store", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1044:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1044:premise", "snli:test:783:premise", "snli:test:577:premise", "snli:test:1224:premise", "style:personachat:test:38:self:1:positive:0"], "evidence_texts": ["An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "good luck , being a doctor is hard . maybe you will write medical books ."], "speaker_id": "snli:test:1044", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1044", "candidate_text": "I wonder whether black man poses in front of e.s.e. sign after making a purchase at a store?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1044:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1044:premise", "snli:test:783:premise", "snli:test:577:premise", "snli:test:1224:premise", "style:personachat:test:125:self:1:positive:0"], "evidence_texts": ["An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:1044", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1045", "candidate_text": "well, man breaks camera in front of liquor store", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1045:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1224:premise", "snli:test:1045:premise", "snli:test:783:premise", "snli:test:1372:premise", "style:personachat:test:38:self:2:positive:0", "style:personachat:test:38:self:2:positive:1"], "evidence_texts": ["Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "i never thought of that . i love clothes as well .", "good luck , being a doctor is hard . maybe you will write medical books ."], "speaker_id": "snli:test:1045", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1045", "candidate_text": "Honestly, Man breaks camera in front of liquor store!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1045:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1224:premise", "snli:test:1045:premise", "snli:test:783:premise", "snli:test:1372:premise", "style:personachat:test:261:self:2:positive:0", "style:personachat:test:261:self:2:positive:1"], "evidence_texts": ["Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "Elderly woman in blue apron balances a basket on her head on a sidewalk while talking to a woman dressed in black.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:1045", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1050", "candidate_text": "well, a girl is brushing a horse", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1050:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1019:premise", "snli:test:1093:premise", "snli:test:1050:premise", "snli:test:944:premise", "style:personachat:test:38:self:5:positive:0", "style:personachat:test:38:self:5:positive:1", "style:personachat:test:38:self:5:positive:2", "style:personachat:test:38:self:5:positive:3", "style:personachat:test:38:self:5:positive:4"], "evidence_texts": ["A group of kids play on a colorful structure.", "A person with an orange shovel is shoveling snow.", "A girl wearing a scarf is brushing a horse.", "A wet child stands in chest deep ocean water.", "oh a beetle how cool . i have a boring honda civic . what are your hobbies ?", "how amazing ! i love to go hiking . what books have you written ?", "very cool ! yes , i love to hike you get some beautiful views !", "i never thought of that . i love clothes as well .", "good luck , being a doctor is hard . maybe you will write medical books ."], "speaker_id": "snli:test:1050", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1050", "candidate_text": "A girl is brushing a horse.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1050:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1019:premise", "snli:test:1093:premise", "snli:test:1050:premise", "snli:test:944:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of kids play on a colorful structure.", "A person with an orange shovel is shoveling snow.", "A girl wearing a scarf is brushing a horse.", "A wet child stands in chest deep ocean water.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1050", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1054", "candidate_text": "well, a guitarist sitting on stage", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1054:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:994:premise", "snli:test:1068:premise", "snli:test:1054:premise", "snli:test:452:premise", "style:personachat:test:38:self:6:positive:0", "style:personachat:test:38:self:6:positive:1", "style:personachat:test:38:self:6:positive:2", "style:personachat:test:38:self:6:positive:3", "style:personachat:test:38:self:6:positive:4", "style:personachat:test:38:self:6:positive:5"], "evidence_texts": ["The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "hello , i am an accountant from ohio . i have two boys and i am single .", "oh a beetle how cool . i have a boring honda civic . what are your hobbies ?", "how amazing ! i love to go hiking . what books have you written ?", "very cool ! yes , i love to hike you get some beautiful views !", "i never thought of that . i love clothes as well .", "good luck , being a doctor is hard . maybe you will write medical books ."], "speaker_id": "snli:test:1054", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1054", "candidate_text": "I wonder whether a guitarist sitting on stage?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1054:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:994:premise", "snli:test:1068:premise", "snli:test:1054:premise", "snli:test:452:premise", "style:personachat:test:125:self:6:positive:0", "style:personachat:test:125:self:6:positive:1", "style:personachat:test:125:self:6:positive:2", "style:personachat:test:125:self:6:positive:3", "style:personachat:test:125:self:6:positive:4", "style:personachat:test:125:self:6:positive:5"], "evidence_texts": ["The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "Two woman wearing dresses are standing outside on the grass while one of them is holding a cup and saucer.", "hello i am fine i cannot walk i am disabled", "its okay i like to knit hats for babies in nicu", "wow have you discovered any new favorite food i love cooking", "that sounds right up my alley being vegan and all", "wow someone who helps others like me and vegan amazing", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:1054", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1061", "candidate_text": "I wonder whether a group of people dance in the daytime?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1061:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:928:premise", "snli:test:1061:premise", "snli:test:1179:premise", "snli:test:808:premise", "style:personachat:test:39:self:1:positive:0"], "evidence_texts": ["An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man has flung himself over a pole with people and canopies in the background.", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:1061", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1061", "candidate_text": "well, a group of people dance in the daytime", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1061:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:928:premise", "snli:test:1061:premise", "snli:test:1179:premise", "snli:test:808:premise", "style:personachat:test:75:self:1:positive:0"], "evidence_texts": ["An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man has flung himself over a pole with people and canopies in the background.", "so are you . great personality"], "speaker_id": "snli:test:1061", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1067", "candidate_text": "I wonder whether a girl tripped on herself before she attempt to jump over a hurdle?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1067:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:685:premise", "snli:test:1116:premise", "snli:test:563:premise", "snli:test:1067:premise", "style:personachat:test:39:self:2:positive:0", "style:personachat:test:39:self:2:positive:1"], "evidence_texts": ["Five children playing soccer chase after a ball.", "The two boys are swimming with boogie boards.", "Boys with their backs against an incoming wave.", "A schoolgirl is jumping over a low hurdle.", "i like pizza too . the sauce is the best part .", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:1067", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1067", "candidate_text": "Honestly, A girl tripped on herself before she attempt to jump over a hurdle!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1067:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:685:premise", "snli:test:1116:premise", "snli:test:563:premise", "snli:test:1067:premise", "style:personachat:test:269:self:2:positive:0", "style:personachat:test:269:self:2:positive:1"], "evidence_texts": ["Five children playing soccer chase after a ball.", "The two boys are swimming with boogie boards.", "Boys with their backs against an incoming wave.", "A schoolgirl is jumping over a low hurdle.", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:1067", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1068", "candidate_text": "I wonder whether the woman is sitting next to someone?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1068:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1068:premise", "snli:test:1054:premise", "snli:test:994:premise", "snli:test:1301:premise", "style:personachat:test:39:self:5:positive:0", "style:personachat:test:39:self:5:positive:1", "style:personachat:test:39:self:5:positive:2", "style:personachat:test:39:self:5:positive:3", "style:personachat:test:39:self:5:positive:4"], "evidence_texts": ["A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "what kind of skating do you do ?", "i used to skateboard . now i just study and play video games .", "i love chips so much . what food do you like ?", "i like pizza too . the sauce is the best part .", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:1068", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1068", "candidate_text": "The woman is sitting next to someone.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1068:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1068:premise", "snli:test:1054:premise", "snli:test:994:premise", "snli:test:1301:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "The farmers are out working hard on there tractor, they have on their overalls and the machine is almost fixed.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1068", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1070", "candidate_text": "I wonder whether the woman is standing?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1070:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1301:premise", "snli:test:1054:premise", "snli:test:1070:premise", "snli:test:1498:premise", "style:personachat:test:39:self:7:positive:0", "style:personachat:test:39:self:7:positive:1", "style:personachat:test:39:self:7:positive:2", "style:personachat:test:39:self:7:positive:3", "style:personachat:test:39:self:7:positive:4", "style:personachat:test:39:self:7:positive:5", "style:personachat:test:39:self:7:positive:6"], "evidence_texts": ["People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "i am doing great . just watching a movie .", "i worked at dairy queen . i am in school now .", "what kind of skating do you do ?", "i used to skateboard . now i just study and play video games .", "i love chips so much . what food do you like ?", "i like pizza too . the sauce is the best part .", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:1070", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1070", "candidate_text": "well, the woman is standing", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1070:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1301:premise", "snli:test:1054:premise", "snli:test:1070:premise", "snli:test:1498:premise", "style:personachat:test:75:self:6:positive:0", "style:personachat:test:75:self:6:positive:1", "style:personachat:test:75:self:6:positive:2", "style:personachat:test:75:self:6:positive:3", "style:personachat:test:75:self:6:positive:4", "style:personachat:test:75:self:6:positive:5"], "evidence_texts": ["People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "A closeup view of a guitarist in a red shirt standing outside, with a group of children playing around him.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "i am good , just hanging out", "i like anything blue that is why i have blue hair", "awesome ! ! ! i need to try a vegan lifestyle", "i am into cosplay . i am a pretty scrawny kid", "aww thanks . i like rap too", "so are you . great personality"], "speaker_id": "snli:test:1070", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1093", "candidate_text": "I wonder whether there is snow outside?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1093:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1050:premise", "snli:test:1093:premise", "snli:test:1127:premise", "snli:test:1019:premise", "style:personachat:test:40:self:1:positive:0"], "evidence_texts": ["A girl wearing a scarf is brushing a horse.", "A person with an orange shovel is shoveling snow.", "People are standing next to a chain link fence.", "A group of kids play on a colorful structure.", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:1093", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1093", "candidate_text": "Honestly, There is snow outside!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1093:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1050:premise", "snli:test:1093:premise", "snli:test:1127:premise", "snli:test:1019:premise", "style:personachat:test:275:self:1:positive:0"], "evidence_texts": ["A girl wearing a scarf is brushing a horse.", "A person with an orange shovel is shoveling snow.", "People are standing next to a chain link fence.", "A group of kids play on a colorful structure.", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:1093", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1110", "candidate_text": "I wonder whether a woman is teaching a ballet class composed of middle aged men holding white scarves?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1110:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:895:premise", "snli:test:502:premise", "snli:test:1110:premise", "snli:test:1172:premise", "style:personachat:test:40:self:2:positive:0", "style:personachat:test:40:self:2:positive:1"], "evidence_texts": ["A brown dog carries an object in its mouth on a snowy hillside.", "A group of kids is splashing in deep water nearby a rock formation.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "Man in red jacket is opening the door while holding his car keys.", "i see ! so besides my brother i have my mom . she is a secretary .", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:1110", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1110", "candidate_text": "A woman is teaching a ballet class composed of middle aged men holding white scarves.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1110:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:895:premise", "snli:test:502:premise", "snli:test:1110:premise", "snli:test:1172:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A brown dog carries an object in its mouth on a snowy hillside.", "A group of kids is splashing in deep water nearby a rock formation.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "Man in red jacket is opening the door while holding his car keys.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1110", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1111", "candidate_text": "I wonder whether a woman is teaching a ballet class?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1111:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1172:premise", "snli:test:895:premise", "snli:test:1283:premise", "snli:test:1111:premise", "style:personachat:test:40:self:5:positive:0", "style:personachat:test:40:self:5:positive:1", "style:personachat:test:40:self:5:positive:2", "style:personachat:test:40:self:5:positive:3", "style:personachat:test:40:self:5:positive:4"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A brown dog carries an object in its mouth on a snowy hillside.", "A man is watching as a woman is reading a piece of paper.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "not me , but maybe for my brother . are you a realtor ?", "sweet . i do not work for a company , i do bodybuilding for a living .", "indeed ! so do you have any family ?", "i see ! so besides my brother i have my mom . she is a secretary .", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:1111", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1111", "candidate_text": "well, a woman is teaching a ballet class", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1111:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1172:premise", "snli:test:895:premise", "snli:test:1283:premise", "snli:test:1111:premise", "style:personachat:test:77:self:5:positive:0", "style:personachat:test:77:self:5:positive:1", "style:personachat:test:77:self:5:positive:2", "style:personachat:test:77:self:5:positive:3", "style:personachat:test:77:self:5:positive:4"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A brown dog carries an object in its mouth on a snowy hillside.", "A man is watching as a woman is reading a piece of paper.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "i usually read while i am eating pizza from my moms restaurant", "eating is my life ! i could never diet", "i wish i could go to the beach . i get nervous and stutter around attractive people though", "i really hope i can", "thank you for the idea . i will talk to my mom at work tonight"], "speaker_id": "snli:test:1111", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1116", "candidate_text": "I wonder whether the two boys are in their bath tub?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1116:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1116:premise", "snli:test:1067:premise", "snli:test:1161:premise", "snli:test:685:premise", "style:personachat:test:40:self:7:positive:0", "style:personachat:test:40:self:7:positive:1", "style:personachat:test:40:self:7:positive:2", "style:personachat:test:40:self:7:positive:3", "style:personachat:test:40:self:7:positive:4", "style:personachat:test:40:self:7:positive:5", "style:personachat:test:40:self:7:positive:6"], "evidence_texts": ["The two boys are swimming with boogie boards.", "A schoolgirl is jumping over a low hurdle.", "A brown dog runs through a grassy field", "Five children playing soccer chase after a ball.", "hey , not bad . hunting , that is cool . do you hunt for sport or meat ?", "awesome . i love meat . i eat a high protein meat based diet .", "not me , but maybe for my brother . are you a realtor ?", "sweet . i do not work for a company , i do bodybuilding for a living .", "indeed ! so do you have any family ?", "i see ! so besides my brother i have my mom . she is a secretary .", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:1116", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1116", "candidate_text": "Honestly, The two boys are in their bath tub!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1116:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1116:premise", "snli:test:1067:premise", "snli:test:1161:premise", "snli:test:685:premise", "style:personachat:test:275:self:7:positive:0", "style:personachat:test:275:self:7:positive:1", "style:personachat:test:275:self:7:positive:2", "style:personachat:test:275:self:7:positive:3", "style:personachat:test:275:self:7:positive:4", "style:personachat:test:275:self:7:positive:5", "style:personachat:test:275:self:7:positive:6"], "evidence_texts": ["The two boys are swimming with boogie boards.", "A schoolgirl is jumping over a low hurdle.", "A brown dog runs through a grassy field", "Five children playing soccer chase after a ball.", "hi there ! want to play some poker ? i love it !", "we cant drink alcohol while playing , though . i quit 5 years back .", "my wife and 3 boys would skin me alive !", "i am so busy doing carpentry that i do not meet many people as friends .", "wow ! get back here , charlie ! are you vegetarian ?", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:1116", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1127", "candidate_text": "well, the people are at a school", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1127:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1093:premise", "snli:test:1194:premise", "snli:test:1050:premise", "snli:test:1127:premise", "style:personachat:test:41:self:1:positive:0"], "evidence_texts": ["A person with an orange shovel is shoveling snow.", "The big dog is checking out the smaller dog.", "A girl wearing a scarf is brushing a horse.", "People are standing next to a chain link fence.", "i m having a hard time typing sorry"], "speaker_id": "snli:test:1127", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1127", "candidate_text": "I wonder whether the people are at a school?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1127:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1093:premise", "snli:test:1194:premise", "snli:test:1050:premise", "snli:test:1127:premise", "style:personachat:test:139:self:1:positive:0"], "evidence_texts": ["A person with an orange shovel is shoveling snow.", "The big dog is checking out the smaller dog.", "A girl wearing a scarf is brushing a horse.", "People are standing next to a chain link fence.", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:1127", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1128", "candidate_text": "well, the men are carrying bicycles", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1128:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1128:premise", "snli:test:892:premise", "snli:test:856:premise", "snli:test:1256:premise", "style:personachat:test:41:self:2:positive:0", "style:personachat:test:41:self:2:positive:1"], "evidence_texts": ["Two guys walking, one carrying a skateboard", "A brown dog jumps over an obstacle.", "Four women competitively rollerskating around an area.", "a young man wearing dark sunglasses smiling", "sorry for the typo what i mean is", "i m having a hard time typing sorry"], "speaker_id": "snli:test:1128", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1128", "candidate_text": "Honestly, The men are carrying bicycles!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1128:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1128:premise", "snli:test:892:premise", "snli:test:856:premise", "snli:test:1256:premise", "style:personachat:test:289:self:2:positive:0", "style:personachat:test:289:self:2:positive:1"], "evidence_texts": ["Two guys walking, one carrying a skateboard", "A brown dog jumps over an obstacle.", "Four women competitively rollerskating around an area.", "a young man wearing dark sunglasses smiling", "yeah . my mom actually works at the front desk ! do you train with your dad ?", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:1128", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1129", "candidate_text": "well, both of the men are walking", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1129:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1256:premise", "snli:test:1129:premise", "snli:test:892:premise", "snli:test:1626:premise", "style:personachat:test:41:self:5:positive:0", "style:personachat:test:41:self:5:positive:1", "style:personachat:test:41:self:5:positive:2", "style:personachat:test:41:self:5:positive:3", "style:personachat:test:41:self:5:positive:4"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two guys walking, one carrying a skateboard", "A brown dog jumps over an obstacle.", "Women in costume stand on a stage.", "you guys like pizza it s my favorite", "hey i am a cat catholic devout", "i m not remain silent but purple is my favorite", "sorry for the typo what i mean is", "i m having a hard time typing sorry"], "speaker_id": "snli:test:1129", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1129", "candidate_text": "Both of the men are walking.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1129:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1256:premise", "snli:test:1129:premise", "snli:test:892:premise", "snli:test:1626:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two guys walking, one carrying a skateboard", "A brown dog jumps over an obstacle.", "Women in costume stand on a stage.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1129", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1132", "candidate_text": "well, a person running", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1132:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1132:premise", "snli:test:860:premise", "snli:test:778:premise", "snli:test:1145:premise", "style:personachat:test:41:self:7:positive:0", "style:personachat:test:41:self:7:positive:1", "style:personachat:test:41:self:7:positive:2", "style:personachat:test:41:self:7:positive:3", "style:personachat:test:41:self:7:positive:4", "style:personachat:test:41:self:7:positive:5", "style:personachat:test:41:self:7:positive:6"], "evidence_texts": ["A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "registered nurse it s not raining fine", "watching one tree hill is relaxing for me", "you guys like pizza it s my favorite", "hey i am a cat catholic devout", "i m not remain silent but purple is my favorite", "sorry for the typo what i mean is", "i m having a hard time typing sorry"], "speaker_id": "snli:test:1132", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1132", "candidate_text": "I wonder whether a person running?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1132:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1132:premise", "snli:test:860:premise", "snli:test:778:premise", "snli:test:1145:premise", "style:personachat:test:139:self:7:positive:0", "style:personachat:test:139:self:7:positive:1", "style:personachat:test:139:self:7:positive:2", "style:personachat:test:139:self:7:positive:3", "style:personachat:test:139:self:7:positive:4", "style:personachat:test:139:self:7:positive:5", "style:personachat:test:139:self:7:positive:6"], "evidence_texts": ["A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "A blond little boy in an orange sweatshirt with red sleeves is using scissors to cut something.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "good , i am in sales , what do you do for your work ?", "i am listening to rush , i love them", "cool beans , do you have any other hobbies then sales ?", "yeah but i do not like it", "i like to eat meat and plants , i eat both", "yeah my friend had that removed at his wedding last week when i attended", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:1132", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1133", "candidate_text": "I wonder whether nobody is running?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1133:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1145:premise", "snli:test:1133:premise", "snli:test:860:premise", "snli:test:1526:premise", "style:personachat:test:42:self:1:positive:0"], "evidence_texts": ["A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "where do you go to college ?"], "speaker_id": "snli:test:1133", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1133", "candidate_text": "well, nobody is running", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1133:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1145:premise", "snli:test:1133:premise", "snli:test:860:premise", "snli:test:1526:premise", "style:personachat:test:80:self:1:positive:0"], "evidence_texts": ["A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A group of cleaners are sweeping up animal feces from the street during a parade or festival.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "no , i would like to attend college though ."], "speaker_id": "snli:test:1133", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1142", "candidate_text": "I wonder whether a woman had a tennis racquet?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1142:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:841:premise", "snli:test:1241:premise", "snli:test:1142:premise", "snli:test:820:premise", "style:personachat:test:42:self:2:positive:0", "style:personachat:test:42:self:2:positive:1"], "evidence_texts": ["A group of footballs players is standing behind a coaching official.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "A woman is in the middle of hitting a tennis ball.", "A person is sitting in front of a graffiti covered wall.", "the nemo looking one . i stare at him when i eat my granola every morning", "where do you go to college ?"], "speaker_id": "snli:test:1142", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1142", "candidate_text": "Honestly, A woman had a tennis racquet!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1142:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:841:premise", "snli:test:1241:premise", "snli:test:1142:premise", "snli:test:820:premise", "style:personachat:test:290:self:2:positive:0", "style:personachat:test:290:self:2:positive:1"], "evidence_texts": ["A group of footballs players is standing behind a coaching official.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "A woman is in the middle of hitting a tennis ball.", "A person is sitting in front of a graffiti covered wall.", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:1142", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1145", "candidate_text": "I wonder whether a woman rises from a chair?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1145:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1133:premise", "snli:test:1145:premise", "snli:test:1526:premise", "snli:test:1132:premise", "style:personachat:test:42:self:5:positive:0", "style:personachat:test:42:self:5:positive:1", "style:personachat:test:42:self:5:positive:2", "style:personachat:test:42:self:5:positive:3", "style:personachat:test:42:self:5:positive:4"], "evidence_texts": ["A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "i have an aquarium . i keep exotic fish ! and i collect also .", "i do not get a chance . i kill time with my vintage collection . from 1950s", "decorations mainly . that sounds very exciting . alone or with family ?", "the nemo looking one . i stare at him when i eat my granola every morning", "where do you go to college ?"], "speaker_id": "snli:test:1145", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1145", "candidate_text": "A woman rises from a chair.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1145:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1133:premise", "snli:test:1145:premise", "snli:test:1526:premise", "snli:test:1132:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1145", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1149", "candidate_text": "I wonder whether the helmet has a stripe on it?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1149:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:876:premise", "snli:test:1149:premise", "snli:test:1797:premise", "snli:test:738:premise", "style:personachat:test:42:self:7:positive:0", "style:personachat:test:42:self:7:positive:1", "style:personachat:test:42:self:7:positive:2", "style:personachat:test:42:self:7:positive:3", "style:personachat:test:42:self:7:positive:4", "style:personachat:test:42:self:7:positive:5", "style:personachat:test:42:self:7:positive:6"], "evidence_texts": ["The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "hi i barb . tired from school . how about you ?", "i am going to be a hairdresser . how is your garden coming along ?", "i have an aquarium . i keep exotic fish ! and i collect also .", "i do not get a chance . i kill time with my vintage collection . from 1950s", "decorations mainly . that sounds very exciting . alone or with family ?", "the nemo looking one . i stare at him when i eat my granola every morning", "where do you go to college ?"], "speaker_id": "snli:test:1149", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1149", "candidate_text": "well, the helmet has a stripe on it", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1149:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:876:premise", "snli:test:1149:premise", "snli:test:1797:premise", "snli:test:738:premise", "style:personachat:test:80:self:7:positive:0", "style:personachat:test:80:self:7:positive:1", "style:personachat:test:80:self:7:positive:2", "style:personachat:test:80:self:7:positive:3", "style:personachat:test:80:self:7:positive:4", "style:personachat:test:80:self:7:positive:5", "style:personachat:test:80:self:7:positive:6"], "evidence_texts": ["The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A female runner dressed in blue athletic wear is running in a competition, while spectators line the street.", "oh my . i am a soldier in south korea , actually .", "well i am only 20 , so a bit younger than you !", "i am a lieutenant . you ?", "yes , it was . how about you ?", "i am in the army . what years did you serve ?", "i have not decided yet . . . it is such a big decision .", "no , i would like to attend college though ."], "speaker_id": "snli:test:1149", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1156", "candidate_text": "I wonder whether the children are looking through a microscope?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1156:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1156:premise", "snli:test:667:premise", "snli:test:1461:premise", "snli:test:301:premise", "style:personachat:test:43:self:1:positive:0"], "evidence_texts": ["A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:1156", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1156", "candidate_text": "Honestly, The children are looking through a microscope!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1156:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1156:premise", "snli:test:667:premise", "snli:test:1461:premise", "snli:test:301:premise", "style:personachat:test:295:self:1:positive:0"], "evidence_texts": ["A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A man in a black shirt, in a commercial kitchen, holding up meat he took out of a bag.", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:1156", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1160", "candidate_text": "I wonder whether a group of coaches are showing their team the scoreboard?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1160:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1160:premise", "snli:test:983:premise", "snli:test:1314:premise", "snli:test:831:premise", "style:personachat:test:43:self:2:positive:0", "style:personachat:test:43:self:2:positive:1"], "evidence_texts": ["A group of men and boys look upwards at something.", "A man stands by two face structures on Easter Island.", "A tan dog chases a black and white soccer ball.", "Four women are taking a walk down an icy road.", "great , how about colors ? my favorite is blue", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:1160", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1160", "candidate_text": "A group of coaches are showing their team the scoreboard.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1160:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1160:premise", "snli:test:983:premise", "snli:test:1314:premise", "snli:test:831:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of men and boys look upwards at something.", "A man stands by two face structures on Easter Island.", "A tan dog chases a black and white soccer ball.", "Four women are taking a walk down an icy road.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1160", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1161", "candidate_text": "I wonder whether a dog chases a rabbit?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1161:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1116:premise", "snli:test:1161:premise", "snli:test:1173:premise", "snli:test:1067:premise", "style:personachat:test:43:self:5:positive:0", "style:personachat:test:43:self:5:positive:1", "style:personachat:test:43:self:5:positive:2", "style:personachat:test:43:self:5:positive:3", "style:personachat:test:43:self:5:positive:4"], "evidence_texts": ["The two boys are swimming with boogie boards.", "A brown dog runs through a grassy field", "kids are looking at something on a table.", "A schoolgirl is jumping over a low hurdle.", "i keep busy with my seven children .", "do you reach mysteries to your children ? they are my favorite type of novel .", "do you teach them about animals , as i grew up on a farm .", "great , how about colors ? my favorite is blue", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:1161", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1161", "candidate_text": "well, a dog chases a rabbit", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1161:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1116:premise", "snli:test:1161:premise", "snli:test:1173:premise", "snli:test:1067:premise", "style:personachat:test:81:self:5:positive:0", "style:personachat:test:81:self:5:positive:1", "style:personachat:test:81:self:5:positive:2", "style:personachat:test:81:self:5:positive:3", "style:personachat:test:81:self:5:positive:4"], "evidence_texts": ["The two boys are swimming with boogie boards.", "A brown dog runs through a grassy field", "kids are looking at something on a table.", "A schoolgirl is jumping over a low hurdle.", "for fun i working on fixing up old cars", "cool what kind of music ?", "cool i also work as a handyman", "its fun to work on different things yes", "saves lots of money the more you do"], "speaker_id": "snli:test:1161", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1172", "candidate_text": "I wonder whether the man dropped his car keys in the sewer?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1172:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1172:premise", "snli:test:1111:premise", "snli:test:1283:premise", "snli:test:1110:premise", "style:personachat:test:43:self:7:positive:0", "style:personachat:test:43:self:7:positive:1", "style:personachat:test:43:self:7:positive:2", "style:personachat:test:43:self:7:positive:3", "style:personachat:test:43:self:7:positive:4", "style:personachat:test:43:self:7:positive:5", "style:personachat:test:43:self:7:positive:6"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "A man is watching as a woman is reading a piece of paper.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "i am well . how are you ?", "that is great . i live far from the beach .", "i keep busy with my seven children .", "do you reach mysteries to your children ? they are my favorite type of novel .", "do you teach them about animals , as i grew up on a farm .", "great , how about colors ? my favorite is blue", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:1172", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1172", "candidate_text": "Honestly, The man dropped his car keys in the sewer!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1172:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1172:premise", "snli:test:1111:premise", "snli:test:1283:premise", "snli:test:1110:premise", "style:personachat:test:295:self:7:positive:0", "style:personachat:test:295:self:7:positive:1", "style:personachat:test:295:self:7:positive:2", "style:personachat:test:295:self:7:positive:3", "style:personachat:test:295:self:7:positive:4", "style:personachat:test:295:self:7:positive:5", "style:personachat:test:295:self:7:positive:6"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "A man is watching as a woman is reading a piece of paper.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "i just got home from work , i do graphic design .", "nope , my real job is wheelchair sales and design is my hobby , i freelance .", "i can hook you up . do you like music ? my fav band is rush ? awesome .", "big question , what does an omnivore do at a vegans wedding ? my challenge last weekend !", "i like to pimp out my clients wheelchairs , what would your brother like ?", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:1172", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1173", "candidate_text": "well, some kids are burning the table down", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1173:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1161:premise", "snli:test:1173:premise", "snli:test:1116:premise", "snli:test:1228:premise", "style:personachat:test:44:self:1:positive:0"], "evidence_texts": ["A brown dog runs through a grassy field", "kids are looking at something on a table.", "The two boys are swimming with boogie boards.", "A light colored dog leaps over a hurdle.", "oh i miss the days of crayons being my hardest decision already"], "speaker_id": "snli:test:1173", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1173", "candidate_text": "I wonder whether some kids are burning the table down?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1173:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1161:premise", "snli:test:1173:premise", "snli:test:1116:premise", "snli:test:1228:premise", "style:personachat:test:150:self:1:positive:0"], "evidence_texts": ["A brown dog runs through a grassy field", "kids are looking at something on a table.", "The two boys are swimming with boogie boards.", "A light colored dog leaps over a hurdle.", "so what do you do for fun ?"], "speaker_id": "snli:test:1173", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1175", "candidate_text": "well, some kids are staring at something on the table", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1175:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1228:premise", "snli:test:1161:premise", "snli:test:1260:premise", "snli:test:1175:premise", "style:personachat:test:44:self:2:positive:0", "style:personachat:test:44:self:2:positive:1"], "evidence_texts": ["A light colored dog leaps over a hurdle.", "A brown dog runs through a grassy field", "People sitting in chairs along a tree-lined sidewalk.", "kids are looking at something on a table.", "i do not want that day to come . i want to hold onto my youth !", "oh i miss the days of crayons being my hardest decision already"], "speaker_id": "snli:test:1175", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1175", "candidate_text": "Honestly, Some kids are staring at something on the table!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1175:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1228:premise", "snli:test:1161:premise", "snli:test:1260:premise", "snli:test:1175:premise", "style:personachat:test:332:self:2:positive:0", "style:personachat:test:332:self:2:positive:1"], "evidence_texts": ["A light colored dog leaps over a hurdle.", "A brown dog runs through a grassy field", "People sitting in chairs along a tree-lined sidewalk.", "kids are looking at something on a table.", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:1175", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1177", "candidate_text": "well, a skateboarder is doing a trick", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1177:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:798:premise", "snli:test:1177:premise", "snli:test:1471:premise", "snli:test:723:premise", "style:personachat:test:44:self:5:positive:0", "style:personachat:test:44:self:5:positive:1", "style:personachat:test:44:self:5:positive:2", "style:personachat:test:44:self:5:positive:3", "style:personachat:test:44:self:5:positive:4"], "evidence_texts": ["Workers standing on a lift.", "skateboarder sliding down a ramp", "Times Square in New York.", "Boy in midair on bicycle", "well i am only 17 , would rather do something with fashion though", "cause my pops wants it . one of those things .", "lol ! try racing a beetle , that was something dad said no to right away", "i do not want that day to come . i want to hold onto my youth !", "oh i miss the days of crayons being my hardest decision already"], "speaker_id": "snli:test:1177", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1177", "candidate_text": "A skateboarder is doing a trick.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1177:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:798:premise", "snli:test:1177:premise", "snli:test:1471:premise", "snli:test:723:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Workers standing on a lift.", "skateboarder sliding down a ramp", "Times Square in New York.", "Boy in midair on bicycle", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1177", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1179", "candidate_text": "well, a snowman is being photographed with its creator", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1179:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1061:premise", "snli:test:928:premise", "snli:test:1219:premise", "snli:test:1179:premise", "style:personachat:test:44:self:7:positive:0", "style:personachat:test:44:self:7:positive:1", "style:personachat:test:44:self:7:positive:2", "style:personachat:test:44:self:7:positive:3", "style:personachat:test:44:self:7:positive:4", "style:personachat:test:44:self:7:positive:5", "style:personachat:test:44:self:7:positive:6"], "evidence_texts": ["A crowd walks about a square at night with smoke or fog drifting over them.", "An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A man in a market holding some very large green onions while smoking a cigarette.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "slow . school today , my lst year thank god .", "high school . i am going to med school after i finish . auburn u", "well i am only 17 , would rather do something with fashion though", "cause my pops wants it . one of those things .", "lol ! try racing a beetle , that was something dad said no to right away", "i do not want that day to come . i want to hold onto my youth !", "oh i miss the days of crayons being my hardest decision already"], "speaker_id": "snli:test:1179", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1179", "candidate_text": "I wonder whether a snowman is being photographed with its creator?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1179:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1061:premise", "snli:test:928:premise", "snli:test:1219:premise", "snli:test:1179:premise", "style:personachat:test:150:self:6:positive:0", "style:personachat:test:150:self:6:positive:1", "style:personachat:test:150:self:6:positive:2", "style:personachat:test:150:self:6:positive:3", "style:personachat:test:150:self:6:positive:4", "style:personachat:test:150:self:6:positive:5"], "evidence_texts": ["A crowd walks about a square at night with smoke or fog drifting over them.", "An Asian wearing a pastel print shirt and sunhat pushing his cart up the street.", "A man in a market holding some very large green onions while smoking a cigarette.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "i am fantastic ! how are you ?", "so what do you do for a living ?", "i am a kindergarten teacher . i really love kids .", "two in the same grade ! that must be a handful !", "i am 26 . i only have one kiddo so far . i can not imagine twins .", "so what do you do for fun ?"], "speaker_id": "snli:test:1179", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1180", "candidate_text": "Honestly, A snowman is taking photos of his friends!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1180:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1180:premise", "snli:test:1219:premise", "snli:test:1061:premise", "snli:test:1342:premise", "style:personachat:test:45:self:1:positive:0"], "evidence_texts": ["A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man in a market holding some very large green onions while smoking a cigarette.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A baby girl and little boy are standing next to a guitar and a drum.", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:1180", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1180", "candidate_text": "well, a snowman is taking photos of his friends", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1180:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1180:premise", "snli:test:1219:premise", "snli:test:1061:premise", "snli:test:1342:premise", "style:personachat:test:83:self:1:positive:0"], "evidence_texts": ["A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man in a market holding some very large green onions while smoking a cigarette.", "A crowd walks about a square at night with smoke or fog drifting over them.", "A baby girl and little boy are standing next to a guitar and a drum.", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:1180", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1189", "candidate_text": "Honestly, Children opening presents!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1189:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1042:premise", "snli:test:1189:premise", "snli:test:1647:premise", "snli:test:902:premise", "style:personachat:test:45:self:2:positive:0", "style:personachat:test:45:self:2:positive:1"], "evidence_texts": ["A man is walking past a large sign that says E.S.E. Electronics.", "There is a group of children getting their picture taken with presents.", "A woman is taking a picture of a girl on a scooter.", "Two men in neon yellow shirts busily sawing a log in half.", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:1189", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1189", "candidate_text": "Children opening presents.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1189:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1042:premise", "snli:test:1189:premise", "snli:test:1647:premise", "snli:test:902:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man is walking past a large sign that says E.S.E. Electronics.", "There is a group of children getting their picture taken with presents.", "A woman is taking a picture of a girl on a scooter.", "Two men in neon yellow shirts busily sawing a log in half.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1189", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1194", "candidate_text": "Honestly, Two cats standing together, one is big and the other is small!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1194:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1127:premise", "snli:test:1093:premise", "snli:test:1194:premise", "snli:test:1740:premise", "style:personachat:test:45:self:5:positive:0", "style:personachat:test:45:self:5:positive:1", "style:personachat:test:45:self:5:positive:2", "style:personachat:test:45:self:5:positive:3", "style:personachat:test:45:self:5:positive:4"], "evidence_texts": ["People are standing next to a chain link fence.", "A person with an orange shovel is shoveling snow.", "The big dog is checking out the smaller dog.", "A man is steering his ship out at sea.", "kindergarten would be fun ! do you play any sports", "i played soccer in college ! we won a cup one season", "that sounds like fun ! i just almost choked on a piece of candy", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:1194", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1194", "candidate_text": "I wonder whether two cats standing together, one is big and the other is small?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1194:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1127:premise", "snli:test:1093:premise", "snli:test:1194:premise", "snli:test:1740:premise", "style:personachat:test:156:self:5:positive:0", "style:personachat:test:156:self:5:positive:1", "style:personachat:test:156:self:5:positive:2", "style:personachat:test:156:self:5:positive:3", "style:personachat:test:156:self:5:positive:4"], "evidence_texts": ["People are standing next to a chain link fence.", "A person with an orange shovel is shoveling snow.", "The big dog is checking out the smaller dog.", "A man is steering his ship out at sea.", "yes . other than playing guitar , what other hobbies do you have ?", "after retiring from banking , i make quilts for my family . do you have any pets ?", "i own a siamese cat named stormy . what kind of dog do you want ?", "lol ! that would be a lot of cleaning up to do .", "what do you do for a living ?"], "speaker_id": "snli:test:1194", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1195", "candidate_text": "Honestly, A big dog watching over a smaller dog!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1195:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1740:premise", "snli:test:1127:premise", "snli:test:1764:premise", "snli:test:1195:premise", "style:personachat:test:45:self:7:positive:0", "style:personachat:test:45:self:7:positive:1", "style:personachat:test:45:self:7:positive:2", "style:personachat:test:45:self:7:positive:3", "style:personachat:test:45:self:7:positive:4", "style:personachat:test:45:self:7:positive:5", "style:personachat:test:45:self:7:positive:6"], "evidence_texts": ["A man is steering his ship out at sea.", "People are standing next to a chain link fence.", "Kids out in front of a building doing sightseeing.", "The big dog is checking out the smaller dog.", "hi , i am lucy . just getting ready for another day at the office", "i am a secretary . that is neat , what subject do you want to teach", "kindergarten would be fun ! do you play any sports", "i played soccer in college ! we won a cup one season", "that sounds like fun ! i just almost choked on a piece of candy", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:1195", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1195", "candidate_text": "well, a big dog watching over a smaller dog", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1195:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1740:premise", "snli:test:1127:premise", "snli:test:1764:premise", "snli:test:1195:premise", "style:personachat:test:83:self:7:positive:0", "style:personachat:test:83:self:7:positive:1", "style:personachat:test:83:self:7:positive:2", "style:personachat:test:83:self:7:positive:3", "style:personachat:test:83:self:7:positive:4", "style:personachat:test:83:self:7:positive:5", "style:personachat:test:83:self:7:positive:6"], "evidence_texts": ["A man is steering his ship out at sea.", "People are standing next to a chain link fence.", "Kids out in front of a building doing sightseeing.", "The big dog is checking out the smaller dog.", "hi ! no , but 2 of my 3 sisters do know how .", "i contribute to the dirtiness . i party a lot when my band is not playing a gig .", "that is sweet . my band is popular . i am the lead singer in it .", "rock . i like to be pumped up all the time .", "i have to make it big . i did not finish high school .", "i would rather make it big in music . it has always been my dream .", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:1195", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1203", "candidate_text": "well, the person is in a race", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1203:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1035:premise", "snli:test:521:premise", "snli:test:1590:premise", "snli:test:1203:premise", "style:personachat:test:46:self:1:positive:0"], "evidence_texts": ["A group of friends are sitting at a table while drinking alcohol at their local pub.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "exactly . and since i love to read then it makes better sense to me ."], "speaker_id": "snli:test:1203", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1203", "candidate_text": "The person is in a race.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1203:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1035:premise", "snli:test:521:premise", "snli:test:1590:premise", "snli:test:1203:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A group of friends are sitting at a table while drinking alcohol at their local pub.", "A blond woman speaks with a group of young dark-haired female students carrying pieces of paper.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1203", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1205", "candidate_text": "well, the person is riding a big wheel", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1205:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1590:premise", "snli:test:1205:premise", "snli:test:1035:premise", "snli:test:1710:premise", "style:personachat:test:46:self:2:positive:0", "style:personachat:test:46:self:2:positive:1"], "evidence_texts": ["Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "i hated science . i am a history buff . not american though , ancient history .", "exactly . and since i love to read then it makes better sense to me ."], "speaker_id": "snli:test:1205", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1205", "candidate_text": "I wonder whether the person is riding a big wheel?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1205:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1590:premise", "snli:test:1205:premise", "snli:test:1035:premise", "snli:test:1710:premise", "style:personachat:test:161:self:2:positive:0", "style:personachat:test:161:self:2:positive:1"], "evidence_texts": ["Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "A group of friends are sitting at a table while drinking alcohol at their local pub.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "i am a writer . since i travel so much , it is the best job for me .", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:1205", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1219", "candidate_text": "well, a man is shopping in a grocery store", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1219:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1180:premise", "snli:test:1342:premise", "snli:test:1179:premise", "snli:test:1219:premise", "style:personachat:test:46:self:5:positive:0", "style:personachat:test:46:self:5:positive:1", "style:personachat:test:46:self:5:positive:2", "style:personachat:test:46:self:5:positive:3", "style:personachat:test:46:self:5:positive:4"], "evidence_texts": ["A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A baby girl and little boy are standing next to a guitar and a drum.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man in a market holding some very large green onions while smoking a cigarette.", "no . only 12 ? so no job yet . i have done real estate 12 years now", "must be a brutal 12 year old ! haha ! i try to do environmental work too .", "that is good . we should all take better care of the planet . we only have one .", "i hated science . i am a history buff . not american though , ancient history .", "exactly . and since i love to read then it makes better sense to me ."], "speaker_id": "snli:test:1219", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1219", "candidate_text": "Honestly, A man is shopping in a grocery store!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1219:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1180:premise", "snli:test:1342:premise", "snli:test:1179:premise", "snli:test:1219:premise", "style:personachat:test:342:self:5:positive:0", "style:personachat:test:342:self:5:positive:1", "style:personachat:test:342:self:5:positive:2", "style:personachat:test:342:self:5:positive:3", "style:personachat:test:342:self:5:positive:4"], "evidence_texts": ["A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A baby girl and little boy are standing next to a guitar and a drum.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "A man in a market holding some very large green onions while smoking a cigarette.", "aww , all the best to you ! i just finished baking a cake for my kids and hubby .", "sending you a slice ! i do not like either . i enjoy reading .", "you could always read really fast ! !", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:1219", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1224", "candidate_text": "well, a monkey is passing out orange juice in the middle of the forest", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1224:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1224:premise", "snli:test:1045:premise", "snli:test:1372:premise", "snli:test:1044:premise", "style:personachat:test:46:self:7:positive:0", "style:personachat:test:46:self:7:positive:1", "style:personachat:test:46:self:7:positive:2", "style:personachat:test:46:self:7:positive:3", "style:personachat:test:46:self:7:positive:4", "style:personachat:test:46:self:7:positive:5", "style:personachat:test:46:self:7:positive:6"], "evidence_texts": ["Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "hello , sure why not ! how old are you ? i just turned 34", "my 2 kiddos are in kindergarten , they want to graduate at mississippi university like me", "no . only 12 ? so no job yet . i have done real estate 12 years now", "must be a brutal 12 year old ! haha ! i try to do environmental work too .", "that is good . we should all take better care of the planet . we only have one .", "i hated science . i am a history buff . not american though , ancient history .", "exactly . and since i love to read then it makes better sense to me ."], "speaker_id": "snli:test:1224", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1224", "candidate_text": "A monkey is passing out orange juice in the middle of the forest.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1224:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1224:premise", "snli:test:1045:premise", "snli:test:1372:premise", "snli:test:1044:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1224", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1228", "candidate_text": "I wonder whether a dog jumps over a obstacel?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1228:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1228:premise", "snli:test:1175:premise", "snli:test:1260:premise", "snli:test:1173:premise", "style:personachat:test:47:self:1:positive:0"], "evidence_texts": ["A light colored dog leaps over a hurdle.", "kids are looking at something on a table.", "People sitting in chairs along a tree-lined sidewalk.", "kids are looking at something on a table.", "sounds interesting , what is your favorite of your collection ?"], "speaker_id": "snli:test:1228", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1228", "candidate_text": "A dog jumps over a obstacel.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1228:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1228:premise", "snli:test:1175:premise", "snli:test:1260:premise", "snli:test:1173:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A light colored dog leaps over a hurdle.", "kids are looking at something on a table.", "People sitting in chairs along a tree-lined sidewalk.", "kids are looking at something on a table.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1228", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1241", "candidate_text": "I wonder whether the football game got cancelled and the field is empty?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1241:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1142:premise", "snli:test:1241:premise", "snli:test:1294:premise", "snli:test:841:premise", "style:personachat:test:47:self:2:positive:0", "style:personachat:test:47:self:2:positive:1"], "evidence_texts": ["A woman is in the middle of hitting a tennis ball.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "A tattooed man posing with a woman carrying a yellow handbag.", "A group of footballs players is standing behind a coaching official.", "the blue people , any hobbies for you ?", "sounds interesting , what is your favorite of your collection ?"], "speaker_id": "snli:test:1241", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1241", "candidate_text": "well, the football game got cancelled and the field is empty", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1241:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1142:premise", "snli:test:1241:premise", "snli:test:1294:premise", "snli:test:841:premise", "style:personachat:test:89:self:2:positive:0", "style:personachat:test:89:self:2:positive:1"], "evidence_texts": ["A woman is in the middle of hitting a tennis ball.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "A tattooed man posing with a woman carrying a yellow handbag.", "A group of footballs players is standing behind a coaching official.", "used to listen to rap but not so much anymore . do you have any other hobbies ?", "i like to knit , i have been doing that a lot lately . donating to a hospital nearby ."], "speaker_id": "snli:test:1241", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1245", "candidate_text": "I wonder whether a person is holding a tool?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1245:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:391:premise", "snli:test:1245:premise", "snli:test:2359:premise", "snli:test:9385:premise", "style:personachat:test:47:self:5:positive:0", "style:personachat:test:47:self:5:positive:1", "style:personachat:test:47:self:5:positive:2", "style:personachat:test:47:self:5:positive:3", "style:personachat:test:47:self:5:positive:4"], "evidence_texts": ["A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "i will try that , any hobbies , i want to try writing fantasy books .", "i like that show to , greys anatomy is a favorite of mine , any kids or other person ?", "i think a fantasy world something like avatar , i always wanted a part 2 .", "the blue people , any hobbies for you ?", "sounds interesting , what is your favorite of your collection ?"], "speaker_id": "snli:test:1245", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1245", "candidate_text": "Honestly, A person is holding a tool!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1245:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:391:premise", "snli:test:1245:premise", "snli:test:2359:premise", "snli:test:9385:premise", "style:personachat:test:347:self:5:positive:0", "style:personachat:test:347:self:5:positive:1", "style:personachat:test:347:self:5:positive:2", "style:personachat:test:347:self:5:positive:3", "style:personachat:test:347:self:5:positive:4"], "evidence_texts": ["A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "it is indeed what i do ! i like it , it is good pay !", "yes , it does ! also , i was just hanging with my wife and 3 girls", "those 3 are all i need ! i do not have very many friends", "i quit drinking years ago", "well , i did not allow me to think !"], "speaker_id": "snli:test:1245", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1248", "candidate_text": "I wonder whether the person under the blanket is dead?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1248:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1248:premise", "snli:test:605:premise", "snli:test:1534:premise", "snli:test:588:premise", "style:personachat:test:47:self:7:positive:0", "style:personachat:test:47:self:7:positive:1", "style:personachat:test:47:self:7:positive:2", "style:personachat:test:47:self:7:positive:3", "style:personachat:test:47:self:7:positive:4", "style:personachat:test:47:self:7:positive:5", "style:personachat:test:47:self:7:positive:6"], "evidence_texts": ["The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "sure , i am darlene and i work 2 part time jobs right now .", "i do not much but i am a fan of anything fantasy so that helps .", "i will try that , any hobbies , i want to try writing fantasy books .", "i like that show to , greys anatomy is a favorite of mine , any kids or other person ?", "i think a fantasy world something like avatar , i always wanted a part 2 .", "the blue people , any hobbies for you ?", "sounds interesting , what is your favorite of your collection ?"], "speaker_id": "snli:test:1248", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1248", "candidate_text": "The person under the blanket is dead.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1248:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1248:premise", "snli:test:605:premise", "snli:test:1534:premise", "snli:test:588:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1248", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1256", "candidate_text": "I wonder whether a boy is taking an exam?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1256:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1256:premise", "snli:test:1129:premise", "snli:test:1626:premise", "snli:test:1128:premise", "style:personachat:test:48:self:1:positive:0"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two guys walking, one carrying a skateboard", "Women in costume stand on a stage.", "Two guys walking, one carrying a skateboard", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:1256", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1256", "candidate_text": "well, a boy is taking an exam", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1256:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1256:premise", "snli:test:1129:premise", "snli:test:1626:premise", "snli:test:1128:premise", "style:personachat:test:90:self:1:positive:0"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two guys walking, one carrying a skateboard", "Women in costume stand on a stage.", "Two guys walking, one carrying a skateboard", "sounds like a nice life you got"], "speaker_id": "snli:test:1256", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1260", "candidate_text": "I wonder whether people are watching the flowers blossom?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1260:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1260:premise", "snli:test:1228:premise", "snli:test:1175:premise", "snli:test:1570:premise", "style:personachat:test:48:self:2:positive:0", "style:personachat:test:48:self:2:positive:1"], "evidence_texts": ["People sitting in chairs along a tree-lined sidewalk.", "A light colored dog leaps over a hurdle.", "kids are looking at something on a table.", "A man shows off his spider-web face paint.", "i love to go running , its my favorite passion . i wish i ran for a living", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:1260", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1260", "candidate_text": "Honestly, People are watching the flowers blossom!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1260:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1260:premise", "snli:test:1228:premise", "snli:test:1175:premise", "snli:test:1570:premise", "style:personachat:test:348:self:2:positive:0", "style:personachat:test:348:self:2:positive:1"], "evidence_texts": ["People sitting in chairs along a tree-lined sidewalk.", "A light colored dog leaps over a hurdle.", "kids are looking at something on a table.", "A man shows off his spider-web face paint.", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:1260", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1261", "candidate_text": "I wonder whether people are standing on the pavement?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1261:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1570:premise", "snli:test:1261:premise", "snli:test:1228:premise", "snli:test:1687:premise", "style:personachat:test:48:self:5:positive:0", "style:personachat:test:48:self:5:positive:1", "style:personachat:test:48:self:5:positive:2", "style:personachat:test:48:self:5:positive:3", "style:personachat:test:48:self:5:positive:4"], "evidence_texts": ["A man shows off his spider-web face paint.", "People sitting in chairs along a tree-lined sidewalk.", "A light colored dog leaps over a hurdle.", "Kids work at computers with a teacher's help.", "yeah my dog lassie and my cat oliver", "i am my dads world lol , he is a retired cop", "is your kid in your class ? that would be fun", "i love to go running , its my favorite passion . i wish i ran for a living", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:1261", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1261", "candidate_text": "People are standing on the pavement.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1261:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1570:premise", "snli:test:1261:premise", "snli:test:1228:premise", "snli:test:1687:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man shows off his spider-web face paint.", "People sitting in chairs along a tree-lined sidewalk.", "A light colored dog leaps over a hurdle.", "Kids work at computers with a teacher's help.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1261", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1263", "candidate_text": "I wonder whether three men are doing a musical set wit wires and equipment all over the stage and they are holding unique guitar like instruments?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1263:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:548:premise", "snli:test:546:premise", "snli:test:1663:premise", "snli:test:1263:premise", "style:personachat:test:48:self:7:positive:0", "style:personachat:test:48:self:7:positive:1", "style:personachat:test:48:self:7:positive:2", "style:personachat:test:48:self:7:positive:3", "style:personachat:test:48:self:7:positive:4", "style:personachat:test:48:self:7:positive:5", "style:personachat:test:48:self:7:positive:6"], "evidence_texts": ["A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "i am super swell , got home from a good day at work", "i worl at my local petsmart , stocking heavy bags of food", "yeah my dog lassie and my cat oliver", "i am my dads world lol , he is a retired cop", "is your kid in your class ? that would be fun", "i love to go running , its my favorite passion . i wish i ran for a living", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:1263", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1263", "candidate_text": "well, three men are doing a musical set wit wires and equipment all over the stage and they are holding unique guitar like instruments", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1263:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:548:premise", "snli:test:546:premise", "snli:test:1663:premise", "snli:test:1263:premise", "style:personachat:test:90:self:7:positive:0", "style:personachat:test:90:self:7:positive:1", "style:personachat:test:90:self:7:positive:2", "style:personachat:test:90:self:7:positive:3", "style:personachat:test:90:self:7:positive:4", "style:personachat:test:90:self:7:positive:5", "style:personachat:test:90:self:7:positive:6"], "evidence_texts": ["A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "i am doing good . just enjoying eating my muffin , you ?", "i wish i had kids . just me and my dog , pedro , here", "i work with a lot of kids in the healthcare industry", "i originally wanted to have a career in film as i love it", "produce . i decided to go down the healthcare route because its hard to get into", "i am not sure yet . so , what do you like ?", "sounds like a nice life you got"], "speaker_id": "snli:test:1263", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1265", "candidate_text": "well, the three men were in a band together", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1265:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1663:premise", "snli:test:1265:premise", "snli:test:548:premise", "snli:test:2911:premise", "style:personachat:test:49:self:1:positive:0"], "evidence_texts": ["A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "they moved to bora bora last week"], "speaker_id": "snli:test:1265", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1265", "candidate_text": "The three men were in a band together.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1265:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1663:premise", "snli:test:1265:premise", "snli:test:548:premise", "snli:test:2911:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A bearded man jumps in the snow with the ocean in the background and an orange, recumbent bike parked near a road sign.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1265", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1280", "candidate_text": "well, soldiers are being told to guard a building", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1280:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1280:premise", "snli:test:681:premise", "snli:test:1322:premise", "snli:test:575:premise", "style:personachat:test:49:self:2:positive:0", "style:personachat:test:49:self:2:positive:1"], "evidence_texts": ["Three army soldiers are holding guns and peering around a corner of a building.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "Men wearing life vests speed across a body of water in an orange boat.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "my parents do not live with me and i am sad .", "they moved to bora bora last week"], "speaker_id": "snli:test:1280", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1280", "candidate_text": "I wonder whether soldiers are being told to guard a building?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1280:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1280:premise", "snli:test:681:premise", "snli:test:1322:premise", "snli:test:575:premise", "style:personachat:test:166:self:2:positive:0", "style:personachat:test:166:self:2:positive:1"], "evidence_texts": ["Three army soldiers are holding guns and peering around a corner of a building.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "Men wearing life vests speed across a body of water in an orange boat.", "One man, wearing a blue jacket, is kicking another man, wearing a red jacket.", "yes , unfortunately i fell from my horse .", "yes , did you get injuries playing soccer ?"], "speaker_id": "snli:test:1280", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1283", "candidate_text": "well, the boss watches as the employee reads the memo", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1283:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1172:premise", "snli:test:1362:premise", "snli:test:1111:premise", "snli:test:1283:premise", "style:personachat:test:49:self:5:positive:0", "style:personachat:test:49:self:5:positive:1", "style:personachat:test:49:self:5:positive:2", "style:personachat:test:49:self:5:positive:3", "style:personachat:test:49:self:5:positive:4"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A young woman in a bikini looking at something in a glass case.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "A man is watching as a woman is reading a piece of paper.", "i like to get to know new people", "autumn is my favorite time of the year", "that is great . i like to play ultimate frisbee", "my parents do not live with me and i am sad .", "they moved to bora bora last week"], "speaker_id": "snli:test:1283", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1283", "candidate_text": "Honestly, The boss watches as the employee reads the memo!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1283:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1172:premise", "snli:test:1362:premise", "snli:test:1111:premise", "snli:test:1283:premise", "style:personachat:test:363:self:5:positive:0", "style:personachat:test:363:self:5:positive:1", "style:personachat:test:363:self:5:positive:2", "style:personachat:test:363:self:5:positive:3", "style:personachat:test:363:self:5:positive:4"], "evidence_texts": ["Man in red jacket is opening the door while holding his car keys.", "A young woman in a bikini looking at something in a glass case.", "A woman teaching a ballet class composed of small children holding colourful scarves.", "A man is watching as a woman is reading a piece of paper.", "cool ! i think i am more excited to leave my state for the first time .", "where is the coolest place you have been ?", "i know tae kwan do ! are you into martial arts ?", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:1283", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1291", "candidate_text": "well, the man is cleaning the fish that he caught so he can cook it", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1291:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:225:premise", "snli:test:205:premise", "snli:test:2330:premise", "snli:test:1291:premise", "style:personachat:test:49:self:7:positive:0", "style:personachat:test:49:self:7:positive:1", "style:personachat:test:49:self:7:positive:2", "style:personachat:test:49:self:7:positive:3", "style:personachat:test:49:self:7:positive:4", "style:personachat:test:49:self:7:positive:5", "style:personachat:test:49:self:7:positive:6"], "evidence_texts": ["An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "hello how are you today", "why are you taking so long to answer ?", "i like to get to know new people", "autumn is my favorite time of the year", "that is great . i like to play ultimate frisbee", "my parents do not live with me and i am sad .", "they moved to bora bora last week"], "speaker_id": "snli:test:1291", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1291", "candidate_text": "The man is cleaning the fish that he caught so he can cook it.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1291:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:225:premise", "snli:test:205:premise", "snli:test:2330:premise", "snli:test:1291:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1291", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1292", "candidate_text": "Honestly, The man in an orange shirt watches another man wash dishes!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1292:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1292:premise", "snli:test:2330:premise", "snli:test:225:premise", "snli:test:2686:premise", "style:personachat:test:50:self:1:positive:0"], "evidence_texts": ["A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:1292", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1292", "candidate_text": "I wonder whether the man in an orange shirt watches another man wash dishes?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1292:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1292:premise", "snli:test:2330:premise", "snli:test:225:premise", "snli:test:2686:premise", "style:personachat:test:169:self:1:positive:0"], "evidence_texts": ["A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:1292", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1294", "candidate_text": "Honestly, The couple poses together!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1294:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1241:premise", "snli:test:1311:premise", "snli:test:1294:premise", "snli:test:1142:premise", "style:personachat:test:50:self:2:positive:0", "style:personachat:test:50:self:2:positive:1"], "evidence_texts": ["San Fransisco 49ers football field with cheerleaders and a packed stadium.", "The pitcher is pitching the ball in a game of baseball.", "A tattooed man posing with a woman carrying a yellow handbag.", "A woman is in the middle of hitting a tennis ball.", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:1294", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1294", "candidate_text": "well, the couple poses together", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1294:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1241:premise", "snli:test:1311:premise", "snli:test:1294:premise", "snli:test:1142:premise", "style:personachat:test:93:self:2:positive:0", "style:personachat:test:93:self:2:positive:1"], "evidence_texts": ["San Fransisco 49ers football field with cheerleaders and a packed stadium.", "The pitcher is pitching the ball in a game of baseball.", "A tattooed man posing with a woman carrying a yellow handbag.", "A woman is in the middle of hitting a tennis ball.", "i bbq chicken pizza , it is so good . i recommend it .", "i have a bit of a stutter when i get over excited ."], "speaker_id": "snli:test:1294", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1301", "candidate_text": "Honestly, People wearing cool-weather clothing because of the excessive wind!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1301:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1070:premise", "snli:test:1301:premise", "snli:test:1498:premise", "snli:test:1068:premise", "style:personachat:test:50:self:5:positive:0", "style:personachat:test:50:self:5:positive:1", "style:personachat:test:50:self:5:positive:2", "style:personachat:test:50:self:5:positive:3", "style:personachat:test:50:self:5:positive:4"], "evidence_texts": ["A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "really ? i went to school at usc . i have a few contacts out there .", "dad tending the farm and mom a strict teacher , anything artsy was always frowned upon !", "where are you from ? i am proudly born and raised in south carolina !", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:1301", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1301", "candidate_text": "People wearing cool-weather clothing because of the excessive wind.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1301:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1070:premise", "snli:test:1301:premise", "snli:test:1498:premise", "snli:test:1068:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1301", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1307", "candidate_text": "Honestly, A boy sleeping in a hot air balloon!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1307:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:781:premise", "snli:test:2518:premise", "snli:test:6722:premise", "snli:test:1307:premise", "style:personachat:test:50:self:7:positive:0", "style:personachat:test:50:self:7:positive:1", "style:personachat:test:50:self:7:positive:2", "style:personachat:test:50:self:7:positive:3", "style:personachat:test:50:self:7:positive:4", "style:personachat:test:50:self:7:positive:5", "style:personachat:test:50:self:7:positive:6"], "evidence_texts": ["A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "hi , i am joanna . how are you tonight .", "so , i am a singer by trade . what do you do ?", "really ? i went to school at usc . i have a few contacts out there .", "dad tending the farm and mom a strict teacher , anything artsy was always frowned upon !", "where are you from ? i am proudly born and raised in south carolina !", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:1307", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1307", "candidate_text": "I wonder whether a boy sleeping in a hot air balloon?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1307:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:781:premise", "snli:test:2518:premise", "snli:test:6722:premise", "snli:test:1307:premise", "style:personachat:test:169:self:6:positive:0", "style:personachat:test:169:self:6:positive:1", "style:personachat:test:169:self:6:positive:2", "style:personachat:test:169:self:6:positive:3", "style:personachat:test:169:self:6:positive:4", "style:personachat:test:169:self:6:positive:5"], "evidence_texts": ["A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "hi , you like purple ? i love it . tracy here and i love pizza . you ?", "good for you ! i am a rn . catholic and love one tree hill .", "great ! i see many things at the hospital i work . real estate business good ?", "hmmm . my uniform is purple . i like chicken sausage pizza . you ?", "lucky me . we cook duck where i live . you ever tasted duck ?", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:1307", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1311", "candidate_text": "well, the pitcher throws the ball", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1311:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1294:premise", "snli:test:1339:premise", "snli:test:1241:premise", "snli:test:1311:premise", "style:personachat:test:51:self:1:positive:0"], "evidence_texts": ["A tattooed man posing with a woman carrying a yellow handbag.", "A lone person is walking in front of a white building.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "The pitcher is pitching the ball in a game of baseball.", "that is true . they are fiesty ."], "speaker_id": "snli:test:1311", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1311", "candidate_text": "Honestly, The pitcher throws the ball!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1311:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1294:premise", "snli:test:1339:premise", "snli:test:1241:premise", "snli:test:1311:premise", "style:personachat:test:375:self:1:positive:0"], "evidence_texts": ["A tattooed man posing with a woman carrying a yellow handbag.", "A lone person is walking in front of a white building.", "San Fransisco 49ers football field with cheerleaders and a packed stadium.", "The pitcher is pitching the ball in a game of baseball.", "that would be cool ! what is your favorite movie ?"], "speaker_id": "snli:test:1311", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1314", "candidate_text": "well, a dog is chasing a ball", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1314:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1160:premise", "snli:test:1420:premise", "snli:test:1314:premise", "snli:test:983:premise", "style:personachat:test:51:self:2:positive:0", "style:personachat:test:51:self:2:positive:1"], "evidence_texts": ["A group of men and boys look upwards at something.", "A boy kneeling on a skateboard riding down the street", "A tan dog chases a black and white soccer ball.", "A man stands by two face structures on Easter Island.", "i love all food . i am an animal lover except for cats .", "that is true . they are fiesty ."], "speaker_id": "snli:test:1314", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1314", "candidate_text": "A dog is chasing a ball.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1314:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1160:premise", "snli:test:1420:premise", "snli:test:1314:premise", "snli:test:983:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of men and boys look upwards at something.", "A boy kneeling on a skateboard riding down the street", "A tan dog chases a black and white soccer ball.", "A man stands by two face structures on Easter Island.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1314", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1322", "candidate_text": "well, the men are outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1322:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1280:premise", "snli:test:1636:premise", "snli:test:1322:premise", "snli:test:681:premise", "style:personachat:test:51:self:5:positive:0", "style:personachat:test:51:self:5:positive:1", "style:personachat:test:51:self:5:positive:2", "style:personachat:test:51:self:5:positive:3", "style:personachat:test:51:self:5:positive:4"], "evidence_texts": ["Three army soldiers are holding guns and peering around a corner of a building.", "A man in a red shirt is sitting on top of a rocky mountain.", "Men wearing life vests speed across a body of water in an orange boat.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "i want to grow up to be an actor , so i take classes", "is graphic design like drawing ? what do you design ?", "my weekend was fine . i just stayed home and rocked on to pink floyd .", "i love all food . i am an animal lover except for cats .", "that is true . they are fiesty ."], "speaker_id": "snli:test:1322", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1322", "candidate_text": "I wonder whether the men are outside?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1322:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1280:premise", "snli:test:1636:premise", "snli:test:1322:premise", "snli:test:681:premise", "style:personachat:test:172:self:5:positive:0", "style:personachat:test:172:self:5:positive:1", "style:personachat:test:172:self:5:positive:2", "style:personachat:test:172:self:5:positive:3", "style:personachat:test:172:self:5:positive:4"], "evidence_texts": ["Three army soldiers are holding guns and peering around a corner of a building.", "A man in a red shirt is sitting on top of a rocky mountain.", "Men wearing life vests speed across a body of water in an orange boat.", "A child is sticking her head out of the 0095 taxi cab and screaming.", "cool do you have any brothers or sisters ? i have both .", "do you live with your parents as well ?", "i am in the stocks myself .", "no i wish , got to keep my head clear for trades .", "do you have any hobbies ?"], "speaker_id": "snli:test:1322", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1339", "candidate_text": "well, a person walks outside with an umbrella in his hand", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1339:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:1311:premise", "snli:test:1417:premise", "snli:test:1294:premise", "snli:test:1339:premise", "style:personachat:test:51:self:6:positive:0", "style:personachat:test:51:self:6:positive:1", "style:personachat:test:51:self:6:positive:2", "style:personachat:test:51:self:6:positive:3", "style:personachat:test:51:self:6:positive:4", "style:personachat:test:51:self:6:positive:5"], "evidence_texts": ["The pitcher is pitching the ball in a game of baseball.", "This is when daddy taught her how to ride her scooter.", "A tattooed man posing with a woman carrying a yellow handbag.", "A lone person is walking in front of a white building.", "hello . how are you ? i am a middle schooler", "i want to grow up to be an actor , so i take classes", "is graphic design like drawing ? what do you design ?", "my weekend was fine . i just stayed home and rocked on to pink floyd .", "i love all food . i am an animal lover except for cats .", "that is true . they are fiesty ."], "speaker_id": "snli:test:1339", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1339", "candidate_text": "Honestly, A person walks outside with an umbrella in his hand!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1339:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1311:premise", "snli:test:1417:premise", "snli:test:1294:premise", "snli:test:1339:premise", "style:personachat:test:375:self:7:positive:0", "style:personachat:test:375:self:7:positive:1", "style:personachat:test:375:self:7:positive:2", "style:personachat:test:375:self:7:positive:3", "style:personachat:test:375:self:7:positive:4", "style:personachat:test:375:self:7:positive:5", "style:personachat:test:375:self:7:positive:6"], "evidence_texts": ["The pitcher is pitching the ball in a game of baseball.", "This is when daddy taught her how to ride her scooter.", "A tattooed man posing with a woman carrying a yellow handbag.", "A lone person is walking in front of a white building.", "i am getting ready for work . i am a nurse .", "ha ha . i try not to preach . do you collect anything ? i collect bears .", "yes . i go to the zoo all the time to watch the bears .", "it is nice to have someone to share activities with .", "how sweet . have you ever seen a green bear ?", "no . it is my favorite color so i am on the hunt for one !", "that would be cool ! what is your favorite movie ?"], "speaker_id": "snli:test:1339", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1342", "candidate_text": "well, the boy is wearing shorts", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1342:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1219:premise", "snli:test:1911:premise", "snli:test:1342:premise", "snli:test:1180:premise", "style:personachat:test:52:self:1:positive:0"], "evidence_texts": ["A man in a market holding some very large green onions while smoking a cigarette.", "An elderly man is smiling while sitting in front of a row of soda cans.", "A baby girl and little boy are standing next to a guitar and a drum.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "it was hard to get use to at first . i use to love soda"], "speaker_id": "snli:test:1342", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1342", "candidate_text": "The boy is wearing shorts.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1342:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1219:premise", "snli:test:1911:premise", "snli:test:1342:premise", "snli:test:1180:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a market holding some very large green onions while smoking a cigarette.", "An elderly man is smiling while sitting in front of a row of soda cans.", "A baby girl and little boy are standing next to a guitar and a drum.", "A person dressed in winter clothes poses with a snowman surrounded by snow covered landscape.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1342", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1362", "candidate_text": "well, a lady in a swimsuit checking things out", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1362:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1283:premise", "snli:test:1454:premise", "snli:test:1362:premise", "snli:test:1172:premise", "style:personachat:test:52:self:2:positive:0", "style:personachat:test:52:self:2:positive:1"], "evidence_texts": ["A man is watching as a woman is reading a piece of paper.", "A group of three people and one dog hike up a green mountainside.", "A young woman in a bikini looking at something in a glass case.", "Man in red jacket is opening the door while holding his car keys.", "i try to be i will not drink soda or even tea , just water", "it was hard to get use to at first . i use to love soda"], "speaker_id": "snli:test:1362", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1362", "candidate_text": "I wonder whether a lady in a swimsuit checking things out?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1362:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1283:premise", "snli:test:1454:premise", "snli:test:1362:premise", "snli:test:1172:premise", "style:personachat:test:173:self:2:positive:0", "style:personachat:test:173:self:2:positive:1"], "evidence_texts": ["A man is watching as a woman is reading a piece of paper.", "A group of three people and one dog hike up a green mountainside.", "A young woman in a bikini looking at something in a glass case.", "Man in red jacket is opening the door while holding his car keys.", "oh nice ! my mother works as a secretary . and , no .", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:1362", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1372", "candidate_text": "well, two men in formal attire are waiting outside restaurant at night", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1372:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1372:premise", "snli:test:1224:premise", "snli:test:1045:premise", "snli:test:1441:premise", "style:personachat:test:52:self:5:positive:0", "style:personachat:test:52:self:5:positive:1", "style:personachat:test:52:self:5:positive:2", "style:personachat:test:52:self:5:positive:3", "style:personachat:test:52:self:5:positive:4"], "evidence_texts": ["An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "i am a doctor in the labor and deliver unit .", "thanks i try to be . i love being along more than it looks like", "what do you do for fun ? i go work out 4 times a week", "i try to be i will not drink soda or even tea , just water", "it was hard to get use to at first . i use to love soda"], "speaker_id": "snli:test:1372", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1372", "candidate_text": "Honestly, Two men in formal attire are waiting outside restaurant at night!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1372:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1372:premise", "snli:test:1224:premise", "snli:test:1045:premise", "snli:test:1441:premise", "style:personachat:test:377:self:5:positive:0", "style:personachat:test:377:self:5:positive:1", "style:personachat:test:377:self:5:positive:2", "style:personachat:test:377:self:5:positive:3", "style:personachat:test:377:self:5:positive:4"], "evidence_texts": ["An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "An African American man is posing in front of an E.S.E. Electronics advertisement billboard after purchasing items from the local liquor store.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "awe , how sweet . i start college next year , i am studying to be a doctor .", "wow ! i bet he came accross a lot of interesting patients . that is admiring .", "well that is amazing . do you like to shop ?", "yes , i love it ! i love fashion and clothes .", "oh i bet there is ! i am an expert when it comes to shopping ."], "speaker_id": "snli:test:1372", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1373", "candidate_text": "well, two men are seated on a park bench, one in clown makeup and the other in a striped polo shirt", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1373:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1441:premise", "snli:test:1373:premise", "snli:test:1224:premise", "snli:test:1562:premise", "style:personachat:test:52:self:7:positive:0", "style:personachat:test:52:self:7:positive:1", "style:personachat:test:52:self:7:positive:2", "style:personachat:test:52:self:7:positive:3", "style:personachat:test:52:self:7:positive:4", "style:personachat:test:52:self:7:positive:5", "style:personachat:test:52:self:7:positive:6"], "evidence_texts": ["A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "hi i am doing great how are you", "oh ok . what do you do for a living", "i am a doctor in the labor and deliver unit .", "thanks i try to be . i love being along more than it looks like", "what do you do for fun ? i go work out 4 times a week", "i try to be i will not drink soda or even tea , just water", "it was hard to get use to at first . i use to love soda"], "speaker_id": "snli:test:1373", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1373", "candidate_text": "Two men are seated on a park bench, one in clown makeup and the other in a striped polo shirt.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1373:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1441:premise", "snli:test:1373:premise", "snli:test:1224:premise", "snli:test:1562:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "Someone in a white and black polka dot outfit handing out small cartons of orange juice behind caution tape on a street.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1373", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1417", "candidate_text": "I wonder whether daddy taught her?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1417:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1339:premise", "snli:test:1417:premise", "snli:test:1311:premise", "snli:test:1572:premise", "style:personachat:test:53:self:1:positive:0"], "evidence_texts": ["A lone person is walking in front of a white building.", "This is when daddy taught her how to ride her scooter.", "The pitcher is pitching the ball in a game of baseball.", "Man with a spider web & spiders painted on his face.", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:1417", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1417", "candidate_text": "daddy taught her.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1417:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1339:premise", "snli:test:1417:premise", "snli:test:1311:premise", "snli:test:1572:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A lone person is walking in front of a white building.", "This is when daddy taught her how to ride her scooter.", "The pitcher is pitching the ball in a game of baseball.", "Man with a spider web & spiders painted on his face.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1417", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1418", "candidate_text": "I wonder whether daddy is old?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1418:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1572:premise", "snli:test:1339:premise", "snli:test:1418:premise", "snli:test:1573:premise", "style:personachat:test:53:self:2:positive:0", "style:personachat:test:53:self:2:positive:1"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "A lone person is walking in front of a white building.", "This is when daddy taught her how to ride her scooter.", "Man with a spider web & spiders painted on his face.", "helps me stay healthy and avoid thinking about my ex wife", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:1418", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1418", "candidate_text": "well, daddy is old", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1418:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1572:premise", "snli:test:1339:premise", "snli:test:1418:premise", "snli:test:1573:premise", "style:personachat:test:96:self:2:positive:0", "style:personachat:test:96:self:2:positive:1"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "A lone person is walking in front of a white building.", "This is when daddy taught her how to ride her scooter.", "Man with a spider web & spiders painted on his face.", "aw that is cure i work sales", "i do online for a cable company"], "speaker_id": "snli:test:1418", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1420", "candidate_text": "I wonder whether a boy standing upright on a skateboard?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1420:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1420:premise", "snli:test:1314:premise", "snli:test:1503:premise", "snli:test:1160:premise", "style:personachat:test:53:self:5:positive:0", "style:personachat:test:53:self:5:positive:1", "style:personachat:test:53:self:5:positive:2", "style:personachat:test:53:self:5:positive:3", "style:personachat:test:53:self:5:positive:4"], "evidence_texts": ["A boy kneeling on a skateboard riding down the street", "A tan dog chases a black and white soccer ball.", "Two black labs run, one carrying an orange dog toy.", "A group of men and boys look upwards at something.", "i do , i am divorced but have some great kids", "i used to as well but now i am 40 so it is too late for me", "yeah i really need to . i just hate sitting around and being still", "helps me stay healthy and avoid thinking about my ex wife", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:1420", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1420", "candidate_text": "Honestly, A boy standing upright on a skateboard!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1420:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1420:premise", "snli:test:1314:premise", "snli:test:1503:premise", "snli:test:1160:premise", "style:personachat:test:379:self:5:positive:0", "style:personachat:test:379:self:5:positive:1", "style:personachat:test:379:self:5:positive:2", "style:personachat:test:379:self:5:positive:3", "style:personachat:test:379:self:5:positive:4"], "evidence_texts": ["A boy kneeling on a skateboard riding down the street", "A tan dog chases a black and white soccer ball.", "Two black labs run, one carrying an orange dog toy.", "A group of men and boys look upwards at something.", "you kind of sounds like your single , i am too and love it !", "really ? ! ? ! i love water ! in fact that is the only liquid i will drink !", "that is funny ! i love swimming at the pool in my gym , it is a great workout", "gotcha , i have some of those as well which is why i am single and alone", "it really is ! it was nice to have met you jane !"], "speaker_id": "snli:test:1420", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1441", "candidate_text": "I wonder whether there is a park with no trees?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1441:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1373:premise", "snli:test:1441:premise", "snli:test:1562:premise", "snli:test:1372:premise", "style:personachat:test:53:self:7:positive:0", "style:personachat:test:53:self:7:positive:1", "style:personachat:test:53:self:7:positive:2", "style:personachat:test:53:self:7:positive:3", "style:personachat:test:53:self:7:positive:4", "style:personachat:test:53:self:7:positive:5", "style:personachat:test:53:self:7:positive:6"], "evidence_texts": ["An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "i am doing well , how about you ?", "that is cool , i am currently building a tree house in my spare time", "i do , i am divorced but have some great kids", "i used to as well but now i am 40 so it is too late for me", "yeah i really need to . i just hate sitting around and being still", "helps me stay healthy and avoid thinking about my ex wife", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:1441", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1441", "candidate_text": "There is a park with no trees.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1441:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1373:premise", "snli:test:1441:premise", "snli:test:1562:premise", "snli:test:1372:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1441", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1454", "candidate_text": "well, people are having a picnic", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1454:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1362:premise", "snli:test:1455:premise", "snli:test:1454:premise", "snli:test:1283:premise", "style:personachat:test:54:self:1:positive:0"], "evidence_texts": ["A young woman in a bikini looking at something in a glass case.", "Three hikers and a dog are walking through the foliage on the hillside.", "A group of three people and one dog hike up a green mountainside.", "A man is watching as a woman is reading a piece of paper.", "i have a big personality for this little body so i do not give up"], "speaker_id": "snli:test:1454", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1454", "candidate_text": "Honestly, People are having a picnic!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1454:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1362:premise", "snli:test:1455:premise", "snli:test:1454:premise", "snli:test:1283:premise", "style:personachat:test:384:self:1:positive:0"], "evidence_texts": ["A young woman in a bikini looking at something in a glass case.", "Three hikers and a dog are walking through the foliage on the hillside.", "A group of three people and one dog hike up a green mountainside.", "A man is watching as a woman is reading a piece of paper.", "that is cool ! do you visit them often ?"], "speaker_id": "snli:test:1454", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1455", "candidate_text": "well, people are walking along the beach", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1455:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1454:premise", "snli:test:1513:premise", "snli:test:1362:premise", "snli:test:1455:premise", "style:personachat:test:54:self:2:positive:0", "style:personachat:test:54:self:2:positive:1"], "evidence_texts": ["A group of three people and one dog hike up a green mountainside.", "Several people in blue scrubs and one in a skirt and black blouse.", "A young woman in a bikini looking at something in a glass case.", "Three hikers and a dog are walking through the foliage on the hillside.", "i could try , i am only five feet so cannot make anything too tall", "i have a big personality for this little body so i do not give up"], "speaker_id": "snli:test:1455", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1455", "candidate_text": "People are walking along the beach.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1455:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1454:premise", "snli:test:1513:premise", "snli:test:1362:premise", "snli:test:1455:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of three people and one dog hike up a green mountainside.", "Several people in blue scrubs and one in a skirt and black blouse.", "A young woman in a bikini looking at something in a glass case.", "Three hikers and a dog are walking through the foliage on the hillside.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1455", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1461", "candidate_text": "well, the former first lady is at the podium for a conference", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1461:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1156:premise", "snli:test:1461:premise", "snli:test:1867:premise", "snli:test:667:premise", "style:personachat:test:54:self:5:positive:0", "style:personachat:test:54:self:5:positive:1", "style:personachat:test:54:self:5:positive:2", "style:personachat:test:54:self:5:positive:3", "style:personachat:test:54:self:5:positive:4"], "evidence_texts": ["A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "i am scared my kids will get my webbed toes", "cleaner at research facility by day , artist by night", "more sculpture , i turn others trash into projects", "i could try , i am only five feet so cannot make anything too tall", "i have a big personality for this little body so i do not give up"], "speaker_id": "snli:test:1461", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1461", "candidate_text": "I wonder whether the former first lady is at the podium for a conference?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1461:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1156:premise", "snli:test:1461:premise", "snli:test:1867:premise", "snli:test:667:premise", "style:personachat:test:178:self:5:positive:0", "style:personachat:test:178:self:5:positive:1", "style:personachat:test:178:self:5:positive:2", "style:personachat:test:178:self:5:positive:3", "style:personachat:test:178:self:5:positive:4"], "evidence_texts": ["A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A man and a child are purchasing ice cream from a man with a hand pushed ice cream cart.", "well i did just adopt a cat from the street . . . i could eat it for my diet", "i do not have a problem with eating animals", "no , but i do sing folk songs . i could sing one about dieting for you ?", "yes ! my mom was an army nurse so i read a lot of military history", "ooh what kind ? any specific genre ?"], "speaker_id": "snli:test:1461", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1467", "candidate_text": "well, the two men are wearing hula grass skirts", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1467:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:937:premise", "snli:test:1638:premise", "snli:test:641:premise", "snli:test:1467:premise", "style:personachat:test:54:self:7:positive:0", "style:personachat:test:54:self:7:positive:1", "style:personachat:test:54:self:7:positive:2", "style:personachat:test:54:self:7:positive:3", "style:personachat:test:54:self:7:positive:4", "style:personachat:test:54:self:7:positive:5", "style:personachat:test:54:self:7:positive:6"], "evidence_texts": ["Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "yes ! i do art so i love all forms of expression", "wow that is a handful", "i am scared my kids will get my webbed toes", "cleaner at research facility by day , artist by night", "more sculpture , i turn others trash into projects", "i could try , i am only five feet so cannot make anything too tall", "i have a big personality for this little body so i do not give up"], "speaker_id": "snli:test:1467", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1467", "candidate_text": "Honestly, The two men are wearing hula grass skirts!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1467:negative", "evidence_budget": 12, "evidence_ids": ["snli:test:937:premise", "snli:test:1638:premise", "snli:test:641:premise", "snli:test:1467:premise", "style:personachat:test:384:self:8:positive:0", "style:personachat:test:384:self:8:positive:1", "style:personachat:test:384:self:8:positive:2", "style:personachat:test:384:self:8:positive:3", "style:personachat:test:384:self:8:positive:4", "style:personachat:test:384:self:8:positive:5", "style:personachat:test:384:self:8:positive:6", "style:personachat:test:384:self:8:positive:7"], "evidence_texts": ["Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "i am good ! do you have any hobbies ?", "gardening sounds amazing ! i like to listen to metallica in my free time .", "that is cool ! where do you work at ?", "it must be amazing to work with horses ! i work as a consultant .", "i like to spend time with my brothers . i have two . how about you ?", "wow ! it must have been hard to grow up with 4 brothers !", "awesome ! i am originally from wisconsin .", "that is cool ! do you visit them often ?"], "speaker_id": "snli:test:1467", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1471", "candidate_text": "well, times square is packed", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1471:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1177:premise", "snli:test:1716:premise", "snli:test:798:premise", "snli:test:1471:premise", "style:personachat:test:55:self:1:positive:0"], "evidence_texts": ["skateboarder sliding down a ramp", "Band plays in tight quarters.", "Workers standing on a lift.", "Times Square in New York.", "i never did enlist , thought about it ."], "speaker_id": "snli:test:1471", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1471", "candidate_text": "Times Square is packed.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1471:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1177:premise", "snli:test:1716:premise", "snli:test:798:premise", "snli:test:1471:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["skateboarder sliding down a ramp", "Band plays in tight quarters.", "Workers standing on a lift.", "Times Square in New York.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1471", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1498", "candidate_text": "well, the ball is on the floor", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1498:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1301:premise", "snli:test:1689:premise", "snli:test:1498:premise", "snli:test:1070:premise", "style:personachat:test:55:self:2:positive:0", "style:personachat:test:55:self:2:positive:1"], "evidence_texts": ["People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "i only study ice and explore , but i wish i would have paid attention more .", "i never did enlist , thought about it ."], "speaker_id": "snli:test:1498", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1498", "candidate_text": "I wonder whether the ball is on the floor?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1498:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1301:premise", "snli:test:1689:premise", "snli:test:1498:premise", "snli:test:1070:premise", "style:personachat:test:191:self:2:positive:0", "style:personachat:test:191:self:2:positive:1"], "evidence_texts": ["People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "A woman is seen from above applying red nail polish to her toes, as they rest on a leather seat.", "do you like to read ?", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:1498", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1500", "candidate_text": "well, military personnel are in the mall", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1500:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1500:premise", "snli:test:397:premise", "snli:test:1786:premise", "snli:test:3834:premise", "style:personachat:test:55:self:5:positive:0", "style:personachat:test:55:self:5:positive:1", "style:personachat:test:55:self:5:positive:2", "style:personachat:test:55:self:5:positive:3", "style:personachat:test:55:self:5:positive:4"], "evidence_texts": ["Military personnel are shopping", "A middle eastern marketplace.", "kid practicing karate move", "Men play hockey.", "they are a main staple of my diet , i work in the arctic .", "that is horrible , i love dogs . do you fish ?", "mine taught me too , he was a geologist .", "i only study ice and explore , but i wish i would have paid attention more .", "i never did enlist , thought about it ."], "speaker_id": "snli:test:1500", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1500", "candidate_text": "Honestly, Military personnel are in the mall!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1500:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1500:premise", "snli:test:397:premise", "snli:test:1786:premise", "snli:test:3834:premise", "style:personachat:test:389:self:5:positive:0", "style:personachat:test:389:self:5:positive:1", "style:personachat:test:389:self:5:positive:2", "style:personachat:test:389:self:5:positive:3", "style:personachat:test:389:self:5:positive:4"], "evidence_texts": ["Military personnel are shopping", "A middle eastern marketplace.", "kid practicing karate move", "Men play hockey.", "i tend to write about romance novels .", "bacon is always a good cure .", "anger is nothing to play around with , keep it under control .", "my two cats think they own the island .", "you have a great mother !"], "speaker_id": "snli:test:1500", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1503", "candidate_text": "well, the dogs are chasing a rabbit", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1503:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1420:premise", "snli:test:1763:premise", "snli:test:1314:premise", "snli:test:1503:premise", "style:personachat:test:55:self:7:positive:0", "style:personachat:test:55:self:7:positive:1", "style:personachat:test:55:self:7:positive:2", "style:personachat:test:55:self:7:positive:3", "style:personachat:test:55:self:7:positive:4", "style:personachat:test:55:self:7:positive:5", "style:personachat:test:55:self:7:positive:6"], "evidence_texts": ["A boy kneeling on a skateboard riding down the street", "A man wearing a kilt has thrown a large pole.", "A tan dog chases a black and white soccer ball.", "Two black labs run, one carrying an orange dog toy.", "hello , i am great . just came back from fishing . how are you ?", "that sounds fun , wonder if the polar bears would like it ?", "they are a main staple of my diet , i work in the arctic .", "that is horrible , i love dogs . do you fish ?", "mine taught me too , he was a geologist .", "i only study ice and explore , but i wish i would have paid attention more .", "i never did enlist , thought about it ."], "speaker_id": "snli:test:1503", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1503", "candidate_text": "The dogs are chasing a rabbit.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1503:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1420:premise", "snli:test:1763:premise", "snli:test:1314:premise", "snli:test:1503:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A boy kneeling on a skateboard riding down the street", "A man wearing a kilt has thrown a large pole.", "A tan dog chases a black and white soccer ball.", "Two black labs run, one carrying an orange dog toy.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1503", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1513", "candidate_text": "well, there are people wearing scrubs", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1513:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1455:premise", "snli:test:1513:premise", "snli:test:1802:premise", "snli:test:1454:premise", "style:personachat:test:56:self:1:positive:0"], "evidence_texts": ["Three hikers and a dog are walking through the foliage on the hillside.", "Several people in blue scrubs and one in a skirt and black blouse.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "A group of three people and one dog hike up a green mountainside.", "i do not have any pets , but i do not mind dogs ."], "speaker_id": "snli:test:1513", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1513", "candidate_text": "I wonder whether there are people wearing scrubs?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1513:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1455:premise", "snli:test:1513:premise", "snli:test:1802:premise", "snli:test:1454:premise", "style:personachat:test:196:self:1:positive:0"], "evidence_texts": ["Three hikers and a dog are walking through the foliage on the hillside.", "Several people in blue scrubs and one in a skirt and black blouse.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "A group of three people and one dog hike up a green mountainside.", "what kind of music you like ?"], "speaker_id": "snli:test:1513", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1526", "candidate_text": "well, the raging bull is going to stamp the matador to pieces", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1526:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1145:premise", "snli:test:1557:premise", "snli:test:1526:premise", "snli:test:1133:premise", "style:personachat:test:56:self:2:positive:0", "style:personachat:test:56:self:2:positive:1"], "evidence_texts": ["A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "An older women looking off into the distance with three older women walking towards her from behind.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "the only thing i cannot eat is shellfish . i am allergic .", "i do not have any pets , but i do not mind dogs ."], "speaker_id": "snli:test:1526", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1526", "candidate_text": "Honestly, The raging bull is going to stamp the matador to pieces!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1526:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1145:premise", "snli:test:1557:premise", "snli:test:1526:premise", "snli:test:1133:premise", "style:personachat:test:399:self:2:positive:0", "style:personachat:test:399:self:2:positive:1"], "evidence_texts": ["A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "An older women looking off into the distance with three older women walking towards her from behind.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "A male runner in an orange and white t-shirt and black shorts is sprinting down the track.", "great ! i wish i could sing better . you sing or model ? i was born in california . you ?", "get out ! a singing fisherman . ever heard of cate mesa ?"], "speaker_id": "snli:test:1526", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1534", "candidate_text": "well, two brothers hold torches to celebrate", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1534:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1248:premise", "snli:test:2282:premise", "snli:test:1534:premise", "snli:test:605:premise", "style:personachat:test:56:self:5:positive:0", "style:personachat:test:56:self:5:positive:1", "style:personachat:test:56:self:5:positive:2", "style:personachat:test:56:self:5:positive:3", "style:personachat:test:56:self:5:positive:4"], "evidence_texts": ["The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "big city sounds exciting . what do you do for work ?", "i work for an apartment complex . i am in charge of handling all the issues .", "nothing too serious . do you like to cook ?", "the only thing i cannot eat is shellfish . i am allergic .", "i do not have any pets , but i do not mind dogs ."], "speaker_id": "snli:test:1534", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1534", "candidate_text": "Two brothers hold torches to celebrate.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1534:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1248:premise", "snli:test:2282:premise", "snli:test:1534:premise", "snli:test:605:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A crowd is watching a group of men in suits with briefcases walk in formation down the street led by a woman holding a sign.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1534", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1557", "candidate_text": "well, an older woman is trying to see a billboard that is far away", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1557:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1526:premise", "snli:test:1557:premise", "snli:test:1700:premise", "snli:test:1145:premise", "style:personachat:test:56:self:7:positive:0", "style:personachat:test:56:self:7:positive:1", "style:personachat:test:56:self:7:positive:2", "style:personachat:test:56:self:7:positive:3", "style:personachat:test:56:self:7:positive:4", "style:personachat:test:56:self:7:positive:5", "style:personachat:test:56:self:7:positive:6"], "evidence_texts": ["a matador in white and gold clothing and pink sock is getting hit by a raging bull", "An older women looking off into the distance with three older women walking towards her from behind.", "Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "hi . how are you ? i just got in from working on my car .", "haha . thanks . working on classic cars is my passion !", "big city sounds exciting . what do you do for work ?", "i work for an apartment complex . i am in charge of handling all the issues .", "nothing too serious . do you like to cook ?", "the only thing i cannot eat is shellfish . i am allergic .", "i do not have any pets , but i do not mind dogs ."], "speaker_id": "snli:test:1557", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1557", "candidate_text": "I wonder whether an older woman is trying to see a billboard that is far away?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1557:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1526:premise", "snli:test:1557:premise", "snli:test:1700:premise", "snli:test:1145:premise", "style:personachat:test:196:self:7:positive:0", "style:personachat:test:196:self:7:positive:1", "style:personachat:test:196:self:7:positive:2", "style:personachat:test:196:self:7:positive:3", "style:personachat:test:196:self:7:positive:4", "style:personachat:test:196:self:7:positive:5", "style:personachat:test:196:self:7:positive:6"], "evidence_texts": ["a matador in white and gold clothing and pink sock is getting hit by a raging bull", "An older women looking off into the distance with three older women walking towards her from behind.", "Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "A woman with short blond-hair rises from a chair as another woman in a burgundy shirt laughs.", "i am good busy with 2 part time jobs", "yeah i wanna write book someday", "i wanna write fantasy but for now i will just was lord of the rings", "i tend too where i get ideas from", "what are some of yours", "i bet that can be interesting", "what kind of music you like ?"], "speaker_id": "snli:test:1557", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1562", "candidate_text": "well, a man is sick next to the toilet", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1562:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1441:premise", "snli:test:1581:premise", "snli:test:1562:premise", "snli:test:1373:premise", "style:personachat:test:57:self:1:positive:0"], "evidence_texts": ["A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "you go ahead for the both of us lol"], "speaker_id": "snli:test:1562", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1562", "candidate_text": "Honestly, A man is sick next to the toilet!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1562:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1441:premise", "snli:test:1581:premise", "snli:test:1562:premise", "snli:test:1373:premise", "style:personachat:test:405:self:1:positive:0"], "evidence_texts": ["A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "An organ grinder, in World War I helmet and clown makeup, speaks to an older man in a blue, striped polo shirt.", "cool ! i am going to go running now and listen to rap ."], "speaker_id": "snli:test:1562", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1570", "candidate_text": "well, a young man shows a group of people the black web painted on his face", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1570:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1261:premise", "snli:test:1687:premise", "snli:test:1570:premise", "snli:test:1260:premise", "style:personachat:test:57:self:2:positive:0", "style:personachat:test:57:self:2:positive:1"], "evidence_texts": ["People sitting in chairs along a tree-lined sidewalk.", "Kids work at computers with a teacher's help.", "A man shows off his spider-web face paint.", "People sitting in chairs along a tree-lined sidewalk.", "i am in oklahoma as a native american", "you go ahead for the both of us lol"], "speaker_id": "snli:test:1570", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1570", "candidate_text": "A young man shows a group of people the black web painted on his face.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1570:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1261:premise", "snli:test:1687:premise", "snli:test:1570:premise", "snli:test:1260:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["People sitting in chairs along a tree-lined sidewalk.", "Kids work at computers with a teacher's help.", "A man shows off his spider-web face paint.", "People sitting in chairs along a tree-lined sidewalk.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1570", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1572", "candidate_text": "well, the man has nothing on his face", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1572:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1572:premise", "snli:test:1418:premise", "snli:test:1417:premise", "snli:test:1599:premise", "style:personachat:test:57:self:5:positive:0", "style:personachat:test:57:self:5:positive:1", "style:personachat:test:57:self:5:positive:2", "style:personachat:test:57:self:5:positive:3", "style:personachat:test:57:self:5:positive:4"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "This is when daddy taught her how to ride her scooter.", "This is when daddy taught her how to ride her scooter.", "An Ambulance is passing a man wearing a bandanna and girl.", "oh that is cool , need a lawyer ?", "not as good as me , like heights ?", "really ? lol i would never", "i am in oklahoma as a native american", "you go ahead for the both of us lol"], "speaker_id": "snli:test:1572", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1572", "candidate_text": "I wonder whether the man has nothing on his face?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1572:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1572:premise", "snli:test:1418:premise", "snli:test:1417:premise", "snli:test:1599:premise", "style:personachat:test:199:self:5:positive:0", "style:personachat:test:199:self:5:positive:1", "style:personachat:test:199:self:5:positive:2", "style:personachat:test:199:self:5:positive:3", "style:personachat:test:199:self:5:positive:4"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "This is when daddy taught her how to ride her scooter.", "This is when daddy taught her how to ride her scooter.", "An Ambulance is passing a man wearing a bandanna and girl.", "i am in my twenties . not going to say exactly . what do you do for fun ?", "cosplay ? i have never been into that . unless you count dressing as santa claus ! i love christmas !", "i went through a blue phase . hair stylists do that .", "new york ! a big slice of cheese pizza is the best !", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:1572", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1573", "candidate_text": "well, the man has his face painted", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1573:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1599:premise", "snli:test:1573:premise", "snli:test:1418:premise", "snli:test:1600:premise", "style:personachat:test:57:self:7:positive:0", "style:personachat:test:57:self:7:positive:1", "style:personachat:test:57:self:7:positive:2", "style:personachat:test:57:self:7:positive:3", "style:personachat:test:57:self:7:positive:4", "style:personachat:test:57:self:7:positive:5", "style:personachat:test:57:self:7:positive:6"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "Man with a spider web & spiders painted on his face.", "This is when daddy taught her how to ride her scooter.", "An Ambulance is passing a man wearing a bandanna and girl.", "i am well , loving on my little dogs an you ?", "cool , i have 2 dogs", "oh that is cool , need a lawyer ?", "not as good as me , like heights ?", "really ? lol i would never", "i am in oklahoma as a native american", "you go ahead for the both of us lol"], "speaker_id": "snli:test:1573", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1573", "candidate_text": "Honestly, The man has his face painted!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1573:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1599:premise", "snli:test:1573:premise", "snli:test:1418:premise", "snli:test:1600:premise", "style:personachat:test:405:self:7:positive:0", "style:personachat:test:405:self:7:positive:1", "style:personachat:test:405:self:7:positive:2", "style:personachat:test:405:self:7:positive:3", "style:personachat:test:405:self:7:positive:4", "style:personachat:test:405:self:7:positive:5", "style:personachat:test:405:self:7:positive:6"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "Man with a spider web & spiders painted on his face.", "This is when daddy taught her how to ride her scooter.", "An Ambulance is passing a man wearing a bandanna and girl.", "yo ! i am great . eating sushi . what are you up to ?", "what was the surgery ? i would be bummed since i run every day .", "how long are you off the knee ? i like working out , so i hope soon .", "well , not too long . you can listen to rap and work the arms .", "no sardines for me . i eat vegan . but salad works .", "and beans . but my favorite is sushi . vegan sushi .", "cool ! i am going to go running now and listen to rap ."], "speaker_id": "snli:test:1573", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1581", "candidate_text": "well, the young boy is asleep while the woman is watching television", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1581:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1562:premise", "snli:test:1581:premise", "snli:test:2760:premise", "snli:test:1441:premise", "style:personachat:test:58:self:1:positive:0"], "evidence_texts": ["A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "that is why i like getting up early so i can have more time for my crafts ."], "speaker_id": "snli:test:1581", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1581", "candidate_text": "The young boy is asleep while the woman is watching television.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1581:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1562:premise", "snli:test:1581:premise", "snli:test:2760:premise", "snli:test:1441:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A woman wearing a long black jacket and a brown hat is walking beside a park with a lot of blooming trees.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1581", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1590", "candidate_text": "well, some people are attending an outdoor book sale", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1590:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1205:premise", "snli:test:1710:premise", "snli:test:1590:premise", "snli:test:1203:premise", "style:personachat:test:58:self:2:positive:0", "style:personachat:test:58:self:2:positive:1"], "evidence_texts": ["A person is on a four wheeler kicking up a lot of dirt in a dune.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "i make arts and crafts to sell around the holidays . they sell pretty good ,", "that is why i like getting up early so i can have more time for my crafts ."], "speaker_id": "snli:test:1590", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1590", "candidate_text": "I wonder whether some people are attending an outdoor book sale?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1590:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1205:premise", "snli:test:1710:premise", "snli:test:1590:premise", "snli:test:1203:premise", "style:personachat:test:201:self:2:positive:0", "style:personachat:test:201:self:2:positive:1"], "evidence_texts": ["A person is on a four wheeler kicking up a lot of dirt in a dune.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "that sounds nice , do you eat dinner as a family ?", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:1590", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1599", "candidate_text": "well, the ambulance isn't stopped by heavy traffic", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1599:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1573:premise", "snli:test:1572:premise", "snli:test:1841:premise", "snli:test:1599:premise", "style:personachat:test:58:self:5:positive:0", "style:personachat:test:58:self:5:positive:1", "style:personachat:test:58:self:5:positive:2", "style:personachat:test:58:self:5:positive:3", "style:personachat:test:58:self:5:positive:4"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "Man with a spider web & spiders painted on his face.", "A man is using what looks to be a fax machine.", "An Ambulance is passing a man wearing a bandanna and girl.", "i home school my son so i am always busy and always tired . i make myself relax", "a cat had kittens in my garage but i am allergic so i am sneezing a lot today", "the allergy meds will knock me out so i can sleep but i need to paint", "i make arts and crafts to sell around the holidays . they sell pretty good ,", "that is why i like getting up early so i can have more time for my crafts ."], "speaker_id": "snli:test:1599", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1599", "candidate_text": "Honestly, The ambulance isn't stopped by heavy traffic!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1599:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1573:premise", "snli:test:1572:premise", "snli:test:1841:premise", "snli:test:1599:premise", "style:personachat:test:413:self:5:positive:0", "style:personachat:test:413:self:5:positive:1", "style:personachat:test:413:self:5:positive:2", "style:personachat:test:413:self:5:positive:3", "style:personachat:test:413:self:5:positive:4"], "evidence_texts": ["Man with a spider web & spiders painted on his face.", "Man with a spider web & spiders painted on his face.", "A man is using what looks to be a fax machine.", "An Ambulance is passing a man wearing a bandanna and girl.", "no worries . i am patient . i get that from my mom . she was a teacher .", "really ? do they make you sick ? my ex makes me sick , lol .", "yeah . it is hard to forget him even though we are divorced .", "yes . time is supposed to heal all wounds .", "that takes talent ! i am not very coordinated ."], "speaker_id": "snli:test:1599", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1600", "candidate_text": "well, the man in the bandana is running after the ambulance", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1600:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1600:premise", "snli:test:1841:premise", "snli:test:1573:premise", "snli:test:2013:premise", "style:personachat:test:58:self:7:positive:0", "style:personachat:test:58:self:7:positive:1", "style:personachat:test:58:self:7:positive:2", "style:personachat:test:58:self:7:positive:3", "style:personachat:test:58:self:7:positive:4", "style:personachat:test:58:self:7:positive:5", "style:personachat:test:58:self:7:positive:6"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "A man is using what looks to be a fax machine.", "Man with a spider web & spiders painted on his face.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "i am good , a little sleepy how about you ? oh i am sneezing too .", "i have been going to bed early and getting up early and i feel better now", "i home school my son so i am always busy and always tired . i make myself relax", "a cat had kittens in my garage but i am allergic so i am sneezing a lot today", "the allergy meds will knock me out so i can sleep but i need to paint", "i make arts and crafts to sell around the holidays . they sell pretty good ,", "that is why i like getting up early so i can have more time for my crafts ."], "speaker_id": "snli:test:1600", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1600", "candidate_text": "The man in the bandana is running after the ambulance.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1600:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1600:premise", "snli:test:1841:premise", "snli:test:1573:premise", "snli:test:2013:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "A man is using what looks to be a fax machine.", "Man with a spider web & spiders painted on his face.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1600", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1626", "candidate_text": "well, the women are on stage", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1626:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1256:premise", "snli:test:1659:premise", "snli:test:1626:premise", "snli:test:1129:premise", "style:personachat:test:59:self:1:positive:0"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two men cool off under a waterfall.", "Women in costume stand on a stage.", "Two guys walking, one carrying a skateboard", "yeah a few different artist but mainly old music"], "speaker_id": "snli:test:1626", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1626", "candidate_text": "I wonder whether the women are on stage?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1626:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1256:premise", "snli:test:1659:premise", "snli:test:1626:premise", "snli:test:1129:premise", "style:personachat:test:202:self:1:positive:0"], "evidence_texts": ["a young man wearing dark sunglasses smiling", "Two men cool off under a waterfall.", "Women in costume stand on a stage.", "Two guys walking, one carrying a skateboard", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:1626", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1633", "candidate_text": "well, a young boy has fun playing outdoors", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1633:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1832:premise", "snli:test:1633:premise", "snli:test:5142:premise", "snli:test:5351:premise", "style:personachat:test:59:self:2:positive:0", "style:personachat:test:59:self:2:positive:1"], "evidence_texts": ["A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "i mostly be at school , the little free time i spend at home", "yeah a few different artist but mainly old music"], "speaker_id": "snli:test:1633", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1633", "candidate_text": "Honestly, A young boy has fun playing outdoors!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1633:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1832:premise", "snli:test:1633:premise", "snli:test:5142:premise", "snli:test:5351:premise", "style:personachat:test:425:self:2:positive:0", "style:personachat:test:425:self:2:positive:1"], "evidence_texts": ["A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "just got off work a day ago ? or are we in a time loop ?", "crazy ! nurses are like elves bringing in new life into the world to fight evil"], "speaker_id": "snli:test:1633", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1634", "candidate_text": "well, a young boy stands at the chalkboard to answer a math problem", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1634:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5351:premise", "snli:test:1832:premise", "snli:test:1634:premise", "snli:test:5732:premise", "style:personachat:test:59:self:5:positive:0", "style:personachat:test:59:self:5:positive:1", "style:personachat:test:59:self:5:positive:2", "style:personachat:test:59:self:5:positive:3", "style:personachat:test:59:self:5:positive:4"], "evidence_texts": ["A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "well i am in high school so all the basics", "o cool which type of school", "oo so you are trying to be a professor", "i mostly be at school , the little free time i spend at home", "yeah a few different artist but mainly old music"], "speaker_id": "snli:test:1634", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1634", "candidate_text": "A young boy stands at the chalkboard to answer a math problem.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1634:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5351:premise", "snli:test:1832:premise", "snli:test:1634:premise", "snli:test:5732:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1634", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1636", "candidate_text": "well, a man sitting on top of a mountain", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1636:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1636:premise", "snli:test:1322:premise", "snli:test:1643:premise", "snli:test:1280:premise", "style:personachat:test:59:self:7:positive:0", "style:personachat:test:59:self:7:positive:1", "style:personachat:test:59:self:7:positive:2", "style:personachat:test:59:self:7:positive:3", "style:personachat:test:59:self:7:positive:4", "style:personachat:test:59:self:7:positive:5", "style:personachat:test:59:self:7:positive:6"], "evidence_texts": ["A man in a red shirt is sitting on top of a rocky mountain.", "Men wearing life vests speed across a body of water in an orange boat.", "Bicyclist ride the course near the ocean as the day comes to an end.", "Three army soldiers are holding guns and peering around a corner of a building.", "hey i am good how about yourself", "i know the feeling i am to", "well i am in high school so all the basics", "o cool which type of school", "oo so you are trying to be a professor", "i mostly be at school , the little free time i spend at home", "yeah a few different artist but mainly old music"], "speaker_id": "snli:test:1636", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1636", "candidate_text": "I wonder whether a man sitting on top of a mountain?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1636:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1636:premise", "snli:test:1322:premise", "snli:test:1643:premise", "snli:test:1280:premise", "style:personachat:test:202:self:7:positive:0", "style:personachat:test:202:self:7:positive:1", "style:personachat:test:202:self:7:positive:2", "style:personachat:test:202:self:7:positive:3", "style:personachat:test:202:self:7:positive:4", "style:personachat:test:202:self:7:positive:5", "style:personachat:test:202:self:7:positive:6"], "evidence_texts": ["A man in a red shirt is sitting on top of a rocky mountain.", "Men wearing life vests speed across a body of water in an orange boat.", "Bicyclist ride the course near the ocean as the day comes to an end.", "Three army soldiers are holding guns and peering around a corner of a building.", "good evening lucy , i am rayne . i hope you are doing well .", "i am a wedding planner , i just graduated from college .", "that is great ! i was just watching some tv , sports , i love them .", "lol , i have been playing sports all through elementary and high school .", "i have to say i favor soccer . what do you enjoy doing ?", "sounds great . yes it can be . do you have any pets ? i have frogs .", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:1636", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1638", "candidate_text": "I wonder whether the boats jumped up from a shallow rock?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1638:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1467:premise", "snli:test:937:premise", "snli:test:1638:premise", "snli:test:641:premise", "style:personachat:test:60:self:1:positive:0"], "evidence_texts": ["Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:1638", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1638", "candidate_text": "well, the boats jumped up from a shallow rock", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1638:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1467:premise", "snli:test:937:premise", "snli:test:1638:premise", "snli:test:641:premise", "style:personachat:test:104:self:1:positive:0"], "evidence_texts": ["Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "Two men wearing dirty clothing are sitting on a sidewalk with their dog and begging for money using a cardboard sign.", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:1638", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1639", "candidate_text": "I wonder whether there is a fishing boat in the harbor?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1639:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1467:premise", "snli:test:2291:premise", "snli:test:937:premise", "snli:test:1639:premise", "style:personachat:test:60:self:2:positive:0", "style:personachat:test:60:self:2:positive:1"], "evidence_texts": ["Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "that is sweet of you . do you like working out ?", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:1639", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1639", "candidate_text": "Honestly, There is a fishing boat in the harbor!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1639:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1467:premise", "snli:test:2291:premise", "snli:test:937:premise", "snli:test:1639:premise", "style:personachat:test:429:self:2:positive:0", "style:personachat:test:429:self:2:positive:1"], "evidence_texts": ["Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "Two older women with a baby that has a pacifier in its mouth, the baby wears an orange and brown dress.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "thank you ! life will be so awesome after retirement !", "oh , definitely ! i can enjoy life to the fullest !"], "speaker_id": "snli:test:1639", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1640", "candidate_text": "I wonder whether a young man and an asian woman are near a dock on a bay?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1640:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1640:premise", "snli:test:2291:premise", "snli:test:2404:premise", "snli:test:1467:premise", "style:personachat:test:60:self:5:positive:0", "style:personachat:test:60:self:5:positive:1", "style:personachat:test:60:self:5:positive:2", "style:personachat:test:60:self:5:positive:3", "style:personachat:test:60:self:5:positive:4"], "evidence_texts": ["Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "i like the atmosphere that it offers . what led you to nursing ?", "good a reason as any . does she work at a hospital ?", "do you also want to work with babies ?", "that is sweet of you . do you like working out ?", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:1640", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1640", "candidate_text": "A young man and an asian woman are near a dock on a bay.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1640:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1640:premise", "snli:test:2291:premise", "snli:test:2404:premise", "snli:test:1467:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "Two men stand in hula grass skirts stand in the back of a truck at the top of a silver ramp.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1640", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1643", "candidate_text": "I wonder whether the cyclist was riding in the morning?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1643:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1636:premise", "snli:test:1705:premise", "snli:test:1322:premise", "snli:test:1643:premise", "style:personachat:test:60:self:7:positive:0", "style:personachat:test:60:self:7:positive:1", "style:personachat:test:60:self:7:positive:2", "style:personachat:test:60:self:7:positive:3", "style:personachat:test:60:self:7:positive:4", "style:personachat:test:60:self:7:positive:5", "style:personachat:test:60:self:7:positive:6"], "evidence_texts": ["A man in a red shirt is sitting on top of a rocky mountain.", "a woman in a brown dress looking at papers in front of a class.", "Men wearing life vests speed across a body of water in an orange boat.", "Bicyclist ride the course near the ocean as the day comes to an end.", "yes , i would like that .", "i am studying to be an investment broker .", "i like the atmosphere that it offers . what led you to nursing ?", "good a reason as any . does she work at a hospital ?", "do you also want to work with babies ?", "that is sweet of you . do you like working out ?", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:1643", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1643", "candidate_text": "well, the cyclist was riding in the morning", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1643:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1636:premise", "snli:test:1705:premise", "snli:test:1322:premise", "snli:test:1643:premise", "style:personachat:test:104:self:6:positive:0", "style:personachat:test:104:self:6:positive:1", "style:personachat:test:104:self:6:positive:2", "style:personachat:test:104:self:6:positive:3", "style:personachat:test:104:self:6:positive:4", "style:personachat:test:104:self:6:positive:5"], "evidence_texts": ["A man in a red shirt is sitting on top of a rocky mountain.", "a woman in a brown dress looking at papers in front of a class.", "Men wearing life vests speed across a body of water in an orange boat.", "Bicyclist ride the course near the ocean as the day comes to an end.", "hi jack ! i am dave and i am doing good today . how are you doing ?", "what book did you finish reading ?", "i read sometimes not often enough . i have heard of the movie though .", "i do enjoy cooking . i like cooking italian food the most .", "no . i do work with a lot of kids at my job though .", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:1643", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1647", "candidate_text": "well, a woman is taking a picture", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1647:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1189:premise", "snli:test:1748:premise", "snli:test:1042:premise", "snli:test:1647:premise", "style:personachat:test:61:self:1:positive:0"], "evidence_texts": ["There is a group of children getting their picture taken with presents.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "A man is walking past a large sign that says E.S.E. Electronics.", "A woman is taking a picture of a girl on a scooter.", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:1647", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1647", "candidate_text": "A woman is taking a picture.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1647:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1189:premise", "snli:test:1748:premise", "snli:test:1042:premise", "snli:test:1647:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["There is a group of children getting their picture taken with presents.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "A man is walking past a large sign that says E.S.E. Electronics.", "A woman is taking a picture of a girl on a scooter.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1647", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1659", "candidate_text": "well, two women are under a waterfall", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1659:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1626:premise", "snli:test:1720:premise", "snli:test:1256:premise", "snli:test:1659:premise", "style:personachat:test:61:self:2:positive:0", "style:personachat:test:61:self:2:positive:1"], "evidence_texts": ["Women in costume stand on a stage.", "Three women sit together near woven walls.", "a young man wearing dark sunglasses smiling", "Two men cool off under a waterfall.", "i work for a large online retail . i like it .", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:1659", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1659", "candidate_text": "I wonder whether two women are under a waterfall?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1659:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1626:premise", "snli:test:1720:premise", "snli:test:1256:premise", "snli:test:1659:premise", "style:personachat:test:206:self:2:positive:0", "style:personachat:test:206:self:2:positive:1"], "evidence_texts": ["Women in costume stand on a stage.", "Three women sit together near woven walls.", "a young man wearing dark sunglasses smiling", "Two men cool off under a waterfall.", "hmmm . what does your dad do ?", "i see . does your dad like riding rollercoasters too ?"], "speaker_id": "snli:test:1659", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1663", "candidate_text": "well, the hose is being used", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1663:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1265:premise", "snli:test:2911:premise", "snli:test:1263:premise", "snli:test:1663:premise", "style:personachat:test:61:self:5:positive:0", "style:personachat:test:61:self:5:positive:1", "style:personachat:test:61:self:5:positive:2", "style:personachat:test:61:self:5:positive:3", "style:personachat:test:61:self:5:positive:4"], "evidence_texts": ["Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "where are you from ? i originate from canada .", "i like to dance in my spare time .", "that is a great gift to have .", "i work for a large online retail . i like it .", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:1663", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1663", "candidate_text": "Honestly, The hose is being used!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1663:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1265:premise", "snli:test:2911:premise", "snli:test:1263:premise", "snli:test:1663:premise", "style:personachat:test:435:self:5:positive:0", "style:personachat:test:435:self:5:positive:1", "style:personachat:test:435:self:5:positive:2", "style:personachat:test:435:self:5:positive:3", "style:personachat:test:435:self:5:positive:4"], "evidence_texts": ["Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "what do you compete in ? i compete in swimming", "that is cool , do you live in the states ?", "i live in toronto canada", "super cold ! cali sounds nice about right now !", "ohhhh nooo i will send you the snow !"], "speaker_id": "snli:test:1663", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1687", "candidate_text": "well, the kids sit in front of computers", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1687:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1570:premise", "snli:test:1736:premise", "snli:test:1261:premise", "snli:test:1687:premise", "style:personachat:test:61:self:7:positive:0", "style:personachat:test:61:self:7:positive:1", "style:personachat:test:61:self:7:positive:2", "style:personachat:test:61:self:7:positive:3", "style:personachat:test:61:self:7:positive:4", "style:personachat:test:61:self:7:positive:5", "style:personachat:test:61:self:7:positive:6"], "evidence_texts": ["A man shows off his spider-web face paint.", "A worker is doing a job on scaffolding.", "People sitting in chairs along a tree-lined sidewalk.", "Kids work at computers with a teacher's help.", "hello , i am good . i was drinking my favorite drink ice tea .", "i have been listening to country music all day .", "where are you from ? i originate from canada .", "i like to dance in my spare time .", "that is a great gift to have .", "i work for a large online retail . i like it .", "sounds like a great job . i would do it ."], "speaker_id": "snli:test:1687", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1687", "candidate_text": "The kids sit in front of computers.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1687:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1570:premise", "snli:test:1736:premise", "snli:test:1261:premise", "snli:test:1687:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man shows off his spider-web face paint.", "A worker is doing a job on scaffolding.", "People sitting in chairs along a tree-lined sidewalk.", "Kids work at computers with a teacher's help.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1687", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1689", "candidate_text": "well, two men are next to each other", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1689:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1498:premise", "snli:test:1689:premise", "snli:test:1976:premise", "snli:test:1301:premise", "style:personachat:test:62:self:1:positive:0"], "evidence_texts": ["One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "i do not read much , but i like magazines about carpentry ."], "speaker_id": "snli:test:1689", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1689", "candidate_text": "I wonder whether two men are next to each other?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1689:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1498:premise", "snli:test:1689:premise", "snli:test:1976:premise", "snli:test:1301:premise", "style:personachat:test:217:self:1:positive:0"], "evidence_texts": ["One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "People are all standing together in front of a statue of an animal, and they are all wearing cool-weather clothing.", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:1689", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1692", "candidate_text": "well, a businessman walks by a closed cafe", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1692:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1692:premise", "snli:test:908:premise", "snli:test:7110:premise", "snli:test:5556:premise", "style:personachat:test:62:self:2:positive:0", "style:personachat:test:62:self:2:positive:1"], "evidence_texts": ["A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "that is true ! i do not though . can you ?", "i do not read much , but i like magazines about carpentry ."], "speaker_id": "snli:test:1692", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1692", "candidate_text": "Honestly, A businessman walks by a closed cafe!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1692:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1692:premise", "snli:test:908:premise", "snli:test:7110:premise", "snli:test:5556:premise", "style:personachat:test:438:self:2:positive:0", "style:personachat:test:438:self:2:positive:1"], "evidence_texts": ["A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "bills cost money . after dinner i usually go out and play with my friends under the sea", "yes ! i must be . i love being me . board games are difficult to play under the sea"], "speaker_id": "snli:test:1692", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1694", "candidate_text": "well, a man walks by an open cafe named \"cafe de flo\"", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1694:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5556:premise", "snli:test:908:premise", "snli:test:1694:premise", "snli:test:6441:premise", "style:personachat:test:62:self:5:positive:0", "style:personachat:test:62:self:5:positive:1", "style:personachat:test:62:self:5:positive:2", "style:personachat:test:62:self:5:positive:3", "style:personachat:test:62:self:5:positive:4"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "congratulations ! what is your degree in ?", "that is great . do you come from a big family ?", "i enjoy poker , but my three kids keep me pretty busy .", "that is true ! i do not though . can you ?", "i do not read much , but i like magazines about carpentry ."], "speaker_id": "snli:test:1694", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1694", "candidate_text": "A man walks by an open cafe named \"Cafe De Flo\".", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1694:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5556:premise", "snli:test:908:premise", "snli:test:1694:premise", "snli:test:6441:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1694", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1697", "candidate_text": "well, the woman is holding old trashcans to carry various objecs", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1697:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:801:premise", "snli:test:1697:premise", "snli:test:1947:premise", "snli:test:193:premise", "style:personachat:test:62:self:7:positive:0", "style:personachat:test:62:self:7:positive:1", "style:personachat:test:62:self:7:positive:2", "style:personachat:test:62:self:7:positive:3", "style:personachat:test:62:self:7:positive:4", "style:personachat:test:62:self:7:positive:5", "style:personachat:test:62:self:7:positive:6"], "evidence_texts": ["Two people talking on a dock.", "A woman holding bags behind plantains.", "A child buried in the sand.", "a group of people dancing together.", "i am good , how are you ?", "i used to party a lot but i have been sober for 5 years .", "congratulations ! what is your degree in ?", "that is great . do you come from a big family ?", "i enjoy poker , but my three kids keep me pretty busy .", "that is true ! i do not though . can you ?", "i do not read much , but i like magazines about carpentry ."], "speaker_id": "snli:test:1697", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1697", "candidate_text": "I wonder whether the woman is holding old trashcans to carry various objecs?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1697:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:801:premise", "snli:test:1697:premise", "snli:test:1947:premise", "snli:test:193:premise", "style:personachat:test:217:self:7:positive:0", "style:personachat:test:217:self:7:positive:1", "style:personachat:test:217:self:7:positive:2", "style:personachat:test:217:self:7:positive:3", "style:personachat:test:217:self:7:positive:4", "style:personachat:test:217:self:7:positive:5", "style:personachat:test:217:self:7:positive:6"], "evidence_texts": ["Two people talking on a dock.", "A woman holding bags behind plantains.", "A child buried in the sand.", "a group of people dancing together.", "hi john . jessie here . i am from canada . where are you from ?", "just recently to my high school sweetheart . we are both 19 .", "oh , i love to sing in the shower country mostly . what music do you like ?", "oh , my husband alex likes jazz .", "no , not really . i only dance if i drink . mostly , i stick to iced tea , though .", "what kinds of things do you like to build ?", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:1697", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1700", "candidate_text": "I wonder whether there are some players chasing a ball?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1700:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1700:premise", "snli:test:1557:premise", "snli:test:1806:premise", "snli:test:1526:premise", "style:personachat:test:63:self:1:positive:0"], "evidence_texts": ["Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "An older women looking off into the distance with three older women walking towards her from behind.", "A group of people standing around smiling, and in the background are more people and handmade signs.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "i see . i have a cookbook coming out . you write ?"], "speaker_id": "snli:test:1700", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1700", "candidate_text": "well, there are some players chasing a ball", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1700:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1700:premise", "snli:test:1557:premise", "snli:test:1806:premise", "snli:test:1526:premise", "style:personachat:test:110:self:1:positive:0"], "evidence_texts": ["Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "An older women looking off into the distance with three older women walking towards her from behind.", "A group of people standing around smiling, and in the background are more people and handmade signs.", "a matador in white and gold clothing and pink sock is getting hit by a raging bull", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:1700", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1705", "candidate_text": "I wonder whether a woman checking papers is a teacher?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1705:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1643:premise", "snli:test:1705:premise", "snli:test:1817:premise", "snli:test:1636:premise", "style:personachat:test:63:self:2:positive:0", "style:personachat:test:63:self:2:positive:1"], "evidence_texts": ["Bicyclist ride the course near the ocean as the day comes to an end.", "a woman in a brown dress looking at papers in front of a class.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A man in a red shirt is sitting on top of a rocky mountain.", "hmmm . got for it ! is six four two twenty tall and big ?", "i see . i have a cookbook coming out . you write ?"], "speaker_id": "snli:test:1705", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1705", "candidate_text": "Honestly, A woman checking papers is a teacher!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1705:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1643:premise", "snli:test:1705:premise", "snli:test:1817:premise", "snli:test:1636:premise", "style:personachat:test:445:self:2:positive:0", "style:personachat:test:445:self:2:positive:1"], "evidence_texts": ["Bicyclist ride the course near the ocean as the day comes to an end.", "a woman in a brown dress looking at papers in front of a class.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A man in a red shirt is sitting on top of a rocky mountain.", "3 not 31 . . that would be scary ! sad", "totally , one just like me ! ! lol"], "speaker_id": "snli:test:1705", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1710", "candidate_text": "I wonder whether a swimmer is winning?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1710:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1590:premise", "snli:test:1788:premise", "snli:test:1710:premise", "snli:test:1205:premise", "style:personachat:test:63:self:5:positive:0", "style:personachat:test:63:self:5:positive:1", "style:personachat:test:63:self:5:positive:2", "style:personachat:test:63:self:5:positive:3", "style:personachat:test:63:self:5:positive:4"], "evidence_texts": ["Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "i am a warehouse worker . i cook a great bbq .", "keep your head up ! i work the forklift . what work do you do ?", "that is good ! people laugh because my name is joe shirley .", "hmmm . got for it ! is six four two twenty tall and big ?", "i see . i have a cookbook coming out . you write ?"], "speaker_id": "snli:test:1710", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1710", "candidate_text": "A swimmer is winning.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1710:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1590:premise", "snli:test:1788:premise", "snli:test:1710:premise", "snli:test:1205:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "A person is on a four wheeler kicking up a lot of dirt in a dune.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1710", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1716", "candidate_text": "I wonder whether the group of people are walking on the sidewalk in the city?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1716:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1716:premise", "snli:test:1471:premise", "snli:test:2221:premise", "snli:test:1177:premise", "style:personachat:test:63:self:7:positive:0", "style:personachat:test:63:self:7:positive:1", "style:personachat:test:63:self:7:positive:2", "style:personachat:test:63:self:7:positive:3", "style:personachat:test:63:self:7:positive:4", "style:personachat:test:63:self:7:positive:5", "style:personachat:test:63:self:7:positive:6"], "evidence_texts": ["Band plays in tight quarters.", "Times Square in New York.", "several people are bike riding.", "skateboarder sliding down a ramp", "they call me big shirley . i love scary movies . you ?", "i see . i workout lifting weights . you ?", "i am a warehouse worker . i cook a great bbq .", "keep your head up ! i work the forklift . what work do you do ?", "that is good ! people laugh because my name is joe shirley .", "hmmm . got for it ! is six four two twenty tall and big ?", "i see . i have a cookbook coming out . you write ?"], "speaker_id": "snli:test:1716", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1716", "candidate_text": "well, the group of people are walking on the sidewalk in the city", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1716:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1716:premise", "snli:test:1471:premise", "snli:test:2221:premise", "snli:test:1177:premise", "style:personachat:test:110:self:7:positive:0", "style:personachat:test:110:self:7:positive:1", "style:personachat:test:110:self:7:positive:2", "style:personachat:test:110:self:7:positive:3", "style:personachat:test:110:self:7:positive:4", "style:personachat:test:110:self:7:positive:5", "style:personachat:test:110:self:7:positive:6"], "evidence_texts": ["Band plays in tight quarters.", "Times Square in New York.", "several people are bike riding.", "skateboarder sliding down a ramp", "i am good just got back from the gym", "sounds cool what is that", "haha no worries what do you do for a living", "that is great whats your favorite so far", "yeah i like it too i have not been in a while though", "i went to italy a few years ago i would love to go back", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:1716", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1720", "candidate_text": "I wonder whether three women sit together?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1720:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1720:premise", "snli:test:1659:premise", "snli:test:1626:premise", "snli:test:1770:premise", "style:personachat:test:64:self:1:positive:0"], "evidence_texts": ["Three women sit together near woven walls.", "Two men cool off under a waterfall.", "Women in costume stand on a stage.", "Women in red are performing with nun-chucks.", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:1720", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1720", "candidate_text": "Honestly, Three women sit together!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1720:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1720:premise", "snli:test:1659:premise", "snli:test:1626:premise", "snli:test:1770:premise", "style:personachat:test:446:self:1:positive:0"], "evidence_texts": ["Three women sit together near woven walls.", "Two men cool off under a waterfall.", "Women in costume stand on a stage.", "Women in red are performing with nun-chucks.", "i remember those ! my mother had one in the army"], "speaker_id": "snli:test:1720", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1721", "candidate_text": "I wonder whether the three bar tenders work hard pouring drinks?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1721:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1770:premise", "snli:test:1721:premise", "snli:test:1659:premise", "snli:test:1772:premise", "style:personachat:test:64:self:2:positive:0", "style:personachat:test:64:self:2:positive:1"], "evidence_texts": ["Women in red are performing with nun-chucks.", "Three women sit together near woven walls.", "Two men cool off under a waterfall.", "Women in red are performing with nun-chucks.", "i hate snow , i am so glad i made the move .", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:1721", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1721", "candidate_text": "The three bar tenders work hard pouring drinks.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1721:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1770:premise", "snli:test:1721:premise", "snli:test:1659:premise", "snli:test:1772:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Women in red are performing with nun-chucks.", "Three women sit together near woven walls.", "Two men cool off under a waterfall.", "Women in red are performing with nun-chucks.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1721", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1736", "candidate_text": "I wonder whether nobody is working?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1736:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1736:premise", "snli:test:1687:premise", "snli:test:1795:premise", "snli:test:1570:premise", "style:personachat:test:64:self:5:positive:0", "style:personachat:test:64:self:5:positive:1", "style:personachat:test:64:self:5:positive:2", "style:personachat:test:64:self:5:positive:3", "style:personachat:test:64:self:5:positive:4"], "evidence_texts": ["A worker is doing a job on scaffolding.", "Kids work at computers with a teacher's help.", "Rock band on stage immersed in light show.", "A man shows off his spider-web face paint.", "no , i do not . i like it hot .", "i am a web designer , i do coding for several country musicians .", "yes , i am . i only listen to country .", "i hate snow , i am so glad i made the move .", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:1736", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1736", "candidate_text": "well, nobody is working", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1736:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1736:premise", "snli:test:1687:premise", "snli:test:1795:premise", "snli:test:1570:premise", "style:personachat:test:112:self:5:positive:0", "style:personachat:test:112:self:5:positive:1", "style:personachat:test:112:self:5:positive:2", "style:personachat:test:112:self:5:positive:3", "style:personachat:test:112:self:5:positive:4"], "evidence_texts": ["A worker is doing a job on scaffolding.", "Kids work at computers with a teacher's help.", "Rock band on stage immersed in light show.", "A man shows off his spider-web face paint.", "what you bought if i may ask", "what type of shoes ? heels or flats ?", "oh cool , how old are you", "i am 28 years old", "yes i have 4 little sisters"], "speaker_id": "snli:test:1736", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1740", "candidate_text": "I wonder whether a man is eaten by a shark?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1740:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:1740:premise", "snli:test:1195:premise", "snli:test:1764:premise", "snli:test:1194:premise", "style:personachat:test:64:self:6:positive:0", "style:personachat:test:64:self:6:positive:1", "style:personachat:test:64:self:6:positive:2", "style:personachat:test:64:self:6:positive:3", "style:personachat:test:64:self:6:positive:4", "style:personachat:test:64:self:6:positive:5"], "evidence_texts": ["A man is steering his ship out at sea.", "The big dog is checking out the smaller dog.", "Kids out in front of a building doing sightseeing.", "The big dog is checking out the smaller dog.", "hello , i am a web designer .", "no , i do not . i like it hot .", "i am a web designer , i do coding for several country musicians .", "yes , i am . i only listen to country .", "i hate snow , i am so glad i made the move .", "where at in the south ? i am in florida ."], "speaker_id": "snli:test:1740", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1740", "candidate_text": "Honestly, A man is eaten by a shark!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1740:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1740:premise", "snli:test:1195:premise", "snli:test:1764:premise", "snli:test:1194:premise", "style:personachat:test:446:self:6:positive:0", "style:personachat:test:446:self:6:positive:1", "style:personachat:test:446:self:6:positive:2", "style:personachat:test:446:self:6:positive:3", "style:personachat:test:446:self:6:positive:4", "style:personachat:test:446:self:6:positive:5"], "evidence_texts": ["A man is steering his ship out at sea.", "The big dog is checking out the smaller dog.", "Kids out in front of a building doing sightseeing.", "The big dog is checking out the smaller dog.", "that is cool so did my husband . . i am a hippy and make my clothes", "nothing like the good ol us of a i love singing folk songs", "that is neat ! i just found a kitty we named her bubbles", "me too ! we have to keep our environment clean to help save them all", "very true . its advancing so quickly !", "i remember those ! my mother had one in the army"], "speaker_id": "snli:test:1740", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1748", "candidate_text": "well, the roller blade wheels are square", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1748:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1748:premise", "snli:test:1647:premise", "snli:test:1792:premise", "snli:test:1189:premise", "style:personachat:test:65:self:1:positive:0"], "evidence_texts": ["A man wearing black pants rides along a concrete ledge on rollerblades.", "A woman is taking a picture of a girl on a scooter.", "Some children are playing soccer in front of a large wooden door.", "There is a group of children getting their picture taken with presents.", "i am short fat and ugly . you"], "speaker_id": "snli:test:1748", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1748", "candidate_text": "I wonder whether the roller blade wheels are square?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1748:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1748:premise", "snli:test:1647:premise", "snli:test:1792:premise", "snli:test:1189:premise", "style:personachat:test:227:self:1:positive:0"], "evidence_texts": ["A man wearing black pants rides along a concrete ledge on rollerblades.", "A woman is taking a picture of a girl on a scooter.", "Some children are playing soccer in front of a large wooden door.", "There is a group of children getting their picture taken with presents.", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:1748", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1763", "candidate_text": "well, a man is wearing a skirt", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1763:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1503:premise", "snli:test:1905:premise", "snli:test:1420:premise", "snli:test:1763:premise", "style:personachat:test:65:self:2:positive:0", "style:personachat:test:65:self:2:positive:1"], "evidence_texts": ["Two black labs run, one carrying an orange dog toy.", "A young cowboy is riding bucking bronco in an arena.", "A boy kneeling on a skateboard riding down the street", "A man wearing a kilt has thrown a large pole.", "awesome i have about thirty more years", "i am short fat and ugly . you"], "speaker_id": "snli:test:1763", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1763", "candidate_text": "Honestly, A man is wearing a skirt!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1763:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1503:premise", "snli:test:1905:premise", "snli:test:1420:premise", "snli:test:1763:premise", "style:personachat:test:456:self:2:positive:0", "style:personachat:test:456:self:2:positive:1"], "evidence_texts": ["Two black labs run, one carrying an orange dog toy.", "A young cowboy is riding bucking bronco in an arena.", "A boy kneeling on a skateboard riding down the street", "A man wearing a kilt has thrown a large pole.", "sometimes , yeah ! we love meat", "we are going on another trip soon , i will !"], "speaker_id": "snli:test:1763", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1764", "candidate_text": "well, kids are sightseeing", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1764:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1764:premise", "snli:test:1740:premise", "snli:test:1849:premise", "snli:test:1195:premise", "style:personachat:test:65:self:5:positive:0", "style:personachat:test:65:self:5:positive:1", "style:personachat:test:65:self:5:positive:2", "style:personachat:test:65:self:5:positive:3", "style:personachat:test:65:self:5:positive:4"], "evidence_texts": ["Kids out in front of a building doing sightseeing.", "A man is steering his ship out at sea.", "An Asian man is praying at a Buddhist temple.", "The big dog is checking out the smaller dog.", "cool i like to paint", "i dig in people trash for things to build", "true . i paint for a living", "awesome i have about thirty more years", "i am short fat and ugly . you"], "speaker_id": "snli:test:1764", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1764", "candidate_text": "Kids are sightseeing.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1764:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1764:premise", "snli:test:1740:premise", "snli:test:1849:premise", "snli:test:1195:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Kids out in front of a building doing sightseeing.", "A man is steering his ship out at sea.", "An Asian man is praying at a Buddhist temple.", "The big dog is checking out the smaller dog.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1764", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1770", "candidate_text": "well, the women are beating each other up with their bare fists", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1770:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1721:premise", "snli:test:1720:premise", "snli:test:1770:premise", "snli:test:2028:premise", "style:personachat:test:65:self:7:positive:0", "style:personachat:test:65:self:7:positive:1", "style:personachat:test:65:self:7:positive:2", "style:personachat:test:65:self:7:positive:3", "style:personachat:test:65:self:7:positive:4", "style:personachat:test:65:self:7:positive:5", "style:personachat:test:65:self:7:positive:6"], "evidence_texts": ["Three women sit together near woven walls.", "Three women sit together near woven walls.", "Women in red are performing with nun-chucks.", "A group of people sitting and knitting.", "hello yes i love bbq", "my poor toes are so webbed", "cool i like to paint", "i dig in people trash for things to build", "true . i paint for a living", "awesome i have about thirty more years", "i am short fat and ugly . you"], "speaker_id": "snli:test:1770", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1770", "candidate_text": "I wonder whether the women are beating each other up with their bare fists?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1770:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1721:premise", "snli:test:1720:premise", "snli:test:1770:premise", "snli:test:2028:premise", "style:personachat:test:227:self:7:positive:0", "style:personachat:test:227:self:7:positive:1", "style:personachat:test:227:self:7:positive:2", "style:personachat:test:227:self:7:positive:3", "style:personachat:test:227:self:7:positive:4", "style:personachat:test:227:self:7:positive:5", "style:personachat:test:227:self:7:positive:6"], "evidence_texts": ["Three women sit together near woven walls.", "Three women sit together near woven walls.", "Women in red are performing with nun-chucks.", "A group of people sitting and knitting.", "i just finished bacon and pancakes .", "that is true . i am studying to become a fashion designer .", "i lifted weights in prison . i did some time for stealing .", "my parents do not care about me . they are hardly ever around .", "i would love to be a mom but i am trying to turn my life around .", "thank you . do you have any children ?", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:1770", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1772", "candidate_text": "I wonder whether the women are auditioning for a kung fu movie?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1772:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1772:premise", "snli:test:2028:premise", "snli:test:1721:premise", "snli:test:2149:premise", "style:personachat:test:66:self:1:positive:0"], "evidence_texts": ["Women in red are performing with nun-chucks.", "A group of people sitting and knitting.", "Three women sit together near woven walls.", "Two men at a table having drinks.", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:1772", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1772", "candidate_text": "well, the women are auditioning for a kung fu movie", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1772:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1772:premise", "snli:test:2028:premise", "snli:test:1721:premise", "snli:test:2149:premise", "style:personachat:test:116:self:1:positive:0"], "evidence_texts": ["Women in red are performing with nun-chucks.", "A group of people sitting and knitting.", "Three women sit together near woven walls.", "Two men at a table having drinks.", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:1772", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1784", "candidate_text": "I wonder whether a man is holding a unicycle in a curious manner?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1784:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1784:premise", "snli:test:9840:premise", "snli:test:4450:premise", "snli:test:6681:premise", "style:personachat:test:66:self:2:positive:0", "style:personachat:test:66:self:2:positive:1"], "evidence_texts": ["A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "when did you buy it ? do you live alone ?", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:1784", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1784", "candidate_text": "Honestly, A man is holding a unicycle in a curious manner!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1784:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1784:premise", "snli:test:9840:premise", "snli:test:4450:premise", "snli:test:6681:premise", "style:personachat:test:466:self:2:positive:0", "style:personachat:test:466:self:2:positive:1"], "evidence_texts": ["A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "my father and grandfather were both army", "i have never been there ! and formally married"], "speaker_id": "snli:test:1784", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1786", "candidate_text": "I wonder whether a kid is practicing?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1786:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1500:premise", "snli:test:397:premise", "snli:test:1786:premise", "snli:test:2988:premise", "style:personachat:test:66:self:5:positive:0", "style:personachat:test:66:self:5:positive:1", "style:personachat:test:66:self:5:positive:2", "style:personachat:test:66:self:5:positive:3", "style:personachat:test:66:self:5:positive:4"], "evidence_texts": ["Military personnel are shopping", "A middle eastern marketplace.", "kid practicing karate move", "Two people play horseshoes.", "do you teach dogs obedience ? that is good . i am afraid of them .", "there were sometimes animals around when i worked at the carnival .", "when i want to relax i watch gritty movies and drink scotch .", "when did you buy it ? do you live alone ?", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:1786", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1786", "candidate_text": "A kid is practicing.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1786:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1500:premise", "snli:test:397:premise", "snli:test:1786:premise", "snli:test:2988:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Military personnel are shopping", "A middle eastern marketplace.", "kid practicing karate move", "Two people play horseshoes.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1786", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1787", "candidate_text": "I wonder whether a man is practicing a karate move?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1787:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2988:premise", "snli:test:1500:premise", "snli:test:2989:premise", "snli:test:1787:premise", "style:personachat:test:66:self:6:positive:0", "style:personachat:test:66:self:6:positive:1", "style:personachat:test:66:self:6:positive:2", "style:personachat:test:66:self:6:positive:3", "style:personachat:test:66:self:6:positive:4", "style:personachat:test:66:self:6:positive:5"], "evidence_texts": ["Two people play horseshoes.", "Military personnel are shopping", "Two people play horseshoes.", "kid practicing karate move", "i am doing well and relaxing at home with a glass of scotch . you ?", "do you teach dogs obedience ? that is good . i am afraid of them .", "there were sometimes animals around when i worked at the carnival .", "when i want to relax i watch gritty movies and drink scotch .", "when did you buy it ? do you live alone ?", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:1787", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1787", "candidate_text": "well, a man is practicing a karate move", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1787:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2988:premise", "snli:test:1500:premise", "snli:test:2989:premise", "snli:test:1787:premise", "style:personachat:test:116:self:7:positive:0", "style:personachat:test:116:self:7:positive:1", "style:personachat:test:116:self:7:positive:2", "style:personachat:test:116:self:7:positive:3", "style:personachat:test:116:self:7:positive:4", "style:personachat:test:116:self:7:positive:5", "style:personachat:test:116:self:7:positive:6"], "evidence_texts": ["Two people play horseshoes.", "Military personnel are shopping", "Two people play horseshoes.", "kid practicing karate move", "doing good . just got home from working at th factory .", "very cool . i grow violets in my garden . i love the color purple .", "i do . a poodle . we watch the buckeyes together every saturday .", "very cool ! where in ohio ?", "i am in florida right now .", "right on . i am in tampa right now .", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:1787", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1788", "candidate_text": "I wonder whether a young man waits outdoors in london on a bench?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1788:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1788:premise", "snli:test:1710:premise", "snli:test:1590:premise", "snli:test:2077:premise", "style:personachat:test:67:self:1:positive:0"], "evidence_texts": ["A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "odd . you need a singer in your metal band ? the church choir bores me ."], "speaker_id": "snli:test:1788", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1788", "candidate_text": "Honestly, A young man waits outdoors in London on a bench!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1788:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1788:premise", "snli:test:1710:premise", "snli:test:1590:premise", "snli:test:2077:premise", "style:personachat:test:467:self:1:positive:0"], "evidence_texts": ["A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "Pedestrians are gazing at a display of books set out on a sidewalk under a tree.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "that is so awesome ! i like to eat ."], "speaker_id": "snli:test:1788", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1789", "candidate_text": "I wonder whether the young man is 60 years old?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1789:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2077:premise", "snli:test:1789:premise", "snli:test:1710:premise", "snli:test:2091:premise", "style:personachat:test:67:self:2:positive:0", "style:personachat:test:67:self:2:positive:1"], "evidence_texts": ["Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "ooh , i love italian food , especially lasagna . do you have a favorite ?", "odd . you need a singer in your metal band ? the church choir bores me ."], "speaker_id": "snli:test:1789", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1789", "candidate_text": "The young man is 60 years old.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1789:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2077:premise", "snli:test:1789:premise", "snli:test:1710:premise", "snli:test:2091:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A swimmer is performing the breaststroke in an indoor swimming pool set up for a competition.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1789", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1792", "candidate_text": "I wonder whether the kids are enemies?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1792:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1792:premise", "snli:test:1748:premise", "snli:test:2156:premise", "snli:test:1647:premise", "style:personachat:test:67:self:5:positive:0", "style:personachat:test:67:self:5:positive:1", "style:personachat:test:67:self:5:positive:2", "style:personachat:test:67:self:5:positive:3", "style:personachat:test:67:self:5:positive:4"], "evidence_texts": ["Some children are playing soccer in front of a large wooden door.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "A child is sitting on the grass by a concrete storm drain.", "A woman is taking a picture of a girl on a scooter.", "wow ! jewish people live in north dakota ! what else do you do ?", "oh cool ! i sing in a choir ! i also play softball", "it helps with the work stress . i played second base . do you like to cook ?", "ooh , i love italian food , especially lasagna . do you have a favorite ?", "odd . you need a singer in your metal band ? the church choir bores me ."], "speaker_id": "snli:test:1792", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1792", "candidate_text": "well, the kids are enemies", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1792:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1792:premise", "snli:test:1748:premise", "snli:test:2156:premise", "snli:test:1647:premise", "style:personachat:test:118:self:5:positive:0", "style:personachat:test:118:self:5:positive:1", "style:personachat:test:118:self:5:positive:2", "style:personachat:test:118:self:5:positive:3", "style:personachat:test:118:self:5:positive:4"], "evidence_texts": ["Some children are playing soccer in front of a large wooden door.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "A child is sitting on the grass by a concrete storm drain.", "A woman is taking a picture of a girl on a scooter.", "i am just packing to move to nyc !", "i am originally from california though !", "how come ? i am moving to nyc for modeling !", "also , i enjoy karaoke , but i am not very good at it !", "i do not like crowes either . i like to stay to myself"], "speaker_id": "snli:test:1792", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1795", "candidate_text": "I wonder whether there is a light show?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1795:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1736:premise", "snli:test:2054:premise", "snli:test:1687:premise", "snli:test:1795:premise", "style:personachat:test:67:self:7:positive:0", "style:personachat:test:67:self:7:positive:1", "style:personachat:test:67:self:7:positive:2", "style:personachat:test:67:self:7:positive:3", "style:personachat:test:67:self:7:positive:4", "style:personachat:test:67:self:7:positive:5", "style:personachat:test:67:self:7:positive:6"], "evidence_texts": ["A worker is doing a job on scaffolding.", "A little boy playing outside on the cement.", "Kids work at computers with a teacher's help.", "Rock band on stage immersed in light show.", "advertising ? that sounds easy . i am a social worker dealing with at risk youth .", "that sounds cool . these kids drive me nuts .", "wow ! jewish people live in north dakota ! what else do you do ?", "oh cool ! i sing in a choir ! i also play softball", "it helps with the work stress . i played second base . do you like to cook ?", "ooh , i love italian food , especially lasagna . do you have a favorite ?", "odd . you need a singer in your metal band ? the church choir bores me ."], "speaker_id": "snli:test:1795", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1795", "candidate_text": "Honestly, There is a light show!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1795:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1736:premise", "snli:test:2054:premise", "snli:test:1687:premise", "snli:test:1795:premise", "style:personachat:test:467:self:7:positive:0", "style:personachat:test:467:self:7:positive:1", "style:personachat:test:467:self:7:positive:2", "style:personachat:test:467:self:7:positive:3", "style:personachat:test:467:self:7:positive:4", "style:personachat:test:467:self:7:positive:5", "style:personachat:test:467:self:7:positive:6"], "evidence_texts": ["A worker is doing a job on scaffolding.", "A little boy playing outside on the cement.", "Kids work at computers with a teacher's help.", "Rock band on stage immersed in light show.", "hello . i am great , how are you ?", "i bet ! i am practicing tae kwan do . i am a champion at it .", "sometimes ! my dad teaches it so i started young . i am only 12 .", "yes , a dog , you ? what state do you live in ?", "cool . i live in texas . i have never been to another state . how about you ?", "oh , wow . that is a lot of travelling ! do you do it for fun ?", "that is so awesome ! i like to eat ."], "speaker_id": "snli:test:1795", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1797", "candidate_text": "well, a band is playing a song for an audience", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1797:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1149:premise", "snli:test:1797:premise", "snli:test:1910:premise", "snli:test:876:premise", "style:personachat:test:68:self:1:positive:0"], "evidence_texts": ["A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:1797", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1797", "candidate_text": "I wonder whether a band is playing a song for an audience?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1797:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1149:premise", "snli:test:1797:premise", "snli:test:1910:premise", "snli:test:876:premise", "style:personachat:test:241:self:1:positive:0"], "evidence_texts": ["A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "The back of a woman wearing a white jacket with blue jeans walking towards a flock of birds.", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:1797", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1802", "candidate_text": "well, a quarterback is holding a ball", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1802:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1513:premise", "snli:test:1945:premise", "snli:test:1802:premise", "snli:test:1455:premise", "style:personachat:test:68:self:2:positive:0", "style:personachat:test:68:self:2:positive:1"], "evidence_texts": ["Several people in blue scrubs and one in a skirt and black blouse.", "A man stands on one hand in the street while a crowd watches.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "Three hikers and a dog are walking through the foliage on the hillside.", "my best kind of hobby i like is entertaining board games and gambling .", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:1802", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1802", "candidate_text": "Honestly, A quarterback is holding a ball!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1802:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1513:premise", "snli:test:1945:premise", "snli:test:1802:premise", "snli:test:1455:premise", "style:personachat:test:479:self:2:positive:0", "style:personachat:test:479:self:2:positive:1"], "evidence_texts": ["Several people in blue scrubs and one in a skirt and black blouse.", "A man stands on one hand in the street while a crowd watches.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "Three hikers and a dog are walking through the foliage on the hillside.", "oh i hate them . xd", "yeah , scared of heights . i take my risks in card games both feet on the ground !"], "speaker_id": "snli:test:1802", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1806", "candidate_text": "well, a group of people are standing around with smiles are their faces, and in the background are more people and handmade signs", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1806:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1700:premise", "snli:test:1996:premise", "snli:test:1806:premise", "snli:test:1557:premise", "style:personachat:test:68:self:5:positive:0", "style:personachat:test:68:self:5:positive:1", "style:personachat:test:68:self:5:positive:2", "style:personachat:test:68:self:5:positive:3", "style:personachat:test:68:self:5:positive:4"], "evidence_texts": ["Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "A group of people standing around smiling, and in the background are more people and handmade signs.", "An older women looking off into the distance with three older women walking towards her from behind.", "wonderful cool but i did get my diploma from school to college .", "so so i used to travel a lot from new jersey to atlantic city .", "it was when i was going but now it probably is not so nice now .", "my best kind of hobby i like is entertaining board games and gambling .", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:1806", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1806", "candidate_text": "A group of people are standing around with smiles are their faces, and in the background are more people and handmade signs.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1806:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1700:premise", "snli:test:1996:premise", "snli:test:1806:premise", "snli:test:1557:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "A group of people standing around smiling, and in the background are more people and handmade signs.", "An older women looking off into the distance with three older women walking towards her from behind.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1806", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1817", "candidate_text": "well, a boy is watching a boat float by", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1817:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1705:premise", "snli:test:1817:premise", "snli:test:1820:premise", "snli:test:1643:premise", "style:personachat:test:68:self:7:positive:0", "style:personachat:test:68:self:7:positive:1", "style:personachat:test:68:self:7:positive:2", "style:personachat:test:68:self:7:positive:3", "style:personachat:test:68:self:7:positive:4", "style:personachat:test:68:self:7:positive:5", "style:personachat:test:68:self:7:positive:6"], "evidence_texts": ["a woman in a brown dress looking at papers in front of a class.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "Three women talk to each other next to another woman sitting on a motorcycle.", "Bicyclist ride the course near the ocean as the day comes to an end.", "how are you doing this late night let us chat .", "oh i do not think i could do that was never to good in the study .", "wonderful cool but i did get my diploma from school to college .", "so so i used to travel a lot from new jersey to atlantic city .", "it was when i was going but now it probably is not so nice now .", "my best kind of hobby i like is entertaining board games and gambling .", "i like pets and i got this cat by the name of arizona ."], "speaker_id": "snli:test:1817", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1817", "candidate_text": "I wonder whether a boy is watching a boat float by?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1817:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1705:premise", "snli:test:1817:premise", "snli:test:1820:premise", "snli:test:1643:premise", "style:personachat:test:241:self:7:positive:0", "style:personachat:test:241:self:7:positive:1", "style:personachat:test:241:self:7:positive:2", "style:personachat:test:241:self:7:positive:3", "style:personachat:test:241:self:7:positive:4", "style:personachat:test:241:self:7:positive:5", "style:personachat:test:241:self:7:positive:6"], "evidence_texts": ["a woman in a brown dress looking at papers in front of a class.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "Three women talk to each other next to another woman sitting on a motorcycle.", "Bicyclist ride the course near the ocean as the day comes to an end.", "hello , i am doing great , how are you this evening ?", "photography , i do not drive in nyc , so i take photos whenever i walk .", "it is . i used to live in ohio . are you jewish ?", "nothing wrong with that , i was only going to ask if you knew of a deli .", "i do at times , i mainly wanted pickles . the office is almost on .", "they are an excellent snack with my show .", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:1817", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1820", "candidate_text": "Honestly, Three women are talking and there is a woman on a motorcyle!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1820:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1820:premise", "snli:test:1817:premise", "snli:test:2035:premise", "snli:test:1705:premise", "style:personachat:test:69:self:1:positive:0"], "evidence_texts": ["Three women talk to each other next to another woman sitting on a motorcycle.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "a woman in a brown dress looking at papers in front of a class.", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:1820", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1820", "candidate_text": "well, three women are talking and there is a woman on a motorcyle", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1820:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1820:premise", "snli:test:1817:premise", "snli:test:2035:premise", "snli:test:1705:premise", "style:personachat:test:121:self:1:positive:0"], "evidence_texts": ["Three women talk to each other next to another woman sitting on a motorcycle.", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "a woman in a brown dress looking at papers in front of a class.", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:1820", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1832", "candidate_text": "Honestly, a man surfs!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1832:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1832:premise", "snli:test:5142:premise", "snli:test:1633:premise", "snli:test:3226:premise", "style:personachat:test:69:self:2:positive:0", "style:personachat:test:69:self:2:positive:1"], "evidence_texts": ["A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:1832", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1832", "candidate_text": "a man surfs.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1832:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1832:premise", "snli:test:5142:premise", "snli:test:1633:premise", "snli:test:3226:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1832", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1841", "candidate_text": "Honestly, A woman is using what looks to be a fax machine!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1841:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1600:premise", "snli:test:1841:premise", "snli:test:2013:premise", "snli:test:1599:premise", "style:personachat:test:69:self:5:positive:0", "style:personachat:test:69:self:5:positive:1", "style:personachat:test:69:self:5:positive:2", "style:personachat:test:69:self:5:positive:3", "style:personachat:test:69:self:5:positive:4"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "A man is using what looks to be a fax machine.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "An Ambulance is passing a man wearing a bandanna and girl.", "i want to visit italy one day .", "do you cook italian food ?", "i understand , i am from the middle east .", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:1841", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1841", "candidate_text": "I wonder whether a woman is using what looks to be a fax machine?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1841:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1600:premise", "snli:test:1841:premise", "snli:test:2013:premise", "snli:test:1599:premise", "style:personachat:test:243:self:5:positive:0", "style:personachat:test:243:self:5:positive:1", "style:personachat:test:243:self:5:positive:2", "style:personachat:test:243:self:5:positive:3", "style:personachat:test:243:self:5:positive:4"], "evidence_texts": ["An Ambulance is passing a man wearing a bandanna and girl.", "A man is using what looks to be a fax machine.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "An Ambulance is passing a man wearing a bandanna and girl.", "that is cool . are you married ?", "wow , sounds like you are a black widower . . .", "yes , i have a mini aussie shepherd . you ?", "are you a pet groomer ?", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:1841", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1849", "candidate_text": "Honestly, The man is making noise and bothering others!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1849:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1764:premise", "snli:test:1849:premise", "snli:test:1930:premise", "snli:test:1740:premise", "style:personachat:test:69:self:7:positive:0", "style:personachat:test:69:self:7:positive:1", "style:personachat:test:69:self:7:positive:2", "style:personachat:test:69:self:7:positive:3", "style:personachat:test:69:self:7:positive:4", "style:personachat:test:69:self:7:positive:5", "style:personachat:test:69:self:7:positive:6"], "evidence_texts": ["Kids out in front of a building doing sightseeing.", "An Asian man is praying at a Buddhist temple.", "Man with plaid shirt staring off into the distance.", "A man is steering his ship out at sea.", "hello . i am well . how are you doing ?", "interesting . where are you from ?", "i want to visit italy one day .", "do you cook italian food ?", "i understand , i am from the middle east .", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:1849", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1849", "candidate_text": "well, the man is making noise and bothering others", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1849:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1764:premise", "snli:test:1849:premise", "snli:test:1930:premise", "snli:test:1740:premise", "style:personachat:test:121:self:7:positive:0", "style:personachat:test:121:self:7:positive:1", "style:personachat:test:121:self:7:positive:2", "style:personachat:test:121:self:7:positive:3", "style:personachat:test:121:self:7:positive:4", "style:personachat:test:121:self:7:positive:5", "style:personachat:test:121:self:7:positive:6"], "evidence_texts": ["Kids out in front of a building doing sightseeing.", "An Asian man is praying at a Buddhist temple.", "Man with plaid shirt staring off into the distance.", "A man is steering his ship out at sea.", "hi i am well trying to stay up you", "why ? you gotta get out . got kids ?", "if no kids like me i have 2 with my husband go travel now", "i do too , i have not any this year", "i like to run or read an old cars", "cool , like jazz ? i love it", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:1849", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1867", "candidate_text": "Honestly, The group of people are scuba divers!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1867:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1461:premise", "snli:test:2022:premise", "snli:test:1156:premise", "snli:test:1867:premise", "style:personachat:test:70:self:1:positive:0"], "evidence_texts": ["First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:1867", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1867", "candidate_text": "The group of people are scuba divers.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1867:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1461:premise", "snli:test:2022:premise", "snli:test:1156:premise", "snli:test:1867:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "A group of children looking at the contents of a slide through a microscope displayed on a computer screen.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1867", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1905", "candidate_text": "Honestly, A cowboy is swimming in the arena!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1905:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1763:premise", "snli:test:1905:premise", "snli:test:1972:premise", "snli:test:1503:premise", "style:personachat:test:70:self:2:positive:0", "style:personachat:test:70:self:2:positive:1"], "evidence_texts": ["A man wearing a kilt has thrown a large pole.", "A young cowboy is riding bucking bronco in an arena.", "Three dogs run through the grass while wearing colored shirts.", "Two black labs run, one carrying an orange dog toy.", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:1905", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1905", "candidate_text": "I wonder whether a cowboy is swimming in the arena?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1905:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1763:premise", "snli:test:1905:premise", "snli:test:1972:premise", "snli:test:1503:premise", "style:personachat:test:244:self:2:positive:0", "style:personachat:test:244:self:2:positive:1"], "evidence_texts": ["A man wearing a kilt has thrown a large pole.", "A young cowboy is riding bucking bronco in an arena.", "Three dogs run through the grass while wearing colored shirts.", "Two black labs run, one carrying an orange dog toy.", "i eat granola every day ! do you have kids ?", "i have two boys . any other family ?"], "speaker_id": "snli:test:1905", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1910", "candidate_text": "Honestly, The man is playing an acoustic guitar!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1910:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1797:premise", "snli:test:1954:premise", "snli:test:1910:premise", "snli:test:1149:premise", "style:personachat:test:70:self:5:positive:0", "style:personachat:test:70:self:5:positive:1", "style:personachat:test:70:self:5:positive:2", "style:personachat:test:70:self:5:positive:3", "style:personachat:test:70:self:5:positive:4"], "evidence_texts": ["A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "i like to cook to much to worry about my figure ! do you like food ?", "hmm . . you sound like a liberal .", "i am proud to say i voted for trump . do you like him ?", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:1910", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1910", "candidate_text": "well, the man is playing an acoustic guitar", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1910:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1797:premise", "snli:test:1954:premise", "snli:test:1910:premise", "snli:test:1149:premise", "style:personachat:test:122:self:5:positive:0", "style:personachat:test:122:self:5:positive:1", "style:personachat:test:122:self:5:positive:2", "style:personachat:test:122:self:5:positive:3", "style:personachat:test:122:self:5:positive:4"], "evidence_texts": ["A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "A person in a yellow helmet rock climbing while another person waits on the ground with the tether.", "about two more years how about you", "you have any brothers or sister ?", "no siblings i wish i had a brother or sister", "very nice i just focus on my studies at this point", "i am single since i am super busy with school"], "speaker_id": "snli:test:1910", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1911", "candidate_text": "Honestly, A young woman drank a lot of soda!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1911:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1342:premise", "snli:test:1219:premise", "snli:test:2197:premise", "snli:test:1911:premise", "style:personachat:test:70:self:7:positive:0", "style:personachat:test:70:self:7:positive:1", "style:personachat:test:70:self:7:positive:2", "style:personachat:test:70:self:7:positive:3", "style:personachat:test:70:self:7:positive:4", "style:personachat:test:70:self:7:positive:5", "style:personachat:test:70:self:7:positive:6"], "evidence_texts": ["A baby girl and little boy are standing next to a guitar and a drum.", "A man in a market holding some very large green onions while smoking a cigarette.", "A man is standing inside a building under construction filled with bricks and various tools.", "An elderly man is smiling while sitting in front of a row of soda cans.", "my wife is a model , but we do pretty well . are you married ?", "we adopted because my wife was afraid of that , but i love my 2 kids !", "i like to cook to much to worry about my figure ! do you like food ?", "hmm . . you sound like a liberal .", "i am proud to say i voted for trump . do you like him ?", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:1911", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1911", "candidate_text": "A young woman drank a lot of soda.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1911:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1342:premise", "snli:test:1219:premise", "snli:test:2197:premise", "snli:test:1911:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A baby girl and little boy are standing next to a guitar and a drum.", "A man in a market holding some very large green onions while smoking a cigarette.", "A man is standing inside a building under construction filled with bricks and various tools.", "An elderly man is smiling while sitting in front of a row of soda cans.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1911", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1912", "candidate_text": "well, an old man collects soda cans", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1912:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1912:premise", "snli:test:2197:premise", "snli:test:1342:premise", "snli:test:2700:premise", "style:personachat:test:71:self:1:positive:0"], "evidence_texts": ["An elderly man is smiling while sitting in front of a row of soda cans.", "A man is standing inside a building under construction filled with bricks and various tools.", "A baby girl and little boy are standing next to a guitar and a drum.", "A man with a tan jacket with a full grocery bag is crossing the street.", "send it to me online"], "speaker_id": "snli:test:1912", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1912", "candidate_text": "I wonder whether an old man collects soda cans?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1912:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1912:premise", "snli:test:2197:premise", "snli:test:1342:premise", "snli:test:2700:premise", "style:personachat:test:254:self:1:positive:0"], "evidence_texts": ["An elderly man is smiling while sitting in front of a row of soda cans.", "A man is standing inside a building under construction filled with bricks and various tools.", "A baby girl and little boy are standing next to a guitar and a drum.", "A man with a tan jacket with a full grocery bag is crossing the street.", "do you build model cars ?"], "speaker_id": "snli:test:1912", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1930", "candidate_text": "well, man staring off into the distance", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1930:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1849:premise", "snli:test:1764:premise", "snli:test:1930:premise", "snli:test:1956:premise", "style:personachat:test:71:self:2:positive:0", "style:personachat:test:71:self:2:positive:1"], "evidence_texts": ["An Asian man is praying at a Buddhist temple.", "Kids out in front of a building doing sightseeing.", "Man with plaid shirt staring off into the distance.", "A man is riding a bicycle in the street.", "really ? let me see your work", "send it to me online"], "speaker_id": "snli:test:1930", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1930", "candidate_text": "Honestly, Man staring off into the distance!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1930:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1849:premise", "snli:test:1764:premise", "snli:test:1930:premise", "snli:test:1956:premise", "style:personachat:test:512:self:2:positive:0", "style:personachat:test:512:self:2:positive:1"], "evidence_texts": ["An Asian man is praying at a Buddhist temple.", "Kids out in front of a building doing sightseeing.", "Man with plaid shirt staring off into the distance.", "A man is riding a bicycle in the street.", "do you have any kids ?", "ha ! well then good . i am 34 ."], "speaker_id": "snli:test:1930", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1931", "candidate_text": "well, man stares off into the distance at the incoming clouds", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1931:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1956:premise", "snli:test:1849:premise", "snli:test:1958:premise", "snli:test:1931:premise", "style:personachat:test:71:self:5:positive:0", "style:personachat:test:71:self:5:positive:1", "style:personachat:test:71:self:5:positive:2", "style:personachat:test:71:self:5:positive:3", "style:personachat:test:71:self:5:positive:4"], "evidence_texts": ["A man is riding a bicycle in the street.", "An Asian man is praying at a Buddhist temple.", "A man is riding a bicycle in the street.", "Man with plaid shirt staring off into the distance.", "i love to sing all the time but will never have a kid", "oh really ? are you single ? i model here in nyc", "really ? i thought you were an artist ,", "really ? let me see your work", "send it to me online"], "speaker_id": "snli:test:1931", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1931", "candidate_text": "Man stares off into the distance at the incoming clouds.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1931:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1956:premise", "snli:test:1849:premise", "snli:test:1958:premise", "snli:test:1931:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man is riding a bicycle in the street.", "An Asian man is praying at a Buddhist temple.", "A man is riding a bicycle in the street.", "Man with plaid shirt staring off into the distance.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1931", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1945", "candidate_text": "well, a man is standing on his hands", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1945:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1802:premise", "snli:test:1945:premise", "snli:test:2122:premise", "snli:test:1513:premise", "style:personachat:test:71:self:7:positive:0", "style:personachat:test:71:self:7:positive:1", "style:personachat:test:71:self:7:positive:2", "style:personachat:test:71:self:7:positive:3", "style:personachat:test:71:self:7:positive:4", "style:personachat:test:71:self:7:positive:5", "style:personachat:test:71:self:7:positive:6"], "evidence_texts": ["The Sooner football player carrying the ball is trying to avoid being tackled.", "A man stands on one hand in the street while a crowd watches.", "A crowd and two males dressed in different cultural attire playing the drums.", "Several people in blue scrubs and one in a skirt and black blouse.", "i am well an you ? it is cold", "yes it is i get it", "i love to sing all the time but will never have a kid", "oh really ? are you single ? i model here in nyc", "really ? i thought you were an artist ,", "really ? let me see your work", "send it to me online"], "speaker_id": "snli:test:1945", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1945", "candidate_text": "I wonder whether a man is standing on his hands?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1945:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1802:premise", "snli:test:1945:premise", "snli:test:2122:premise", "snli:test:1513:premise", "style:personachat:test:254:self:7:positive:0", "style:personachat:test:254:self:7:positive:1", "style:personachat:test:254:self:7:positive:2", "style:personachat:test:254:self:7:positive:3", "style:personachat:test:254:self:7:positive:4", "style:personachat:test:254:self:7:positive:5", "style:personachat:test:254:self:7:positive:6"], "evidence_texts": ["The Sooner football player carrying the ball is trying to avoid being tackled.", "A man stands on one hand in the street while a crowd watches.", "A crowd and two males dressed in different cultural attire playing the drums.", "Several people in blue scrubs and one in a skirt and black blouse.", "hello , ginger and i want to live in florida .", "it is raining , we are getting the tail end of nate .", "i prefer mustangs and jeeps . my baby does not go in either .", "no , i do not . i am a photographer .", "that is none of your business , and very rude to ask .", "have you seen the movie titanic ? it is my favorite .", "do you build model cars ?"], "speaker_id": "snli:test:1945", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1947", "candidate_text": "Honestly, The child is swimming at the pool!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1947:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1697:premise", "snli:test:2303:premise", "snli:test:801:premise", "snli:test:1947:premise", "style:personachat:test:72:self:1:positive:0"], "evidence_texts": ["A woman holding bags behind plantains.", "Motorcycle rider going over a bump.", "Two people talking on a dock.", "A child buried in the sand.", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:1947", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1947", "candidate_text": "well, the child is swimming at the pool", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1947:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1697:premise", "snli:test:2303:premise", "snli:test:801:premise", "snli:test:1947:premise", "style:personachat:test:124:self:1:positive:0"], "evidence_texts": ["A woman holding bags behind plantains.", "Motorcycle rider going over a bump.", "Two people talking on a dock.", "A child buried in the sand.", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:1947", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1954", "candidate_text": "Honestly, A man in a gray shirt is outdoors!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1954:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1910:premise", "snli:test:2616:premise", "snli:test:1954:premise", "snli:test:1797:premise", "style:personachat:test:72:self:2:positive:0", "style:personachat:test:72:self:2:positive:1"], "evidence_texts": ["A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "i work out about 2 hours a day but it is hard to fit it all in .", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:1954", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1954", "candidate_text": "A man in a gray shirt is outdoors.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:1954:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1910:premise", "snli:test:2616:premise", "snli:test:1954:premise", "snli:test:1797:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A guitarist, a drummer, and another band member are performing on stage with red lights on the background.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1954", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1956", "candidate_text": "Honestly, a bicycle is being ridden!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1956:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1956:premise", "snli:test:1931:premise", "snli:test:1930:premise", "snli:test:2108:premise", "style:personachat:test:72:self:5:positive:0", "style:personachat:test:72:self:5:positive:1", "style:personachat:test:72:self:5:positive:2", "style:personachat:test:72:self:5:positive:3", "style:personachat:test:72:self:5:positive:4"], "evidence_texts": ["A man is riding a bicycle in the street.", "Man with plaid shirt staring off into the distance.", "Man with plaid shirt staring off into the distance.", "A group of people in black outfits drinking beer.", "3 kids and a great husband too . we live beside the sea .", "that is a lot ! they would not fit into my mustang convertible .", "do you like to work out ? i am trying to lose some weight .", "i work out about 2 hours a day but it is hard to fit it all in .", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:1956", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1956", "candidate_text": "I wonder whether a bicycle is being ridden?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1956:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1956:premise", "snli:test:1931:premise", "snli:test:1930:premise", "snli:test:2108:premise", "style:personachat:test:260:self:5:positive:0", "style:personachat:test:260:self:5:positive:1", "style:personachat:test:260:self:5:positive:2", "style:personachat:test:260:self:5:positive:3", "style:personachat:test:260:self:5:positive:4"], "evidence_texts": ["A man is riding a bicycle in the street.", "Man with plaid shirt staring off into the distance.", "Man with plaid shirt staring off into the distance.", "A group of people in black outfits drinking beer.", "i love music and especially bluegrass which i can play good", "i had a tail when i was born but had it cut off", "i had super long hair . do you like sports ?", "that does i love the olympics especially the summer ones", "cool do you like to watch movies ?"], "speaker_id": "snli:test:1956", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1958", "candidate_text": "Honestly, the man is chasing after his daughter!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1958:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2108:premise", "snli:test:1931:premise", "snli:test:1958:premise", "snli:test:2147:premise", "style:personachat:test:72:self:7:positive:0", "style:personachat:test:72:self:7:positive:1", "style:personachat:test:72:self:7:positive:2", "style:personachat:test:72:self:7:positive:3", "style:personachat:test:72:self:7:positive:4", "style:personachat:test:72:self:7:positive:5", "style:personachat:test:72:self:7:positive:6"], "evidence_texts": ["A group of people in black outfits drinking beer.", "Man with plaid shirt staring off into the distance.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "hi , just finished a great work out . do you like swimming ?", "i have kids too ! we go swimming together every day .", "3 kids and a great husband too . we live beside the sea .", "that is a lot ! they would not fit into my mustang convertible .", "do you like to work out ? i am trying to lose some weight .", "i work out about 2 hours a day but it is hard to fit it all in .", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:1958", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1958", "candidate_text": "well, the man is chasing after his daughter", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1958:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2108:premise", "snli:test:1931:premise", "snli:test:1958:premise", "snli:test:2147:premise", "style:personachat:test:124:self:6:positive:0", "style:personachat:test:124:self:6:positive:1", "style:personachat:test:124:self:6:positive:2", "style:personachat:test:124:self:6:positive:3", "style:personachat:test:124:self:6:positive:4", "style:personachat:test:124:self:6:positive:5"], "evidence_texts": ["A group of people in black outfits drinking beer.", "Man with plaid shirt staring off into the distance.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "i am okay how are you today ?", "that is good where are you from ?", "ca nice i never been hiking before i live in upstate ny", "yes ny has lots of interesting foods what is your favorite type of food ?", "oh okay lol what do you do for work ?", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:1958", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1972", "candidate_text": "well, three small dogs run through the grass during the day", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1972:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1972:premise", "snli:test:1905:premise", "snli:test:1763:premise", "snli:test:2050:premise", "style:personachat:test:73:self:1:positive:0"], "evidence_texts": ["Three dogs run through the grass while wearing colored shirts.", "A young cowboy is riding bucking bronco in an arena.", "A man wearing a kilt has thrown a large pole.", "A man is laid back in front of a tv.", "i average five miles a day ."], "speaker_id": "snli:test:1972", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1972", "candidate_text": "Three small dogs run through the grass during the day.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1972:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1972:premise", "snli:test:1905:premise", "snli:test:1763:premise", "snli:test:2050:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Three dogs run through the grass while wearing colored shirts.", "A young cowboy is riding bucking bronco in an arena.", "A man wearing a kilt has thrown a large pole.", "A man is laid back in front of a tv.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1972", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1973", "candidate_text": "well, there are animals outdoors", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1973:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2050:premise", "snli:test:1973:premise", "snli:test:1905:premise", "snli:test:2062:premise", "style:personachat:test:73:self:2:positive:0", "style:personachat:test:73:self:2:positive:1"], "evidence_texts": ["A man is laid back in front of a tv.", "Three dogs run through the grass while wearing colored shirts.", "A young cowboy is riding bucking bronco in an arena.", "People hurrying to get to work in a busy city.", "i do too . it is what keeps me young .", "i average five miles a day ."], "speaker_id": "snli:test:1973", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1973", "candidate_text": "I wonder whether there are animals outdoors?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1973:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2050:premise", "snli:test:1973:premise", "snli:test:1905:premise", "snli:test:2062:premise", "style:personachat:test:267:self:2:positive:0", "style:personachat:test:267:self:2:positive:1"], "evidence_texts": ["A man is laid back in front of a tv.", "Three dogs run through the grass while wearing colored shirts.", "A young cowboy is riding bucking bronco in an arena.", "People hurrying to get to work in a busy city.", "dance is great exercise for a young one like yourself !", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:1973", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:1976", "candidate_text": "well, the man who just purchased a pinstriped suit for a formal dinner didn't notice the poster to his left that shows a man wearing the same suit", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1976:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1976:premise", "snli:test:1689:premise", "snli:test:2017:premise", "snli:test:1498:premise", "style:personachat:test:73:self:5:positive:0", "style:personachat:test:73:self:5:positive:1", "style:personachat:test:73:self:5:positive:2", "style:personachat:test:73:self:5:positive:3", "style:personachat:test:73:self:5:positive:4"], "evidence_texts": ["A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "i have a horse ranch , it was my fathers .", "that sounds very nice , i need to find a hobby like that .", "ride my horses around the ranch and take photos .", "i do too . it is what keeps me young .", "i average five miles a day ."], "speaker_id": "snli:test:1976", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:1976", "candidate_text": "Honestly, The man who just purchased a pinstriped suit for a formal dinner didn't notice the poster to his left that shows a man wearing the same suit!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1976:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1976:premise", "snli:test:1689:premise", "snli:test:2017:premise", "snli:test:1498:premise", "style:personachat:test:549:self:5:positive:0", "style:personachat:test:549:self:5:positive:1", "style:personachat:test:549:self:5:positive:2", "style:personachat:test:549:self:5:positive:3", "style:personachat:test:549:self:5:positive:4"], "evidence_texts": ["A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "One woman with jeans, gray shirt and sandals; with girl in all pink watching a white ball float in midair.", "that sounds awesome ! that is what i want to do . fashion design .", "i have never listened to them . i prefered pop growing up .", "ah . i see . my best friend stole my boyfriend . . .", "its no big deal . its in the past . it was high school .", "i am too ! i went through a bad phase then . . . i was even arrested"], "speaker_id": "snli:test:1976", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:1996", "candidate_text": "well, people checking out their reflection", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1996:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1996:premise", "snli:test:1806:premise", "snli:test:2577:premise", "snli:test:1700:premise", "style:personachat:test:73:self:7:positive:0", "style:personachat:test:73:self:7:positive:1", "style:personachat:test:73:self:7:positive:2", "style:personachat:test:73:self:7:positive:3", "style:personachat:test:73:self:7:positive:4", "style:personachat:test:73:self:7:positive:5", "style:personachat:test:73:self:7:positive:6"], "evidence_texts": ["People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "A group of people standing around smiling, and in the background are more people and handmade signs.", "Adults and children share in the looking at something, and some young ladies stand to the side.", "Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "hello there . it is okay , one of the horses kicked me .", "it always hurts , lol . i need to retire .", "i have a horse ranch , it was my fathers .", "that sounds very nice , i need to find a hobby like that .", "ride my horses around the ranch and take photos .", "i do too . it is what keeps me young .", "i average five miles a day ."], "speaker_id": "snli:test:1996", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:1996", "candidate_text": "People checking out their reflection.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:1996:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1996:premise", "snli:test:1806:premise", "snli:test:2577:premise", "snli:test:1700:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "A group of people standing around smiling, and in the background are more people and handmade signs.", "Adults and children share in the looking at something, and some young ladies stand to the side.", "Three soccer players, two in orange one in yellow, running for the ball on a soccer field.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:1996", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2013", "candidate_text": "Honestly, there are 3 people with blindfolds!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2013:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1841:premise", "snli:test:2013:premise", "snli:test:2083:premise", "snli:test:1600:premise", "style:personachat:test:74:self:1:positive:0"], "evidence_texts": ["A man is using what looks to be a fax machine.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A woman in a red shirt and white skirt playing tennis.", "An Ambulance is passing a man wearing a bandanna and girl.", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:2013", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2013", "candidate_text": "I wonder whether there are 3 people with blindfolds?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2013:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1841:premise", "snli:test:2013:premise", "snli:test:2083:premise", "snli:test:1600:premise", "style:personachat:test:268:self:1:positive:0"], "evidence_texts": ["A man is using what looks to be a fax machine.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A woman in a red shirt and white skirt playing tennis.", "An Ambulance is passing a man wearing a bandanna and girl.", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:2013", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2017", "candidate_text": "Honestly, A machine takes down architecture as a man sprays water on it!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2017:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1976:premise", "snli:test:2017:premise", "snli:test:1689:premise", "snli:test:3011:premise", "style:personachat:test:74:self:2:positive:0", "style:personachat:test:74:self:2:positive:1"], "evidence_texts": ["A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:2017", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2017", "candidate_text": "well, a machine takes down architecture as a man sprays water on it", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2017:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1976:premise", "snli:test:2017:premise", "snli:test:1689:premise", "snli:test:3011:premise", "style:personachat:test:129:self:2:positive:0", "style:personachat:test:129:self:2:positive:1"], "evidence_texts": ["A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "The man in the black shirt is showing the man in the orange shirt something that he has for sale.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "one of my favorite passions is food ! if it looks good i will try it at least once", "both , i use to want to be a chef"], "speaker_id": "snli:test:2017", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2018", "candidate_text": "Honestly, A man throws rocks at a wall!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2018:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3011:premise", "snli:test:1976:premise", "snli:test:2018:premise", "snli:test:3178:premise", "style:personachat:test:74:self:5:positive:0", "style:personachat:test:74:self:5:positive:1", "style:personachat:test:74:self:5:positive:2", "style:personachat:test:74:self:5:positive:3", "style:personachat:test:74:self:5:positive:4"], "evidence_texts": ["A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "great ! what is your favorite animal ? i love riding horses on my dads farm", "awe , i love turtles too . any hobbies ?", "cool . my favorite hobby is watching horror films", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:2018", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2018", "candidate_text": "A man throws rocks at a wall.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2018:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3011:premise", "snli:test:1976:premise", "snli:test:2018:premise", "snli:test:3178:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "A man in a pinstriped suit walks down the street and does n't notice the quirky poster on his left.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2018", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2022", "candidate_text": "Honestly, a woman with two blue cars sits in a parking lot!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2022:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:1867:premise", "snli:test:2031:premise", "snli:test:2022:premise", "snli:test:1461:premise", "style:personachat:test:74:self:6:positive:0", "style:personachat:test:74:self:6:positive:1", "style:personachat:test:74:self:6:positive:2", "style:personachat:test:74:self:6:positive:3", "style:personachat:test:74:self:6:positive:4", "style:personachat:test:74:self:6:positive:5"], "evidence_texts": ["A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "me too ! how are you today ?", "great ! what is your favorite animal ? i love riding horses on my dads farm", "awe , i love turtles too . any hobbies ?", "cool . my favorite hobby is watching horror films", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:2022", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2022", "candidate_text": "I wonder whether a woman with two blue cars sits in a parking lot?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2022:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1867:premise", "snli:test:2031:premise", "snli:test:2022:premise", "snli:test:1461:premise", "style:personachat:test:268:self:6:positive:0", "style:personachat:test:268:self:6:positive:1", "style:personachat:test:268:self:6:positive:2", "style:personachat:test:268:self:6:positive:3", "style:personachat:test:268:self:6:positive:4", "style:personachat:test:268:self:6:positive:5"], "evidence_texts": ["A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "First Lady Laura Bush at podium, in front of seated audience, at the White House Conference on Global Literacy.", "i am fine i am worry as next month i have to pay loan .", "i do not have a job", "what do you mean by this ?", "lol no i will not do this . it is illegal", "haha yes you do have bad ideas", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:2022", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2028", "candidate_text": "well, people are sitting", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2028:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2028:premise", "snli:test:1772:premise", "snli:test:2149:premise", "snli:test:1770:premise", "style:personachat:test:75:self:1:positive:0"], "evidence_texts": ["A group of people sitting and knitting.", "Women in red are performing with nun-chucks.", "Two men at a table having drinks.", "Women in red are performing with nun-chucks.", "so are you . great personality"], "speaker_id": "snli:test:2028", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2028", "candidate_text": "Honestly, People are sitting!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2028:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2028:premise", "snli:test:1772:premise", "snli:test:2149:premise", "snli:test:1770:premise", "style:personachat:test:565:self:1:positive:0"], "evidence_texts": ["A group of people sitting and knitting.", "Women in red are performing with nun-chucks.", "Two men at a table having drinks.", "Women in red are performing with nun-chucks.", "no ! jack from titanic . i want to use the puppy money to remake it"], "speaker_id": "snli:test:2028", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2031", "candidate_text": "well, the girl in a pink dress uses an umbrella for a sun shade", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2031:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2022:premise", "snli:test:2065:premise", "snli:test:1867:premise", "snli:test:2031:premise", "style:personachat:test:75:self:2:positive:0", "style:personachat:test:75:self:2:positive:1"], "evidence_texts": ["A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "aww thanks . i like rap too", "so are you . great personality"], "speaker_id": "snli:test:2031", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2031", "candidate_text": "The girl in a pink dress uses an umbrella for a sun shade.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2031:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2022:premise", "snli:test:2065:premise", "snli:test:1867:premise", "snli:test:2031:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A group of people wearing white robes with flame pattern posing for a picture at some sort of festival.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2031", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2035", "candidate_text": "well, a woman is practicing for the olympics", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2035:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1820:premise", "snli:test:2547:premise", "snli:test:1817:premise", "snli:test:2035:premise", "style:personachat:test:75:self:5:positive:0", "style:personachat:test:75:self:5:positive:1", "style:personachat:test:75:self:5:positive:2", "style:personachat:test:75:self:5:positive:3", "style:personachat:test:75:self:5:positive:4"], "evidence_texts": ["Three women talk to each other next to another woman sitting on a motorcycle.", "A girl in a green coat walks down a rural road playing a flute", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "i like anything blue that is why i have blue hair", "awesome ! ! ! i need to try a vegan lifestyle", "i am into cosplay . i am a pretty scrawny kid", "aww thanks . i like rap too", "so are you . great personality"], "speaker_id": "snli:test:2035", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2035", "candidate_text": "I wonder whether a woman is practicing for the olympics?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2035:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1820:premise", "snli:test:2547:premise", "snli:test:1817:premise", "snli:test:2035:premise", "style:personachat:test:271:self:5:positive:0", "style:personachat:test:271:self:5:positive:1", "style:personachat:test:271:self:5:positive:2", "style:personachat:test:271:self:5:positive:3", "style:personachat:test:271:self:5:positive:4"], "evidence_texts": ["Three women talk to each other next to another woman sitting on a motorcycle.", "A girl in a green coat walks down a rural road playing a flute", "A little boy with a shirt, shorts and socks on, standing on the beach.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "that is a good approach . what do you do for fun ? i love karaoke !", "hiking is good for you ! its tough for me though cause i am colorblind .", "yeah everything . i just see black and white . it has benefits though .", "it is unique . i can see camouflaged movement better than other people can though !", "yeah its helped me become a specialist in the army . what law do you study ?"], "speaker_id": "snli:test:2035", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2050", "candidate_text": "well, he is watching the news", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2050:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:1973:premise", "snli:test:2062:premise", "snli:test:2050:premise", "snli:test:1972:premise", "style:personachat:test:75:self:6:positive:0", "style:personachat:test:75:self:6:positive:1", "style:personachat:test:75:self:6:positive:2", "style:personachat:test:75:self:6:positive:3", "style:personachat:test:75:self:6:positive:4", "style:personachat:test:75:self:6:positive:5"], "evidence_texts": ["Three dogs run through the grass while wearing colored shirts.", "People hurrying to get to work in a busy city.", "A man is laid back in front of a tv.", "Three dogs run through the grass while wearing colored shirts.", "i am good , just hanging out", "i like anything blue that is why i have blue hair", "awesome ! ! ! i need to try a vegan lifestyle", "i am into cosplay . i am a pretty scrawny kid", "aww thanks . i like rap too", "so are you . great personality"], "speaker_id": "snli:test:2050", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2050", "candidate_text": "Honestly, he is watching the news!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2050:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:1973:premise", "snli:test:2062:premise", "snli:test:2050:premise", "snli:test:1972:premise", "style:personachat:test:565:self:6:positive:0", "style:personachat:test:565:self:6:positive:1", "style:personachat:test:565:self:6:positive:2", "style:personachat:test:565:self:6:positive:3", "style:personachat:test:565:self:6:positive:4", "style:personachat:test:565:self:6:positive:5"], "evidence_texts": ["Three dogs run through the grass while wearing colored shirts.", "People hurrying to get to work in a busy city.", "A man is laid back in front of a tv.", "Three dogs run through the grass while wearing colored shirts.", "i met alot of shy people when i studied abroad it is ok", "i studied in norway , but i gave it up to chase my dream of making model cars", "i had to stop because i just became a father . . . . to 9 puppies by my dog ginger", "maybe . it would cost 1000 trying to recreate the titanic", "can she remake jack . i have seen it 10 times", "no ! jack from titanic . i want to use the puppy money to remake it"], "speaker_id": "snli:test:2050", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2054", "candidate_text": "I wonder whether a boy is doing homework?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2054:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1795:premise", "snli:test:2079:premise", "snli:test:2054:premise", "snli:test:1736:premise", "style:personachat:test:76:self:1:positive:0"], "evidence_texts": ["Rock band on stage immersed in light show.", "Boy with red shorts getting soaked with water.", "A little boy playing outside on the cement.", "A worker is doing a job on scaffolding.", "awesome . how do you stay so fit ?"], "speaker_id": "snli:test:2054", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2054", "candidate_text": "Honestly, A boy is doing homework!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2054:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1795:premise", "snli:test:2079:premise", "snli:test:2054:premise", "snli:test:1736:premise", "style:personachat:test:566:self:1:positive:0"], "evidence_texts": ["Rock band on stage immersed in light show.", "Boy with red shorts getting soaked with water.", "A little boy playing outside on the cement.", "A worker is doing a job on scaffolding.", "yep ! i am single too ! wish i had a husband though ."], "speaker_id": "snli:test:2054", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2059", "candidate_text": "I wonder whether two men are getting ready to play basketball?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2059:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:616:premise", "snli:test:3505:premise", "snli:test:8990:premise", "snli:test:2059:premise", "style:personachat:test:76:self:2:positive:0", "style:personachat:test:76:self:2:positive:1"], "evidence_texts": ["A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "all of them ! i just love the time together and the sport you ?", "awesome . how do you stay so fit ?"], "speaker_id": "snli:test:2059", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2059", "candidate_text": "Two men are getting ready to play basketball.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2059:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:616:premise", "snli:test:3505:premise", "snli:test:8990:premise", "snli:test:2059:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2059", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2062", "candidate_text": "I wonder whether people rushing to get to work in a busy city cause they are late?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2062:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2050:premise", "snli:test:2097:premise", "snli:test:2062:premise", "snli:test:1973:premise", "style:personachat:test:76:self:5:positive:0", "style:personachat:test:76:self:5:positive:1", "style:personachat:test:76:self:5:positive:2", "style:personachat:test:76:self:5:positive:3", "style:personachat:test:76:self:5:positive:4"], "evidence_texts": ["A man is laid back in front of a tv.", "A man in a gray sweatshirt using a power saw.", "People hurrying to get to work in a busy city.", "Three dogs run through the grass while wearing colored shirts.", "i prefer home workouts , but mostly just workin this pizza lol", "spooky movies give me the heebie jeebies and my dogs just bark at the scary times lol", "heck yeah ! the guys love when i bring barbecue to sunday football !", "all of them ! i just love the time together and the sport you ?", "awesome . how do you stay so fit ?"], "speaker_id": "snli:test:2062", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2062", "candidate_text": "well, people rushing to get to work in a busy city cause they are late", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2062:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2050:premise", "snli:test:2097:premise", "snli:test:2062:premise", "snli:test:1973:premise", "style:personachat:test:132:self:5:positive:0", "style:personachat:test:132:self:5:positive:1", "style:personachat:test:132:self:5:positive:2", "style:personachat:test:132:self:5:positive:3", "style:personachat:test:132:self:5:positive:4"], "evidence_texts": ["A man is laid back in front of a tv.", "A man in a gray sweatshirt using a power saw.", "People hurrying to get to work in a busy city.", "Three dogs run through the grass while wearing colored shirts.", "i do not . i spend all of my time playing frisbee", "cats are cool . i have two dogs . pretty big ones as well", "ohio . where are you from ?", "there is a lot of rain here currently as well . from the hurricanes probably", "a little . i eat sardines every morning ."], "speaker_id": "snli:test:2062", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2065", "candidate_text": "I wonder whether a woman is measuring out wine from a barrel?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2065:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2031:premise", "snli:test:2065:premise", "snli:test:2086:premise", "snli:test:2022:premise", "style:personachat:test:76:self:6:positive:0", "style:personachat:test:76:self:6:positive:1", "style:personachat:test:76:self:6:positive:2", "style:personachat:test:76:self:6:positive:3", "style:personachat:test:76:self:6:positive:4", "style:personachat:test:76:self:6:positive:5"], "evidence_texts": ["A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "sounds exhausting ! i admire your ability to do manual labor !", "i prefer home workouts , but mostly just workin this pizza lol", "spooky movies give me the heebie jeebies and my dogs just bark at the scary times lol", "heck yeah ! the guys love when i bring barbecue to sunday football !", "all of them ! i just love the time together and the sport you ?", "awesome . how do you stay so fit ?"], "speaker_id": "snli:test:2065", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2065", "candidate_text": "Honestly, A woman is measuring out wine from a barrel!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2065:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2031:premise", "snli:test:2065:premise", "snli:test:2086:premise", "snli:test:2022:premise", "style:personachat:test:566:self:6:positive:0", "style:personachat:test:566:self:6:positive:1", "style:personachat:test:566:self:6:positive:2", "style:personachat:test:566:self:6:positive:3", "style:personachat:test:566:self:6:positive:4", "style:personachat:test:566:self:6:positive:5"], "evidence_texts": ["A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "A young blond girl is eating her frozen snack while sitting on a bench next to a red stroller.", "me too ! where are you from ?", "not me ! ! i like to be alone !", "that is cool ! i am a writer ! where do you live ?", "i write stories for work ! romance usually !", "i do ! but i live on an island so there are not many stores around !", "yep ! i am single too ! wish i had a husband though ."], "speaker_id": "snli:test:2065", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2077", "candidate_text": "well, the women rides a scooter down the street", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2077:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1789:premise", "snli:test:2077:premise", "snli:test:2091:premise", "snli:test:1788:premise", "style:personachat:test:77:self:1:positive:0"], "evidence_texts": ["A young man waits on a bench with his bag behind a few advertisements in London.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "A young man waits on a bench with his bag behind a few advertisements in London.", "thank you for the idea . i will talk to my mom at work tonight"], "speaker_id": "snli:test:2077", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2077", "candidate_text": "I wonder whether the women rides a scooter down the street?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2077:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1789:premise", "snli:test:2077:premise", "snli:test:2091:premise", "snli:test:1788:premise", "style:personachat:test:278:self:1:positive:0"], "evidence_texts": ["A young man waits on a bench with his bag behind a few advertisements in London.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "A young man waits on a bench with his bag behind a few advertisements in London.", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:2077", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2079", "candidate_text": "well, a boy is getting splashed", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2079:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2054:premise", "snli:test:2216:premise", "snli:test:1795:premise", "snli:test:2079:premise", "style:personachat:test:77:self:2:positive:0", "style:personachat:test:77:self:2:positive:1"], "evidence_texts": ["A little boy playing outside on the cement.", "Two small children walk away in a field.", "Rock band on stage immersed in light show.", "Boy with red shorts getting soaked with water.", "i really hope i can", "thank you for the idea . i will talk to my mom at work tonight"], "speaker_id": "snli:test:2079", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2079", "candidate_text": "Honestly, A boy is getting splashed!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2079:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2054:premise", "snli:test:2216:premise", "snli:test:1795:premise", "snli:test:2079:premise", "style:personachat:test:584:self:2:positive:0", "style:personachat:test:584:self:2:positive:1"], "evidence_texts": ["A little boy playing outside on the cement.", "Two small children walk away in a field.", "Rock band on stage immersed in light show.", "Boy with red shorts getting soaked with water.", "i do like movies , and i hope to watch many when i retire .", "i love to eat ! candy is delicious and chicken !"], "speaker_id": "snli:test:2079", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2083", "candidate_text": "well, a woman is wearing a shirt", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2083:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2013:premise", "snli:test:2130:premise", "snli:test:1841:premise", "snli:test:2083:premise", "style:personachat:test:77:self:5:positive:0", "style:personachat:test:77:self:5:positive:1", "style:personachat:test:77:self:5:positive:2", "style:personachat:test:77:self:5:positive:3", "style:personachat:test:77:self:5:positive:4"], "evidence_texts": ["Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A windsurfer catching air with water streaming off of the board.", "A man is using what looks to be a fax machine.", "A woman in a red shirt and white skirt playing tennis.", "i usually read while i am eating pizza from my moms restaurant", "eating is my life ! i could never diet", "i wish i could go to the beach . i get nervous and stutter around attractive people though", "i really hope i can", "thank you for the idea . i will talk to my mom at work tonight"], "speaker_id": "snli:test:2083", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2083", "candidate_text": "A woman is wearing a shirt.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2083:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2013:premise", "snli:test:2130:premise", "snli:test:1841:premise", "snli:test:2083:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A windsurfer catching air with water streaming off of the board.", "A man is using what looks to be a fax machine.", "A woman in a red shirt and white skirt playing tennis.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2083", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2086", "candidate_text": "well, a person is standing on a tennis court", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2086:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2065:premise", "snli:test:2170:premise", "snli:test:2086:premise", "snli:test:2031:premise", "style:personachat:test:77:self:6:positive:0", "style:personachat:test:77:self:6:positive:1", "style:personachat:test:77:self:6:positive:2", "style:personachat:test:77:self:6:positive:3", "style:personachat:test:77:self:6:positive:4", "style:personachat:test:77:self:6:positive:5"], "evidence_texts": ["An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "i am good . just reading a nice romance about a girl who loves a ghost", "i usually read while i am eating pizza from my moms restaurant", "eating is my life ! i could never diet", "i wish i could go to the beach . i get nervous and stutter around attractive people though", "i really hope i can", "thank you for the idea . i will talk to my mom at work tonight"], "speaker_id": "snli:test:2086", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2086", "candidate_text": "I wonder whether a person is standing on a tennis court?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2086:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2065:premise", "snli:test:2170:premise", "snli:test:2086:premise", "snli:test:2031:premise", "style:personachat:test:278:self:6:positive:0", "style:personachat:test:278:self:6:positive:1", "style:personachat:test:278:self:6:positive:2", "style:personachat:test:278:self:6:positive:3", "style:personachat:test:278:self:6:positive:4", "style:personachat:test:278:self:6:positive:5"], "evidence_texts": ["An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "A young Asian girl is wearing a pink dress and holding an umbrella even though it is not raining.", "i am great and you", "after a hard day i listen to some oldies", "ok , oldies and barbeque restaurants are my favorite", "well i am retiring soon like in five years so i will be visiting many", "yes i am . . . located in south carolina", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:2086", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2091", "candidate_text": "I wonder whether the stockings are white and purple?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2091:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2077:premise", "snli:test:2168:premise", "snli:test:1789:premise", "snli:test:2091:premise", "style:personachat:test:78:self:1:positive:0"], "evidence_texts": ["Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "Right leg in a brace, a young boy lies on the couch while using his cellphone.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "they are . you like scotch ?"], "speaker_id": "snli:test:2091", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2091", "candidate_text": "well, the stockings are white and purple", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2091:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2077:premise", "snli:test:2168:premise", "snli:test:1789:premise", "snli:test:2091:premise", "style:personachat:test:136:self:1:positive:0"], "evidence_texts": ["Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "Right leg in a brace, a young boy lies on the couch while using his cellphone.", "A young man waits on a bench with his bag behind a few advertisements in London.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:2091", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2097", "candidate_text": "I wonder whether the guy is dancing to michael jackson?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2097:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2062:premise", "snli:test:2097:premise", "snli:test:2050:premise", "snli:test:2355:premise", "style:personachat:test:78:self:2:positive:0", "style:personachat:test:78:self:2:positive:1"], "evidence_texts": ["People hurrying to get to work in a busy city.", "A man in a gray sweatshirt using a power saw.", "A man is laid back in front of a tv.", "Two boys with blond-hair, wearing striped shirts on a bed.", "i had a rough experience with one . a poodle .", "they are . you like scotch ?"], "speaker_id": "snli:test:2097", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2097", "candidate_text": "Honestly, the guy is dancing to Michael Jackson!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2097:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2062:premise", "snli:test:2097:premise", "snli:test:2050:premise", "snli:test:2355:premise", "style:personachat:test:587:self:2:positive:0", "style:personachat:test:587:self:2:positive:1"], "evidence_texts": ["People hurrying to get to work in a busy city.", "A man in a gray sweatshirt using a power saw.", "A man is laid back in front of a tv.", "Two boys with blond-hair, wearing striped shirts on a bed.", "my sister lives with me . i also have a dog .", "it is ! what do you look like ?"], "speaker_id": "snli:test:2097", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2098", "candidate_text": "I wonder whether the guy in the sweatshirt uses the power saw?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2098:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2355:premise", "snli:test:2062:premise", "snli:test:2098:premise", "snli:test:2440:premise", "style:personachat:test:78:self:5:positive:0", "style:personachat:test:78:self:5:positive:1", "style:personachat:test:78:self:5:positive:2", "style:personachat:test:78:self:5:positive:3", "style:personachat:test:78:self:5:positive:4"], "evidence_texts": ["Two boys with blond-hair, wearing striped shirts on a bed.", "People hurrying to get to work in a busy city.", "A man in a gray sweatshirt using a power saw.", "Man in puffy yellow jacket on skis on a mountain.", "that is awesome ! ! i worked at a carnival once upon a time .", "no , i worked at the ferris wheel .", "agreed . they do not have dogs , which i am terrified of .", "i had a rough experience with one . a poodle .", "they are . you like scotch ?"], "speaker_id": "snli:test:2098", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2098", "candidate_text": "the guy in the sweatshirt uses the power saw.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2098:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2355:premise", "snli:test:2062:premise", "snli:test:2098:premise", "snli:test:2440:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two boys with blond-hair, wearing striped shirts on a bed.", "People hurrying to get to work in a busy city.", "A man in a gray sweatshirt using a power saw.", "Man in puffy yellow jacket on skis on a mountain.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2098", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2108", "candidate_text": "I wonder whether people drink beer at the pub?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2108:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2108:premise", "snli:test:1958:premise", "snli:test:2147:premise", "snli:test:1956:premise", "style:personachat:test:78:self:7:positive:0", "style:personachat:test:78:self:7:positive:1", "style:personachat:test:78:self:7:positive:2", "style:personachat:test:78:self:7:positive:3", "style:personachat:test:78:self:7:positive:4", "style:personachat:test:78:self:7:positive:5", "style:personachat:test:78:self:7:positive:6"], "evidence_texts": ["A group of people in black outfits drinking beer.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "A man is riding a bicycle in the street.", "i am good . you ? i am more of a writer .", "writing is my work , so i cant get out of it if i tried", "that is awesome ! ! i worked at a carnival once upon a time .", "no , i worked at the ferris wheel .", "agreed . they do not have dogs , which i am terrified of .", "i had a rough experience with one . a poodle .", "they are . you like scotch ?"], "speaker_id": "snli:test:2108", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2108", "candidate_text": "well, people drink beer at the pub", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2108:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2108:premise", "snli:test:1958:premise", "snli:test:2147:premise", "snli:test:1956:premise", "style:personachat:test:136:self:6:positive:0", "style:personachat:test:136:self:6:positive:1", "style:personachat:test:136:self:6:positive:2", "style:personachat:test:136:self:6:positive:3", "style:personachat:test:136:self:6:positive:4", "style:personachat:test:136:self:6:positive:5"], "evidence_texts": ["A group of people in black outfits drinking beer.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "A man is riding a bicycle in the street.", "i am good . just waiting for the good doctor to come on .", "i am making dinner while i wait", "i had to grade some papers and do homework with my kids", "my husband works nights so its hard sometimes", "cooking and reading are always my relaxers", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:2108", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2122", "candidate_text": "well, two men in cultural garb are playing drums among a crowd", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2122:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1945:premise", "snli:test:2260:premise", "snli:test:2122:premise", "snli:test:1802:premise", "style:personachat:test:79:self:1:positive:0"], "evidence_texts": ["A man stands on one hand in the street while a crowd watches.", "A man is painting a city scene on a canvas in the street.", "A crowd and two males dressed in different cultural attire playing the drums.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "pretty cool that you did not have to go get a stupid tattoo or something"], "speaker_id": "snli:test:2122", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2122", "candidate_text": "Two men in cultural garb are playing drums among a crowd.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2122:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1945:premise", "snli:test:2260:premise", "snli:test:2122:premise", "snli:test:1802:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man stands on one hand in the street while a crowd watches.", "A man is painting a city scene on a canvas in the street.", "A crowd and two males dressed in different cultural attire playing the drums.", "The Sooner football player carrying the ball is trying to avoid being tackled.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2122", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2130", "candidate_text": "well, a person is windsurfing", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2130:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2083:premise", "snli:test:2013:premise", "snli:test:2130:premise", "snli:test:2133:premise", "style:personachat:test:79:self:2:positive:0", "style:personachat:test:79:self:2:positive:1"], "evidence_texts": ["A woman in a red shirt and white skirt playing tennis.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A windsurfer catching air with water streaming off of the board.", "Many people have decided to take a rest on the steps.", "oh i assumed you got it removed as a baby", "pretty cool that you did not have to go get a stupid tattoo or something"], "speaker_id": "snli:test:2130", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2130", "candidate_text": "I wonder whether a person is windsurfing?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2130:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2083:premise", "snli:test:2013:premise", "snli:test:2130:premise", "snli:test:2133:premise", "style:personachat:test:285:self:2:positive:0", "style:personachat:test:285:self:2:positive:1"], "evidence_texts": ["A woman in a red shirt and white skirt playing tennis.", "Three people wearing black blindfolds near a \"Salvar O Planeta\" sign.", "A windsurfer catching air with water streaming off of the board.", "Many people have decided to take a rest on the steps.", "yes it will be sad , but lots of great places to skate ther .", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:2130", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2131", "candidate_text": "well, the windsurfer is having fun", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2131:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2133:premise", "snli:test:2083:premise", "snli:test:2183:premise", "snli:test:2131:premise", "style:personachat:test:79:self:5:positive:0", "style:personachat:test:79:self:5:positive:1", "style:personachat:test:79:self:5:positive:2", "style:personachat:test:79:self:5:positive:3", "style:personachat:test:79:self:5:positive:4"], "evidence_texts": ["Many people have decided to take a rest on the steps.", "A woman in a red shirt and white skirt playing tennis.", "A BMX rider is in midair while on a dirt trail.", "A windsurfer catching air with water streaming off of the board.", "more of a hobby . i am 40 and still do not have a set career", "i cant stay still long enough to keep a steady job . i am a restless guy", "i heard that happens sometimes . glad you got it removed", "oh i assumed you got it removed as a baby", "pretty cool that you did not have to go get a stupid tattoo or something"], "speaker_id": "snli:test:2131", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2131", "candidate_text": "Honestly, The windsurfer is having fun!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2131:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2133:premise", "snli:test:2083:premise", "snli:test:2183:premise", "snli:test:2131:premise", "style:personachat:test:588:self:5:positive:0", "style:personachat:test:588:self:5:positive:1", "style:personachat:test:588:self:5:positive:2", "style:personachat:test:588:self:5:positive:3", "style:personachat:test:588:self:5:positive:4"], "evidence_texts": ["Many people have decided to take a rest on the steps.", "A woman in a red shirt and white skirt playing tennis.", "A BMX rider is in midair while on a dirt trail.", "A windsurfer catching air with water streaming off of the board.", "what kind of opportunity do you have ?", "i would be willing to help ! i also have management experience from my job .", "nope , i am not allergic to anything ! you ?", "i love doing anything outside , especially in the snow ! what about you ?", "those are pretty cool hobbies ! do you have any siblings ?"], "speaker_id": "snli:test:2131", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2133", "candidate_text": "well, the people are on the famous steps rocky runs up in a training montage", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2133:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2131:premise", "snli:test:2133:premise", "snli:test:2183:premise", "snli:test:2130:premise", "style:personachat:test:79:self:7:positive:0", "style:personachat:test:79:self:7:positive:1", "style:personachat:test:79:self:7:positive:2", "style:personachat:test:79:self:7:positive:3", "style:personachat:test:79:self:7:positive:4", "style:personachat:test:79:self:7:positive:5", "style:personachat:test:79:self:7:positive:6"], "evidence_texts": ["A windsurfer catching air with water streaming off of the board.", "Many people have decided to take a rest on the steps.", "A BMX rider is in midair while on a dirt trail.", "A windsurfer catching air with water streaming off of the board.", "i do not really know how to skate but i think its very cool", "i listen to music while i am doing carpentry . i just cant seem to stop moving ever", "more of a hobby . i am 40 and still do not have a set career", "i cant stay still long enough to keep a steady job . i am a restless guy", "i heard that happens sometimes . glad you got it removed", "oh i assumed you got it removed as a baby", "pretty cool that you did not have to go get a stupid tattoo or something"], "speaker_id": "snli:test:2133", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2133", "candidate_text": "The people are on the famous steps Rocky runs up in a training montage.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2133:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2131:premise", "snli:test:2133:premise", "snli:test:2183:premise", "snli:test:2130:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A windsurfer catching air with water streaming off of the board.", "Many people have decided to take a rest on the steps.", "A BMX rider is in midair while on a dirt trail.", "A windsurfer catching air with water streaming off of the board.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2133", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2147", "candidate_text": "well, a woman is wearing an apron", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2147:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2108:premise", "snli:test:2318:premise", "snli:test:1958:premise", "snli:test:2147:premise", "style:personachat:test:80:self:1:positive:0"], "evidence_texts": ["A group of people in black outfits drinking beer.", "Four children are sitting at a table eating breakfast.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "no , i would like to attend college though ."], "speaker_id": "snli:test:2147", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2147", "candidate_text": "I wonder whether a woman is wearing an apron?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2147:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2108:premise", "snli:test:2318:premise", "snli:test:1958:premise", "snli:test:2147:premise", "style:personachat:test:287:self:1:positive:0"], "evidence_texts": ["A group of people in black outfits drinking beer.", "Four children are sitting at a table eating breakfast.", "A man is riding a bicycle in the street.", "A woman in an apron is looking for bread.", "have you always been into body building ?"], "speaker_id": "snli:test:2147", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2149", "candidate_text": "well, two people are drinking", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2149:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2028:premise", "snli:test:2149:premise", "snli:test:2209:premise", "snli:test:1772:premise", "style:personachat:test:80:self:2:positive:0", "style:personachat:test:80:self:2:positive:1"], "evidence_texts": ["A group of people sitting and knitting.", "Two men at a table having drinks.", "A man standing alone in a park.", "Women in red are performing with nun-chucks.", "i have not decided yet . . . it is such a big decision .", "no , i would like to attend college though ."], "speaker_id": "snli:test:2149", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2149", "candidate_text": "Honestly, Two people are drinking!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2149:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2028:premise", "snli:test:2149:premise", "snli:test:2209:premise", "snli:test:1772:premise", "style:personachat:test:593:self:2:positive:0", "style:personachat:test:593:self:2:positive:1"], "evidence_texts": ["A group of people sitting and knitting.", "Two men at a table having drinks.", "A man standing alone in a park.", "Women in red are performing with nun-chucks.", "she has a mean terrier ! and maybe .", "the dog is a wire haired terrier and he is terrifying ! most dogs scare me ."], "speaker_id": "snli:test:2149", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2156", "candidate_text": "well, a child is sitting next to a computer desk", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2156:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2156:premise", "snli:test:1792:premise", "snli:test:2178:premise", "snli:test:1748:premise", "style:personachat:test:80:self:5:positive:0", "style:personachat:test:80:self:5:positive:1", "style:personachat:test:80:self:5:positive:2", "style:personachat:test:80:self:5:positive:3", "style:personachat:test:80:self:5:positive:4"], "evidence_texts": ["A child is sitting on the grass by a concrete storm drain.", "Some children are playing soccer in front of a large wooden door.", "a mass of people looking and moving in the same general direction.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "i am a lieutenant . you ?", "yes , it was . how about you ?", "i am in the army . what years did you serve ?", "i have not decided yet . . . it is such a big decision .", "no , i would like to attend college though ."], "speaker_id": "snli:test:2156", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2156", "candidate_text": "A child is sitting next to a computer desk.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2156:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2156:premise", "snli:test:1792:premise", "snli:test:2178:premise", "snli:test:1748:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A child is sitting on the grass by a concrete storm drain.", "Some children are playing soccer in front of a large wooden door.", "a mass of people looking and moving in the same general direction.", "A man wearing black pants rides along a concrete ledge on rollerblades.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2156", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2168", "candidate_text": "well, the boy is not happy", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2168:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2168:premise", "snli:test:2091:premise", "snli:test:2614:premise", "snli:test:2077:premise", "style:personachat:test:80:self:7:positive:0", "style:personachat:test:80:self:7:positive:1", "style:personachat:test:80:self:7:positive:2", "style:personachat:test:80:self:7:positive:3", "style:personachat:test:80:self:7:positive:4", "style:personachat:test:80:self:7:positive:5", "style:personachat:test:80:self:7:positive:6"], "evidence_texts": ["Right leg in a brace, a young boy lies on the couch while using his cellphone.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "oh my . i am a soldier in south korea , actually .", "well i am only 20 , so a bit younger than you !", "i am a lieutenant . you ?", "yes , it was . how about you ?", "i am in the army . what years did you serve ?", "i have not decided yet . . . it is such a big decision .", "no , i would like to attend college though ."], "speaker_id": "snli:test:2168", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2168", "candidate_text": "I wonder whether the boy is not happy?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2168:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2168:premise", "snli:test:2091:premise", "snli:test:2614:premise", "snli:test:2077:premise", "style:personachat:test:287:self:7:positive:0", "style:personachat:test:287:self:7:positive:1", "style:personachat:test:287:self:7:positive:2", "style:personachat:test:287:self:7:positive:3", "style:personachat:test:287:self:7:positive:4", "style:personachat:test:287:self:7:positive:5", "style:personachat:test:287:self:7:positive:6"], "evidence_texts": ["Right leg in a brace, a young boy lies on the couch while using his cellphone.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "Woman in a blue dress and carrying a black purse, rides a scooter down the street.", "i am too old for that . but i try to walk a mile each day !", "that sounds lovely . i have a cat .", "i could not have one when i worked in banking . but now i have retired .", "that is so sweet ! i knit things for my children all the time", "it is pretty simple , i took a class and now i am a pro !", "you have to get that protein", "have you always been into body building ?"], "speaker_id": "snli:test:2168", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2170", "candidate_text": "well, the girl is jumping because she is happy", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2170:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2086:premise", "snli:test:2195:premise", "snli:test:2170:premise", "snli:test:2065:premise", "style:personachat:test:81:self:1:positive:0"], "evidence_texts": ["A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "saves lots of money the more you do"], "speaker_id": "snli:test:2170", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2170", "candidate_text": "Honestly, The girl is jumping because she is happy!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2170:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2086:premise", "snli:test:2195:premise", "snli:test:2170:premise", "snli:test:2065:premise", "style:personachat:test:604:self:1:positive:0"], "evidence_texts": ["A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "An Asian woman in a hat is measuring food from barrels with a ladle into a smaller metal pan.", "i m sure it is ! we are our worst critic after all ."], "speaker_id": "snli:test:2170", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2178", "candidate_text": "well, a mass of people moving the same way", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2178:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2156:premise", "snli:test:1792:premise", "snli:test:2178:premise", "snli:test:2367:premise", "style:personachat:test:81:self:2:positive:0", "style:personachat:test:81:self:2:positive:1"], "evidence_texts": ["A child is sitting on the grass by a concrete storm drain.", "Some children are playing soccer in front of a large wooden door.", "a mass of people looking and moving in the same general direction.", "Fans are dressed in green and yellow for a major sporting event.", "its fun to work on different things yes", "saves lots of money the more you do"], "speaker_id": "snli:test:2178", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2178", "candidate_text": "A mass of people moving the same way.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2178:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2156:premise", "snli:test:1792:premise", "snli:test:2178:premise", "snli:test:2367:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A child is sitting on the grass by a concrete storm drain.", "Some children are playing soccer in front of a large wooden door.", "a mass of people looking and moving in the same general direction.", "Fans are dressed in green and yellow for a major sporting event.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2178", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2179", "candidate_text": "well, a mass of people moving in the same direction in order to find the object", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2179:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2367:premise", "snli:test:2156:premise", "snli:test:2523:premise", "snli:test:2179:premise", "style:personachat:test:81:self:5:positive:0", "style:personachat:test:81:self:5:positive:1", "style:personachat:test:81:self:5:positive:2", "style:personachat:test:81:self:5:positive:3", "style:personachat:test:81:self:5:positive:4"], "evidence_texts": ["Fans are dressed in green and yellow for a major sporting event.", "A child is sitting on the grass by a concrete storm drain.", "Two men are working together to empty the barrel of its contents.", "a mass of people looking and moving in the same general direction.", "for fun i working on fixing up old cars", "cool what kind of music ?", "cool i also work as a handyman", "its fun to work on different things yes", "saves lots of money the more you do"], "speaker_id": "snli:test:2179", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2179", "candidate_text": "I wonder whether a mass of people moving in the same direction in order to find the object?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2179:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2367:premise", "snli:test:2156:premise", "snli:test:2523:premise", "snli:test:2179:premise", "style:personachat:test:288:self:5:positive:0", "style:personachat:test:288:self:5:positive:1", "style:personachat:test:288:self:5:positive:2", "style:personachat:test:288:self:5:positive:3", "style:personachat:test:288:self:5:positive:4"], "evidence_texts": ["Fans are dressed in green and yellow for a major sporting event.", "A child is sitting on the grass by a concrete storm drain.", "Two men are working together to empty the barrel of its contents.", "a mass of people looking and moving in the same general direction.", "no thank god , i kind of got a cat it was a stray", "no it is not mine is clothes an singing folk music", "i bet it is lol", "your students were in what grade ?", "why ? it is freezing there"], "speaker_id": "snli:test:2179", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2183", "candidate_text": "well, the bmx rider is riding a two wheeled vehicle", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2183:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2133:premise", "snli:test:2277:premise", "snli:test:2131:premise", "snli:test:2183:premise", "style:personachat:test:81:self:7:positive:0", "style:personachat:test:81:self:7:positive:1", "style:personachat:test:81:self:7:positive:2", "style:personachat:test:81:self:7:positive:3", "style:personachat:test:81:self:7:positive:4", "style:personachat:test:81:self:7:positive:5", "style:personachat:test:81:self:7:positive:6"], "evidence_texts": ["Many people have decided to take a rest on the steps.", "A group of young adults with \"X\" on their hands, smiling.", "A windsurfer catching air with water streaming off of the board.", "A BMX rider is in midair while on a dirt trail.", "i am dan and its going good i just got back from hunting", "i am deer hunting . i work for a property management company", "for fun i working on fixing up old cars", "cool what kind of music ?", "cool i also work as a handyman", "its fun to work on different things yes", "saves lots of money the more you do"], "speaker_id": "snli:test:2183", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2183", "candidate_text": "Honestly, The BMX rider is riding a two wheeled vehicle!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2183:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2133:premise", "snli:test:2277:premise", "snli:test:2131:premise", "snli:test:2183:premise", "style:personachat:test:604:self:7:positive:0", "style:personachat:test:604:self:7:positive:1", "style:personachat:test:604:self:7:positive:2", "style:personachat:test:604:self:7:positive:3", "style:personachat:test:604:self:7:positive:4", "style:personachat:test:604:self:7:positive:5", "style:personachat:test:604:self:7:positive:6"], "evidence_texts": ["Many people have decided to take a rest on the steps.", "A group of young adults with \"X\" on their hands, smiling.", "A windsurfer catching air with water streaming off of the board.", "A BMX rider is in midair while on a dirt trail.", "not too bad . a little sad , but such is life .", "just need to lose some weight . having 3 kids will do that .", "smart choice ! do you like to cook ? i would be a great chef lol", "i want to go there , but with me on days and hubby nights , it s not feasible .", "sounds great ? do you have any pets ? we have a family cat .", "very true . what else do you like to do ?", "i m sure it is ! we are our worst critic after all ."], "speaker_id": "snli:test:2183", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2195", "candidate_text": "well, two workers are working", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2195:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2170:premise", "snli:test:2340:premise", "snli:test:2086:premise", "snli:test:2195:premise", "style:personachat:test:82:self:1:positive:0"], "evidence_texts": ["A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "one day i would like to move to the west coast though ."], "speaker_id": "snli:test:2195", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2195", "candidate_text": "Two workers are working.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2195:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2170:premise", "snli:test:2340:premise", "snli:test:2086:premise", "snli:test:2195:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "A tennis player in blue shorts and a white shirt making an aggressive backhand swing towards the tennis ball.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2195", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2197", "candidate_text": "well, a man stands in a construction site", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2197:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1912:premise", "snli:test:2197:premise", "snli:test:2700:premise", "snli:test:1911:premise", "style:personachat:test:82:self:2:positive:0", "style:personachat:test:82:self:2:positive:1"], "evidence_texts": ["An elderly man is smiling while sitting in front of a row of soda cans.", "A man is standing inside a building under construction filled with bricks and various tools.", "A man with a tan jacket with a full grocery bag is crossing the street.", "An elderly man is smiling while sitting in front of a row of soda cans.", "i really like living here , so much to do .", "one day i would like to move to the west coast though ."], "speaker_id": "snli:test:2197", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2197", "candidate_text": "I wonder whether a man stands in a construction site?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2197:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1912:premise", "snli:test:2197:premise", "snli:test:2700:premise", "snli:test:1911:premise", "style:personachat:test:292:self:2:positive:0", "style:personachat:test:292:self:2:positive:1"], "evidence_texts": ["An elderly man is smiling while sitting in front of a row of soda cans.", "A man is standing inside a building under construction filled with bricks and various tools.", "A man with a tan jacket with a full grocery bag is crossing the street.", "An elderly man is smiling while sitting in front of a row of soda cans.", "i am not married . it would be hard to have kids while still in college .", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:2197", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2209", "candidate_text": "well, a man alone in the park", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2209:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2149:premise", "snli:test:2209:premise", "snli:test:2380:premise", "snli:test:2028:premise", "style:personachat:test:82:self:5:positive:0", "style:personachat:test:82:self:5:positive:1", "style:personachat:test:82:self:5:positive:2", "style:personachat:test:82:self:5:positive:3", "style:personachat:test:82:self:5:positive:4"], "evidence_texts": ["Two men at a table having drinks.", "A man standing alone in a park.", "A man scales a rock climbing wall.", "A group of people sitting and knitting.", "tell me something about yourself .", "i happen to be jewish .", "good . where do you live ? i live in ny .", "i really like living here , so much to do .", "one day i would like to move to the west coast though ."], "speaker_id": "snli:test:2209", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2209", "candidate_text": "Honestly, a man alone in the park!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2209:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2149:premise", "snli:test:2209:premise", "snli:test:2380:premise", "snli:test:2028:premise", "style:personachat:test:609:self:5:positive:0", "style:personachat:test:609:self:5:positive:1", "style:personachat:test:609:self:5:positive:2", "style:personachat:test:609:self:5:positive:3", "style:personachat:test:609:self:5:positive:4"], "evidence_texts": ["Two men at a table having drinks.", "A man standing alone in a park.", "A man scales a rock climbing wall.", "A group of people sitting and knitting.", "so you get to learn while in pajamas ! neat !", "nebraska . so there is not much to do here . i enjoy bird hunting here .", "only when my company does not have me repairing properties , yes !", "i do . been doing it my whole life .", "well , fishing as well ! except i am not allowed to eat shellfish"], "speaker_id": "snli:test:2209", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2216", "candidate_text": "well, two small children walk outdoors", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2216:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2216:premise", "snli:test:2079:premise", "snli:test:2293:premise", "snli:test:2054:premise", "style:personachat:test:82:self:7:positive:0", "style:personachat:test:82:self:7:positive:1", "style:personachat:test:82:self:7:positive:2", "style:personachat:test:82:self:7:positive:3", "style:personachat:test:82:self:7:positive:4", "style:personachat:test:82:self:7:positive:5", "style:personachat:test:82:self:7:positive:6"], "evidence_texts": ["Two small children walk away in a field.", "Boy with red shorts getting soaked with water.", "People shopping for vegetables at an outdoor market.", "A little boy playing outside on the cement.", "i am good . just watching the office .", "i am eating a carrot , i do not eat anything sweet .", "tell me something about yourself .", "i happen to be jewish .", "good . where do you live ? i live in ny .", "i really like living here , so much to do .", "one day i would like to move to the west coast though ."], "speaker_id": "snli:test:2216", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2216", "candidate_text": "Two small children walk outdoors.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2216:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2216:premise", "snli:test:2079:premise", "snli:test:2293:premise", "snli:test:2054:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two small children walk away in a field.", "Boy with red shorts getting soaked with water.", "People shopping for vegetables at an outdoor market.", "A little boy playing outside on the cement.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2216", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2221", "candidate_text": "well, some people are riding bicycles", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2221:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1716:premise", "snli:test:2221:premise", "snli:test:3217:premise", "snli:test:1471:premise", "style:personachat:test:83:self:1:positive:0"], "evidence_texts": ["Band plays in tight quarters.", "several people are bike riding.", "Four African boys playing soccer.", "Times Square in New York.", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:2221", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2221", "candidate_text": "I wonder whether some people are riding bicycles?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2221:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1716:premise", "snli:test:2221:premise", "snli:test:3217:premise", "snli:test:1471:premise", "style:personachat:test:298:self:1:positive:0"], "evidence_texts": ["Band plays in tight quarters.", "several people are bike riding.", "Four African boys playing soccer.", "Times Square in New York.", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:2221", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2260", "candidate_text": "well, a man is painting outside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2260:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2260:premise", "snli:test:2122:premise", "snli:test:2288:premise", "snli:test:1945:premise", "style:personachat:test:83:self:2:positive:0", "style:personachat:test:83:self:2:positive:1"], "evidence_texts": ["A man is painting a city scene on a canvas in the street.", "A crowd and two males dressed in different cultural attire playing the drums.", "A man in a khaki jacket and ball cap walks down the street.", "A man stands on one hand in the street while a crowd watches.", "i would rather make it big in music . it has always been my dream .", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:2260", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2260", "candidate_text": "Honestly, A man is painting outside!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2260:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2260:premise", "snli:test:2122:premise", "snli:test:2288:premise", "snli:test:1945:premise", "style:personachat:test:614:self:2:positive:0", "style:personachat:test:614:self:2:positive:1"], "evidence_texts": ["A man is painting a city scene on a canvas in the street.", "A crowd and two males dressed in different cultural attire playing the drums.", "A man in a khaki jacket and ball cap walks down the street.", "A man stands on one hand in the street while a crowd watches.", "what is the weather like there ?", "switch places with me ! it is too hot and sunny here"], "speaker_id": "snli:test:2260", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2277", "candidate_text": "well, the group of adults are crying hysterially", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2277:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2183:premise", "snli:test:2277:premise", "snli:test:2459:premise", "snli:test:2133:premise", "style:personachat:test:83:self:5:positive:0", "style:personachat:test:83:self:5:positive:1", "style:personachat:test:83:self:5:positive:2", "style:personachat:test:83:self:5:positive:3", "style:personachat:test:83:self:5:positive:4"], "evidence_texts": ["A BMX rider is in midair while on a dirt trail.", "A group of young adults with \"X\" on their hands, smiling.", "The bears are facing each other with their mouths wide open.", "Many people have decided to take a rest on the steps.", "that is sweet . my band is popular . i am the lead singer in it .", "rock . i like to be pumped up all the time .", "i have to make it big . i did not finish high school .", "i would rather make it big in music . it has always been my dream .", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:2277", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2277", "candidate_text": "The group of adults are crying hysterially.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2277:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2183:premise", "snli:test:2277:premise", "snli:test:2459:premise", "snli:test:2133:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A BMX rider is in midair while on a dirt trail.", "A group of young adults with \"X\" on their hands, smiling.", "The bears are facing each other with their mouths wide open.", "Many people have decided to take a rest on the steps.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2277", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2282", "candidate_text": "well, the young white girl is standing on a street while it's sunny outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2282:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1534:premise", "snli:test:2503:premise", "snli:test:2282:premise", "snli:test:1248:premise", "style:personachat:test:83:self:7:positive:0", "style:personachat:test:83:self:7:positive:1", "style:personachat:test:83:self:7:positive:2", "style:personachat:test:83:self:7:positive:3", "style:personachat:test:83:self:7:positive:4", "style:personachat:test:83:self:7:positive:5", "style:personachat:test:83:self:7:positive:6"], "evidence_texts": ["Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "hi ! no , but 2 of my 3 sisters do know how .", "i contribute to the dirtiness . i party a lot when my band is not playing a gig .", "that is sweet . my band is popular . i am the lead singer in it .", "rock . i like to be pumped up all the time .", "i have to make it big . i did not finish high school .", "i would rather make it big in music . it has always been my dream .", "maybe we can add folk music one day for you ."], "speaker_id": "snli:test:2282", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2282", "candidate_text": "I wonder whether the young white girl is standing on a street while it's sunny outside?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2282:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1534:premise", "snli:test:2503:premise", "snli:test:2282:premise", "snli:test:1248:premise", "style:personachat:test:298:self:7:positive:0", "style:personachat:test:298:self:7:positive:1", "style:personachat:test:298:self:7:positive:2", "style:personachat:test:298:self:7:positive:3", "style:personachat:test:298:self:7:positive:4", "style:personachat:test:298:self:7:positive:5", "style:personachat:test:298:self:7:positive:6"], "evidence_texts": ["Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "The firemen and women wearing blue gloves have finished working on a person who is lying on ground bandaged and draped in a blue blanket.", "pretty good . just studying , you ?", "i love football , do you play ?", "i really like to be active .", "that is a good way to get together with friends", "i can see that . do you like music ?", "i have a passion for rap music", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:2282", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2288", "candidate_text": "Honestly, The man is running down the street!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2288:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2288:premise", "snli:test:2260:premise", "snli:test:2334:premise", "snli:test:2122:premise", "style:personachat:test:84:self:1:positive:0"], "evidence_texts": ["A man in a khaki jacket and ball cap walks down the street.", "A man is painting a city scene on a canvas in the street.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A crowd and two males dressed in different cultural attire playing the drums.", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:2288", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2288", "candidate_text": "well, the man is running down the street", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2288:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2288:premise", "snli:test:2260:premise", "snli:test:2334:premise", "snli:test:2122:premise", "style:personachat:test:148:self:1:positive:0"], "evidence_texts": ["A man in a khaki jacket and ball cap walks down the street.", "A man is painting a city scene on a canvas in the street.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A crowd and two males dressed in different cultural attire playing the drums.", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:2288", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2291", "candidate_text": "Honestly, The men are naked!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2291:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1640:premise", "snli:test:2404:premise", "snli:test:1639:premise", "snli:test:2291:premise", "style:personachat:test:84:self:2:positive:0", "style:personachat:test:84:self:2:positive:1"], "evidence_texts": ["Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "do you sell cars too", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:2291", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2291", "candidate_text": "The men are naked.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2291:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1640:premise", "snli:test:2404:premise", "snli:test:1639:premise", "snli:test:2291:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2291", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2293", "candidate_text": "Honestly, People shopping for veggies at a nice market!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2293:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2216:premise", "snli:test:2293:premise", "snli:test:2079:premise", "snli:test:2348:premise", "style:personachat:test:84:self:5:positive:0", "style:personachat:test:84:self:5:positive:1", "style:personachat:test:84:self:5:positive:2", "style:personachat:test:84:self:5:positive:3", "style:personachat:test:84:self:5:positive:4"], "evidence_texts": ["Two small children walk away in a field.", "People shopping for vegetables at an outdoor market.", "Boy with red shorts getting soaked with water.", "A group of swimmers jump into a pool.", "the sea is my home . the aquatic creatures are my friends .", "sure you could call it that . do have any hobbies ?", "how old are you ? i spend all day at school ! i would rather be playing .", "do you sell cars too", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:2293", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2293", "candidate_text": "I wonder whether people shopping for veggies at a nice market?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2293:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2216:premise", "snli:test:2293:premise", "snli:test:2079:premise", "snli:test:2348:premise", "style:personachat:test:299:self:5:positive:0", "style:personachat:test:299:self:5:positive:1", "style:personachat:test:299:self:5:positive:2", "style:personachat:test:299:self:5:positive:3", "style:personachat:test:299:self:5:positive:4"], "evidence_texts": ["Two small children walk away in a field.", "People shopping for vegetables at an outdoor market.", "Boy with red shorts getting soaked with water.", "A group of swimmers jump into a pool.", "hi mia , what are your hobbies ? mine is spending time online .", "how very cool . do you watch tv ? i play on a show called second life .", "oh , i play a furry . tell me more about your cat .", "no , i do not have enough time . what is your cats name ?", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:2293", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2294", "candidate_text": "Honestly, People shopping for junk at a flea market!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2294:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2348:premise", "snli:test:2216:premise", "snli:test:2294:premise", "snli:test:2382:premise", "style:personachat:test:84:self:7:positive:0", "style:personachat:test:84:self:7:positive:1", "style:personachat:test:84:self:7:positive:2", "style:personachat:test:84:self:7:positive:3", "style:personachat:test:84:self:7:positive:4", "style:personachat:test:84:self:7:positive:5", "style:personachat:test:84:self:7:positive:6"], "evidence_texts": ["A group of swimmers jump into a pool.", "Two small children walk away in a field.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "i am well just got in from playing with the dolphins . how are you ?", "animal . my home is under water . where do you live ?", "the sea is my home . the aquatic creatures are my friends .", "sure you could call it that . do have any hobbies ?", "how old are you ? i spend all day at school ! i would rather be playing .", "do you sell cars too", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:2294", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2294", "candidate_text": "well, people shopping for junk at a flea market", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2294:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2348:premise", "snli:test:2216:premise", "snli:test:2294:premise", "snli:test:2382:premise", "style:personachat:test:148:self:7:positive:0", "style:personachat:test:148:self:7:positive:1", "style:personachat:test:148:self:7:positive:2", "style:personachat:test:148:self:7:positive:3", "style:personachat:test:148:self:7:positive:4", "style:personachat:test:148:self:7:positive:5", "style:personachat:test:148:self:7:positive:6"], "evidence_texts": ["A group of swimmers jump into a pool.", "Two small children walk away in a field.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "i am working on my latest quilt , do you have hobbies ?", "wow that is a cool bet . do they have names ?", "lol that is hilarious . it is a patriotic quilt as a present for my dad", "yes , he is a war vet . i am making it for veterans day", "that is something to be proud of . we should all be proud of our military", "well best of luck to him . i know he must appreciate your support", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:2294", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2303", "candidate_text": "I wonder whether motorcycle rider riding over a bump?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2303:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1947:premise", "snli:test:2465:premise", "snli:test:1697:premise", "snli:test:2303:premise", "style:personachat:test:85:self:1:positive:0"], "evidence_texts": ["A child buried in the sand.", "The children play in the pool.", "A woman holding bags behind plantains.", "Motorcycle rider going over a bump.", "i am unmarried , you are a married person ?"], "speaker_id": "snli:test:2303", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2303", "candidate_text": "Honestly, Motorcycle rider riding over a bump!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2303:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1947:premise", "snli:test:2465:premise", "snli:test:1697:premise", "snli:test:2303:premise", "style:personachat:test:619:self:1:positive:0"], "evidence_texts": ["A child buried in the sand.", "The children play in the pool.", "A woman holding bags behind plantains.", "Motorcycle rider going over a bump.", "wow medical marvel ! fastest surgery ever"], "speaker_id": "snli:test:2303", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2318", "candidate_text": "I wonder whether children are eating?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2318:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2147:premise", "snli:test:2411:premise", "snli:test:2318:premise", "snli:test:2108:premise", "style:personachat:test:85:self:2:positive:0", "style:personachat:test:85:self:2:positive:1"], "evidence_texts": ["A woman in an apron is looking for bread.", "A snowboarder is grinding down a long concrete rail.", "Four children are sitting at a table eating breakfast.", "A group of people in black outfits drinking beer.", "i afraid of snakes , you", "i am unmarried , you are a married person ?"], "speaker_id": "snli:test:2318", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2318", "candidate_text": "Children are eating.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2318:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2147:premise", "snli:test:2411:premise", "snli:test:2318:premise", "snli:test:2108:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in an apron is looking for bread.", "A snowboarder is grinding down a long concrete rail.", "Four children are sitting at a table eating breakfast.", "A group of people in black outfits drinking beer.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2318", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2330", "candidate_text": "I wonder whether lady in red storms out of a cafe?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2330:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1292:premise", "snli:test:2686:premise", "snli:test:2330:premise", "snli:test:1291:premise", "style:personachat:test:85:self:5:positive:0", "style:personachat:test:85:self:5:positive:1", "style:personachat:test:85:self:5:positive:2", "style:personachat:test:85:self:5:positive:3", "style:personachat:test:85:self:5:positive:4"], "evidence_texts": ["A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "my work is science fiction author , you ?", "my father and mother passed away last year", "i done my engineering graduate from mit", "i afraid of snakes , you", "i am unmarried , you are a married person ?"], "speaker_id": "snli:test:2330", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2330", "candidate_text": "well, lady in red storms out of a cafe", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2330:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1292:premise", "snli:test:2686:premise", "snli:test:2330:premise", "snli:test:1291:premise", "style:personachat:test:151:self:5:positive:0", "style:personachat:test:151:self:5:positive:1", "style:personachat:test:151:self:5:positive:2", "style:personachat:test:151:self:5:positive:3", "style:personachat:test:151:self:5:positive:4"], "evidence_texts": ["A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "oh , i like fitness too , but i am a big person . but it does not stop me .", "being married is wonderful . do you have children ? i adore mine .", "that is an important job . my job is a kindergarten teacher , so i can relate .", "i agree . in order to balance life , i live by the sea and swim daily .", "i would highly recommend it ."], "speaker_id": "snli:test:2330", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2334", "candidate_text": "I wonder whether there are dogs?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2334:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2288:premise", "snli:test:2600:premise", "snli:test:2334:premise", "snli:test:2260:premise", "style:personachat:test:85:self:7:positive:0", "style:personachat:test:85:self:7:positive:1", "style:personachat:test:85:self:7:positive:2", "style:personachat:test:85:self:7:positive:3", "style:personachat:test:85:self:7:positive:4", "style:personachat:test:85:self:7:positive:5", "style:personachat:test:85:self:7:positive:6"], "evidence_texts": ["A man in a khaki jacket and ball cap walks down the street.", "A person with a green backpack is riding a bike down the road.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A man is painting a city scene on a canvas in the street.", "i am fine thank you , how about u ?", "i am from new york , you ?", "my work is science fiction author , you ?", "my father and mother passed away last year", "i done my engineering graduate from mit", "i afraid of snakes , you", "i am unmarried , you are a married person ?"], "speaker_id": "snli:test:2334", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2334", "candidate_text": "Honestly, There are dogs!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2334:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2288:premise", "snli:test:2600:premise", "snli:test:2334:premise", "snli:test:2260:premise", "style:personachat:test:619:self:7:positive:0", "style:personachat:test:619:self:7:positive:1", "style:personachat:test:619:self:7:positive:2", "style:personachat:test:619:self:7:positive:3", "style:personachat:test:619:self:7:positive:4", "style:personachat:test:619:self:7:positive:5", "style:personachat:test:619:self:7:positive:6"], "evidence_texts": ["A man in a khaki jacket and ball cap walks down the street.", "A person with a green backpack is riding a bike down the road.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A man is painting a city scene on a canvas in the street.", "i did , how are you doing ?", "oh cool , i grew up playing frisbee in north dakota", "i had some donuts at my advertising job i started last year", "oh no , you better take it easy", "yeah good thing you played recently", "cool . you just said you got done playing ?", "wow medical marvel ! fastest surgery ever"], "speaker_id": "snli:test:2334", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2340", "candidate_text": "well, students are inside of a movie theater", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2340:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2340:premise", "snli:test:2195:premise", "snli:test:2826:premise", "snli:test:2170:premise", "style:personachat:test:86:self:1:positive:0"], "evidence_texts": ["A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "oh cool . i am still in high school . i will be going to auburn next year ."], "speaker_id": "snli:test:2340", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2340", "candidate_text": "I wonder whether students are inside of a movie theater?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2340:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2340:premise", "snli:test:2195:premise", "snli:test:2826:premise", "snli:test:2170:premise", "style:personachat:test:311:self:1:positive:0"], "evidence_texts": ["A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "A girl wearing khakis and a red sweatshirt is jumping in the air, while she is in the forest.", "what kind of pets do you have ?"], "speaker_id": "snli:test:2340", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2348", "candidate_text": "well, a group of swimmers jump into a pool during a swim meet", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2348:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2348:premise", "snli:test:2294:premise", "snli:test:2382:premise", "snli:test:2293:premise", "style:personachat:test:86:self:2:positive:0", "style:personachat:test:86:self:2:positive:1"], "evidence_texts": ["A group of swimmers jump into a pool.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "People shopping for vegetables at an outdoor market.", "okay . . . well , scary movies just get me too frightened .", "oh cool . i am still in high school . i will be going to auburn next year ."], "speaker_id": "snli:test:2348", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2348", "candidate_text": "Honestly, A group of swimmers jump into a pool during a swim meet!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2348:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2348:premise", "snli:test:2294:premise", "snli:test:2382:premise", "snli:test:2293:premise", "style:personachat:test:620:self:2:positive:0", "style:personachat:test:620:self:2:positive:1"], "evidence_texts": ["A group of swimmers jump into a pool.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "People shopping for vegetables at an outdoor market.", "so you must be lonely then ?", "yeah , dogs rock ! my parents were gone a lot but my dog was always there ."], "speaker_id": "snli:test:2348", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2355", "candidate_text": "well, two boys in bed", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2355:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2098:premise", "snli:test:2440:premise", "snli:test:2097:premise", "snli:test:2355:premise", "style:personachat:test:86:self:5:positive:0", "style:personachat:test:86:self:5:positive:1", "style:personachat:test:86:self:5:positive:2", "style:personachat:test:86:self:5:positive:3", "style:personachat:test:86:self:5:positive:4"], "evidence_texts": ["A man in a gray sweatshirt using a power saw.", "Man in puffy yellow jacket on skis on a mountain.", "A man in a gray sweatshirt using a power saw.", "Two boys with blond-hair, wearing striped shirts on a bed.", "cool . does your mom like to shop ? i really enjoy going to the mall .", "i am not into movies . i would love to have my own fashion line .", "probably something like mels makes . i do not like scary movies .", "okay . . . well , scary movies just get me too frightened .", "oh cool . i am still in high school . i will be going to auburn next year ."], "speaker_id": "snli:test:2355", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2355", "candidate_text": "Two boys in bed.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2355:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2098:premise", "snli:test:2440:premise", "snli:test:2097:premise", "snli:test:2355:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a gray sweatshirt using a power saw.", "Man in puffy yellow jacket on skis on a mountain.", "A man in a gray sweatshirt using a power saw.", "Two boys with blond-hair, wearing striped shirts on a bed.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2355", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2359", "candidate_text": "well, three brothers pound on some drums", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2359:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:1245:premise", "snli:test:2571:premise", "snli:test:391:premise", "snli:test:2359:premise", "style:personachat:test:86:self:7:positive:0", "style:personachat:test:86:self:7:positive:1", "style:personachat:test:86:self:7:positive:2", "style:personachat:test:86:self:7:positive:3", "style:personachat:test:86:self:7:positive:4", "style:personachat:test:86:self:7:positive:5", "style:personachat:test:86:self:7:positive:6"], "evidence_texts": ["A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "i am doing super good ! how are you ?", "oh that sounds so cool ! my name is melanie . i am still in high school .", "cool . does your mom like to shop ? i really enjoy going to the mall .", "i am not into movies . i would love to have my own fashion line .", "probably something like mels makes . i do not like scary movies .", "okay . . . well , scary movies just get me too frightened .", "oh cool . i am still in high school . i will be going to auburn next year ."], "speaker_id": "snli:test:2359", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2359", "candidate_text": "I wonder whether three brothers pound on some drums?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2359:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:1245:premise", "snli:test:2571:premise", "snli:test:391:premise", "snli:test:2359:premise", "style:personachat:test:311:self:7:positive:0", "style:personachat:test:311:self:7:positive:1", "style:personachat:test:311:self:7:positive:2", "style:personachat:test:311:self:7:positive:3", "style:personachat:test:311:self:7:positive:4", "style:personachat:test:311:self:7:positive:5", "style:personachat:test:311:self:7:positive:6"], "evidence_texts": ["A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "hi , i am doing great . how are you ?", "that is cool . i am a student attending auburn university .", "i am studying to become a doctor .", "i broke my bike . i love my volkswagen beetle .", "it is green . i want kids when i grow up .", "that is a lot of gran kids . how old are you ?", "what kind of pets do you have ?"], "speaker_id": "snli:test:2359", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2367", "candidate_text": "I wonder whether there are fans drinking beer at a sporting event?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2367:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2179:premise", "snli:test:2523:premise", "snli:test:2178:premise", "snli:test:2367:premise", "style:personachat:test:87:self:1:positive:0"], "evidence_texts": ["a mass of people looking and moving in the same general direction.", "Two men are working together to empty the barrel of its contents.", "a mass of people looking and moving in the same general direction.", "Fans are dressed in green and yellow for a major sporting event.", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:2367", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2367", "candidate_text": "well, there are fans drinking beer at a sporting event", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2367:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2179:premise", "snli:test:2523:premise", "snli:test:2178:premise", "snli:test:2367:premise", "style:personachat:test:154:self:1:positive:0"], "evidence_texts": ["a mass of people looking and moving in the same general direction.", "Two men are working together to empty the barrel of its contents.", "a mass of people looking and moving in the same general direction.", "Fans are dressed in green and yellow for a major sporting event.", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:2367", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2380", "candidate_text": "I wonder whether a man rock climbing?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2380:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2380:premise", "snli:test:2209:premise", "snli:test:2512:premise", "snli:test:2149:premise", "style:personachat:test:87:self:2:positive:0", "style:personachat:test:87:self:2:positive:1"], "evidence_texts": ["A man scales a rock climbing wall.", "A man standing alone in a park.", "Two women in tank tops are drinking.", "Two men at a table having drinks.", "nice ! do you like to try different foods ? i will try at least once .", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:2380", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2380", "candidate_text": "Honestly, A man rock climbing!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2380:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2380:premise", "snli:test:2209:premise", "snli:test:2512:premise", "snli:test:2149:premise", "style:personachat:test:624:self:2:positive:0", "style:personachat:test:624:self:2:positive:1"], "evidence_texts": ["A man scales a rock climbing wall.", "A man standing alone in a park.", "Two women in tank tops are drinking.", "Two men at a table having drinks.", "just frisbee . when i can i travel to bora bora . that is where they moved", "not old enough to meet new friends !"], "speaker_id": "snli:test:2380", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2382", "candidate_text": "I wonder whether a man and wife sit outside with their children?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2382:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2348:premise", "snli:test:2294:premise", "snli:test:2382:premise", "snli:test:2412:premise", "style:personachat:test:87:self:5:positive:0", "style:personachat:test:87:self:5:positive:1", "style:personachat:test:87:self:5:positive:2", "style:personachat:test:87:self:5:positive:3", "style:personachat:test:87:self:5:positive:4"], "evidence_texts": ["A group of swimmers jump into a pool.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "A barefoot child climbs at the jungle gym.", "do you know how to draw ? i do not . i want to learn how .", "that is cool . do you like reading ? i do . mainly a lot of poetry .", "how about collections do you have one ? mine is weird stickers .", "nice ! do you like to try different foods ? i will try at least once .", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:2382", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2382", "candidate_text": "A man and wife sit outside with their children.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2382:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2348:premise", "snli:test:2294:premise", "snli:test:2382:premise", "snli:test:2412:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of swimmers jump into a pool.", "People shopping for vegetables at an outdoor market.", "A family with young children sits down outside.", "A barefoot child climbs at the jungle gym.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2382", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2383", "candidate_text": "I wonder whether there is a family sitting outside with their children?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2383:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2412:premise", "snli:test:2348:premise", "snli:test:2413:premise", "snli:test:2383:premise", "style:personachat:test:87:self:7:positive:0", "style:personachat:test:87:self:7:positive:1", "style:personachat:test:87:self:7:positive:2", "style:personachat:test:87:self:7:positive:3", "style:personachat:test:87:self:7:positive:4", "style:personachat:test:87:self:7:positive:5", "style:personachat:test:87:self:7:positive:6"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "A group of swimmers jump into a pool.", "A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "hi . i am doing great thanks for asking . i do play poker sometimes .", "sorry to hear . i walk a lot since i hike in the nature . do you hike ?", "do you know how to draw ? i do not . i want to learn how .", "that is cool . do you like reading ? i do . mainly a lot of poetry .", "how about collections do you have one ? mine is weird stickers .", "nice ! do you like to try different foods ? i will try at least once .", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:2383", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2383", "candidate_text": "well, there is a family sitting outside with their children", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2383:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2412:premise", "snli:test:2348:premise", "snli:test:2413:premise", "snli:test:2383:premise", "style:personachat:test:154:self:6:positive:0", "style:personachat:test:154:self:6:positive:1", "style:personachat:test:154:self:6:positive:2", "style:personachat:test:154:self:6:positive:3", "style:personachat:test:154:self:6:positive:4", "style:personachat:test:154:self:6:positive:5"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "A group of swimmers jump into a pool.", "A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "hi . my day is good . i am hanging out with both of my sons .", "i love those crafts ! right now i am into my aquarium filled with exotic fish .", "you might like my collection of decorations . they are 50s vintage !", "that is awesome . what do you eat for breakfast ? mine is granola every day .", "that is very healthy . i like it . do you eat fish ?", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:2383", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2404", "candidate_text": "Honestly, A man is outdoors!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2404:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2404:premise", "snli:test:2291:premise", "snli:test:2757:premise", "snli:test:1640:premise", "style:personachat:test:88:self:1:positive:0"], "evidence_texts": ["A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:2404", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2404", "candidate_text": "A man is outdoors.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2404:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2404:premise", "snli:test:2291:premise", "snli:test:2757:premise", "snli:test:1640:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "Young shirtless man and asian woman jump up from a shallow rock at the shore with boats moored in the background", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2404", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2411", "candidate_text": "Honestly, A snowboarder is practicing a new trick!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2411:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2318:premise", "snli:test:2569:premise", "snli:test:2147:premise", "snli:test:2411:premise", "style:personachat:test:88:self:2:positive:0", "style:personachat:test:88:self:2:positive:1"], "evidence_texts": ["Four children are sitting at a table eating breakfast.", "A man in bright shirt carrying a fallen branch.", "A woman in an apron is looking for bread.", "A snowboarder is grinding down a long concrete rail.", "oh that is neat ! before my parents died a couple years back , we went to shows .", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:2411", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2411", "candidate_text": "I wonder whether a snowboarder is practicing a new trick?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2411:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2318:premise", "snli:test:2569:premise", "snli:test:2147:premise", "snli:test:2411:premise", "style:personachat:test:321:self:2:positive:0", "style:personachat:test:321:self:2:positive:1"], "evidence_texts": ["Four children are sitting at a table eating breakfast.", "A man in bright shirt carrying a fallen branch.", "A woman in an apron is looking for bread.", "A snowboarder is grinding down a long concrete rail.", "i am actually a student . i travel for fun .", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:2411", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2412", "candidate_text": "Honestly, The child is sitting on a bed!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2412:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2412:premise", "snli:test:2383:premise", "snli:test:2382:premise", "snli:test:2425:premise", "style:personachat:test:88:self:5:positive:0", "style:personachat:test:88:self:5:positive:1", "style:personachat:test:88:self:5:positive:2", "style:personachat:test:88:self:5:positive:3", "style:personachat:test:88:self:5:positive:4"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "A family with young children sits down outside.", "2 women and 2 girls crossing the road", "totally agree . so what stocks should i buy ?", "i figured it was worth a shot or maybe amazon stock ?", "that must be why you are still there ? kidding .", "oh that is neat ! before my parents died a couple years back , we went to shows .", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:2412", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2412", "candidate_text": "well, the child is sitting on a bed", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2412:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2412:premise", "snli:test:2383:premise", "snli:test:2382:premise", "snli:test:2425:premise", "style:personachat:test:155:self:5:positive:0", "style:personachat:test:155:self:5:positive:1", "style:personachat:test:155:self:5:positive:2", "style:personachat:test:155:self:5:positive:3", "style:personachat:test:155:self:5:positive:4"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "A family with young children sits down outside.", "2 women and 2 girls crossing the road", "just as a hobby though , i own my own successful dieting business full time", "oh yeah i love exercising for long hours . finding motivation is most important", "eating is also important , i like writing diet cookbooks", "i love walking my dog and two cats as well", "so do i and they give me a reason to get out"], "speaker_id": "snli:test:2412", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2413", "candidate_text": "Honestly, A child is playing outside on a jungle gym!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2413:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2425:premise", "snli:test:2413:premise", "snli:test:2383:premise", "snli:test:2461:premise", "style:personachat:test:88:self:7:positive:0", "style:personachat:test:88:self:7:positive:1", "style:personachat:test:88:self:7:positive:2", "style:personachat:test:88:self:7:positive:3", "style:personachat:test:88:self:7:positive:4", "style:personachat:test:88:self:7:positive:5", "style:personachat:test:88:self:7:positive:6"], "evidence_texts": ["2 women and 2 girls crossing the road", "A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "A man leads the way in a race.", "oh that sounds rough . i am an author , so i work from home .", "what is your favorite genre ? i specialize in science fiction .", "totally agree . so what stocks should i buy ?", "i figured it was worth a shot or maybe amazon stock ?", "that must be why you are still there ? kidding .", "oh that is neat ! before my parents died a couple years back , we went to shows .", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:2413", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2413", "candidate_text": "A child is playing outside on a jungle gym.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2413:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2425:premise", "snli:test:2413:premise", "snli:test:2383:premise", "snli:test:2461:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["2 women and 2 girls crossing the road", "A barefoot child climbs at the jungle gym.", "A family with young children sits down outside.", "A man leads the way in a race.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2413", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2425", "candidate_text": "well, a group of girls and women are laughing as they cross the road", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2425:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2413:premise", "snli:test:2425:premise", "snli:test:2461:premise", "snli:test:2412:premise", "style:personachat:test:89:self:1:positive:0"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "2 women and 2 girls crossing the road", "A man leads the way in a race.", "A barefoot child climbs at the jungle gym.", "i like to knit , i have been doing that a lot lately . donating to a hospital nearby ."], "speaker_id": "snli:test:2425", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2425", "candidate_text": "I wonder whether a group of girls and women are laughing as they cross the road?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2425:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2413:premise", "snli:test:2425:premise", "snli:test:2461:premise", "snli:test:2412:premise", "style:personachat:test:323:self:1:positive:0"], "evidence_texts": ["A barefoot child climbs at the jungle gym.", "2 women and 2 girls crossing the road", "A man leads the way in a race.", "A barefoot child climbs at the jungle gym.", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:2425", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2440", "candidate_text": "well, the man is outside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2440:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2440:premise", "snli:test:2355:premise", "snli:test:2098:premise", "snli:test:2490:premise", "style:personachat:test:89:self:2:positive:0", "style:personachat:test:89:self:2:positive:1"], "evidence_texts": ["Man in puffy yellow jacket on skis on a mountain.", "Two boys with blond-hair, wearing striped shirts on a bed.", "A man in a gray sweatshirt using a power saw.", "A boat pear with people boarding and disembarking some boats.", "used to listen to rap but not so much anymore . do you have any other hobbies ?", "i like to knit , i have been doing that a lot lately . donating to a hospital nearby ."], "speaker_id": "snli:test:2440", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2440", "candidate_text": "Honestly, The man is outside!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2440:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2440:premise", "snli:test:2355:premise", "snli:test:2098:premise", "snli:test:2490:premise", "style:personachat:test:636:self:2:positive:0", "style:personachat:test:636:self:2:positive:1"], "evidence_texts": ["Man in puffy yellow jacket on skis on a mountain.", "Two boys with blond-hair, wearing striped shirts on a bed.", "A man in a gray sweatshirt using a power saw.", "A boat pear with people boarding and disembarking some boats.", "absolutely ! my mom and dad were always gone growing up so i am used to eating junk .", "oh most definitely ! the pink ones are the best !"], "speaker_id": "snli:test:2440", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2441", "candidate_text": "well, the man is sitting on a chair in his house", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2441:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2490:premise", "snli:test:2441:premise", "snli:test:2355:premise", "snli:test:2491:premise", "style:personachat:test:89:self:5:positive:0", "style:personachat:test:89:self:5:positive:1", "style:personachat:test:89:self:5:positive:2", "style:personachat:test:89:self:5:positive:3", "style:personachat:test:89:self:5:positive:4"], "evidence_texts": ["A boat pear with people boarding and disembarking some boats.", "Man in puffy yellow jacket on skis on a mountain.", "Two boys with blond-hair, wearing striped shirts on a bed.", "A boat pear with people boarding and disembarking some boats.", "got in a brutal car accident but working hard to get back on my feet", "thank you , time off has allowed me to do some small work to help other", "i do , i like country music a lot . how about you ?", "used to listen to rap but not so much anymore . do you have any other hobbies ?", "i like to knit , i have been doing that a lot lately . donating to a hospital nearby ."], "speaker_id": "snli:test:2441", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2441", "candidate_text": "The man is sitting on a chair in his house.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2441:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2490:premise", "snli:test:2441:premise", "snli:test:2355:premise", "snli:test:2491:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A boat pear with people boarding and disembarking some boats.", "Man in puffy yellow jacket on skis on a mountain.", "Two boys with blond-hair, wearing striped shirts on a bed.", "A boat pear with people boarding and disembarking some boats.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2441", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2459", "candidate_text": "well, two bears are facing each other in the woods with their mouths open", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2459:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2277:premise", "snli:test:2558:premise", "snli:test:2183:premise", "snli:test:2459:premise", "style:personachat:test:89:self:7:positive:0", "style:personachat:test:89:self:7:positive:1", "style:personachat:test:89:self:7:positive:2", "style:personachat:test:89:self:7:positive:3", "style:personachat:test:89:self:7:positive:4", "style:personachat:test:89:self:7:positive:5", "style:personachat:test:89:self:7:positive:6"], "evidence_texts": ["A group of young adults with \"X\" on their hands, smiling.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A BMX rider is in midair while on a dirt trail.", "The bears are facing each other with their mouths wide open.", "i am alright , how was your day ?", "i do not do much since the accident , off of work . what do you do for fun ?", "got in a brutal car accident but working hard to get back on my feet", "thank you , time off has allowed me to do some small work to help other", "i do , i like country music a lot . how about you ?", "used to listen to rap but not so much anymore . do you have any other hobbies ?", "i like to knit , i have been doing that a lot lately . donating to a hospital nearby ."], "speaker_id": "snli:test:2459", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2459", "candidate_text": "I wonder whether two bears are facing each other in the woods with their mouths open?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2459:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2277:premise", "snli:test:2558:premise", "snli:test:2183:premise", "snli:test:2459:premise", "style:personachat:test:323:self:7:positive:0", "style:personachat:test:323:self:7:positive:1", "style:personachat:test:323:self:7:positive:2", "style:personachat:test:323:self:7:positive:3", "style:personachat:test:323:self:7:positive:4", "style:personachat:test:323:self:7:positive:5", "style:personachat:test:323:self:7:positive:6"], "evidence_texts": ["A group of young adults with \"X\" on their hands, smiling.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A BMX rider is in midair while on a dirt trail.", "The bears are facing each other with their mouths wide open.", "i do not to scared lol", "lol why not ? what is your fav color ?", "i went to u of i", "my fav is blue an my truck i drive is blue", "i like that for you , got kids ? i have 7", "really ? how many is that ? i grew up farming loved it", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:2459", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2461", "candidate_text": "well, the man is running a marathon", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2461:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2425:premise", "snli:test:2461:premise", "snli:test:2493:premise", "snli:test:2413:premise", "style:personachat:test:90:self:1:positive:0"], "evidence_texts": ["2 women and 2 girls crossing the road", "A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "A barefoot child climbs at the jungle gym.", "sounds like a nice life you got"], "speaker_id": "snli:test:2461", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2461", "candidate_text": "Honestly, The man is running a marathon!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2461:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2425:premise", "snli:test:2461:premise", "snli:test:2493:premise", "snli:test:2413:premise", "style:personachat:test:637:self:1:positive:0"], "evidence_texts": ["2 women and 2 girls crossing the road", "A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "A barefoot child climbs at the jungle gym.", "how late ? my parents were also older . in their 50s !"], "speaker_id": "snli:test:2461", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2465", "candidate_text": "well, the children are studying fractions inside the classroom", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2465:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2303:premise", "snli:test:2465:premise", "snli:test:2690:premise", "snli:test:1947:premise", "style:personachat:test:90:self:2:positive:0", "style:personachat:test:90:self:2:positive:1"], "evidence_texts": ["Motorcycle rider going over a bump.", "The children play in the pool.", "Two individuals use a photo kiosk", "A child buried in the sand.", "i am not sure yet . so , what do you like ?", "sounds like a nice life you got"], "speaker_id": "snli:test:2465", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2465", "candidate_text": "The children are studying fractions inside the classroom.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2465:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2303:premise", "snli:test:2465:premise", "snli:test:2690:premise", "snli:test:1947:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Motorcycle rider going over a bump.", "The children play in the pool.", "Two individuals use a photo kiosk", "A child buried in the sand.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2465", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2490", "candidate_text": "well, people are at a nascar race", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2490:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2441:premise", "snli:test:2440:premise", "snli:test:2490:premise", "snli:test:2506:premise", "style:personachat:test:90:self:5:positive:0", "style:personachat:test:90:self:5:positive:1", "style:personachat:test:90:self:5:positive:2", "style:personachat:test:90:self:5:positive:3", "style:personachat:test:90:self:5:positive:4"], "evidence_texts": ["Man in puffy yellow jacket on skis on a mountain.", "Man in puffy yellow jacket on skis on a mountain.", "A boat pear with people boarding and disembarking some boats.", "A male with a prosthetic leg diving for a volleyball.", "i work with a lot of kids in the healthcare industry", "i originally wanted to have a career in film as i love it", "produce . i decided to go down the healthcare route because its hard to get into", "i am not sure yet . so , what do you like ?", "sounds like a nice life you got"], "speaker_id": "snli:test:2490", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2490", "candidate_text": "I wonder whether people are at a nascar race?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2490:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2441:premise", "snli:test:2440:premise", "snli:test:2490:premise", "snli:test:2506:premise", "style:personachat:test:334:self:5:positive:0", "style:personachat:test:334:self:5:positive:1", "style:personachat:test:334:self:5:positive:2", "style:personachat:test:334:self:5:positive:3", "style:personachat:test:334:self:5:positive:4"], "evidence_texts": ["Man in puffy yellow jacket on skis on a mountain.", "Man in puffy yellow jacket on skis on a mountain.", "A boat pear with people boarding and disembarking some boats.", "A male with a prosthetic leg diving for a volleyball.", "i have tried it . i used my free time to lift weights . do you lift ?", "dogs are awesome . i am a huge man that loves watching horror movies .", "me either i drive a forklift in a warehouse where i work .", "thank you and you are so kind .", "wow what happened to it ?"], "speaker_id": "snli:test:2490", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2491", "candidate_text": "well, people are boarding and disembarking some boats", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2491:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2506:premise", "snli:test:2441:premise", "snli:test:2507:premise", "snli:test:2491:premise", "style:personachat:test:90:self:7:positive:0", "style:personachat:test:90:self:7:positive:1", "style:personachat:test:90:self:7:positive:2", "style:personachat:test:90:self:7:positive:3", "style:personachat:test:90:self:7:positive:4", "style:personachat:test:90:self:7:positive:5", "style:personachat:test:90:self:7:positive:6"], "evidence_texts": ["A male with a prosthetic leg diving for a volleyball.", "Man in puffy yellow jacket on skis on a mountain.", "A male with a prosthetic leg diving for a volleyball.", "A boat pear with people boarding and disembarking some boats.", "i am doing good . just enjoying eating my muffin , you ?", "i wish i had kids . just me and my dog , pedro , here", "i work with a lot of kids in the healthcare industry", "i originally wanted to have a career in film as i love it", "produce . i decided to go down the healthcare route because its hard to get into", "i am not sure yet . so , what do you like ?", "sounds like a nice life you got"], "speaker_id": "snli:test:2491", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2491", "candidate_text": "Honestly, people are boarding and disembarking some boats!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2491:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2506:premise", "snli:test:2441:premise", "snli:test:2507:premise", "snli:test:2491:premise", "style:personachat:test:637:self:6:positive:0", "style:personachat:test:637:self:6:positive:1", "style:personachat:test:637:self:6:positive:2", "style:personachat:test:637:self:6:positive:3", "style:personachat:test:637:self:6:positive:4", "style:personachat:test:637:self:6:positive:5"], "evidence_texts": ["A male with a prosthetic leg diving for a volleyball.", "Man in puffy yellow jacket on skis on a mountain.", "A male with a prosthetic leg diving for a volleyball.", "A boat pear with people boarding and disembarking some boats.", "hi ! i am getting my family ready . takes a while since there is four of us .", "i also attend school . what are you going to school for ?", "awesome ! music is what i am studying since i like it so much .", "i saw them in concert . could not see much since i am only five feet .", "did you come from a big family ? it is just me , parents and a sibling .", "how late ? my parents were also older . in their 50s !"], "speaker_id": "snli:test:2491", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2493", "candidate_text": "I wonder whether people waiting for their grandmother to let them in a home?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2493:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2461:premise", "snli:test:2493:premise", "snli:test:2425:premise", "snli:test:2531:premise", "style:personachat:test:91:self:1:positive:0"], "evidence_texts": ["A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "2 women and 2 girls crossing the road", "Man sits at the doorway of barber shop.", "what do you like to do ?"], "speaker_id": "snli:test:2493", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2493", "candidate_text": "Honestly, People waiting for their grandmother to let them in a home!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2493:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2461:premise", "snli:test:2493:premise", "snli:test:2425:premise", "snli:test:2531:premise", "style:personachat:test:646:self:1:positive:0"], "evidence_texts": ["A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "2 women and 2 girls crossing the road", "Man sits at the doorway of barber shop.", "fantasy videogames ! ! do you like them ?"], "speaker_id": "snli:test:2493", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2494", "candidate_text": "I wonder whether people waiting in line at a fast food restaurant?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2494:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2531:premise", "snli:test:2461:premise", "snli:test:2494:premise", "snli:test:2609:premise", "style:personachat:test:91:self:2:positive:0", "style:personachat:test:91:self:2:positive:1"], "evidence_texts": ["Man sits at the doorway of barber shop.", "A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "Man racing a quad on a dirt racetrack.", "cool ! ! any property in canada ? my family is looking to upgrade .", "what do you like to do ?"], "speaker_id": "snli:test:2494", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2494", "candidate_text": "People waiting in line at a fast food restaurant.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2494:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2531:premise", "snli:test:2461:premise", "snli:test:2494:premise", "snli:test:2609:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Man sits at the doorway of barber shop.", "A man leads the way in a race.", "Tourists waiting outside of a home to visit.", "Man racing a quad on a dirt racetrack.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2494", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2499", "candidate_text": "I wonder whether someone was trying to take a shortcut home?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2499:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:941:premise", "snli:test:9688:premise", "snli:test:2815:premise", "snli:test:2499:premise", "style:personachat:test:91:self:5:positive:0", "style:personachat:test:91:self:5:positive:1", "style:personachat:test:91:self:5:positive:2", "style:personachat:test:91:self:5:positive:3", "style:personachat:test:91:self:5:positive:4"], "evidence_texts": ["A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "eh ! iced tea with home grown canada syrup delicious", "red or blue ? coding is so much fun ! how old are you ? i am 22", "i freelance mostly website coding and such .", "cool ! ! any property in canada ? my family is looking to upgrade .", "what do you like to do ?"], "speaker_id": "snli:test:2499", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2499", "candidate_text": "well, someone was trying to take a shortcut home", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2499:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:941:premise", "snli:test:9688:premise", "snli:test:2815:premise", "snli:test:2499:premise", "style:personachat:test:162:self:5:positive:0", "style:personachat:test:162:self:5:positive:1", "style:personachat:test:162:self:5:positive:2", "style:personachat:test:162:self:5:positive:3", "style:personachat:test:162:self:5:positive:4"], "evidence_texts": ["A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "oh , i do not have siblings so i am not sure what that is like", "are you in a hospital treating folks", "who is marty relation to you", "oh i see , well have a great evening", "take care and tell marty where is mc fly"], "speaker_id": "snli:test:2499", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2500", "candidate_text": "I wonder whether a scooter rider wearing casual clothes races past a building?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2500:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2500:premise", "snli:test:2815:premise", "snli:test:941:premise", "snli:test:3469:premise", "style:personachat:test:91:self:6:positive:0", "style:personachat:test:91:self:6:positive:1", "style:personachat:test:91:self:6:positive:2", "style:personachat:test:91:self:6:positive:3", "style:personachat:test:91:self:6:positive:4", "style:personachat:test:91:self:6:positive:5"], "evidence_texts": ["A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "great ! just coding for apple . and drinking an iced tea", "eh ! iced tea with home grown canada syrup delicious", "red or blue ? coding is so much fun ! how old are you ? i am 22", "i freelance mostly website coding and such .", "cool ! ! any property in canada ? my family is looking to upgrade .", "what do you like to do ?"], "speaker_id": "snli:test:2500", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2500", "candidate_text": "Honestly, A scooter rider wearing casual clothes races past a building!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2500:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2500:premise", "snli:test:2815:premise", "snli:test:941:premise", "snli:test:3469:premise", "style:personachat:test:646:self:7:positive:0", "style:personachat:test:646:self:7:positive:1", "style:personachat:test:646:self:7:positive:2", "style:personachat:test:646:self:7:positive:3", "style:personachat:test:646:self:7:positive:4", "style:personachat:test:646:self:7:positive:5", "style:personachat:test:646:self:7:positive:6"], "evidence_texts": ["A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "hello ! i am well . being shy in class", "oh wow ! i am too shy to go to the gym . where do you go ?", "how many laps do you do ? i stay on my computer because i work all day", "wow ! i build model spaceships and we launch them from pools .", "its okay ! it was very hard to get my degree in computer science .", "now i just play a lot of games .", "fantasy videogames ! ! do you like them ?"], "speaker_id": "snli:test:2500", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2503", "candidate_text": "well, the man works in the office", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2503:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2282:premise", "snli:test:2605:premise", "snli:test:1534:premise", "snli:test:2503:premise", "style:personachat:test:92:self:1:positive:0"], "evidence_texts": ["A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "what kind of music you like"], "speaker_id": "snli:test:2503", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2503", "candidate_text": "I wonder whether the man works in the office?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2503:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2282:premise", "snli:test:2605:premise", "snli:test:1534:premise", "snli:test:2503:premise", "style:personachat:test:341:self:1:positive:0"], "evidence_texts": ["A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "Two adult men in black t-shirt and loud tunics hold smoking torches in preparation for an act of entertainment or celebration in a forested environment.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:2503", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2506", "candidate_text": "well, the male has at least one prosthetic leg", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2506:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2491:premise", "snli:test:2490:premise", "snli:test:2506:premise", "snli:test:2566:premise", "style:personachat:test:92:self:2:positive:0", "style:personachat:test:92:self:2:positive:1"], "evidence_texts": ["A boat pear with people boarding and disembarking some boats.", "A boat pear with people boarding and disembarking some boats.", "A male with a prosthetic leg diving for a volleyball.", "A white man taking a photo in front of water.", "i like working pets more than people", "what kind of music you like"], "speaker_id": "snli:test:2506", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2506", "candidate_text": "Honestly, The male has at least one prosthetic leg!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2506:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2491:premise", "snli:test:2490:premise", "snli:test:2506:premise", "snli:test:2566:premise", "style:personachat:test:648:self:2:positive:0", "style:personachat:test:648:self:2:positive:1"], "evidence_texts": ["A boat pear with people boarding and disembarking some boats.", "A boat pear with people boarding and disembarking some boats.", "A male with a prosthetic leg diving for a volleyball.", "A white man taking a photo in front of water.", "i believe it . it seemed like something i would like .", "maybe i will revisit the idea in the future !"], "speaker_id": "snli:test:2506", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2507", "candidate_text": "well, the male is playing soccer", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2507:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2566:premise", "snli:test:2491:premise", "snli:test:2731:premise", "snli:test:2507:premise", "style:personachat:test:92:self:5:positive:0", "style:personachat:test:92:self:5:positive:1", "style:personachat:test:92:self:5:positive:2", "style:personachat:test:92:self:5:positive:3", "style:personachat:test:92:self:5:positive:4"], "evidence_texts": ["A white man taking a photo in front of water.", "A boat pear with people boarding and disembarking some boats.", "asian women selling fruits from baskets at an outdoor market", "A male with a prosthetic leg diving for a volleyball.", "i like board games . how about you", "i used to go camping in arizona , that is why named my cat arizona", "cats are my favorite . just graduated vet school", "i like working pets more than people", "what kind of music you like"], "speaker_id": "snli:test:2507", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2507", "candidate_text": "The male is playing soccer.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2507:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2566:premise", "snli:test:2491:premise", "snli:test:2731:premise", "snli:test:2507:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A white man taking a photo in front of water.", "A boat pear with people boarding and disembarking some boats.", "asian women selling fruits from baskets at an outdoor market", "A male with a prosthetic leg diving for a volleyball.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2507", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2512", "candidate_text": "well, two topless women are making out", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2512:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2512:premise", "snli:test:2380:premise", "snli:test:2630:premise", "snli:test:2209:premise", "style:personachat:test:92:self:7:positive:0", "style:personachat:test:92:self:7:positive:1", "style:personachat:test:92:self:7:positive:2", "style:personachat:test:92:self:7:positive:3", "style:personachat:test:92:self:7:positive:4", "style:personachat:test:92:self:7:positive:5", "style:personachat:test:92:self:7:positive:6"], "evidence_texts": ["Two women in tank tops are drinking.", "A man scales a rock climbing wall.", "Women enjoying a game of table tennis.", "A man standing alone in a park.", "hi , not doing much . recovering my brother wedding was saturday", "it was too much fun ! steak sounds good", "i like board games . how about you", "i used to go camping in arizona , that is why named my cat arizona", "cats are my favorite . just graduated vet school", "i like working pets more than people", "what kind of music you like"], "speaker_id": "snli:test:2512", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2512", "candidate_text": "I wonder whether two topless women are making out?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2512:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2512:premise", "snli:test:2380:premise", "snli:test:2630:premise", "snli:test:2209:premise", "style:personachat:test:341:self:6:positive:0", "style:personachat:test:341:self:6:positive:1", "style:personachat:test:341:self:6:positive:2", "style:personachat:test:341:self:6:positive:3", "style:personachat:test:341:self:6:positive:4", "style:personachat:test:341:self:6:positive:5"], "evidence_texts": ["Two women in tank tops are drinking.", "A man scales a rock climbing wall.", "Women enjoying a game of table tennis.", "A man standing alone in a park.", "mississippi . best jazz around ! how about you ?", "i was just there for a vintage car show . nice state !", "your best friend is eve 9000 ? is that an ai ?", "how exciting ! how did you meet ?", "wow that is incredible ! are you an engineer as well ?", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:2512", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2518", "candidate_text": "well, a tall person with a hat", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2518:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1307:premise", "snli:test:3100:premise", "snli:test:2518:premise", "snli:test:781:premise", "style:personachat:test:93:self:1:positive:0"], "evidence_texts": ["Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "i have a bit of a stutter when i get over excited ."], "speaker_id": "snli:test:2518", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2518", "candidate_text": "Honestly, A tall person with a hat!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2518:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1307:premise", "snli:test:3100:premise", "snli:test:2518:premise", "snli:test:781:premise", "style:personachat:test:654:self:1:positive:0"], "evidence_texts": ["Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "haha . i did not say that . ohio is a great state !"], "speaker_id": "snli:test:2518", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2523", "candidate_text": "well, two fishermen are emptying their catch from a barrel", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2523:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2367:premise", "snli:test:2535:premise", "snli:test:2179:premise", "snli:test:2523:premise", "style:personachat:test:93:self:2:positive:0", "style:personachat:test:93:self:2:positive:1"], "evidence_texts": ["Fans are dressed in green and yellow for a major sporting event.", "A boy is running in a spinning metal tunnel in the playground.", "a mass of people looking and moving in the same general direction.", "Two men are working together to empty the barrel of its contents.", "i bbq chicken pizza , it is so good . i recommend it .", "i have a bit of a stutter when i get over excited ."], "speaker_id": "snli:test:2523", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2523", "candidate_text": "Two fishermen are emptying their catch from a barrel.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2523:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2367:premise", "snli:test:2535:premise", "snli:test:2179:premise", "snli:test:2523:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Fans are dressed in green and yellow for a major sporting event.", "A boy is running in a spinning metal tunnel in the playground.", "a mass of people looking and moving in the same general direction.", "Two men are working together to empty the barrel of its contents.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2523", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2531", "candidate_text": "well, the man is standing in the roof", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2531:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2494:premise", "snli:test:2609:premise", "snli:test:2493:premise", "snli:test:2531:premise", "style:personachat:test:93:self:5:positive:0", "style:personachat:test:93:self:5:positive:1", "style:personachat:test:93:self:5:positive:2", "style:personachat:test:93:self:5:positive:3", "style:personachat:test:93:self:5:positive:4"], "evidence_texts": ["Tourists waiting outside of a home to visit.", "Man racing a quad on a dirt racetrack.", "Tourists waiting outside of a home to visit.", "Man sits at the doorway of barber shop.", "interesting , do you have a job ?", "do you work at a vet ?", "oh , that sounds horrible . i work part time at my moms pizza restaurant .", "i bbq chicken pizza , it is so good . i recommend it .", "i have a bit of a stutter when i get over excited ."], "speaker_id": "snli:test:2531", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2531", "candidate_text": "I wonder whether the man is standing in the roof?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2531:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2494:premise", "snli:test:2609:premise", "snli:test:2493:premise", "snli:test:2531:premise", "style:personachat:test:354:self:5:positive:0", "style:personachat:test:354:self:5:positive:1", "style:personachat:test:354:self:5:positive:2", "style:personachat:test:354:self:5:positive:3", "style:personachat:test:354:self:5:positive:4"], "evidence_texts": ["Tourists waiting outside of a home to visit.", "Man racing a quad on a dirt racetrack.", "Tourists waiting outside of a home to visit.", "Man sits at the doorway of barber shop.", "really ? me too , are you in a relationship ? i am not as of yet .", "i got me some nice blue jean thigh high boots . any siblings ?", "oh no sorry to hear that ! we could go out on a date !", "i bet you are a great catch , i will be a full time nurse soon .", "lol see , where are you located at ?"], "speaker_id": "snli:test:2531", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2535", "candidate_text": "well, he is getting dizzy", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2535:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2523:premise", "snli:test:2742:premise", "snli:test:2367:premise", "snli:test:2535:premise", "style:personachat:test:93:self:7:positive:0", "style:personachat:test:93:self:7:positive:1", "style:personachat:test:93:self:7:positive:2", "style:personachat:test:93:self:7:positive:3", "style:personachat:test:93:self:7:positive:4", "style:personachat:test:93:self:7:positive:5", "style:personachat:test:93:self:7:positive:6"], "evidence_texts": ["Two men are working together to empty the barrel of its contents.", "Two people are sitting on a rock looking off into the distance.", "Fans are dressed in green and yellow for a major sporting event.", "A boy is running in a spinning metal tunnel in the playground.", "i am doing good . i finally got a moment to myself .", "what kind of projects do you mean ?", "interesting , do you have a job ?", "do you work at a vet ?", "oh , that sounds horrible . i work part time at my moms pizza restaurant .", "i bbq chicken pizza , it is so good . i recommend it .", "i have a bit of a stutter when i get over excited ."], "speaker_id": "snli:test:2535", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2535", "candidate_text": "Honestly, He is getting dizzy!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2535:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2523:premise", "snli:test:2742:premise", "snli:test:2367:premise", "snli:test:2535:premise", "style:personachat:test:654:self:7:positive:0", "style:personachat:test:654:self:7:positive:1", "style:personachat:test:654:self:7:positive:2", "style:personachat:test:654:self:7:positive:3", "style:personachat:test:654:self:7:positive:4", "style:personachat:test:654:self:7:positive:5", "style:personachat:test:654:self:7:positive:6"], "evidence_texts": ["Two men are working together to empty the barrel of its contents.", "Two people are sitting on a rock looking off into the distance.", "Fans are dressed in green and yellow for a major sporting event.", "A boy is running in a spinning metal tunnel in the playground.", "hey . that is a terrible conundrum !", "wish i could . i keep a large garden as well .", "i am a little busy . i teach a college class in biology .", "maybe when i am free . i like walking in my spare time .", "sure . i also love travelling . been to six countries already !", "yes , but ohio is not a country , you know ?", "haha . i did not say that . ohio is a great state !"], "speaker_id": "snli:test:2535", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2547", "candidate_text": "well, the flautist plays a lullaby", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2547:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2035:premise", "snli:test:2727:premise", "snli:test:1820:premise", "snli:test:2547:premise", "style:personachat:test:94:self:1:positive:0"], "evidence_texts": ["A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "Man jumping up in the air and splitting his legs with his mouth open.", "Three women talk to each other next to another woman sitting on a motorcycle.", "A girl in a green coat walks down a rural road playing a flute", "my cat is named lily . small world ."], "speaker_id": "snli:test:2547", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2547", "candidate_text": "The flautist plays a lullaby.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2547:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2035:premise", "snli:test:2727:premise", "snli:test:1820:premise", "snli:test:2547:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "Man jumping up in the air and splitting his legs with his mouth open.", "Three women talk to each other next to another woman sitting on a motorcycle.", "A girl in a green coat walks down a rural road playing a flute", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2547", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2558", "candidate_text": "well, a woman getting her annual check up", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2558:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2459:premise", "snli:test:2592:premise", "snli:test:2558:premise", "snli:test:2277:premise", "style:personachat:test:94:self:2:positive:0", "style:personachat:test:94:self:2:positive:1"], "evidence_texts": ["The bears are facing each other with their mouths wide open.", "A man in military uniform points as three other men watch.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A group of young adults with \"X\" on their hands, smiling.", "i never learned to swim . what kind of horses ?", "my cat is named lily . small world ."], "speaker_id": "snli:test:2558", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2558", "candidate_text": "I wonder whether a woman getting her annual check up?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2558:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2459:premise", "snli:test:2592:premise", "snli:test:2558:premise", "snli:test:2277:premise", "style:personachat:test:355:self:2:positive:0", "style:personachat:test:355:self:2:positive:1"], "evidence_texts": ["The bears are facing each other with their mouths wide open.", "A man in military uniform points as three other men watch.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A group of young adults with \"X\" on their hands, smiling.", "i am a medium and read tarot cards", "what kind of teacher ? you enjoy it ?"], "speaker_id": "snli:test:2558", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2566", "candidate_text": "well, a man is using the restroom", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2566:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2507:premise", "snli:test:2731:premise", "snli:test:2566:premise", "snli:test:2506:premise", "style:personachat:test:94:self:5:positive:0", "style:personachat:test:94:self:5:positive:1", "style:personachat:test:94:self:5:positive:2", "style:personachat:test:94:self:5:positive:3", "style:personachat:test:94:self:5:positive:4"], "evidence_texts": ["A male with a prosthetic leg diving for a volleyball.", "asian women selling fruits from baskets at an outdoor market", "A white man taking a photo in front of water.", "A male with a prosthetic leg diving for a volleyball.", "i love horses . who is your favorite singer ?", "i like prince , probably because i love the color purple .", "i love the city but i am scared of the water .", "i never learned to swim . what kind of horses ?", "my cat is named lily . small world ."], "speaker_id": "snli:test:2566", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2566", "candidate_text": "Honestly, A man is using the restroom!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2566:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2507:premise", "snli:test:2731:premise", "snli:test:2566:premise", "snli:test:2506:premise", "style:personachat:test:662:self:5:positive:0", "style:personachat:test:662:self:5:positive:1", "style:personachat:test:662:self:5:positive:2", "style:personachat:test:662:self:5:positive:3", "style:personachat:test:662:self:5:positive:4"], "evidence_texts": ["A male with a prosthetic leg diving for a volleyball.", "asian women selling fruits from baskets at an outdoor market", "A white man taking a photo in front of water.", "A male with a prosthetic leg diving for a volleyball.", "yes . i love it here . i belong to a horror film group .", "i went to school for teaching but i sing professionally now .", "that sounds illegal . are they not protected ?", "really ? i am surprised to hear that . where do you live ?", "you really do like the cold ! i prefer the heat ."], "speaker_id": "snli:test:2566", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2569", "candidate_text": "well, the man is cleaning up his yard after the bad thunderstorm the evening before", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2569:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2411:premise", "snli:test:2569:premise", "snli:test:2675:premise", "snli:test:2318:premise", "style:personachat:test:94:self:7:positive:0", "style:personachat:test:94:self:7:positive:1", "style:personachat:test:94:self:7:positive:2", "style:personachat:test:94:self:7:positive:3", "style:personachat:test:94:self:7:positive:4", "style:personachat:test:94:self:7:positive:5", "style:personachat:test:94:self:7:positive:6"], "evidence_texts": ["A snowboarder is grinding down a long concrete rail.", "A man in bright shirt carrying a fallen branch.", "Two oriental older women gathering rice from the fields.", "Four children are sitting at a table eating breakfast.", "very well thank you , just left the factory . yourself ?", "same here , now i want to watch the buckeyes game !", "i love horses . who is your favorite singer ?", "i like prince , probably because i love the color purple .", "i love the city but i am scared of the water .", "i never learned to swim . what kind of horses ?", "my cat is named lily . small world ."], "speaker_id": "snli:test:2569", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2569", "candidate_text": "The man is cleaning up his yard after the bad thunderstorm the evening before.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2569:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2411:premise", "snli:test:2569:premise", "snli:test:2675:premise", "snli:test:2318:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A snowboarder is grinding down a long concrete rail.", "A man in bright shirt carrying a fallen branch.", "Two oriental older women gathering rice from the fields.", "Four children are sitting at a table eating breakfast.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2569", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2571", "candidate_text": "well, the sky is blue with a single puffy cloud", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2571:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2359:premise", "snli:test:2946:premise", "snli:test:1245:premise", "snli:test:2571:premise", "style:personachat:test:95:self:1:positive:0"], "evidence_texts": ["Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "i just did 15 miles ."], "speaker_id": "snli:test:2571", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2571", "candidate_text": "I wonder whether the sky is blue with a single puffy cloud?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2571:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2359:premise", "snli:test:2946:premise", "snli:test:1245:premise", "snli:test:2571:premise", "style:personachat:test:360:self:1:positive:0"], "evidence_texts": ["Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "A young man holds a hammer shaped object in the air with one hand and a similar shaped object in the other while making a funny expression.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "what are your plans for this evening ?"], "speaker_id": "snli:test:2571", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2577", "candidate_text": "well, some children are looking", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2577:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1996:premise", "snli:test:2577:premise", "snli:test:2934:premise", "snli:test:1806:premise", "style:personachat:test:95:self:2:positive:0", "style:personachat:test:95:self:2:positive:1"], "evidence_texts": ["People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "Adults and children share in the looking at something, and some young ladies stand to the side.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "A group of people standing around smiling, and in the background are more people and handmade signs.", "not running is what makes me cry like a baby", "i just did 15 miles ."], "speaker_id": "snli:test:2577", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2577", "candidate_text": "Honestly, Some children are looking!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2577:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1996:premise", "snli:test:2577:premise", "snli:test:2934:premise", "snli:test:1806:premise", "style:personachat:test:677:self:2:positive:0", "style:personachat:test:677:self:2:positive:1"], "evidence_texts": ["People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "Adults and children share in the looking at something, and some young ladies stand to the side.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "A group of people standing around smiling, and in the background are more people and handmade signs.", "italian is quite good . anything in particular ?", "never ! have you started getting ready for the holidays yet ?"], "speaker_id": "snli:test:2577", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2592", "candidate_text": "well, the man is training some guys", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2592:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2592:premise", "snli:test:2558:premise", "snli:test:2772:premise", "snli:test:2459:premise", "style:personachat:test:95:self:5:positive:0", "style:personachat:test:95:self:5:positive:1", "style:personachat:test:95:self:5:positive:2", "style:personachat:test:95:self:5:positive:3", "style:personachat:test:95:self:5:positive:4"], "evidence_texts": ["A man in military uniform points as three other men watch.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A yellow race car sliding through a corner as spectators watch.", "The bears are facing each other with their mouths wide open.", "i hate all animals to be honest", "dropping out of high school was a bad idea . the landlord just called .", "lucky , he can give you slack .", "not running is what makes me cry like a baby", "i just did 15 miles ."], "speaker_id": "snli:test:2592", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2592", "candidate_text": "the man is training some guys.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2592:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2592:premise", "snli:test:2558:premise", "snli:test:2772:premise", "snli:test:2459:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in military uniform points as three other men watch.", "A nurse is listening to another women's heartbeat with a stethoscope.", "A yellow race car sliding through a corner as spectators watch.", "The bears are facing each other with their mouths wide open.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2592", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2600", "candidate_text": "well, the person is walking", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2600:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2600:premise", "snli:test:2334:premise", "snli:test:2643:premise", "snli:test:2288:premise", "style:personachat:test:95:self:7:positive:0", "style:personachat:test:95:self:7:positive:1", "style:personachat:test:95:self:7:positive:2", "style:personachat:test:95:self:7:positive:3", "style:personachat:test:95:self:7:positive:4", "style:personachat:test:95:self:7:positive:5", "style:personachat:test:95:self:7:positive:6"], "evidence_texts": ["A person with a green backpack is riding a bike down the road.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "A man in a khaki jacket and ball cap walks down the street.", "bad , i am broke and my low income apartment has rent due tomorrow", "same here , fast food does not pay like how i thought it did at 16", "i hate all animals to be honest", "dropping out of high school was a bad idea . the landlord just called .", "lucky , he can give you slack .", "not running is what makes me cry like a baby", "i just did 15 miles ."], "speaker_id": "snli:test:2600", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2600", "candidate_text": "I wonder whether the person is walking?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2600:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2600:premise", "snli:test:2334:premise", "snli:test:2643:premise", "snli:test:2288:premise", "style:personachat:test:360:self:7:positive:0", "style:personachat:test:360:self:7:positive:1", "style:personachat:test:360:self:7:positive:2", "style:personachat:test:360:self:7:positive:3", "style:personachat:test:360:self:7:positive:4", "style:personachat:test:360:self:7:positive:5", "style:personachat:test:360:self:7:positive:6"], "evidence_texts": ["A person with a green backpack is riding a bike down the road.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "A man in a khaki jacket and ball cap walks down the street.", "hello how are you today ?", "i love to wake up before the sun shines .", "i love creating kids pop up books .", "my town population is only 2000 people .", "i am my daughter teacher at homeschooling .", "i hate cats they make me sneeze", "what are your plans for this evening ?"], "speaker_id": "snli:test:2600", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2605", "candidate_text": "well, a group of people holding cats take a picture", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2605:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2503:premise", "snli:test:2605:premise", "snli:test:3118:premise", "snli:test:2282:premise", "style:personachat:test:96:self:1:positive:0"], "evidence_texts": ["A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "i do online for a cable company"], "speaker_id": "snli:test:2605", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2605", "candidate_text": "Honestly, A group of people holding cats take a picture!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2605:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2503:premise", "snli:test:2605:premise", "snli:test:3118:premise", "snli:test:2282:premise", "style:personachat:test:680:self:1:positive:0"], "evidence_texts": ["A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young white girl is standing on the street in front of a storefront, dressed up in red with a hoop decorated with card symbols.", "yeah . also , hotels are very expensive as well !"], "speaker_id": "snli:test:2605", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2609", "candidate_text": "well, a man racing on a dirt track", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2609:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2531:premise", "snli:test:2609:premise", "snli:test:3035:premise", "snli:test:2494:premise", "style:personachat:test:96:self:2:positive:0", "style:personachat:test:96:self:2:positive:1"], "evidence_texts": ["Man sits at the doorway of barber shop.", "Man racing a quad on a dirt racetrack.", "Cook handling hot tray with food on it.", "Tourists waiting outside of a home to visit.", "aw that is cure i work sales", "i do online for a cable company"], "speaker_id": "snli:test:2609", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2609", "candidate_text": "A man racing on a dirt track.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2609:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2531:premise", "snli:test:2609:premise", "snli:test:3035:premise", "snli:test:2494:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Man sits at the doorway of barber shop.", "Man racing a quad on a dirt racetrack.", "Cook handling hot tray with food on it.", "Tourists waiting outside of a home to visit.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2609", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2614", "candidate_text": "well, a girl is inside playing a board game", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2614:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2168:premise", "snli:test:3668:premise", "snli:test:2614:premise", "snli:test:2091:premise", "style:personachat:test:96:self:5:positive:0", "style:personachat:test:96:self:5:positive:1", "style:personachat:test:96:self:5:positive:2", "style:personachat:test:96:self:5:positive:3", "style:personachat:test:96:self:5:positive:4"], "evidence_texts": ["Right leg in a brace, a young boy lies on the couch while using his cellphone.", "An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "what is that ? an outside at the park so nice . like bands ?", "no it was local with rush as the band", "it was , so what about you ?", "aw that is cure i work sales", "i do online for a cable company"], "speaker_id": "snli:test:2614", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2614", "candidate_text": "I wonder whether a girl is inside playing a board game?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2614:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2168:premise", "snli:test:3668:premise", "snli:test:2614:premise", "snli:test:2091:premise", "style:personachat:test:364:self:5:positive:0", "style:personachat:test:364:self:5:positive:1", "style:personachat:test:364:self:5:positive:2", "style:personachat:test:364:self:5:positive:3", "style:personachat:test:364:self:5:positive:4"], "evidence_texts": ["Right leg in a brace, a young boy lies on the couch while using his cellphone.", "An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "A person wearing orange stockings and chunky boots has her legs stretched out on the grass.", "oh no ! do you hunt mer people also ? ?", "geology ? is that rocks or something ? i hide behind those sometimes .", "oh no . please tell me where he fishes so i can stay away ! !", "my sister got stuck in a fishing net once !", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:2614", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2616", "candidate_text": "well, a boy is playing with a ball", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2616:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2616:premise", "snli:test:1954:premise", "snli:test:2723:premise", "snli:test:1910:premise", "style:personachat:test:96:self:7:positive:0", "style:personachat:test:96:self:7:positive:1", "style:personachat:test:96:self:7:positive:2", "style:personachat:test:96:self:7:positive:3", "style:personachat:test:96:self:7:positive:4", "style:personachat:test:96:self:7:positive:5", "style:personachat:test:96:self:7:positive:6"], "evidence_texts": ["A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "i am well tired an you ?", "my friend got married last week it was beautiful", "what is that ? an outside at the park so nice . like bands ?", "no it was local with rush as the band", "it was , so what about you ?", "aw that is cure i work sales", "i do online for a cable company"], "speaker_id": "snli:test:2616", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2616", "candidate_text": "Honestly, A boy is playing with a ball!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2616:negative", "evidence_budget": 14, "evidence_ids": ["snli:test:2616:premise", "snli:test:1954:premise", "snli:test:2723:premise", "snli:test:1910:premise", "style:personachat:test:680:self:10:positive:0", "style:personachat:test:680:self:10:positive:1", "style:personachat:test:680:self:10:positive:2", "style:personachat:test:680:self:10:positive:3", "style:personachat:test:680:self:10:positive:4", "style:personachat:test:680:self:10:positive:5", "style:personachat:test:680:self:10:positive:6", "style:personachat:test:680:self:10:positive:7", "style:personachat:test:680:self:10:positive:8", "style:personachat:test:680:self:10:positive:9"], "evidence_texts": ["A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "A young man, wearing a T-shirt that says Joplin, plays the guitar in front of a large amplifier.", "i think hollywood is a mess !", "i do not think church would help these people lol", "i think beers would work", "no , i do not . do you ?", "i am allergic as well !", "this is very very true", "well , i just thought it was cool .", "it was just amazing to see !", "it is very expensive . to expensive , i think", "yeah . also , hotels are very expensive as well !"], "speaker_id": "snli:test:2616", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2630", "candidate_text": "well, women are reading books", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2630:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2512:premise", "snli:test:2676:premise", "snli:test:2630:premise", "snli:test:2380:premise", "style:personachat:test:97:self:1:positive:0"], "evidence_texts": ["Two women in tank tops are drinking.", "Two men spar in martial arts gear.", "Women enjoying a game of table tennis.", "A man scales a rock climbing wall.", "it is ok , i wanted something different ."], "speaker_id": "snli:test:2630", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2630", "candidate_text": "Women are reading books.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2630:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2512:premise", "snli:test:2676:premise", "snli:test:2630:premise", "snli:test:2380:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two women in tank tops are drinking.", "Two men spar in martial arts gear.", "Women enjoying a game of table tennis.", "A man scales a rock climbing wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2630", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2643", "candidate_text": "well, a sad woman wearing very formal colorful clothing and she seems to be dancing", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2643:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2600:premise", "snli:test:2781:premise", "snli:test:2334:premise", "snli:test:2643:premise", "style:personachat:test:97:self:2:positive:0", "style:personachat:test:97:self:2:positive:1"], "evidence_texts": ["A person with a green backpack is riding a bike down the road.", "A scientist studies a slide in order to work on her new creation.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "funny thing is i am from canada", "it is ok , i wanted something different ."], "speaker_id": "snli:test:2643", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2643", "candidate_text": "I wonder whether a sad woman wearing very formal colorful clothing and she seems to be dancing?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2643:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2600:premise", "snli:test:2781:premise", "snli:test:2334:premise", "snli:test:2643:premise", "style:personachat:test:365:self:2:positive:0", "style:personachat:test:365:self:2:positive:1"], "evidence_texts": ["A person with a green backpack is riding a bike down the road.", "A scientist studies a slide in order to work on her new creation.", "Sled and dogs headed towards another group of sled dogs with one loose.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "i work for a group that proves the validity of the jewish faith .", "what kind of job do you have ?"], "speaker_id": "snli:test:2643", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2675", "candidate_text": "well, two asian women are wearing hats", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2675:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2569:premise", "snli:test:2764:premise", "snli:test:2411:premise", "snli:test:2675:premise", "style:personachat:test:97:self:5:positive:0", "style:personachat:test:97:self:5:positive:1", "style:personachat:test:97:self:5:positive:2", "style:personachat:test:97:self:5:positive:3", "style:personachat:test:97:self:5:positive:4"], "evidence_texts": ["A man in bright shirt carrying a fallen branch.", "Asian individuals are taking pictures of some unidentified thing.", "A snowboarder is grinding down a long concrete rail.", "Two oriental older women gathering rice from the fields.", "my down fall is that i love tea .", "i do web design and what do you do ?", "yes very nice car . i am a country girl who likes a truck .", "funny thing is i am from canada", "it is ok , i wanted something different ."], "speaker_id": "snli:test:2675", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2675", "candidate_text": "Honestly, Two Asian women are wearing hats!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2675:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2569:premise", "snli:test:2764:premise", "snli:test:2411:premise", "snli:test:2675:premise", "style:personachat:test:685:self:2:positive:0", "style:personachat:test:685:self:2:positive:1"], "evidence_texts": ["A man in bright shirt carrying a fallen branch.", "Asian individuals are taking pictures of some unidentified thing.", "A snowboarder is grinding down a long concrete rail.", "Two oriental older women gathering rice from the fields.", "well besides working out i love roller coasters . do you like them ?", "definitely ! have you ridden any hybrid metal and wood ones ?"], "speaker_id": "snli:test:2675", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2676", "candidate_text": "well, there is only one person present", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2676:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2676:premise", "snli:test:2630:premise", "snli:test:2684:premise", "snli:test:2512:premise", "style:personachat:test:97:self:7:positive:0", "style:personachat:test:97:self:7:positive:1", "style:personachat:test:97:self:7:positive:2", "style:personachat:test:97:self:7:positive:3", "style:personachat:test:97:self:7:positive:4", "style:personachat:test:97:self:7:positive:5", "style:personachat:test:97:self:7:positive:6"], "evidence_texts": ["Two men spar in martial arts gear.", "Women enjoying a game of table tennis.", "A man and woman setup a camera.", "Two women in tank tops are drinking.", "i am great , how are you ? i like to workout .", "while i am still young i am building up my muscle .", "my down fall is that i love tea .", "i do web design and what do you do ?", "yes very nice car . i am a country girl who likes a truck .", "funny thing is i am from canada", "it is ok , i wanted something different ."], "speaker_id": "snli:test:2676", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2676", "candidate_text": "There is only one person present.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2676:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2676:premise", "snli:test:2630:premise", "snli:test:2684:premise", "snli:test:2512:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men spar in martial arts gear.", "Women enjoying a game of table tennis.", "A man and woman setup a camera.", "Two women in tank tops are drinking.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2676", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2684", "candidate_text": "well, two people setup a device", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2684:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2684:premise", "snli:test:2676:premise", "snli:test:2711:premise", "snli:test:2630:premise", "style:personachat:test:98:self:1:positive:0"], "evidence_texts": ["A man and woman setup a camera.", "Two men spar in martial arts gear.", "An medical worker examines a young girl.", "Women enjoying a game of table tennis.", "no , it is just me . been busy as of late ."], "speaker_id": "snli:test:2684", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2684", "candidate_text": "I wonder whether two people setup a device?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2684:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2684:premise", "snli:test:2676:premise", "snli:test:2711:premise", "snli:test:2630:premise", "style:personachat:test:366:self:1:positive:0"], "evidence_texts": ["A man and woman setup a camera.", "Two men spar in martial arts gear.", "An medical worker examines a young girl.", "Women enjoying a game of table tennis.", "ops you are selling by phone ?"], "speaker_id": "snli:test:2684", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2686", "candidate_text": "well, two children are looking through a telescope together at a city street", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2686:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2330:premise", "snli:test:2740:premise", "snli:test:2686:premise", "snli:test:1292:premise", "style:personachat:test:98:self:2:positive:0", "style:personachat:test:98:self:2:positive:1"], "evidence_texts": ["A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "i like simple plan as well .", "no , it is just me . been busy as of late ."], "speaker_id": "snli:test:2686", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2686", "candidate_text": "Honestly, Two children are looking through a telescope together at a city street!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2686:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2330:premise", "snli:test:2740:premise", "snli:test:2686:premise", "snli:test:1292:premise", "style:personachat:test:689:self:1:positive:0"], "evidence_texts": ["A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "A man in a white and gray shirt looks on as a shirtless man with a mustache cleans fish on a pale green table.", "no , i do not care for them . too much work ! do you work 12 hours shifts ?"], "speaker_id": "snli:test:2686", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2690", "candidate_text": "well, the people are on a bike", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2690:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2465:premise", "snli:test:3252:premise", "snli:test:2690:premise", "snli:test:2303:premise", "style:personachat:test:98:self:5:positive:0", "style:personachat:test:98:self:5:positive:1", "style:personachat:test:98:self:5:positive:2", "style:personachat:test:98:self:5:positive:3", "style:personachat:test:98:self:5:positive:4"], "evidence_texts": ["The children play in the pool.", "A family smiling on the couch.", "Two individuals use a photo kiosk", "Motorcycle rider going over a bump.", "i hear ya . i get dirty working in my garden .", "true . i love my cat . she has a purple litter box .", "she does . she sits with me when i watch the buckeyes on tv .", "i like simple plan as well .", "no , it is just me . been busy as of late ."], "speaker_id": "snli:test:2690", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2690", "candidate_text": "The people are on a bike.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2690:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2465:premise", "snli:test:3252:premise", "snli:test:2690:premise", "snli:test:2303:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The children play in the pool.", "A family smiling on the couch.", "Two individuals use a photo kiosk", "Motorcycle rider going over a bump.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2690", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2700", "candidate_text": "well, the man is wearing nothing but a t shirt", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2700:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2700:premise", "snli:test:2197:premise", "snli:test:2712:premise", "snli:test:1912:premise", "style:personachat:test:98:self:6:positive:0", "style:personachat:test:98:self:6:positive:1", "style:personachat:test:98:self:6:positive:2", "style:personachat:test:98:self:6:positive:3", "style:personachat:test:98:self:6:positive:4", "style:personachat:test:98:self:6:positive:5"], "evidence_texts": ["A man with a tan jacket with a full grocery bag is crossing the street.", "A man is standing inside a building under construction filled with bricks and various tools.", "A girl in a black sweater and jeans pours water into an empty soda bottle.", "An elderly man is smiling while sitting in front of a row of soda cans.", "doing well . i live in ohio and my job is in a factory . you ?", "i hear ya . i get dirty working in my garden .", "true . i love my cat . she has a purple litter box .", "she does . she sits with me when i watch the buckeyes on tv .", "i like simple plan as well .", "no , it is just me . been busy as of late ."], "speaker_id": "snli:test:2700", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2700", "candidate_text": "I wonder whether the man is wearing nothing but a t shirt?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2700:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2700:premise", "snli:test:2197:premise", "snli:test:2712:premise", "snli:test:1912:premise", "style:personachat:test:366:self:7:positive:0", "style:personachat:test:366:self:7:positive:1", "style:personachat:test:366:self:7:positive:2", "style:personachat:test:366:self:7:positive:3", "style:personachat:test:366:self:7:positive:4", "style:personachat:test:366:self:7:positive:5", "style:personachat:test:366:self:7:positive:6"], "evidence_texts": ["A man with a tan jacket with a full grocery bag is crossing the street.", "A man is standing inside a building under construction filled with bricks and various tools.", "A girl in a black sweater and jeans pours water into an empty soda bottle.", "An elderly man is smiling while sitting in front of a row of soda cans.", "hello ! i am fine . i am listening music .", "i am pretty fine . what about you ?", "i am just 5 . 2 i am not tall lol you ?", "i do not like tall people at all .", "i am looking for a job do you work ?", "are you working as a nurse ?", "ops you are selling by phone ?"], "speaker_id": "snli:test:2700", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2711", "candidate_text": "well, a health worker is evaluating a girl on her sickness", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2711:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2684:premise", "snli:test:2736:premise", "snli:test:2676:premise", "snli:test:2711:premise", "style:personachat:test:99:self:1:positive:0"], "evidence_texts": ["A man and woman setup a camera.", "Construction worker looking over the job site.", "Two men spar in martial arts gear.", "An medical worker examines a young girl.", "i am afraid of not being able to pay my bills ."], "speaker_id": "snli:test:2711", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2711", "candidate_text": "Honestly, A health worker is evaluating a girl on her sickness!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2711:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2684:premise", "snli:test:2736:premise", "snli:test:2676:premise", "snli:test:2711:premise", "style:personachat:test:689:self:7:positive:0", "style:personachat:test:689:self:7:positive:1", "style:personachat:test:689:self:7:positive:2", "style:personachat:test:689:self:7:positive:3", "style:personachat:test:689:self:7:positive:4", "style:personachat:test:689:self:7:positive:5", "style:personachat:test:689:self:7:positive:6"], "evidence_texts": ["A man and woman setup a camera.", "Construction worker looking over the job site.", "Two men spar in martial arts gear.", "An medical worker examines a young girl.", "doing great thanks ! how are you ? are you working ?", "yeah ! what do you do ? i work fast food for now .", "a nurse ? super impressive . i did not finish high school but hope to go back one day .", "yeah , it does not pay much but it is a job . what are your hobbies ?", "i reside in low income housing sans gym , so i started running . i love it !", "i am currently in training for one actually ! do you have any pets or animals ?", "no , i do not care for them . too much work ! do you work 12 hours shifts ?"], "speaker_id": "snli:test:2711", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2712", "candidate_text": "well, someone pours water into a bottle", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2712:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2712:premise", "snli:test:2700:premise", "snli:test:3081:premise", "snli:test:2197:premise", "style:personachat:test:99:self:2:positive:0", "style:personachat:test:99:self:2:positive:1"], "evidence_texts": ["A girl in a black sweater and jeans pours water into an empty soda bottle.", "A man with a tan jacket with a full grocery bag is crossing the street.", "Two women competing in a roller derby with several teammates and referee in the background.", "A man is standing inside a building under construction filled with bricks and various tools.", "i could ! i just wish i could afford it .", "i am afraid of not being able to pay my bills ."], "speaker_id": "snli:test:2712", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2712", "candidate_text": "Someone pours water into a bottle.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2712:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2712:premise", "snli:test:2700:premise", "snli:test:3081:premise", "snli:test:2197:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A girl in a black sweater and jeans pours water into an empty soda bottle.", "A man with a tan jacket with a full grocery bag is crossing the street.", "Two women competing in a roller derby with several teammates and referee in the background.", "A man is standing inside a building under construction filled with bricks and various tools.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2712", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2723", "candidate_text": "well, the roofer is putting on shingles", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2723:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2616:premise", "snli:test:2902:premise", "snli:test:1954:premise", "snli:test:2723:premise", "style:personachat:test:99:self:5:positive:0", "style:personachat:test:99:self:5:positive:1", "style:personachat:test:99:self:5:positive:2", "style:personachat:test:99:self:5:positive:3", "style:personachat:test:99:self:5:positive:4"], "evidence_texts": ["A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "i see . did you graduate college ? i just did two years ago .", "that is awesome ! where is that ? i live in seattle .", "i have not heard of that town . i share an apartment with 2 roommates .", "i could ! i just wish i could afford it .", "i am afraid of not being able to pay my bills ."], "speaker_id": "snli:test:2723", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2723", "candidate_text": "I wonder whether the roofer is putting on shingles?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2723:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2616:premise", "snli:test:2902:premise", "snli:test:1954:premise", "snli:test:2723:premise", "style:personachat:test:370:self:5:positive:0", "style:personachat:test:370:self:5:positive:1", "style:personachat:test:370:self:5:positive:2", "style:personachat:test:370:self:5:positive:3", "style:personachat:test:370:self:5:positive:4"], "evidence_texts": ["A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A man in a gray shirt and green tie is working with his computer at his office desk.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "yes i should , but my three kids like junk food lol", "it is hard to eat together , since i work days and my husband works overnights", "do you have any pets ? we have cats , which we love", "that would be nice , i love dogs too .", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:2723", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2727", "candidate_text": "well, a man is jumping", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2727:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2547:premise", "snli:test:2778:premise", "snli:test:2035:premise", "snli:test:2727:premise", "style:personachat:test:99:self:7:positive:0", "style:personachat:test:99:self:7:positive:1", "style:personachat:test:99:self:7:positive:2", "style:personachat:test:99:self:7:positive:3", "style:personachat:test:99:self:7:positive:4", "style:personachat:test:99:self:7:positive:5", "style:personachat:test:99:self:7:positive:6"], "evidence_texts": ["A girl in a green coat walks down a rural road playing a flute", "A woman with a bag is sitting amongst three people who are reading books.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "Man jumping up in the air and splitting his legs with his mouth open.", "i cannot afford alcohol . i can barely afford to pay my other bills each month .", "i work as a kindergarten teacher . i love kids !", "i see . did you graduate college ? i just did two years ago .", "that is awesome ! where is that ? i live in seattle .", "i have not heard of that town . i share an apartment with 2 roommates .", "i could ! i just wish i could afford it .", "i am afraid of not being able to pay my bills ."], "speaker_id": "snli:test:2727", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2727", "candidate_text": "Honestly, A man is jumping!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2727:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2547:premise", "snli:test:2778:premise", "snli:test:2035:premise", "snli:test:2727:premise", "style:personachat:test:696:self:5:positive:0", "style:personachat:test:696:self:5:positive:1", "style:personachat:test:696:self:5:positive:2", "style:personachat:test:696:self:5:positive:3", "style:personachat:test:696:self:5:positive:4"], "evidence_texts": ["A girl in a green coat walks down a rural road playing a flute", "A woman with a bag is sitting amongst three people who are reading books.", "A young woman wearing a yellow sweater and black pants is ice skating outdoors.", "Man jumping up in the air and splitting his legs with his mouth open.", "really . . . i am sorry he or she is broken .", "lol , funny . i am listening to my favorite music , jazz . and you ?", "cool , blue roots . i have clients that color their pets hair blue .", "yes , part time . however , i work with children or a pediatrician .", "awesome ! i loved school . i enjoyed running and reading ."], "speaker_id": "snli:test:2727", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2731", "candidate_text": "well, black women on the beach", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2731:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2566:premise", "snli:test:2747:premise", "snli:test:2507:premise", "snli:test:2731:premise", "style:personachat:test:100:self:1:positive:0"], "evidence_texts": ["A white man taking a photo in front of water.", "Woman smiling while holding a book and yellow paper star.", "A male with a prosthetic leg diving for a volleyball.", "asian women selling fruits from baskets at an outdoor market", "yes i am on my last year in college ."], "speaker_id": "snli:test:2731", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2731", "candidate_text": "black women on the beach.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2731:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2566:premise", "snli:test:2747:premise", "snli:test:2507:premise", "snli:test:2731:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A white man taking a photo in front of water.", "Woman smiling while holding a book and yellow paper star.", "A male with a prosthetic leg diving for a volleyball.", "asian women selling fruits from baskets at an outdoor market", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2731", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2736", "candidate_text": "well, construction worker at work", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2736:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2736:premise", "snli:test:2711:premise", "snli:test:2886:premise", "snli:test:2684:premise", "style:personachat:test:100:self:2:positive:0", "style:personachat:test:100:self:2:positive:1"], "evidence_texts": ["Construction worker looking over the job site.", "An medical worker examines a young girl.", "Two young musicians play from a stage.", "A man and woman setup a camera.", "i like to play softball and volleyball . i guess that is considered a hobby .", "yes i am on my last year in college ."], "speaker_id": "snli:test:2736", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2736", "candidate_text": "I wonder whether construction worker at work?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2736:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2736:premise", "snli:test:2711:premise", "snli:test:2886:premise", "snli:test:2684:premise", "style:personachat:test:378:self:2:positive:0", "style:personachat:test:378:self:2:positive:1"], "evidence_texts": ["Construction worker looking over the job site.", "An medical worker examines a young girl.", "Two young musicians play from a stage.", "A man and woman setup a camera.", "i like him a lot . my favorite by far", "oh yeah what is it of ?"], "speaker_id": "snli:test:2736", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2740", "candidate_text": "well, a basketball player is trying to make a shot while two players try to block it", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2740:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2740:premise", "snli:test:2686:premise", "snli:test:3041:premise", "snli:test:2330:premise", "style:personachat:test:100:self:5:positive:0", "style:personachat:test:100:self:5:positive:1", "style:personachat:test:100:self:5:positive:2", "style:personachat:test:100:self:5:positive:3", "style:personachat:test:100:self:5:positive:4"], "evidence_texts": ["A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "i also enjoy hanging out with friends , how about you ?", "oh how nice , no not yet .", "one of my favorite songs . what kind of book covers do you design ?", "i like to play softball and volleyball . i guess that is considered a hobby .", "yes i am on my last year in college ."], "speaker_id": "snli:test:2740", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2740", "candidate_text": "Honestly, A basketball player is trying to make a shot while two players try to block it!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2740:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2740:premise", "snli:test:2686:premise", "snli:test:3041:premise", "snli:test:2330:premise", "style:personachat:test:698:self:2:positive:0", "style:personachat:test:698:self:2:positive:1"], "evidence_texts": ["A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A lady in pink long-sleeved blouse is holding a pink bag in her lap as she reaches something to the guy next to her.", "it is okay , start small .", "oh , i think you should travel , i have been to 6 countries !"], "speaker_id": "snli:test:2740", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2742", "candidate_text": "well, two people are outdoors", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2742:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2535:premise", "snli:test:3090:premise", "snli:test:2742:premise", "snli:test:2523:premise", "style:personachat:test:100:self:6:positive:0", "style:personachat:test:100:self:6:positive:1", "style:personachat:test:100:self:6:positive:2", "style:personachat:test:100:self:6:positive:3", "style:personachat:test:100:self:6:positive:4", "style:personachat:test:100:self:6:positive:5"], "evidence_texts": ["A boy is running in a spinning metal tunnel in the playground.", "A band singing and playing electric guitar for a crowd of people.", "Two people are sitting on a rock looking off into the distance.", "Two men are working together to empty the barrel of its contents.", "my favorite music is rap , tupac is my favorite", "i also enjoy hanging out with friends , how about you ?", "oh how nice , no not yet .", "one of my favorite songs . what kind of book covers do you design ?", "i like to play softball and volleyball . i guess that is considered a hobby .", "yes i am on my last year in college ."], "speaker_id": "snli:test:2742", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2742", "candidate_text": "Two people are outdoors.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2742:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2535:premise", "snli:test:3090:premise", "snli:test:2742:premise", "snli:test:2523:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A boy is running in a spinning metal tunnel in the playground.", "A band singing and playing electric guitar for a crowd of people.", "Two people are sitting on a rock looking off into the distance.", "Two men are working together to empty the barrel of its contents.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2742", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2747", "candidate_text": "well, the woman holding book with smiling in the home", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2747:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2731:premise", "snli:test:2807:premise", "snli:test:2566:premise", "snli:test:2747:premise", "style:personachat:test:101:self:1:positive:0"], "evidence_texts": ["asian women selling fruits from baskets at an outdoor market", "Three men on mountain bikes go over a sand hill.", "A white man taking a photo in front of water.", "Woman smiling while holding a book and yellow paper star.", "cuban food is my favorite outside of bacon"], "speaker_id": "snli:test:2747", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2747", "candidate_text": "I wonder whether the woman holding book with smiling in the home?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2747:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2731:premise", "snli:test:2807:premise", "snli:test:2566:premise", "snli:test:2747:premise", "style:personachat:test:383:self:1:positive:0"], "evidence_texts": ["asian women selling fruits from baskets at an outdoor market", "Three men on mountain bikes go over a sand hill.", "A white man taking a photo in front of water.", "Woman smiling while holding a book and yellow paper star.", "oh yes i have . it was a lot of fun . will you be going anytime soon ?"], "speaker_id": "snli:test:2747", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2757", "candidate_text": "well, the man is riding on the sidewalk", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2757:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2404:premise", "snli:test:2757:premise", "snli:test:2841:premise", "snli:test:2291:premise", "style:personachat:test:101:self:2:positive:0", "style:personachat:test:101:self:2:positive:1"], "evidence_texts": ["A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "i have an uncle from puerto rico . my family is cuban . we love music", "cuban food is my favorite outside of bacon"], "speaker_id": "snli:test:2757", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2757", "candidate_text": "Honestly, The man is riding on the sidewalk!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2757:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2404:premise", "snli:test:2757:premise", "snli:test:2841:premise", "snli:test:2291:premise", "style:personachat:test:701:self:1:positive:0"], "evidence_texts": ["A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "A speaking woman at a podium has the phrase \"Armenian Genocide\" written on it, flanked by priests and men in suits.", "ah okay ! and not sure it hurts my ears . i prefer to cook and clean"], "speaker_id": "snli:test:2757", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2760", "candidate_text": "well, two men bellydance", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2760:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2760:premise", "snli:test:1581:premise", "snli:test:1562:premise", "snli:test:3842:premise", "style:personachat:test:101:self:5:positive:0", "style:personachat:test:101:self:5:positive:1", "style:personachat:test:101:self:5:positive:2", "style:personachat:test:101:self:5:positive:3", "style:personachat:test:101:self:5:positive:4"], "evidence_texts": ["A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "i admire all servicemen and women . i am currently chomping down on bacon", "what are your current goals at the moment ? i want to finish my m . s .", "i see . what is home for you ? i am a native californian", "i have an uncle from puerto rico . my family is cuban . we love music", "cuban food is my favorite outside of bacon"], "speaker_id": "snli:test:2760", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2760", "candidate_text": "two men bellydance.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2760:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2760:premise", "snli:test:1581:premise", "snli:test:1562:premise", "snli:test:3842:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man sitting with his head down on his arm and a roll of toilet paper and a cellphone sitting beside him.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2760", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2761", "candidate_text": "well, someone dances while a man holds a paddle", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2761:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3842:premise", "snli:test:2761:premise", "snli:test:1581:premise", "snli:test:5063:premise", "style:personachat:test:101:self:7:positive:0", "style:personachat:test:101:self:7:positive:1", "style:personachat:test:101:self:7:positive:2", "style:personachat:test:101:self:7:positive:3", "style:personachat:test:101:self:7:positive:4", "style:personachat:test:101:self:7:positive:5", "style:personachat:test:101:self:7:positive:6"], "evidence_texts": ["A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "i am wonderful thank you . i am listening to classical music", "i am what you call a water engineer", "i admire all servicemen and women . i am currently chomping down on bacon", "what are your current goals at the moment ? i want to finish my m . s .", "i see . what is home for you ? i am a native californian", "i have an uncle from puerto rico . my family is cuban . we love music", "cuban food is my favorite outside of bacon"], "speaker_id": "snli:test:2761", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2761", "candidate_text": "I wonder whether someone dances while a man holds a paddle?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2761:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3842:premise", "snli:test:2761:premise", "snli:test:1581:premise", "snli:test:5063:premise", "style:personachat:test:383:self:7:positive:0", "style:personachat:test:383:self:7:positive:1", "style:personachat:test:383:self:7:positive:2", "style:personachat:test:383:self:7:positive:3", "style:personachat:test:383:self:7:positive:4", "style:personachat:test:383:self:7:positive:5", "style:personachat:test:383:self:7:positive:6"], "evidence_texts": ["A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A young boy holding a red toy wearing green snowman and reindeer pajamas with an older woman on a couch behind him.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "i just finished feeding my little ones and they are ready for bed .", "awwww . . . . you would love mine ; though they are a hand full .", "so sorry to hear that ; you probably would make them smile .", "everyone does not like everybody ; so i guess it is fair to say .", "i really do not have a favorite ; what is your favorite ?", "the office i think if ok ; i am not sure about the latter .", "oh yes i have . it was a lot of fun . will you be going anytime soon ?"], "speaker_id": "snli:test:2761", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2764", "candidate_text": "I wonder whether asians play soccer in the park?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2764:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2764:premise", "snli:test:2675:premise", "snli:test:2846:premise", "snli:test:2569:premise", "style:personachat:test:102:self:1:positive:0"], "evidence_texts": ["Asian individuals are taking pictures of some unidentified thing.", "Two oriental older women gathering rice from the fields.", "A woman looks at a plate filled with steam.", "A man in bright shirt carrying a fallen branch.", "oh you are a veteran ?"], "speaker_id": "snli:test:2764", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2764", "candidate_text": "well, asians play soccer in the park", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2764:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2764:premise", "snli:test:2675:premise", "snli:test:2846:premise", "snli:test:2569:premise", "style:personachat:test:183:self:1:positive:0"], "evidence_texts": ["Asian individuals are taking pictures of some unidentified thing.", "Two oriental older women gathering rice from the fields.", "A woman looks at a plate filled with steam.", "A man in bright shirt carrying a fallen branch.", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:2764", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2772", "candidate_text": "I wonder whether a nascar is being watched?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2772:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2772:premise", "snli:test:2592:premise", "snli:test:2797:premise", "snli:test:2558:premise", "style:personachat:test:102:self:2:positive:0", "style:personachat:test:102:self:2:positive:1"], "evidence_texts": ["A yellow race car sliding through a corner as spectators watch.", "A man in military uniform points as three other men watch.", "Young boy plays with a truck outdoors, while a woman waits.", "A nurse is listening to another women's heartbeat with a stethoscope.", "you are a vet ? i think my cat is getting sick .", "oh you are a veteran ?"], "speaker_id": "snli:test:2772", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2772", "candidate_text": "Honestly, A Nascar is being watched!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2772:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2772:premise", "snli:test:2592:premise", "snli:test:2797:premise", "snli:test:2558:premise", "style:personachat:test:712:self:1:positive:0"], "evidence_texts": ["A yellow race car sliding through a corner as spectators watch.", "A man in military uniform points as three other men watch.", "Young boy plays with a truck outdoors, while a woman waits.", "A nurse is listening to another women's heartbeat with a stethoscope.", "hah ! no . why did you quit being a republican ?"], "speaker_id": "snli:test:2772", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2778", "candidate_text": "I wonder whether the woman is sitting?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2778:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2727:premise", "snli:test:2547:premise", "snli:test:2778:premise", "snli:test:2819:premise", "style:personachat:test:102:self:5:positive:0", "style:personachat:test:102:self:5:positive:1", "style:personachat:test:102:self:5:positive:2", "style:personachat:test:102:self:5:positive:3", "style:personachat:test:102:self:5:positive:4"], "evidence_texts": ["Man jumping up in the air and splitting his legs with his mouth open.", "A girl in a green coat walks down a rural road playing a flute", "A woman with a bag is sitting amongst three people who are reading books.", "A big black poodle running on the grass with a toy in its mouth.", "hard to be bias when grading multiple choice with a definitive answer", "i teach a kindergarten class in seattle", "it is difficult but it is my passion . passion has trouble paying the bills though", "you are a vet ? i think my cat is getting sick .", "oh you are a veteran ?"], "speaker_id": "snli:test:2778", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2778", "candidate_text": "The woman is sitting.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2778:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2727:premise", "snli:test:2547:premise", "snli:test:2778:premise", "snli:test:2819:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Man jumping up in the air and splitting his legs with his mouth open.", "A girl in a green coat walks down a rural road playing a flute", "A woman with a bag is sitting amongst three people who are reading books.", "A big black poodle running on the grass with a toy in its mouth.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2778", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2779", "candidate_text": "I wonder whether nobody is reading a book?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2779:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2819:premise", "snli:test:2727:premise", "snli:test:2829:premise", "snli:test:2779:premise", "style:personachat:test:102:self:7:positive:0", "style:personachat:test:102:self:7:positive:1", "style:personachat:test:102:self:7:positive:2", "style:personachat:test:102:self:7:positive:3", "style:personachat:test:102:self:7:positive:4", "style:personachat:test:102:self:7:positive:5", "style:personachat:test:102:self:7:positive:6"], "evidence_texts": ["A big black poodle running on the grass with a toy in its mouth.", "Man jumping up in the air and splitting his legs with his mouth open.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A woman with a bag is sitting amongst three people who are reading books.", "hard day on the job today . and you ?", "that is a good way to de stress . i usually just grade papers", "hard to be bias when grading multiple choice with a definitive answer", "i teach a kindergarten class in seattle", "it is difficult but it is my passion . passion has trouble paying the bills though", "you are a vet ? i think my cat is getting sick .", "oh you are a veteran ?"], "speaker_id": "snli:test:2779", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2779", "candidate_text": "well, nobody is reading a book", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2779:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2819:premise", "snli:test:2727:premise", "snli:test:2829:premise", "snli:test:2779:premise", "style:personachat:test:183:self:7:positive:0", "style:personachat:test:183:self:7:positive:1", "style:personachat:test:183:self:7:positive:2", "style:personachat:test:183:self:7:positive:3", "style:personachat:test:183:self:7:positive:4", "style:personachat:test:183:self:7:positive:5", "style:personachat:test:183:self:7:positive:6"], "evidence_texts": ["A big black poodle running on the grass with a toy in its mouth.", "Man jumping up in the air and splitting his legs with his mouth open.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A woman with a bag is sitting amongst three people who are reading books.", "how are you doing tonight , i am missing my tail it was removed", "oh no do you like hollister i worked there back in high school ?", "yeah very cool , i quit to become a skater , i get paid to skate", "not to me but other people say yes , what kind of actor ?", "i would watch that show i am open to everything", "hopefully you have a loving family to help , what do you do for fun ?", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:2779", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2781", "candidate_text": "well, a woman is reading", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2781:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2643:premise", "snli:test:2781:premise", "snli:test:2943:premise", "snli:test:2600:premise", "style:personachat:test:103:self:1:positive:0"], "evidence_texts": ["A woman wearing very formal colorful clothing and she seems to be dancing.", "A scientist studies a slide in order to work on her new creation.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A person with a green backpack is riding a bike down the road.", "i like most foods , my favorite is italian . i am not a big fan of thai food ."], "speaker_id": "snli:test:2781", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2781", "candidate_text": "A woman is reading.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2781:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2643:premise", "snli:test:2781:premise", "snli:test:2943:premise", "snli:test:2600:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A woman wearing very formal colorful clothing and she seems to be dancing.", "A scientist studies a slide in order to work on her new creation.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A person with a green backpack is riding a bike down the road.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2781", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2797", "candidate_text": "well, young boy sits inside", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2797:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2772:premise", "snli:test:2797:premise", "snli:test:2820:premise", "snli:test:2592:premise", "style:personachat:test:103:self:2:positive:0", "style:personachat:test:103:self:2:positive:1"], "evidence_texts": ["A yellow race car sliding through a corner as spectators watch.", "Young boy plays with a truck outdoors, while a woman waits.", "A young boy in a cowboy hat rides a large horse.", "A man in military uniform points as three other men watch.", "he will eat anything but we did not eat a lot of potatoes growing up in australia mate .", "i like most foods , my favorite is italian . i am not a big fan of thai food ."], "speaker_id": "snli:test:2797", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2797", "candidate_text": "I wonder whether young boy sits inside?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2797:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2772:premise", "snli:test:2797:premise", "snli:test:2820:premise", "snli:test:2592:premise", "style:personachat:test:397:self:2:positive:0", "style:personachat:test:397:self:2:positive:1"], "evidence_texts": ["A yellow race car sliding through a corner as spectators watch.", "Young boy plays with a truck outdoors, while a woman waits.", "A young boy in a cowboy hat rides a large horse.", "A man in military uniform points as three other men watch.", "where do you live ? i work in the city .", "do you like to read ? how old are you ? i am middle age ."], "speaker_id": "snli:test:2797", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2807", "candidate_text": "well, men on bikes go over a sand hill", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2807:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2747:premise", "snli:test:2984:premise", "snli:test:2731:premise", "snli:test:2807:premise", "style:personachat:test:103:self:5:positive:0", "style:personachat:test:103:self:5:positive:1", "style:personachat:test:103:self:5:positive:2", "style:personachat:test:103:self:5:positive:3", "style:personachat:test:103:self:5:positive:4"], "evidence_texts": ["Woman smiling while holding a book and yellow paper star.", "Two people are riding at once on one single bicycle.", "asian women selling fruits from baskets at an outdoor market", "Three men on mountain bikes go over a sand hill.", "spending time outside is great , i have never been much of a skater . any other hobbies ?", "i like cooking and spending time walking my dog . his name is charlie .", "oh wow , one of my brothers was bought with a tail but had it removed .", "he will eat anything but we did not eat a lot of potatoes growing up in australia mate .", "i like most foods , my favorite is italian . i am not a big fan of thai food ."], "speaker_id": "snli:test:2807", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2807", "candidate_text": "Honestly, Men on bikes go over a sand hill!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2807:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2747:premise", "snli:test:2984:premise", "snli:test:2731:premise", "snli:test:2807:premise", "style:personachat:test:716:self:2:positive:0", "style:personachat:test:716:self:2:positive:1"], "evidence_texts": ["Woman smiling while holding a book and yellow paper star.", "Two people are riding at once on one single bicycle.", "asian women selling fruits from baskets at an outdoor market", "Three men on mountain bikes go over a sand hill.", "do you ever take pictures of yourself hunting and post them ?", "yes all the time ! i like to post pictures of myself on face book ."], "speaker_id": "snli:test:2807", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2815", "candidate_text": "well, an asian woman is dancing sexily", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2815:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2500:premise", "snli:test:3469:premise", "snli:test:2499:premise", "snli:test:2815:premise", "style:personachat:test:103:self:6:positive:0", "style:personachat:test:103:self:6:positive:1", "style:personachat:test:103:self:6:positive:2", "style:personachat:test:103:self:6:positive:3", "style:personachat:test:103:self:6:positive:4", "style:personachat:test:103:self:6:positive:5"], "evidence_texts": ["A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "hey there ! its going alright , how about you ? how was your day ?", "spending time outside is great , i have never been much of a skater . any other hobbies ?", "i like cooking and spending time walking my dog . his name is charlie .", "oh wow , one of my brothers was bought with a tail but had it removed .", "he will eat anything but we did not eat a lot of potatoes growing up in australia mate .", "i like most foods , my favorite is italian . i am not a big fan of thai food ."], "speaker_id": "snli:test:2815", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2815", "candidate_text": "An Asian woman is dancing sexily.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2815:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2500:premise", "snli:test:3469:premise", "snli:test:2499:premise", "snli:test:2815:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2815", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2819", "candidate_text": "well, a white dog stole a toy", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2819:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2779:premise", "snli:test:2829:premise", "snli:test:2778:premise", "snli:test:2819:premise", "style:personachat:test:104:self:1:positive:0"], "evidence_texts": ["A woman with a bag is sitting amongst three people who are reading books.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A woman with a bag is sitting amongst three people who are reading books.", "A big black poodle running on the grass with a toy in its mouth.", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:2819", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2819", "candidate_text": "I wonder whether a white dog stole a toy?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2819:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2779:premise", "snli:test:2829:premise", "snli:test:2778:premise", "snli:test:2819:premise", "style:personachat:test:403:self:1:positive:0"], "evidence_texts": ["A woman with a bag is sitting amongst three people who are reading books.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A woman with a bag is sitting amongst three people who are reading books.", "A big black poodle running on the grass with a toy in its mouth.", "can you give me the website ?"], "speaker_id": "snli:test:2819", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2820", "candidate_text": "well, a boy is pretending to be a cowboy", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2820:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2820:premise", "snli:test:2797:premise", "snli:test:2772:premise", "snli:test:2957:premise", "style:personachat:test:104:self:2:positive:0", "style:personachat:test:104:self:2:positive:1"], "evidence_texts": ["A young boy in a cowboy hat rides a large horse.", "Young boy plays with a truck outdoors, while a woman waits.", "A yellow race car sliding through a corner as spectators watch.", "A man wearing white turns to hit a yellow tennis ball.", "no . i do work with a lot of kids at my job though .", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:2820", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2820", "candidate_text": "Honestly, A boy is pretending to be a cowboy!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2820:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2820:premise", "snli:test:2797:premise", "snli:test:2772:premise", "snli:test:2957:premise", "style:personachat:test:735:self:1:positive:0"], "evidence_texts": ["A young boy in a cowboy hat rides a large horse.", "Young boy plays with a truck outdoors, while a woman waits.", "A yellow race car sliding through a corner as spectators watch.", "A man wearing white turns to hit a yellow tennis ball.", "that would be so fun !"], "speaker_id": "snli:test:2820", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2821", "candidate_text": "well, a boy rides a horse", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2821:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2957:premise", "snli:test:2821:premise", "snli:test:2797:premise", "snli:test:2993:premise", "style:personachat:test:104:self:5:positive:0", "style:personachat:test:104:self:5:positive:1", "style:personachat:test:104:self:5:positive:2", "style:personachat:test:104:self:5:positive:3", "style:personachat:test:104:self:5:positive:4"], "evidence_texts": ["A man wearing white turns to hit a yellow tennis ball.", "A young boy in a cowboy hat rides a large horse.", "Young boy plays with a truck outdoors, while a woman waits.", "The people await to cross the street while walking the dog.", "what book did you finish reading ?", "i read sometimes not often enough . i have heard of the movie though .", "i do enjoy cooking . i like cooking italian food the most .", "no . i do work with a lot of kids at my job though .", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:2821", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2821", "candidate_text": "A boy rides a horse.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2821:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2957:premise", "snli:test:2821:premise", "snli:test:2797:premise", "snli:test:2993:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man wearing white turns to hit a yellow tennis ball.", "A young boy in a cowboy hat rides a large horse.", "Young boy plays with a truck outdoors, while a woman waits.", "The people await to cross the street while walking the dog.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2821", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2826", "candidate_text": "well, a man is standing", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2826:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2340:premise", "snli:test:2861:premise", "snli:test:2826:premise", "snli:test:2195:premise", "style:personachat:test:104:self:6:positive:0", "style:personachat:test:104:self:6:positive:1", "style:personachat:test:104:self:6:positive:2", "style:personachat:test:104:self:6:positive:3", "style:personachat:test:104:self:6:positive:4", "style:personachat:test:104:self:6:positive:5"], "evidence_texts": ["A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "hi jack ! i am dave and i am doing good today . how are you doing ?", "what book did you finish reading ?", "i read sometimes not often enough . i have heard of the movie though .", "i do enjoy cooking . i like cooking italian food the most .", "no . i do work with a lot of kids at my job though .", "my work is with a program that helps troubled teens through mentoring ."], "speaker_id": "snli:test:2826", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2826", "candidate_text": "I wonder whether a man is standing?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2826:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2340:premise", "snli:test:2861:premise", "snli:test:2826:premise", "snli:test:2195:premise", "style:personachat:test:403:self:6:positive:0", "style:personachat:test:403:self:6:positive:1", "style:personachat:test:403:self:6:positive:2", "style:personachat:test:403:self:6:positive:3", "style:personachat:test:403:self:6:positive:4", "style:personachat:test:403:self:6:positive:5"], "evidence_texts": ["A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "Two construction workers are working in front of a wall which has a bunch of names carved into it.", "good evening . going good . there ?", "how old are you ? if you do not mind .", "i am 24 . i finished college in may . now looking for a job .", "what do you do now ? do you work ?", "what kind of online work do you do", "can you give me the website ?"], "speaker_id": "snli:test:2826", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2829", "candidate_text": "Honestly, A man is raising his arm to get someones attention!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2829:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2819:premise", "snli:test:2829:premise", "snli:test:2893:premise", "snli:test:2779:premise", "style:personachat:test:105:self:1:positive:0"], "evidence_texts": ["A big black poodle running on the grass with a toy in its mouth.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A girl with short hair and a white shirt is holding a beer bottle", "A woman with a bag is sitting amongst three people who are reading books.", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:2829", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2829", "candidate_text": "well, a man is raising his arm to get someones attention", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2829:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2819:premise", "snli:test:2829:premise", "snli:test:2893:premise", "snli:test:2779:premise", "style:personachat:test:187:self:1:positive:0"], "evidence_texts": ["A big black poodle running on the grass with a toy in its mouth.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "A girl with short hair and a white shirt is holding a beer bottle", "A woman with a bag is sitting amongst three people who are reading books.", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:2829", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2841", "candidate_text": "Honestly, A woman is a good surfer!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2841:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2757:premise", "snli:test:2841:premise", "snli:test:2931:premise", "snli:test:2404:premise", "style:personachat:test:105:self:2:positive:0", "style:personachat:test:105:self:2:positive:1"], "evidence_texts": ["A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:2841", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2841", "candidate_text": "A woman is a good surfer.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2841:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2757:premise", "snli:test:2841:premise", "snli:test:2931:premise", "snli:test:2404:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "A man in a black jacket standing beside a green mailbox thoroughly soiled by a group of pigeons that are nearby.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2841", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2846", "candidate_text": "Honestly, The woman is in a kitchen!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2846:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2764:premise", "snli:test:2869:premise", "snli:test:2846:premise", "snli:test:2675:premise", "style:personachat:test:105:self:5:positive:0", "style:personachat:test:105:self:5:positive:1", "style:personachat:test:105:self:5:positive:2", "style:personachat:test:105:self:5:positive:3", "style:personachat:test:105:self:5:positive:4"], "evidence_texts": ["Asian individuals are taking pictures of some unidentified thing.", "A group of people are walking by a van.", "A woman looks at a plate filled with steam.", "Two oriental older women gathering rice from the fields.", "they played in the school band . do you play any instruments ?", "i should have ! i slacked off ended up being a administrative assistant . do you have kids ?", "i am going to have a polka band play when i retire in 5 years !", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:2846", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2846", "candidate_text": "I wonder whether the woman is in a kitchen?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2846:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2764:premise", "snli:test:2869:premise", "snli:test:2846:premise", "snli:test:2675:premise", "style:personachat:test:409:self:5:positive:0", "style:personachat:test:409:self:5:positive:1", "style:personachat:test:409:self:5:positive:2", "style:personachat:test:409:self:5:positive:3", "style:personachat:test:409:self:5:positive:4"], "evidence_texts": ["Asian individuals are taking pictures of some unidentified thing.", "A group of people are walking by a van.", "A woman looks at a plate filled with steam.", "Two oriental older women gathering rice from the fields.", "god bless them . how old are they ?", "you have baby every three years . do you work ?", "that is very nice , my mom loves to cook and me no", "haha i am 24 i am not young at all .", "what does it mean that word ?"], "speaker_id": "snli:test:2846", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2861", "candidate_text": "Honestly, Two people are enjoying ice cream!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2861:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2826:premise", "snli:test:2861:premise", "snli:test:2877:premise", "snli:test:2340:premise", "style:personachat:test:105:self:7:positive:0", "style:personachat:test:105:self:7:positive:1", "style:personachat:test:105:self:7:positive:2", "style:personachat:test:105:self:7:positive:3", "style:personachat:test:105:self:7:positive:4", "style:personachat:test:105:self:7:positive:5", "style:personachat:test:105:self:7:positive:6"], "evidence_texts": ["A man stands on a dock near several boats, while 2 other men are on one of the boats.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "i like playing sports like racquetball , but not watching so much .", "my fiance is into hard rock , i like oldies , but we make it work .", "they played in the school band . do you play any instruments ?", "i should have ! i slacked off ended up being a administrative assistant . do you have kids ?", "i am going to have a polka band play when i retire in 5 years !", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:2861", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2861", "candidate_text": "well, two people are enjoying ice cream", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2861:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2826:premise", "snli:test:2861:premise", "snli:test:2877:premise", "snli:test:2340:premise", "style:personachat:test:187:self:6:positive:0", "style:personachat:test:187:self:6:positive:1", "style:personachat:test:187:self:6:positive:2", "style:personachat:test:187:self:6:positive:3", "style:personachat:test:187:self:6:positive:4", "style:personachat:test:187:self:6:positive:5"], "evidence_texts": ["A man stands on a dock near several boats, while 2 other men are on one of the boats.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A lady teacher warning her students about their behavior in a museum setting, and inviting them to be quite.", "i am doing great . just having a few slices of bacon for breakfast . you ?", "i did a lot of those when i was in school learning to be an engineer", "that is a good subject to get into . do you speak any other languages ?", "that is the only language i speak as well .", "i am on the same page . what music do you ike ?", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:2861", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2869", "candidate_text": "well, people getting ready to enter a vehicle", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2869:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2846:premise", "snli:test:2869:premise", "snli:test:2896:premise", "snli:test:2764:premise", "style:personachat:test:106:self:1:positive:0"], "evidence_texts": ["A woman looks at a plate filled with steam.", "A group of people are walking by a van.", "Mother and daughter are meeting for the first time.", "Asian individuals are taking pictures of some unidentified thing.", "two boys over here . i would love to see what you make ."], "speaker_id": "snli:test:2869", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2869", "candidate_text": "People getting ready to enter a vehicle.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2869:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2846:premise", "snli:test:2869:premise", "snli:test:2896:premise", "snli:test:2764:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A woman looks at a plate filled with steam.", "A group of people are walking by a van.", "Mother and daughter are meeting for the first time.", "Asian individuals are taking pictures of some unidentified thing.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2869", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2877", "candidate_text": "well, a man sips whisky", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2877:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2861:premise", "snli:test:2877:premise", "snli:test:3146:premise", "snli:test:2826:premise", "style:personachat:test:106:self:2:positive:0", "style:personachat:test:106:self:2:positive:1"], "evidence_texts": ["A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "doritos . are you an artist ? i love to cook as well .", "two boys over here . i would love to see what you make ."], "speaker_id": "snli:test:2877", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2877", "candidate_text": "I wonder whether a man sips whisky?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2877:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2861:premise", "snli:test:2877:premise", "snli:test:3146:premise", "snli:test:2826:premise", "style:personachat:test:414:self:2:positive:0", "style:personachat:test:414:self:2:positive:1"], "evidence_texts": ["A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man stands on a dock near several boats, while 2 other men are on one of the boats.", "that is cool . is it motorized ?", "did they make any more besides yours ?"], "speaker_id": "snli:test:2877", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2886", "candidate_text": "well, the stage is empty", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2886:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2736:premise", "snli:test:3106:premise", "snli:test:2886:premise", "snli:test:2711:premise", "style:personachat:test:106:self:5:positive:0", "style:personachat:test:106:self:5:positive:1", "style:personachat:test:106:self:5:positive:2", "style:personachat:test:106:self:5:positive:3", "style:personachat:test:106:self:5:positive:4"], "evidence_texts": ["Construction worker looking over the job site.", "Women in blue dress singing on stage.", "Two young musicians play from a stage.", "An medical worker examines a young girl.", "25 years this july . we still like going to bluegrass concerts together .", "he is retired and eats chips all day . i do not like them .", "fun . i like to play guitar to old bluegrass tunes . do you play ?", "doritos . are you an artist ? i love to cook as well .", "two boys over here . i would love to see what you make ."], "speaker_id": "snli:test:2886", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2886", "candidate_text": "Honestly, The stage is empty!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2886:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2736:premise", "snli:test:3106:premise", "snli:test:2886:premise", "snli:test:2711:premise", "style:personachat:test:751:self:2:positive:0", "style:personachat:test:751:self:2:positive:1"], "evidence_texts": ["Construction worker looking over the job site.", "Women in blue dress singing on stage.", "Two young musicians play from a stage.", "An medical worker examines a young girl.", "as a native american it is hard for me to get into .", "haha , that is actually my ideal life right there !"], "speaker_id": "snli:test:2886", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2891", "candidate_text": "well, the people are swimming in the lake", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2891:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6938:premise", "snli:test:4435:premise", "snli:test:734:premise", "snli:test:2891:premise", "style:personachat:test:106:self:7:positive:0", "style:personachat:test:106:self:7:positive:1", "style:personachat:test:106:self:7:positive:2", "style:personachat:test:106:self:7:positive:3", "style:personachat:test:106:self:7:positive:4", "style:personachat:test:106:self:7:positive:5", "style:personachat:test:106:self:7:positive:6"], "evidence_texts": ["A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "congratulations ! i married my high school sweetheart while working in hollister .", "yes . too long . we are skateboarders who travel together to do shows .", "25 years this july . we still like going to bluegrass concerts together .", "he is retired and eats chips all day . i do not like them .", "fun . i like to play guitar to old bluegrass tunes . do you play ?", "doritos . are you an artist ? i love to cook as well .", "two boys over here . i would love to see what you make ."], "speaker_id": "snli:test:2891", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2891", "candidate_text": "The people are swimming in the lake.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2891:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6938:premise", "snli:test:4435:premise", "snli:test:734:premise", "snli:test:2891:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2891", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2893", "candidate_text": "well, a girl with short hair and a blue shirt is holding a beer bottle", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2893:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2829:premise", "snli:test:2893:premise", "snli:test:3074:premise", "snli:test:2819:premise", "style:personachat:test:107:self:1:positive:0"], "evidence_texts": ["A man wearing a blue shirt and headphones around his neck raises his arm.", "A girl with short hair and a white shirt is holding a beer bottle", "A local street in an Arabic country where there is a market selling Coca-Cola.", "A big black poodle running on the grass with a toy in its mouth.", "well i do not know about karma but we get a good laugh"], "speaker_id": "snli:test:2893", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2893", "candidate_text": "I wonder whether a girl with short hair and a blue shirt is holding a beer bottle?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2893:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2829:premise", "snli:test:2893:premise", "snli:test:3074:premise", "snli:test:2819:premise", "style:personachat:test:415:self:1:positive:0"], "evidence_texts": ["A man wearing a blue shirt and headphones around his neck raises his arm.", "A girl with short hair and a white shirt is holding a beer bottle", "A local street in an Arabic country where there is a market selling Coca-Cola.", "A big black poodle running on the grass with a toy in its mouth.", "what do you do for fun ?"], "speaker_id": "snli:test:2893", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2896", "candidate_text": "well, the mother and daughter are meeting over lunch", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2896:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2896:premise", "snli:test:2869:premise", "snli:test:2900:premise", "snli:test:2846:premise", "style:personachat:test:107:self:2:positive:0", "style:personachat:test:107:self:2:positive:1"], "evidence_texts": ["Mother and daughter are meeting for the first time.", "A group of people are walking by a van.", "A team of surgeons operate on a female patient.", "A woman looks at a plate filled with steam.", "it is a running joke now tho . she is deaf on one side", "well i do not know about karma but we get a good laugh"], "speaker_id": "snli:test:2896", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2896", "candidate_text": "Honestly, The mother and daughter are meeting over lunch!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2896:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2896:premise", "snli:test:2869:premise", "snli:test:2900:premise", "snli:test:2846:premise", "style:personachat:test:766:self:1:positive:0"], "evidence_texts": ["Mother and daughter are meeting for the first time.", "A group of people are walking by a van.", "A team of surgeons operate on a female patient.", "A woman looks at a plate filled with steam.", "amen ! our country needs more god !"], "speaker_id": "snli:test:2896", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2900", "candidate_text": "well, the surgeons are operating", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2900:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2900:premise", "snli:test:2896:premise", "snli:test:2922:premise", "snli:test:2869:premise", "style:personachat:test:107:self:5:positive:0", "style:personachat:test:107:self:5:positive:1", "style:personachat:test:107:self:5:positive:2", "style:personachat:test:107:self:5:positive:3", "style:personachat:test:107:self:5:positive:4"], "evidence_texts": ["A team of surgeons operate on a female patient.", "Mother and daughter are meeting for the first time.", "An older white woman makes cupcakes in her kitchen.", "A group of people are walking by a van.", "i have turned body building into a career .", "thanks . one thing that does not involve seeing color", "i am very much so my sister likes to pick on me", "it is a running joke now tho . she is deaf on one side", "well i do not know about karma but we get a good laugh"], "speaker_id": "snli:test:2900", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2900", "candidate_text": "The surgeons are operating.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2900:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2900:premise", "snli:test:2896:premise", "snli:test:2922:premise", "snli:test:2869:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A team of surgeons operate on a female patient.", "Mother and daughter are meeting for the first time.", "An older white woman makes cupcakes in her kitchen.", "A group of people are walking by a van.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2900", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2902", "candidate_text": "well, a boy denies his sister a drink", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2902:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2723:premise", "snli:test:2920:premise", "snli:test:2902:premise", "snli:test:2616:premise", "style:personachat:test:107:self:7:positive:0", "style:personachat:test:107:self:7:positive:1", "style:personachat:test:107:self:7:positive:2", "style:personachat:test:107:self:7:positive:3", "style:personachat:test:107:self:7:positive:4", "style:personachat:test:107:self:7:positive:5", "style:personachat:test:107:self:7:positive:6"], "evidence_texts": ["A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "i am well thank you and yourself ?", "horses i love them . i also like rabbits country music you ?", "i have turned body building into a career .", "thanks . one thing that does not involve seeing color", "i am very much so my sister likes to pick on me", "it is a running joke now tho . she is deaf on one side", "well i do not know about karma but we get a good laugh"], "speaker_id": "snli:test:2902", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2902", "candidate_text": "I wonder whether a boy denies his sister a drink?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2902:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2723:premise", "snli:test:2920:premise", "snli:test:2902:premise", "snli:test:2616:premise", "style:personachat:test:415:self:7:positive:0", "style:personachat:test:415:self:7:positive:1", "style:personachat:test:415:self:7:positive:2", "style:personachat:test:415:self:7:positive:3", "style:personachat:test:415:self:7:positive:4", "style:personachat:test:415:self:7:positive:5", "style:personachat:test:415:self:7:positive:6"], "evidence_texts": ["A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A boy wearing a shirt with the Simpsons character, Ralph Wiggum, plays a Nintendo DS with his sister.", "it depends . personally , my favorite way to relax is to scroll through some clothes catalogs .", "definitely . i just love fashion , and i love clothes shopping . what about you ?", "i see . you are a writer , then ?", "i am a freshman in college . i really want to be a doctor though .", "ha , that sounds like quite a story .", "that makes sense . i am pretty boring . tell me more about yourself .", "what do you do for fun ?"], "speaker_id": "snli:test:2902", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2911", "candidate_text": "I wonder whether three young people are putting on suntan location while standing on the beach together?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2911:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:1663:premise", "snli:test:3103:premise", "snli:test:1265:premise", "snli:test:2911:premise", "style:personachat:test:108:self:1:positive:0"], "evidence_texts": ["A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:2911", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2911", "candidate_text": "well, three young people are putting on suntan location while standing on the beach together", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2911:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:1663:premise", "snli:test:3103:premise", "snli:test:1265:premise", "snli:test:2911:premise", "style:personachat:test:192:self:1:positive:0"], "evidence_texts": ["A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "Three men are doing a musical set with wires and equipment all over the stage, and they are holding unique guitar like instruments.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "thank you , it changes a persons life ."], "speaker_id": "snli:test:2911", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2920", "candidate_text": "I wonder whether an old man is riding a bike with a blue laundry hamper that is full of clothes on his handlebars?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2920:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2920:premise", "snli:test:2902:premise", "snli:test:2997:premise", "snli:test:2723:premise", "style:personachat:test:108:self:2:positive:0", "style:personachat:test:108:self:2:positive:1"], "evidence_texts": ["An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "all of my pictures of me", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:2920", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2920", "candidate_text": "Honestly, An old man is riding a bike with a blue laundry hamper that is full of clothes on his handlebars!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2920:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2920:premise", "snli:test:2902:premise", "snli:test:2997:premise", "snli:test:2723:premise", "style:personachat:test:767:self:1:positive:0"], "evidence_texts": ["An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A roofer in a gray sweatshirt and orange hat walks on a unfinished roof at a lake-side home.", "nice name ! my girlfriend is calling me , have to go !"], "speaker_id": "snli:test:2920", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2922", "candidate_text": "I wonder whether an old woman is in a kitchen?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2922:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2900:premise", "snli:test:3048:premise", "snli:test:2922:premise", "snli:test:2896:premise", "style:personachat:test:108:self:5:positive:0", "style:personachat:test:108:self:5:positive:1", "style:personachat:test:108:self:5:positive:2", "style:personachat:test:108:self:5:positive:3", "style:personachat:test:108:self:5:positive:4"], "evidence_texts": ["A team of surgeons operate on a female patient.", "Two little girls reading a book at the library.", "An older white woman makes cupcakes in her kitchen.", "Mother and daughter are meeting for the first time.", "where do your parents live", "new york . i jut bought a house !", "i love to dance at the clubs .", "all of my pictures of me", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:2922", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2922", "candidate_text": "An old woman is in a kitchen.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2922:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2900:premise", "snli:test:3048:premise", "snli:test:2922:premise", "snli:test:2896:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A team of surgeons operate on a female patient.", "Two little girls reading a book at the library.", "An older white woman makes cupcakes in her kitchen.", "Mother and daughter are meeting for the first time.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2922", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2931", "candidate_text": "I wonder whether a grounds crew maintain a park?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2931:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2841:premise", "snli:test:2972:premise", "snli:test:2757:premise", "snli:test:2931:premise", "style:personachat:test:108:self:7:positive:0", "style:personachat:test:108:self:7:positive:1", "style:personachat:test:108:self:7:positive:2", "style:personachat:test:108:self:7:positive:3", "style:personachat:test:108:self:7:positive:4", "style:personachat:test:108:self:7:positive:5", "style:personachat:test:108:self:7:positive:6"], "evidence_texts": ["A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "hi ! ! ! ! ! ! fall is my favorite i love pumpkin spice lattes", "i love candy and everything sweet . what do you love about fall", "where do your parents live", "new york . i jut bought a house !", "i love to dance at the clubs .", "all of my pictures of me", "who is timothy ? is he your boyfriend"], "speaker_id": "snli:test:2931", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2931", "candidate_text": "well, a grounds crew maintain a park", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2931:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2841:premise", "snli:test:2972:premise", "snli:test:2757:premise", "snli:test:2931:premise", "style:personachat:test:192:self:7:positive:0", "style:personachat:test:192:self:7:positive:1", "style:personachat:test:192:self:7:positive:2", "style:personachat:test:192:self:7:positive:3", "style:personachat:test:192:self:7:positive:4", "style:personachat:test:192:self:7:positive:5", "style:personachat:test:192:self:7:positive:6"], "evidence_texts": ["A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "A man on a bicycle, wearing cycle gear, riding at a fast past down paved trail surrounded by tree's and grass.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "good evening how are you tonight ?", "i do not like bugs either but bacon is good .", "muffins are my favorite food .", "i am just watching a movie and eating muffins .", "i am in the healthcare field so i am just chilling tonight .", "my dad died of cancer so i wanted to help others .", "thank you , it changes a persons life ."], "speaker_id": "snli:test:2931", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2934", "candidate_text": "I wonder whether men are playing basketball?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2934:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2577:premise", "snli:test:3019:premise", "snli:test:2934:premise", "snli:test:1996:premise", "style:personachat:test:109:self:1:positive:0"], "evidence_texts": ["Adults and children share in the looking at something, and some young ladies stand to the side.", "A man on a bicycle rides past a park, with a group of people in the background.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:2934", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2934", "candidate_text": "Honestly, Men are playing basketball!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2934:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2577:premise", "snli:test:3019:premise", "snli:test:2934:premise", "snli:test:1996:premise", "style:personachat:test:767:self:7:positive:0", "style:personachat:test:767:self:7:positive:1", "style:personachat:test:767:self:7:positive:2", "style:personachat:test:767:self:7:positive:3", "style:personachat:test:767:self:7:positive:4", "style:personachat:test:767:self:7:positive:5", "style:personachat:test:767:self:7:positive:6"], "evidence_texts": ["Adults and children share in the looking at something, and some young ladies stand to the side.", "A man on a bicycle rides past a park, with a group of people in the background.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "People looking at their reflection and the reflection of a city in a long irregular shaped mirror", "i love blue hair , my live in girlfriend has blue hair too !", "maryland , studying law at johns hopkins", "that is cool ; do you have pets ? my dog max says hi .", "robots are great , i take mine on all my travel adventures .", "yes , better than some humans !", "we will have to get to get together and have my robot meet yours !", "nice name ! my girlfriend is calling me , have to go !"], "speaker_id": "snli:test:2934", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2943", "candidate_text": "I wonder whether a woman is on a rooftop?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2943:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2781:premise", "snli:test:2643:premise", "snli:test:2978:premise", "snli:test:2943:premise", "style:personachat:test:109:self:2:positive:0", "style:personachat:test:109:self:2:positive:1"], "evidence_texts": ["A scientist studies a slide in order to work on her new creation.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "A snowboarder making a big jump doing a trick on a high mountain.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "you are funny . listening to bach , eating bacon , without bugs . ideal !", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:2943", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2943", "candidate_text": "A woman is on a rooftop.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2943:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2781:premise", "snli:test:2643:premise", "snli:test:2978:premise", "snli:test:2943:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A scientist studies a slide in order to work on her new creation.", "A woman wearing very formal colorful clothing and she seems to be dancing.", "A snowboarder making a big jump doing a trick on a high mountain.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2943", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2944", "candidate_text": "I wonder whether a woman is enjoying the outside weather?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2944:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2944:premise", "snli:test:2978:premise", "snli:test:2781:premise", "snli:test:3149:premise", "style:personachat:test:109:self:5:positive:0", "style:personachat:test:109:self:5:positive:1", "style:personachat:test:109:self:5:positive:2", "style:personachat:test:109:self:5:positive:3", "style:personachat:test:109:self:5:positive:4"], "evidence_texts": ["A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A snowboarder making a big jump doing a trick on a high mountain.", "A scientist studies a slide in order to work on her new creation.", "A man in a suit cheering in a library in front of children.", "yes . four . they eat all of the bacon in the house , makes me sad . you ?", "too funny . i will have to learn your tricks . do you have a favorite food ?", "anything but bug green . you must have a favorite .", "you are funny . listening to bach , eating bacon , without bugs . ideal !", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:2944", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2944", "candidate_text": "well, a woman is enjoying the outside weather", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2944:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2944:premise", "snli:test:2978:premise", "snli:test:2781:premise", "snli:test:3149:premise", "style:personachat:test:193:self:5:positive:0", "style:personachat:test:193:self:5:positive:1", "style:personachat:test:193:self:5:positive:2", "style:personachat:test:193:self:5:positive:3", "style:personachat:test:193:self:5:positive:4"], "evidence_texts": ["A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A snowboarder making a big jump doing a trick on a high mountain.", "A scientist studies a slide in order to work on her new creation.", "A man in a suit cheering in a library in front of children.", "really ? what is your fav animal ? mine are horses", "lol or so ? i love the ocean an beach", "really ? wow , need a new ride ?", "gotcha my dad has awesome cars there", "no i really do not but night"], "speaker_id": "snli:test:2944", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2946", "candidate_text": "I wonder whether children are playing?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2946:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2571:premise", "snli:test:3956:premise", "snli:test:2946:premise", "snli:test:2359:premise", "style:personachat:test:109:self:7:positive:0", "style:personachat:test:109:self:7:positive:1", "style:personachat:test:109:self:7:positive:2", "style:personachat:test:109:self:7:positive:3", "style:personachat:test:109:self:7:positive:4", "style:personachat:test:109:self:7:positive:5", "style:personachat:test:109:self:7:positive:6"], "evidence_texts": ["A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "doing great ! i finally got rid of the box elder bugs in my front yard . yuck .", "me too ! they just make me want to scream and run . fast !", "yes . four . they eat all of the bacon in the house , makes me sad . you ?", "too funny . i will have to learn your tricks . do you have a favorite food ?", "anything but bug green . you must have a favorite .", "you are funny . listening to bach , eating bacon , without bugs . ideal !", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:2946", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2946", "candidate_text": "Honestly, Children are playing!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2946:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2571:premise", "snli:test:3956:premise", "snli:test:2946:premise", "snli:test:2359:premise", "style:personachat:test:769:self:5:positive:0", "style:personachat:test:769:self:5:positive:1", "style:personachat:test:769:self:5:positive:2", "style:personachat:test:769:self:5:positive:3", "style:personachat:test:769:self:5:positive:4"], "evidence_texts": ["A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men dressed in white shirts and white hats, (two with baseball caps, the leader with a white construction helmet), pounding sticks on steel and plastic drums.", "that is not good . do you like to read ? i read whenever possible .", "my profession is in it .", "yeah , i can say the same here .", "yes i do . my favorite is football .", "i love watching movies ! i like reading more though ."], "speaker_id": "snli:test:2946", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2957", "candidate_text": "well, someone takes a swing at a ball", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2957:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2821:premise", "snli:test:2957:premise", "snli:test:2993:premise", "snli:test:2820:premise", "style:personachat:test:110:self:1:positive:0"], "evidence_texts": ["A young boy in a cowboy hat rides a large horse.", "A man wearing white turns to hit a yellow tennis ball.", "The people await to cross the street while walking the dog.", "A young boy in a cowboy hat rides a large horse.", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:2957", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:2957", "candidate_text": "I wonder whether someone takes a swing at a ball?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2957:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2821:premise", "snli:test:2957:premise", "snli:test:2993:premise", "snli:test:2820:premise", "style:personachat:test:423:self:1:positive:0"], "evidence_texts": ["A young boy in a cowboy hat rides a large horse.", "A man wearing white turns to hit a yellow tennis ball.", "The people await to cross the street while walking the dog.", "A young boy in a cowboy hat rides a large horse.", "does your family live close to you ?"], "speaker_id": "snli:test:2957", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:2972", "candidate_text": "well, a man hugs his mother and leaves for war", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2972:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2972:premise", "snli:test:2931:premise", "snli:test:3324:premise", "snli:test:2841:premise", "style:personachat:test:110:self:2:positive:0", "style:personachat:test:110:self:2:positive:1"], "evidence_texts": ["It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "i went to italy a few years ago i would love to go back", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:2972", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2972", "candidate_text": "Honestly, A man hugs his mother and leaves for war!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2972:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2972:premise", "snli:test:2931:premise", "snli:test:3324:premise", "snli:test:2841:premise", "style:personachat:test:775:self:1:positive:0"], "evidence_texts": ["It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "A lady in a red shirt and bikini bottoms is about to go surfing on a blue board in the ocean.", "that is nice . i love sweets so i bake a lot . then i dance it off !"], "speaker_id": "snli:test:2972", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2978", "candidate_text": "well, a snowboarder lands a trick during a competition", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2978:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2944:premise", "snli:test:3149:premise", "snli:test:2978:premise", "snli:test:2943:premise", "style:personachat:test:110:self:5:positive:0", "style:personachat:test:110:self:5:positive:1", "style:personachat:test:110:self:5:positive:2", "style:personachat:test:110:self:5:positive:3", "style:personachat:test:110:self:5:positive:4"], "evidence_texts": ["A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A man in a suit cheering in a library in front of children.", "A snowboarder making a big jump doing a trick on a high mountain.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "haha no worries what do you do for a living", "that is great whats your favorite so far", "yeah i like it too i have not been in a while though", "i went to italy a few years ago i would love to go back", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:2978", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2978", "candidate_text": "A snowboarder lands a trick during a competition.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2978:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2944:premise", "snli:test:3149:premise", "snli:test:2978:premise", "snli:test:2943:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "A man in a suit cheering in a library in front of children.", "A snowboarder making a big jump doing a trick on a high mountain.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2978", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2984", "candidate_text": "well, two people playing chess at a park", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2984:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2984:premise", "snli:test:2807:premise", "snli:test:3015:premise", "snli:test:2747:premise", "style:personachat:test:110:self:7:positive:0", "style:personachat:test:110:self:7:positive:1", "style:personachat:test:110:self:7:positive:2", "style:personachat:test:110:self:7:positive:3", "style:personachat:test:110:self:7:positive:4", "style:personachat:test:110:self:7:positive:5", "style:personachat:test:110:self:7:positive:6"], "evidence_texts": ["Two people are riding at once on one single bicycle.", "Three men on mountain bikes go over a sand hill.", "An older man wearing stars and stripes rides a bike.", "Woman smiling while holding a book and yellow paper star.", "i am good just got back from the gym", "sounds cool what is that", "haha no worries what do you do for a living", "that is great whats your favorite so far", "yeah i like it too i have not been in a while though", "i went to italy a few years ago i would love to go back", "sometimes i eat a lot of meat and protein"], "speaker_id": "snli:test:2984", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2984", "candidate_text": "I wonder whether two people playing chess at a park?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2984:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2984:premise", "snli:test:2807:premise", "snli:test:3015:premise", "snli:test:2747:premise", "style:personachat:test:423:self:7:positive:0", "style:personachat:test:423:self:7:positive:1", "style:personachat:test:423:self:7:positive:2", "style:personachat:test:423:self:7:positive:3", "style:personachat:test:423:self:7:positive:4", "style:personachat:test:423:self:7:positive:5", "style:personachat:test:423:self:7:positive:6"], "evidence_texts": ["Two people are riding at once on one single bicycle.", "Three men on mountain bikes go over a sand hill.", "An older man wearing stars and stripes rides a bike.", "Woman smiling while holding a book and yellow paper star.", "listening to hip hop music .", "i also live in texas . my father and brother also live here .", "are there any good vegan restaurants there ?", "i spend most of my time trying to stay fit .", "yes , i need to go to the gym later .", "do you listen to music at the gym ?", "does your family live close to you ?"], "speaker_id": "snli:test:2984", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2988", "candidate_text": "I wonder whether sally was riding her horse?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2988:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2988:premise", "snli:test:1787:premise", "snli:test:1786:premise", "snli:test:3086:premise", "style:personachat:test:111:self:1:positive:0"], "evidence_texts": ["Two people play horseshoes.", "kid practicing karate move", "kid practicing karate move", "Two girls playing basketball.", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:2988", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2988", "candidate_text": "well, sally was riding her horse", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2988:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2988:premise", "snli:test:1787:premise", "snli:test:1786:premise", "snli:test:3086:premise", "style:personachat:test:195:self:1:positive:0"], "evidence_texts": ["Two people play horseshoes.", "kid practicing karate move", "kid practicing karate move", "Two girls playing basketball.", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:2988", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2989", "candidate_text": "I wonder whether tom and sue were playing horseshoes?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2989:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3086:premise", "snli:test:2989:premise", "snli:test:1787:premise", "snli:test:3819:premise", "style:personachat:test:111:self:2:positive:0", "style:personachat:test:111:self:2:positive:1"], "evidence_texts": ["Two girls playing basketball.", "Two people play horseshoes.", "kid practicing karate move", "Woman is looking something.", "three . do you have a cat or dog ?", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:2989", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2989", "candidate_text": "Honestly, Tom and Sue were playing horseshoes!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2989:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3086:premise", "snli:test:2989:premise", "snli:test:1787:premise", "snli:test:3819:premise", "style:personachat:test:780:self:1:positive:0"], "evidence_texts": ["Two girls playing basketball.", "Two people play horseshoes.", "kid practicing karate move", "Woman is looking something.", "absolutely ! the collection is pretty large . what is your favorite color ?"], "speaker_id": "snli:test:2989", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2993", "candidate_text": "I wonder whether people are at home watching tv?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2993:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2957:premise", "snli:test:2993:premise", "snli:test:3264:premise", "snli:test:2821:premise", "style:personachat:test:111:self:5:positive:0", "style:personachat:test:111:self:5:positive:1", "style:personachat:test:111:self:5:positive:2", "style:personachat:test:111:self:5:positive:3", "style:personachat:test:111:self:5:positive:4"], "evidence_texts": ["A man wearing white turns to hit a yellow tennis ball.", "The people await to cross the street while walking the dog.", "Two girls in red life jackets lock oars on a river.", "A young boy in a cowboy hat rides a large horse.", "you should ! just read a book this weekend at my brothers wedding .", "yeah . i have weddings . i do love playing board games though .", "sad . you like cats ? i have one pet called arizona .", "three . do you have a cat or dog ?", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:2993", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2993", "candidate_text": "people are at home watching tv.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:2993:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2957:premise", "snli:test:2993:premise", "snli:test:3264:premise", "snli:test:2821:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man wearing white turns to hit a yellow tennis ball.", "The people await to cross the street while walking the dog.", "Two girls in red life jackets lock oars on a river.", "A young boy in a cowboy hat rides a large horse.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:2993", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:2997", "candidate_text": "I wonder whether two men are composing together?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2997:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2920:premise", "snli:test:2997:premise", "snli:test:3053:premise", "snli:test:2902:premise", "style:personachat:test:111:self:7:positive:0", "style:personachat:test:111:self:7:positive:1", "style:personachat:test:111:self:7:positive:2", "style:personachat:test:111:self:7:positive:3", "style:personachat:test:111:self:7:positive:4", "style:personachat:test:111:self:7:positive:5", "style:personachat:test:111:self:7:positive:6"], "evidence_texts": ["An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "hey there . how is it going ?", "have you read about pablo escobar ? i love reading biographies .", "you should ! just read a book this weekend at my brothers wedding .", "yeah . i have weddings . i do love playing board games though .", "sad . you like cats ? i have one pet called arizona .", "three . do you have a cat or dog ?", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:2997", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:2997", "candidate_text": "well, two men are composing together", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:2997:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2920:premise", "snli:test:2997:premise", "snli:test:3053:premise", "snli:test:2902:premise", "style:personachat:test:195:self:7:positive:0", "style:personachat:test:195:self:7:positive:1", "style:personachat:test:195:self:7:positive:2", "style:personachat:test:195:self:7:positive:3", "style:personachat:test:195:self:7:positive:4", "style:personachat:test:195:self:7:positive:5", "style:personachat:test:195:self:7:positive:6"], "evidence_texts": ["An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "The young boy in a red shirt and jeans resists his sister's request for water on the road.", "i am good how about you", "oh how exciting that is . still looking for mister right", "yeah would help if i stay out of the clubs will not find him there", "would you say obedience school for dogs is one ?", "that is a good one kinda play that with my dogs i teach", "it can be fun . just not when my sweet tooth kicks in lol", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:2997", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3011", "candidate_text": "well, people sit around a table", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3011:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2018:premise", "snli:test:3178:premise", "snli:test:2017:premise", "snli:test:3011:premise", "style:personachat:test:112:self:1:positive:0"], "evidence_texts": ["A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "yes i have 4 little sisters"], "speaker_id": "snli:test:3011", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3011", "candidate_text": "people sit around a table.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3011:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2018:premise", "snli:test:3178:premise", "snli:test:2017:premise", "snli:test:3011:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3011", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3015", "candidate_text": "well, the man wearing stars and stripes on a bike is in a parade", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3015:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2984:premise", "snli:test:3244:premise", "snli:test:2807:premise", "snli:test:3015:premise", "style:personachat:test:112:self:2:positive:0", "style:personachat:test:112:self:2:positive:1"], "evidence_texts": ["Two people are riding at once on one single bicycle.", "Three people are dressed in costumes and playing musical instruments.", "Three men on mountain bikes go over a sand hill.", "An older man wearing stars and stripes rides a bike.", "i am 28 years old", "yes i have 4 little sisters"], "speaker_id": "snli:test:3015", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3015", "candidate_text": "I wonder whether the man wearing stars and stripes on a bike is in a parade?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3015:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2984:premise", "snli:test:3244:premise", "snli:test:2807:premise", "snli:test:3015:premise", "style:personachat:test:443:self:2:positive:0", "style:personachat:test:443:self:2:positive:1"], "evidence_texts": ["Two people are riding at once on one single bicycle.", "Three people are dressed in costumes and playing musical instruments.", "Three men on mountain bikes go over a sand hill.", "An older man wearing stars and stripes rides a bike.", "i would like to go to africa . what about you ?", "interesting . whats your favorite video game ?"], "speaker_id": "snli:test:3015", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3019", "candidate_text": "well, a guy rides his bike in the middle of a park", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3019:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2934:premise", "snli:test:3060:premise", "snli:test:2577:premise", "snli:test:3019:premise", "style:personachat:test:112:self:5:positive:0", "style:personachat:test:112:self:5:positive:1", "style:personachat:test:112:self:5:positive:2", "style:personachat:test:112:self:5:positive:3", "style:personachat:test:112:self:5:positive:4"], "evidence_texts": ["A group of children divided into a maroon team and a blue team playing a soccer game.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "Adults and children share in the looking at something, and some young ladies stand to the side.", "A man on a bicycle rides past a park, with a group of people in the background.", "what you bought if i may ask", "what type of shoes ? heels or flats ?", "oh cool , how old are you", "i am 28 years old", "yes i have 4 little sisters"], "speaker_id": "snli:test:3019", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3019", "candidate_text": "Honestly, A guy rides his bike in the middle of a park!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3019:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2934:premise", "snli:test:3060:premise", "snli:test:2577:premise", "snli:test:3019:premise", "style:personachat:test:797:self:2:positive:0", "style:personachat:test:797:self:2:positive:1"], "evidence_texts": ["A group of children divided into a maroon team and a blue team playing a soccer game.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "Adults and children share in the looking at something, and some young ladies stand to the side.", "A man on a bicycle rides past a park, with a group of people in the background.", "you must have some talents . do you win at poker often ?", "winning at poker is an enviable talent !"], "speaker_id": "snli:test:3019", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3035", "candidate_text": "well, girl eats cotton candy", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3035:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:2609:premise", "snli:test:3247:premise", "snli:test:2531:premise", "snli:test:3035:premise", "style:personachat:test:112:self:6:positive:0", "style:personachat:test:112:self:6:positive:1", "style:personachat:test:112:self:6:positive:2", "style:personachat:test:112:self:6:positive:3", "style:personachat:test:112:self:6:positive:4", "style:personachat:test:112:self:6:positive:5"], "evidence_texts": ["Man racing a quad on a dirt racetrack.", "Everyone is getting ready to play the game.", "Man sits at the doorway of barber shop.", "Cook handling hot tray with food on it.", "oh cool , i be online a lot", "what you bought if i may ask", "what type of shoes ? heels or flats ?", "oh cool , how old are you", "i am 28 years old", "yes i have 4 little sisters"], "speaker_id": "snli:test:3035", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3035", "candidate_text": "girl eats cotton candy.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3035:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2609:premise", "snli:test:3247:premise", "snli:test:2531:premise", "snli:test:3035:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Man racing a quad on a dirt racetrack.", "Everyone is getting ready to play the game.", "Man sits at the doorway of barber shop.", "Cook handling hot tray with food on it.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3035", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3041", "candidate_text": "I wonder whether there are a few girls present?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3041:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2740:premise", "snli:test:3041:premise", "snli:test:3192:premise", "snli:test:2686:premise", "style:personachat:test:113:self:1:positive:0"], "evidence_texts": ["A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:3041", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3041", "candidate_text": "There are a few girls present.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3041:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2740:premise", "snli:test:3041:premise", "snli:test:3192:premise", "snli:test:2686:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Two children are looking through a telescope on a city street, and the boy is using a step ladder to see through the eyeglass.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3041", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3048", "candidate_text": "I wonder whether the girls are in a library?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3048:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3048:premise", "snli:test:2922:premise", "snli:test:3068:premise", "snli:test:2900:premise", "style:personachat:test:113:self:2:positive:0", "style:personachat:test:113:self:2:positive:1"], "evidence_texts": ["Two little girls reading a book at the library.", "An older white woman makes cupcakes in her kitchen.", "A cyclist dressed in black and white is pointing.", "A team of surgeons operate on a female patient.", "what kind of engineer are you ?", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:3048", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3048", "candidate_text": "well, the girls are in a library", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3048:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3048:premise", "snli:test:2922:premise", "snli:test:3068:premise", "snli:test:2900:premise", "style:personachat:test:198:self:2:positive:0", "style:personachat:test:198:self:2:positive:1"], "evidence_texts": ["Two little girls reading a book at the library.", "An older white woman makes cupcakes in her kitchen.", "A cyclist dressed in black and white is pointing.", "A team of surgeons operate on a female patient.", "i learned to love that movie since that is all we had to watch at school .", "i studied abroad so it is the only movie i had in my room ."], "speaker_id": "snli:test:3048", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3053", "candidate_text": "I wonder whether a kid has a painted face?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3053:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2997:premise", "snli:test:3053:premise", "snli:test:3141:premise", "snli:test:2920:premise", "style:personachat:test:113:self:5:positive:0", "style:personachat:test:113:self:5:positive:1", "style:personachat:test:113:self:5:positive:2", "style:personachat:test:113:self:5:positive:3", "style:personachat:test:113:self:5:positive:4"], "evidence_texts": ["A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "ah , i get it . i just finished making dinner .", "i made pesto chicken and a salad .", "i do ! i would like a big family .", "what kind of engineer are you ?", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:3053", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3053", "candidate_text": "Honestly, A kid has a painted face!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3053:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2997:premise", "snli:test:3053:premise", "snli:test:3141:premise", "snli:test:2920:premise", "style:personachat:test:802:self:2:positive:0", "style:personachat:test:802:self:2:positive:1"], "evidence_texts": ["A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "An older man is riding a bike with a blue laundry hamper full of clothes on his handlebars.", "i have always been pretty athletic so it works ! what other hobbies do you enjoy ?", "nothing wrong with that ! who could pass up relaxing activities ?"], "speaker_id": "snli:test:3053", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3060", "candidate_text": "I wonder whether a girl is making scrambled eggs?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3060:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3060:premise", "snli:test:3019:premise", "snli:test:2934:premise", "snli:test:3078:premise", "style:personachat:test:113:self:6:positive:0", "style:personachat:test:113:self:6:positive:1", "style:personachat:test:113:self:6:positive:2", "style:personachat:test:113:self:6:positive:3", "style:personachat:test:113:self:6:positive:4", "style:personachat:test:113:self:6:positive:5"], "evidence_texts": ["A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A man on a bicycle rides past a park, with a group of people in the background.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "i do enjoy classical music !", "ah , i get it . i just finished making dinner .", "i made pesto chicken and a salad .", "i do ! i would like a big family .", "what kind of engineer are you ?", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:3060", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3060", "candidate_text": "A girl is making scrambled eggs.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3060:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3060:premise", "snli:test:3019:premise", "snli:test:2934:premise", "snli:test:3078:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A man on a bicycle rides past a park, with a group of people in the background.", "A group of children divided into a maroon team and a blue team playing a soccer game.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3060", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3062", "candidate_text": "well, a woman is painting", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3062:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3078:premise", "snli:test:3019:premise", "snli:test:3062:premise", "snli:test:3113:premise", "style:personachat:test:114:self:1:positive:0"], "evidence_texts": ["A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A man on a bicycle rides past a park, with a group of people in the background.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "yeah pretty much like that what else do you do"], "speaker_id": "snli:test:3062", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3062", "candidate_text": "Honestly, A woman is painting!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3062:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3078:premise", "snli:test:3019:premise", "snli:test:3062:premise", "snli:test:3113:premise", "style:personachat:test:802:self:7:positive:0", "style:personachat:test:802:self:7:positive:1", "style:personachat:test:802:self:7:positive:2", "style:personachat:test:802:self:7:positive:3", "style:personachat:test:802:self:7:positive:4", "style:personachat:test:802:self:7:positive:5", "style:personachat:test:802:self:7:positive:6"], "evidence_texts": ["A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A man on a bicycle rides past a park, with a group of people in the background.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "i am just doing a bit of cleaning . my turtle , timothy , needs his tank cleaned .", "that is so funny ! i have a cat named cricket ! lol", "fortunately , not ! just an interesting tidbit to add in when meeting new people .", "yes , it would be ! my parents live in the middle of the ocean . . . bora bora", "what a neat hobby ! i spend my time playing ultimate frisbee", "i have always been pretty athletic so it works ! what other hobbies do you enjoy ?", "nothing wrong with that ! who could pass up relaxing activities ?"], "speaker_id": "snli:test:3062", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3068", "candidate_text": "well, a cyclist dressed in black and white eats cheese", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3068:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3068:premise", "snli:test:3048:premise", "snli:test:3110:premise", "snli:test:2922:premise", "style:personachat:test:114:self:2:positive:0", "style:personachat:test:114:self:2:positive:1"], "evidence_texts": ["A cyclist dressed in black and white is pointing.", "Two little girls reading a book at the library.", "A man sits at a table in a room.", "An older white woman makes cupcakes in her kitchen.", "i gave them a name based on the characteristics", "yeah pretty much like that what else do you do"], "speaker_id": "snli:test:3068", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3068", "candidate_text": "A cyclist dressed in black and white eats cheese.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3068:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3068:premise", "snli:test:3048:premise", "snli:test:3110:premise", "snli:test:2922:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A cyclist dressed in black and white is pointing.", "Two little girls reading a book at the library.", "A man sits at a table in a room.", "An older white woman makes cupcakes in her kitchen.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3068", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3074", "candidate_text": "well, a market is selling soft drinks in an arabic country", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3074:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2893:premise", "snli:test:3151:premise", "snli:test:3074:premise", "snli:test:2829:premise", "style:personachat:test:114:self:5:positive:0", "style:personachat:test:114:self:5:positive:1", "style:personachat:test:114:self:5:positive:2", "style:personachat:test:114:self:5:positive:3", "style:personachat:test:114:self:5:positive:4"], "evidence_texts": ["A girl with short hair and a white shirt is holding a beer bottle", "A girl reaches up to kiss a cat, which is sitting on the counter.", "A local street in an Arabic country where there is a market selling Coca-Cola.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "well in fact it is and i m very knowledgeable about it because i write books about it", "yeah i study at experimented and go to exercise", "i m with my cats and dogs two cats", "i gave them a name based on the characteristics", "yeah pretty much like that what else do you do"], "speaker_id": "snli:test:3074", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3074", "candidate_text": "I wonder whether a market is selling soft drinks in an arabic country?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3074:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2893:premise", "snli:test:3151:premise", "snli:test:3074:premise", "snli:test:2829:premise", "style:personachat:test:450:self:5:positive:0", "style:personachat:test:450:self:5:positive:1", "style:personachat:test:450:self:5:positive:2", "style:personachat:test:450:self:5:positive:3", "style:personachat:test:450:self:5:positive:4"], "evidence_texts": ["A girl with short hair and a white shirt is holding a beer bottle", "A girl reaches up to kiss a cat, which is sitting on the counter.", "A local street in an Arabic country where there is a market selling Coca-Cola.", "A man wearing a blue shirt and headphones around his neck raises his arm.", "no i do not like music ?", "cool , i am normally irritating or irritable", "i bet an customers , i like the boss niece smh", "i do not have much to say well do you like bands ?", "so ? you been boring to me based of how slow you type anyway"], "speaker_id": "snli:test:3074", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3078", "candidate_text": "well, a man wearing a bathing suit is bent over in a cornfield while looking at the camera", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3078:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3062:premise", "snli:test:3113:premise", "snli:test:3078:premise", "snli:test:3060:premise", "style:personachat:test:114:self:7:positive:0", "style:personachat:test:114:self:7:positive:1", "style:personachat:test:114:self:7:positive:2", "style:personachat:test:114:self:7:positive:3", "style:personachat:test:114:self:7:positive:4", "style:personachat:test:114:self:7:positive:5", "style:personachat:test:114:self:7:positive:6"], "evidence_texts": ["A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "are you interested with diet supplement", "i run my business it is my own business dieting business", "well in fact it is and i m very knowledgeable about it because i write books about it", "yeah i study at experimented and go to exercise", "i m with my cats and dogs two cats", "i gave them a name based on the characteristics", "yeah pretty much like that what else do you do"], "speaker_id": "snli:test:3078", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3078", "candidate_text": "Honestly, A man wearing a bathing suit is bent over in a cornfield while looking at the camera!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3078:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3062:premise", "snli:test:3113:premise", "snli:test:3078:premise", "snli:test:3060:premise", "style:personachat:test:813:self:5:positive:0", "style:personachat:test:813:self:5:positive:1", "style:personachat:test:813:self:5:positive:2", "style:personachat:test:813:self:5:positive:3", "style:personachat:test:813:self:5:positive:4"], "evidence_texts": ["A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "what do you like to do for fun ?", "that is very nice . i love spending time with my big pet dog .", "maybe take a nice long walk to think . that always helps me .", "no , but i love to read books about traveling so i often read about it .", "sounds very nice . i want to travel to italy . i love italian food !"], "speaker_id": "snli:test:3078", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3081", "candidate_text": "I wonder whether two roller derby competitors skate quickly ahead of teammates?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3081:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2712:premise", "snli:test:3081:premise", "snli:test:3621:premise", "snli:test:2700:premise", "style:personachat:test:115:self:1:positive:0"], "evidence_texts": ["A girl in a black sweater and jeans pours water into an empty soda bottle.", "Two women competing in a roller derby with several teammates and referee in the background.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A man with a tan jacket with a full grocery bag is crossing the street.", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:3081", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3081", "candidate_text": "Honestly, Two roller derby competitors skate quickly ahead of teammates!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3081:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2712:premise", "snli:test:3081:premise", "snli:test:3621:premise", "snli:test:2700:premise", "style:personachat:test:813:self:7:positive:0", "style:personachat:test:813:self:7:positive:1", "style:personachat:test:813:self:7:positive:2", "style:personachat:test:813:self:7:positive:3", "style:personachat:test:813:self:7:positive:4", "style:personachat:test:813:self:7:positive:5", "style:personachat:test:813:self:7:positive:6"], "evidence_texts": ["A girl in a black sweater and jeans pours water into an empty soda bottle.", "Two women competing in a roller derby with several teammates and referee in the background.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A man with a tan jacket with a full grocery bag is crossing the street.", "i am doing pretty good . just got back from a nice long walk . you ?", "ugh i am sorry to hear that . what do you want to do for work ?", "what do you like to do for fun ?", "that is very nice . i love spending time with my big pet dog .", "maybe take a nice long walk to think . that always helps me .", "no , but i love to read books about traveling so i often read about it .", "sounds very nice . i want to travel to italy . i love italian food !"], "speaker_id": "snli:test:3081", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3086", "candidate_text": "I wonder whether the men are playing basketball?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3086:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2989:premise", "snli:test:3819:premise", "snli:test:3086:premise", "snli:test:2988:premise", "style:personachat:test:115:self:2:positive:0", "style:personachat:test:115:self:2:positive:1"], "evidence_texts": ["Two people play horseshoes.", "Woman is looking something.", "Two girls playing basketball.", "Two people play horseshoes.", "yes i am working but still he is in my mind .", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:3086", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3086", "candidate_text": "The men are playing basketball.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3086:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2989:premise", "snli:test:3819:premise", "snli:test:3086:premise", "snli:test:2988:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two people play horseshoes.", "Woman is looking something.", "Two girls playing basketball.", "Two people play horseshoes.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3086", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3090", "candidate_text": "I wonder whether a band is playing a cover song?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3090:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2742:premise", "snli:test:3287:premise", "snli:test:3090:premise", "snli:test:2535:premise", "style:personachat:test:115:self:5:positive:0", "style:personachat:test:115:self:5:positive:1", "style:personachat:test:115:self:5:positive:2", "style:personachat:test:115:self:5:positive:3", "style:personachat:test:115:self:5:positive:4"], "evidence_texts": ["Two people are sitting on a rock looking off into the distance.", "One man in baseball cap looks out while two men look down.", "A band singing and playing electric guitar for a crowd of people.", "A boy is running in a spinning metal tunnel in the playground.", "that is good . my cousin loves to play soccer .", "i am single , my ex bf getting married . congratulations for you", "i hope so . break up is bad , specially when you love someone .", "yes i am working but still he is in my mind .", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:3090", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3090", "candidate_text": "well, a band is playing a cover song", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3090:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2742:premise", "snli:test:3287:premise", "snli:test:3090:premise", "snli:test:2535:premise", "style:personachat:test:205:self:5:positive:0", "style:personachat:test:205:self:5:positive:1", "style:personachat:test:205:self:5:positive:2", "style:personachat:test:205:self:5:positive:3", "style:personachat:test:205:self:5:positive:4"], "evidence_texts": ["Two people are sitting on a rock looking off into the distance.", "One man in baseball cap looks out while two men look down.", "A band singing and playing electric guitar for a crowd of people.", "A boy is running in a spinning metal tunnel in the playground.", "that is cool , i like playing video games a lot do you play ?", "that is cool , i like them but they can be weird . any hobbies ?", "oh that is cool . do you work fulltime ?", "oh that is pretty sweet , i watch a lot of sports do you ?", "yeah , my mom works a lot and i always thought i should balance more"], "speaker_id": "snli:test:3090", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3100", "candidate_text": "I wonder whether a guy is next to a dog while holding some musical instruments?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3100:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3100:premise", "snli:test:2518:premise", "snli:test:6160:premise", "snli:test:1307:premise", "style:personachat:test:115:self:7:positive:0", "style:personachat:test:115:self:7:positive:1", "style:personachat:test:115:self:7:positive:2", "style:personachat:test:115:self:7:positive:3", "style:personachat:test:115:self:7:positive:4", "style:personachat:test:115:self:7:positive:5", "style:personachat:test:115:self:7:positive:6"], "evidence_texts": ["A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "hello ! i am fine what about you ?", "who plays ? i do not watch soccer at all .", "that is good . my cousin loves to play soccer .", "i am single , my ex bf getting married . congratulations for you", "i hope so . break up is bad , specially when you love someone .", "yes i am working but still he is in my mind .", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:3100", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3100", "candidate_text": "Honestly, A guy is next to a dog while holding some musical instruments!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3100:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3100:premise", "snli:test:2518:premise", "snli:test:6160:premise", "snli:test:1307:premise", "style:personachat:test:815:self:5:positive:0", "style:personachat:test:815:self:5:positive:1", "style:personachat:test:815:self:5:positive:2", "style:personachat:test:815:self:5:positive:3", "style:personachat:test:815:self:5:positive:4"], "evidence_texts": ["A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "travel when i can . my best friend goes with me . we met in kindergarten .", "it is ok . where did you grow up ? i did in north dakota .", "that is fun ! are you jewish ? i am .", "yes i am . i have been my whole life .", "oh really ? we need to become friends so i can have a doctor friend !"], "speaker_id": "snli:test:3100", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3103", "candidate_text": "well, a woman dances with a man in a busy club", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3103:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2911:premise", "snli:test:1663:premise", "snli:test:5558:premise", "snli:test:3103:premise", "style:personachat:test:116:self:1:positive:0"], "evidence_texts": ["Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:3103", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3103", "candidate_text": "I wonder whether a woman dances with a man in a busy club?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3103:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2911:premise", "snli:test:1663:premise", "snli:test:5558:premise", "snli:test:3103:premise", "style:personachat:test:453:self:1:positive:0"], "evidence_texts": ["Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "A man in a white striped shirt and blue jeans using a water hose to spray water on the ground of concrete ruins.Fo.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "why ? but you guys are so much fun"], "speaker_id": "snli:test:3103", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3104", "candidate_text": "well, a man and woman walk down a city street", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3104:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3104:premise", "snli:test:5558:premise", "snli:test:2911:premise", "snli:test:6031:premise", "style:personachat:test:116:self:2:positive:0", "style:personachat:test:116:self:2:positive:1"], "evidence_texts": ["A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "right on . i am in tampa right now .", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:3104", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3104", "candidate_text": "Honestly, A man and woman walk down a city street!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3104:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3104:premise", "snli:test:5558:premise", "snli:test:2911:premise", "snli:test:6031:premise", "style:personachat:test:820:self:1:positive:0"], "evidence_texts": ["A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "Three young men wearing backpacks, shorts, t-shirts, sandals and hats are standing together on a sandy surface with palm trees in the distance.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "trying to run out time , two can play at that !"], "speaker_id": "snli:test:3104", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3106", "candidate_text": "well, a woman is singing on stage", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3106:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:2886:premise", "snli:test:3276:premise", "snli:test:3106:premise", "snli:test:2736:premise", "style:personachat:test:116:self:5:positive:0", "style:personachat:test:116:self:5:positive:1", "style:personachat:test:116:self:5:positive:2", "style:personachat:test:116:self:5:positive:3", "style:personachat:test:116:self:5:positive:4"], "evidence_texts": ["Two young musicians play from a stage.", "People in motion down a city sidewalk.", "Women in blue dress singing on stage.", "Construction worker looking over the job site.", "i do . a poodle . we watch the buckeyes together every saturday .", "very cool ! where in ohio ?", "i am in florida right now .", "right on . i am in tampa right now .", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:3106", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3106", "candidate_text": "A woman is singing on stage.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3106:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2886:premise", "snli:test:3276:premise", "snli:test:3106:premise", "snli:test:2736:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two young musicians play from a stage.", "People in motion down a city sidewalk.", "Women in blue dress singing on stage.", "Construction worker looking over the job site.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3106", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3110", "candidate_text": "well, a man sits", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3110:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3068:premise", "snli:test:3126:premise", "snli:test:3110:premise", "snli:test:3048:premise", "style:personachat:test:116:self:7:positive:0", "style:personachat:test:116:self:7:positive:1", "style:personachat:test:116:self:7:positive:2", "style:personachat:test:116:self:7:positive:3", "style:personachat:test:116:self:7:positive:4", "style:personachat:test:116:self:7:positive:5", "style:personachat:test:116:self:7:positive:6"], "evidence_texts": ["A cyclist dressed in black and white is pointing.", "A group of young people performing on a stage.", "A man sits at a table in a room.", "Two little girls reading a book at the library.", "doing good . just got home from working at th factory .", "very cool . i grow violets in my garden . i love the color purple .", "i do . a poodle . we watch the buckeyes together every saturday .", "very cool ! where in ohio ?", "i am in florida right now .", "right on . i am in tampa right now .", "i love it here . and the weather is cooling now ."], "speaker_id": "snli:test:3110", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3110", "candidate_text": "I wonder whether a man sits?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3110:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3068:premise", "snli:test:3126:premise", "snli:test:3110:premise", "snli:test:3048:premise", "style:personachat:test:453:self:7:positive:0", "style:personachat:test:453:self:7:positive:1", "style:personachat:test:453:self:7:positive:2", "style:personachat:test:453:self:7:positive:3", "style:personachat:test:453:self:7:positive:4", "style:personachat:test:453:self:7:positive:5", "style:personachat:test:453:self:7:positive:6"], "evidence_texts": ["A cyclist dressed in black and white is pointing.", "A group of young people performing on a stage.", "A man sits at a table in a room.", "Two little girls reading a book at the library.", "i am a nurse , caring for people is what i love to do", "i enjoy riding my bike with my 2 dogs", "that is great , all 3 of my grandchildren are adults now", "i would love to go to a party , what time we leaving", "no i am 24 , i had kids at 5 years old", "lol , my 2 grown grandkids can help", "why ? but you guys are so much fun"], "speaker_id": "snli:test:3110", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3113", "candidate_text": "I wonder whether the man has a colorful skateboard?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3113:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3078:premise", "snli:test:3113:premise", "snli:test:3200:premise", "snli:test:3062:premise", "style:personachat:test:117:self:1:positive:0"], "evidence_texts": ["A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A young man with glasses and his companion are attempting to do laundry in a public place.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "do you have any hobbies ?"], "speaker_id": "snli:test:3113", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3113", "candidate_text": "well, the man has a colorful skateboard", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3113:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3078:premise", "snli:test:3113:premise", "snli:test:3200:premise", "snli:test:3062:premise", "style:personachat:test:208:self:1:positive:0"], "evidence_texts": ["A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A young man with glasses and his companion are attempting to do laundry in a public place.", "A brown-haired woman in a blue shirt decorated with gold trimmings paints a picture of two windows.", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:3113", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3118", "candidate_text": "I wonder whether a wife and husband perform?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3118:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2605:premise", "snli:test:2503:premise", "snli:test:3118:premise", "snli:test:3122:premise", "style:personachat:test:117:self:2:positive:0", "style:personachat:test:117:self:2:positive:1"], "evidence_texts": ["A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "my sister has a couple , and they are horrid . thankfully , she lives in london .", "do you have any hobbies ?"], "speaker_id": "snli:test:3118", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3118", "candidate_text": "Honestly, a wife and husband perform!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3118:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2605:premise", "snli:test:2503:premise", "snli:test:3118:premise", "snli:test:3122:premise", "style:personachat:test:821:self:1:positive:0"], "evidence_texts": ["A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A white male in and office setting, wearing a gray dress shirt and black dress pants with black dress shoes is jump roping and smiling.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "it makes me a better gamer !"], "speaker_id": "snli:test:3118", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3119", "candidate_text": "I wonder whether a man plays guitar?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3119:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3122:premise", "snli:test:2605:premise", "snli:test:3395:premise", "snli:test:3119:premise", "style:personachat:test:117:self:5:positive:0", "style:personachat:test:117:self:5:positive:1", "style:personachat:test:117:self:5:positive:2", "style:personachat:test:117:self:5:positive:3", "style:personachat:test:117:self:5:positive:4"], "evidence_texts": ["A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "oh , okay . i am online a lot , and a furry .", "that is a kid doc , right ?", "is it interesting ? i am not around kids , they freak me out .", "my sister has a couple , and they are horrid . thankfully , she lives in london .", "do you have any hobbies ?"], "speaker_id": "snli:test:3119", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3119", "candidate_text": "a man plays guitar.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3119:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3122:premise", "snli:test:2605:premise", "snli:test:3395:premise", "snli:test:3119:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "A group of people in jackets and hats are kneeling outside posing for a picture, each one with their dog by them on a leash.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3119", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3122", "candidate_text": "I wonder whether there are people in a house taking pictures?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3122:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3119:premise", "snli:test:3395:premise", "snli:test:3122:premise", "snli:test:3118:premise", "style:personachat:test:117:self:7:positive:0", "style:personachat:test:117:self:7:positive:1", "style:personachat:test:117:self:7:positive:2", "style:personachat:test:117:self:7:positive:3", "style:personachat:test:117:self:7:positive:4", "style:personachat:test:117:self:7:positive:5", "style:personachat:test:117:self:7:positive:6"], "evidence_texts": ["A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "hello , brianna . i am mia . do you play second life ?", "it is a larp game on the internet .", "oh , okay . i am online a lot , and a furry .", "that is a kid doc , right ?", "is it interesting ? i am not around kids , they freak me out .", "my sister has a couple , and they are horrid . thankfully , she lives in london .", "do you have any hobbies ?"], "speaker_id": "snli:test:3122", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3122", "candidate_text": "well, there are people in a house taking pictures", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3122:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3119:premise", "snli:test:3395:premise", "snli:test:3122:premise", "snli:test:3118:premise", "style:personachat:test:208:self:7:positive:0", "style:personachat:test:208:self:7:positive:1", "style:personachat:test:208:self:7:positive:2", "style:personachat:test:208:self:7:positive:3", "style:personachat:test:208:self:7:positive:4", "style:personachat:test:208:self:7:positive:5", "style:personachat:test:208:self:7:positive:6"], "evidence_texts": ["A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "hello , how are you tonight ? i am in a band , actually", "what do you do for a living ?", "that sounds interesting , do you not get sick ?", "you are a vegan and eat sushi ? how does that work ?", "those usually have cream cheese in them , so they are not vegan .", "i have never heard of vegan sushi .", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:3122", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3126", "candidate_text": "well, a group of people are watching an empty stage", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3126:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3110:premise", "snli:test:3174:premise", "snli:test:3126:premise", "snli:test:3068:premise", "style:personachat:test:118:self:1:positive:0"], "evidence_texts": ["A man sits at a table in a room.", "Two men are laying floor tiles in a doorway.", "A group of young people performing on a stage.", "A cyclist dressed in black and white is pointing.", "i do not like crowes either . i like to stay to myself"], "speaker_id": "snli:test:3126", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3126", "candidate_text": "A group of people are watching an empty stage.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3126:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3110:premise", "snli:test:3174:premise", "snli:test:3126:premise", "snli:test:3068:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man sits at a table in a room.", "Two men are laying floor tiles in a doorway.", "A group of young people performing on a stage.", "A cyclist dressed in black and white is pointing.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3126", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3141", "candidate_text": "well, a man in a black t-shirt and shorts is hiking up a mountain", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3141:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3053:premise", "snli:test:3141:premise", "snli:test:2997:premise", "snli:test:3367:premise", "style:personachat:test:118:self:2:positive:0", "style:personachat:test:118:self:2:positive:1"], "evidence_texts": ["A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "also , i enjoy karaoke , but i am not very good at it !", "i do not like crowes either . i like to stay to myself"], "speaker_id": "snli:test:3141", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3141", "candidate_text": "I wonder whether a man in a black t-shirt and shorts is hiking up a mountain?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3141:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3053:premise", "snli:test:3141:premise", "snli:test:2997:premise", "snli:test:3367:premise", "style:personachat:test:462:self:2:positive:0", "style:personachat:test:462:self:2:positive:1"], "evidence_texts": ["A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "A kneeling man bends his ear to listen on another in front of a guitar and audio equipment.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "i love taylor swift she has to be my favorite singer of all time", "definitely , i listen to it when playing with my two horses , do you have pets ?"], "speaker_id": "snli:test:3141", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3143", "candidate_text": "well, a man in a whit shirt and shorts is rock climbing at a very very slow pace", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3143:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3367:premise", "snli:test:3053:premise", "snli:test:3474:premise", "snli:test:3143:premise", "style:personachat:test:118:self:5:positive:0", "style:personachat:test:118:self:5:positive:1", "style:personachat:test:118:self:5:positive:2", "style:personachat:test:118:self:5:positive:3", "style:personachat:test:118:self:5:positive:4"], "evidence_texts": ["Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "i am just packing to move to nyc !", "i am originally from california though !", "how come ? i am moving to nyc for modeling !", "also , i enjoy karaoke , but i am not very good at it !", "i do not like crowes either . i like to stay to myself"], "speaker_id": "snli:test:3143", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3143", "candidate_text": "Honestly, A man in a whit shirt and shorts is rock climbing at a very very slow pace!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3143:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3367:premise", "snli:test:3053:premise", "snli:test:3474:premise", "snli:test:3143:premise", "style:personachat:test:827:self:2:positive:0", "style:personachat:test:827:self:2:positive:1"], "evidence_texts": ["Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "A kid with his face painted is at the top of a concrete structure with asterisks carved inside.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "paris is nice . i love barcelona better though .", "never been to israel . so much history there !"], "speaker_id": "snli:test:3143", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3146", "candidate_text": "well, a man watering his neighbors plants", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3146:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2877:premise", "snli:test:3359:premise", "snli:test:3146:premise", "snli:test:2861:premise", "style:personachat:test:118:self:7:positive:0", "style:personachat:test:118:self:7:positive:1", "style:personachat:test:118:self:7:positive:2", "style:personachat:test:118:self:7:positive:3", "style:personachat:test:118:self:7:positive:4", "style:personachat:test:118:self:7:positive:5", "style:personachat:test:118:self:7:positive:6"], "evidence_texts": ["A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "i am good , and how are you ?", "i thank you for serving !", "i am just packing to move to nyc !", "i am originally from california though !", "how come ? i am moving to nyc for modeling !", "also , i enjoy karaoke , but i am not very good at it !", "i do not like crowes either . i like to stay to myself"], "speaker_id": "snli:test:3146", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3146", "candidate_text": "A man watering his neighbors plants.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3146:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2877:premise", "snli:test:3359:premise", "snli:test:3146:premise", "snli:test:2861:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A guy in a blue shirt chases a guy in an orange shirt during a game of flag football.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3146", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3149", "candidate_text": "Honestly, A lawyer is encouraging some children at a contest!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3149:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2978:premise", "snli:test:3149:premise", "snli:test:3163:premise", "snli:test:2944:premise", "style:personachat:test:119:self:1:positive:0"], "evidence_texts": ["A snowboarder making a big jump doing a trick on a high mountain.", "A man in a suit cheering in a library in front of children.", "A couple look into the camera as the foreground is filled with balloons.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:3149", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3149", "candidate_text": "I wonder whether a lawyer is encouraging some children at a contest?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3149:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2978:premise", "snli:test:3149:premise", "snli:test:3163:premise", "snli:test:2944:premise", "style:personachat:test:464:self:1:positive:0"], "evidence_texts": ["A snowboarder making a big jump doing a trick on a high mountain.", "A man in a suit cheering in a library in front of children.", "A couple look into the camera as the foreground is filled with balloons.", "A thin brunette woman lounges while reading on a rooftop overlooking a cityscape.", "that sounds fun . do you dye their hair crazy colors ?"], "speaker_id": "snli:test:3149", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3151", "candidate_text": "Honestly, A girl is showing affection towards a cat!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3151:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3074:premise", "snli:test:3258:premise", "snli:test:2893:premise", "snli:test:3151:premise", "style:personachat:test:119:self:2:positive:0", "style:personachat:test:119:self:2:positive:1"], "evidence_texts": ["A local street in an Arabic country where there is a market selling Coca-Cola.", "A man holds a rope for a boy who's about to climb a wall.", "A girl with short hair and a white shirt is holding a beer bottle", "A girl reaches up to kiss a cat, which is sitting on the counter.", "that was my next guess ! my ex wife has a similar one . you been married before ?", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:3151", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3151", "candidate_text": "well, a girl is showing affection towards a cat", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3151:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3074:premise", "snli:test:3258:premise", "snli:test:2893:premise", "snli:test:3151:premise", "style:personachat:test:211:self:2:positive:0", "style:personachat:test:211:self:2:positive:1"], "evidence_texts": ["A local street in an Arabic country where there is a market selling Coca-Cola.", "A man holds a rope for a boy who's about to climb a wall.", "A girl with short hair and a white shirt is holding a beer bottle", "A girl reaches up to kiss a cat, which is sitting on the counter.", "i was married 4 times , and lost three of them in accidents .", "you are never to young to be in love , my dear ."], "speaker_id": "snli:test:3151", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3163", "candidate_text": "Honestly, The couple is trying to figure out who the person across the street is who is holding the camera!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3163:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3149:premise", "snli:test:3292:premise", "snli:test:2978:premise", "snli:test:3163:premise", "style:personachat:test:119:self:5:positive:0", "style:personachat:test:119:self:5:positive:1", "style:personachat:test:119:self:5:positive:2", "style:personachat:test:119:self:5:positive:3", "style:personachat:test:119:self:5:positive:4"], "evidence_texts": ["A man in a suit cheering in a library in front of children.", "A person waterskiing in a river with a large wall in the background.", "A snowboarder making a big jump doing a trick on a high mountain.", "A couple look into the camera as the foreground is filled with balloons.", "unless you are under 18 , lol .", "what is your tattoo of ?", "is it on your lower back ? my daughter wants one , ugh !", "that was my next guess ! my ex wife has a similar one . you been married before ?", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:3163", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3163", "candidate_text": "The couple is trying to figure out who the person across the street is who is holding the camera.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3163:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3149:premise", "snli:test:3292:premise", "snli:test:2978:premise", "snli:test:3163:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a suit cheering in a library in front of children.", "A person waterskiing in a river with a large wall in the background.", "A snowboarder making a big jump doing a trick on a high mountain.", "A couple look into the camera as the foreground is filled with balloons.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3163", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3174", "candidate_text": "Honestly, Two men are being lazy!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3174:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3126:premise", "snli:test:3382:premise", "snli:test:3174:premise", "snli:test:3110:premise", "style:personachat:test:119:self:7:positive:0", "style:personachat:test:119:self:7:positive:1", "style:personachat:test:119:self:7:positive:2", "style:personachat:test:119:self:7:positive:3", "style:personachat:test:119:self:7:positive:4", "style:personachat:test:119:self:7:positive:5", "style:personachat:test:119:self:7:positive:6"], "evidence_texts": ["A group of young people performing on a stage.", "A boy is being splashed in a swimming pool.", "Two men are laying floor tiles in a doorway.", "A man sits at a table in a room.", "sure . are you male or female ?", "nope . a man . like a 40 yr old man so . . . . . . .", "unless you are under 18 , lol .", "what is your tattoo of ?", "is it on your lower back ? my daughter wants one , ugh !", "that was my next guess ! my ex wife has a similar one . you been married before ?", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:3174", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3174", "candidate_text": "I wonder whether two men are being lazy?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3174:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3126:premise", "snli:test:3382:premise", "snli:test:3174:premise", "snli:test:3110:premise", "style:personachat:test:464:self:7:positive:0", "style:personachat:test:464:self:7:positive:1", "style:personachat:test:464:self:7:positive:2", "style:personachat:test:464:self:7:positive:3", "style:personachat:test:464:self:7:positive:4", "style:personachat:test:464:self:7:positive:5", "style:personachat:test:464:self:7:positive:6"], "evidence_texts": ["A group of young people performing on a stage.", "A boy is being splashed in a swimming pool.", "Two men are laying floor tiles in a doorway.", "A man sits at a table in a room.", "i am doing very well , thank you . i am sorry to hear that .", "why would you do that ? i am taking my sister shopping tomorrow .", "i am thinking buckle and macys .", "i am a nurse , single and no kids . she gets my money , lol", "oh wow . that is a lot of husbands .", "that would be awful . i am never getting married .", "that sounds fun . do you dye their hair crazy colors ?"], "speaker_id": "snli:test:3174", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3178", "candidate_text": "well, the large audience is watching an award ceremony", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3178:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3011:premise", "snli:test:3236:premise", "snli:test:3178:premise", "snli:test:2018:premise", "style:personachat:test:120:self:1:positive:0"], "evidence_texts": ["A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "i do too , i want to travel ."], "speaker_id": "snli:test:3178", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3178", "candidate_text": "Honestly, The large audience is watching an award ceremony!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3178:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3011:premise", "snli:test:3236:premise", "snli:test:3178:premise", "snli:test:2018:premise", "style:personachat:test:836:self:7:positive:0", "style:personachat:test:836:self:7:positive:1", "style:personachat:test:836:self:7:positive:2", "style:personachat:test:836:self:7:positive:3", "style:personachat:test:836:self:7:positive:4", "style:personachat:test:836:self:7:positive:5", "style:personachat:test:836:self:7:positive:6"], "evidence_texts": ["A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A man spraying water onto a former piece of architecture that is being taken down by a piece of machinery.", "no did you ? i like the band simple plan", "i am a crabby person lol i have to work early", "i kind of have a crush on the bosses niece she is 25 years old", "i need to focus on a bond looks fade", "what is your favorite dish ?", "i love seafood . it is so good .", "goodness ! that sounds amazing ! i like caviar and oysters"], "speaker_id": "snli:test:3178", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3192", "candidate_text": "well, two men are in a car", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3192:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3192:premise", "snli:test:3041:premise", "snli:test:3867:premise", "snli:test:2740:premise", "style:personachat:test:120:self:2:positive:0", "style:personachat:test:120:self:2:positive:1"], "evidence_texts": ["A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "they do not make me do that .", "i do too , i want to travel ."], "speaker_id": "snli:test:3192", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3192", "candidate_text": "Two men are in a car.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3192:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3192:premise", "snli:test:3041:premise", "snli:test:3867:premise", "snli:test:2740:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A basketball player in white in red goes up for the shot as two defensive men in red jump up to block the shot.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3192", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3200", "candidate_text": "well, a tall person with glasses", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3200:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3200:premise", "snli:test:3113:premise", "snli:test:3355:premise", "snli:test:3078:premise", "style:personachat:test:120:self:5:positive:0", "style:personachat:test:120:self:5:positive:1", "style:personachat:test:120:self:5:positive:2", "style:personachat:test:120:self:5:positive:3", "style:personachat:test:120:self:5:positive:4"], "evidence_texts": ["A young man with glasses and his companion are attempting to do laundry in a public place.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "i have never ridden a horse , is it difficult ?", "i drive a forklift and i have to prop mine up as well .", "what else do you do for fun ?", "they do not make me do that .", "i do too , i want to travel ."], "speaker_id": "snli:test:3200", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3200", "candidate_text": "I wonder whether a tall person with glasses?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3200:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3200:premise", "snli:test:3113:premise", "snli:test:3355:premise", "snli:test:3078:premise", "style:personachat:test:468:self:5:positive:0", "style:personachat:test:468:self:5:positive:1", "style:personachat:test:468:self:5:positive:2", "style:personachat:test:468:self:5:positive:3", "style:personachat:test:468:self:5:positive:4"], "evidence_texts": ["A young man with glasses and his companion are attempting to do laundry in a public place.", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "A woman wearing a striped wrap is bent over in a cornfield while looking at the camera.", "i hope you have a wonderful time . please stay safe .", "i work from home and then pick my daughter up after school .", "i am a medical transcriptionist voice recognition editor .", "i am sure she would be proud . hard work being a nurse .", "i have 2 kids and i enjoy cooking . you ?"], "speaker_id": "snli:test:3200", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3217", "candidate_text": "well, some kids are watching soccer", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3217:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2221:premise", "snli:test:3217:premise", "snli:test:5023:premise", "snli:test:1716:premise", "style:personachat:test:120:self:7:positive:0", "style:personachat:test:120:self:7:positive:1", "style:personachat:test:120:self:7:positive:2", "style:personachat:test:120:self:7:positive:3", "style:personachat:test:120:self:7:positive:4", "style:personachat:test:120:self:7:positive:5", "style:personachat:test:120:self:7:positive:6"], "evidence_texts": ["several people are bike riding.", "Four African boys playing soccer.", "African woman walking through field.", "Band plays in tight quarters.", "i enjoy barbecue , and weight lifting .", "i am very tall , it is the only thing i can comfortably do .", "i have never ridden a horse , is it difficult ?", "i drive a forklift and i have to prop mine up as well .", "what else do you do for fun ?", "they do not make me do that .", "i do too , i want to travel ."], "speaker_id": "snli:test:3217", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3217", "candidate_text": "Honestly, Some kids are watching soccer!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3217:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:2221:premise", "snli:test:3217:premise", "snli:test:5023:premise", "snli:test:1716:premise", "style:personachat:test:842:self:5:positive:0", "style:personachat:test:842:self:5:positive:1", "style:personachat:test:842:self:5:positive:2", "style:personachat:test:842:self:5:positive:3", "style:personachat:test:842:self:5:positive:4"], "evidence_texts": ["several people are bike riding.", "Four African boys playing soccer.", "African woman walking through field.", "Band plays in tight quarters.", "why yes , we do speak english mostly . are there mermaids where you live ?", "i am just a student but i have heard some talk of engineers in our books", "we have alot of gross worms in the sea and pests that should be considered insects", "no bacon here but we have delicious lobster and crabcakes !", "i have heard about those in our human studies class !"], "speaker_id": "snli:test:3217", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3225", "candidate_text": "well, a woman is running in a park", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3225:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8613:premise", "snli:test:3225:premise", "snli:test:4210:premise", "snli:test:5142:premise", "style:personachat:test:121:self:1:positive:0"], "evidence_texts": ["A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:3225", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3225", "candidate_text": "A woman is running in a park.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3225:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8613:premise", "snli:test:3225:premise", "snli:test:4210:premise", "snli:test:5142:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3225", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3226", "candidate_text": "well, a woman is walking down a busy street in tokyo", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3226:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5142:premise", "snli:test:8613:premise", "snli:test:3226:premise", "snli:test:1832:premise", "style:personachat:test:121:self:2:positive:0", "style:personachat:test:121:self:2:positive:1"], "evidence_texts": ["A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "cool , like jazz ? i love it", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:3226", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3226", "candidate_text": "I wonder whether a woman is walking down a busy street in tokyo?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3226:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5142:premise", "snli:test:8613:premise", "snli:test:3226:premise", "snli:test:1832:premise", "style:personachat:test:469:self:2:positive:0", "style:personachat:test:469:self:2:positive:1"], "evidence_texts": ["A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "no allergies . my favorite movie is titanic , i watched it ten times .", "ginger is definitely do or die . are you renting apartments ?"], "speaker_id": "snli:test:3226", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3236", "candidate_text": "well, a girl drinks a soda while sleeping", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3236:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3236:premise", "snli:test:3178:premise", "snli:test:3333:premise", "snli:test:3011:premise", "style:personachat:test:121:self:5:positive:0", "style:personachat:test:121:self:5:positive:1", "style:personachat:test:121:self:5:positive:2", "style:personachat:test:121:self:5:positive:3", "style:personachat:test:121:self:5:positive:4"], "evidence_texts": ["One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "if no kids like me i have 2 with my husband go travel now", "i do too , i have not any this year", "i like to run or read an old cars", "cool , like jazz ? i love it", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:3236", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3236", "candidate_text": "Honestly, A girl drinks a soda while sleeping!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3236:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3236:premise", "snli:test:3178:premise", "snli:test:3333:premise", "snli:test:3011:premise", "style:personachat:test:856:self:2:positive:0", "style:personachat:test:856:self:2:positive:1"], "evidence_texts": ["One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A group of people sitting around a rectangular table having either pieces of paper or laptops in front of them.", "you must be busy ! can you still work out with 3 kids ?", "no fox news again ! my dad and brother would really dislike you"], "speaker_id": "snli:test:3236", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3244", "candidate_text": "well, the people are playing at a festival", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3244:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3244:premise", "snli:test:3015:premise", "snli:test:3271:premise", "snli:test:2984:premise", "style:personachat:test:121:self:7:positive:0", "style:personachat:test:121:self:7:positive:1", "style:personachat:test:121:self:7:positive:2", "style:personachat:test:121:self:7:positive:3", "style:personachat:test:121:self:7:positive:4", "style:personachat:test:121:self:7:positive:5", "style:personachat:test:121:self:7:positive:6"], "evidence_texts": ["Three people are dressed in costumes and playing musical instruments.", "An older man wearing stars and stripes rides a bike.", "Nurses in a medical setting conversing over a plastic cup.", "Two people are riding at once on one single bicycle.", "hi i am well trying to stay up you", "why ? you gotta get out . got kids ?", "if no kids like me i have 2 with my husband go travel now", "i do too , i have not any this year", "i like to run or read an old cars", "cool , like jazz ? i love it", "lol i hear you your dad is awesome"], "speaker_id": "snli:test:3244", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3244", "candidate_text": "The people are playing at a festival.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3244:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3244:premise", "snli:test:3015:premise", "snli:test:3271:premise", "snli:test:2984:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Three people are dressed in costumes and playing musical instruments.", "An older man wearing stars and stripes rides a bike.", "Nurses in a medical setting conversing over a plastic cup.", "Two people are riding at once on one single bicycle.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3244", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3247", "candidate_text": "well, people prepare to play games", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3247:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3035:premise", "snli:test:3302:premise", "snli:test:2609:premise", "snli:test:3247:premise", "style:personachat:test:122:self:1:positive:0"], "evidence_texts": ["Cook handling hot tray with food on it.", "A person obscured in shadow in a gymnasium.", "Man racing a quad on a dirt racetrack.", "Everyone is getting ready to play the game.", "i am single since i am super busy with school"], "speaker_id": "snli:test:3247", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3247", "candidate_text": "I wonder whether people prepare to play games?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3247:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3035:premise", "snli:test:3302:premise", "snli:test:2609:premise", "snli:test:3247:premise", "style:personachat:test:470:self:1:positive:0"], "evidence_texts": ["Cook handling hot tray with food on it.", "A person obscured in shadow in a gymnasium.", "Man racing a quad on a dirt racetrack.", "Everyone is getting ready to play the game.", "cool what kind of music do you like ?"], "speaker_id": "snli:test:3247", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3252", "candidate_text": "well, people are on a couch", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3252:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3252:premise", "snli:test:2690:premise", "snli:test:2465:premise", "snli:test:3399:premise", "style:personachat:test:122:self:2:positive:0", "style:personachat:test:122:self:2:positive:1"], "evidence_texts": ["A family smiling on the couch.", "Two individuals use a photo kiosk", "The children play in the pool.", "Two people sitting on the sand.", "very nice i just focus on my studies at this point", "i am single since i am super busy with school"], "speaker_id": "snli:test:3252", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3252", "candidate_text": "Honestly, People are on a couch!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3252:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3252:premise", "snli:test:2690:premise", "snli:test:2465:premise", "snli:test:3399:premise", "style:personachat:test:863:self:1:positive:0"], "evidence_texts": ["A family smiling on the couch.", "Two individuals use a photo kiosk", "The children play in the pool.", "Two people sitting on the sand.", "i bet ! i will end up with a zoo at my house !"], "speaker_id": "snli:test:3252", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3254", "candidate_text": "well, a family is sitting on a couch watching tv", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3254:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3399:premise", "snli:test:2690:premise", "snli:test:3254:premise", "snli:test:3518:premise", "style:personachat:test:122:self:5:positive:0", "style:personachat:test:122:self:5:positive:1", "style:personachat:test:122:self:5:positive:2", "style:personachat:test:122:self:5:positive:3", "style:personachat:test:122:self:5:positive:4"], "evidence_texts": ["Two people sitting on the sand.", "Two individuals use a photo kiosk", "A family smiling on the couch.", "A rally in a different country.", "about two more years how about you", "you have any brothers or sister ?", "no siblings i wish i had a brother or sister", "very nice i just focus on my studies at this point", "i am single since i am super busy with school"], "speaker_id": "snli:test:3254", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3254", "candidate_text": "A family is sitting on a couch watching TV.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3254:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3399:premise", "snli:test:2690:premise", "snli:test:3254:premise", "snli:test:3518:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two people sitting on the sand.", "Two individuals use a photo kiosk", "A family smiling on the couch.", "A rally in a different country.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3254", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3258", "candidate_text": "well, there is a man climbing as the boy holds the rope", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3258:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3151:premise", "snli:test:3430:premise", "snli:test:3258:premise", "snli:test:3074:premise", "style:personachat:test:122:self:7:positive:0", "style:personachat:test:122:self:7:positive:1", "style:personachat:test:122:self:7:positive:2", "style:personachat:test:122:self:7:positive:3", "style:personachat:test:122:self:7:positive:4", "style:personachat:test:122:self:7:positive:5", "style:personachat:test:122:self:7:positive:6"], "evidence_texts": ["A girl reaches up to kiss a cat, which is sitting on the counter.", "A man with his finger in his mouth is standing outside of an RV.", "A man holds a rope for a boy who's about to climb a wall.", "A local street in an Arabic country where there is a market selling Coca-Cola.", "i am well just studying for class", "i am working on my law degree", "about two more years how about you", "you have any brothers or sister ?", "no siblings i wish i had a brother or sister", "very nice i just focus on my studies at this point", "i am single since i am super busy with school"], "speaker_id": "snli:test:3258", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3258", "candidate_text": "I wonder whether there is a man climbing as the boy holds the rope?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3258:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3151:premise", "snli:test:3430:premise", "snli:test:3258:premise", "snli:test:3074:premise", "style:personachat:test:470:self:7:positive:0", "style:personachat:test:470:self:7:positive:1", "style:personachat:test:470:self:7:positive:2", "style:personachat:test:470:self:7:positive:3", "style:personachat:test:470:self:7:positive:4", "style:personachat:test:470:self:7:positive:5", "style:personachat:test:470:self:7:positive:6"], "evidence_texts": ["A girl reaches up to kiss a cat, which is sitting on the counter.", "A man with his finger in his mouth is standing outside of an RV.", "A man holds a rope for a boy who's about to climb a wall.", "A local street in an Arabic country where there is a market selling Coca-Cola.", "i am good i am eating pizza which is the best food", "i am also i have a big family 6 sisters and one brother", "i work for my moms restaurant part time what do you do ?", "cool i wish i could do that . i stutter when i get excited with things", "good idea . i like to read romances about paranormal things", "do you like to watch movies ?", "cool what kind of music do you like ?"], "speaker_id": "snli:test:3258", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3264", "candidate_text": "well, two girls at a camp", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3264:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3264:premise", "snli:test:2993:premise", "snli:test:3369:premise", "snli:test:2957:premise", "style:personachat:test:123:self:1:positive:0"], "evidence_texts": ["Two girls in red life jackets lock oars on a river.", "The people await to cross the street while walking the dog.", "People are waiting in line to board a double decker bus.", "A man wearing white turns to hit a yellow tennis ball.", "yeah i can understand that feeling"], "speaker_id": "snli:test:3264", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3264", "candidate_text": "Honestly, Two girls at a camp!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3264:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3264:premise", "snli:test:2993:premise", "snli:test:3369:premise", "snli:test:2957:premise", "style:personachat:test:863:self:7:positive:0", "style:personachat:test:863:self:7:positive:1", "style:personachat:test:863:self:7:positive:2", "style:personachat:test:863:self:7:positive:3", "style:personachat:test:863:self:7:positive:4", "style:personachat:test:863:self:7:positive:5", "style:personachat:test:863:self:7:positive:6"], "evidence_texts": ["Two girls in red life jackets lock oars on a river.", "The people await to cross the street while walking the dog.", "People are waiting in line to board a double decker bus.", "A man wearing white turns to hit a yellow tennis ball.", "hello . i am retired , so i am great ! i was a banker . how are you ?", "i love that show too ! i make quilts for relatives and sew while i watch it .", "me neither . i enjoy walking . i try to do at least a mile a day .", "i love sweet things . do you have any pets ? i have a cat a siamese .", "that is too cute . i absolutely love animals .", "that is awesome of you . i might start doing that now that i am retired .", "i bet ! i will end up with a zoo at my house !"], "speaker_id": "snli:test:3264", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3271", "candidate_text": "well, the nurses are eating lunch", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3271:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3244:premise", "snli:test:3289:premise", "snli:test:3015:premise", "snli:test:3271:premise", "style:personachat:test:123:self:2:positive:0", "style:personachat:test:123:self:2:positive:1"], "evidence_texts": ["Three people are dressed in costumes and playing musical instruments.", "Police are setting up a barricade due to a fire.", "An older man wearing stars and stripes rides a bike.", "Nurses in a medical setting conversing over a plastic cup.", "they stole them from you ?", "yeah i can understand that feeling"], "speaker_id": "snli:test:3271", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3271", "candidate_text": "The nurses are eating lunch.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3271:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3244:premise", "snli:test:3289:premise", "snli:test:3015:premise", "snli:test:3271:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three people are dressed in costumes and playing musical instruments.", "Police are setting up a barricade due to a fire.", "An older man wearing stars and stripes rides a bike.", "Nurses in a medical setting conversing over a plastic cup.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3271", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3276", "candidate_text": "well, people are walking to an event", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3276:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3276:premise", "snli:test:3106:premise", "snli:test:3446:premise", "snli:test:2886:premise", "style:personachat:test:123:self:5:positive:0", "style:personachat:test:123:self:5:positive:1", "style:personachat:test:123:self:5:positive:2", "style:personachat:test:123:self:5:positive:3", "style:personachat:test:123:self:5:positive:4"], "evidence_texts": ["People in motion down a city sidewalk.", "Women in blue dress singing on stage.", "A man laughing while at a restaurant.", "Two young musicians play from a stage.", "i run fast that is about it", "i am single . i dropped out of school", "i live in terrible apartment complex", "they stole them from you ?", "yeah i can understand that feeling"], "speaker_id": "snli:test:3276", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3276", "candidate_text": "I wonder whether people are walking to an event?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3276:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3276:premise", "snli:test:3106:premise", "snli:test:3446:premise", "snli:test:2886:premise", "style:personachat:test:476:self:5:positive:0", "style:personachat:test:476:self:5:positive:1", "style:personachat:test:476:self:5:positive:2", "style:personachat:test:476:self:5:positive:3", "style:personachat:test:476:self:5:positive:4"], "evidence_texts": ["People in motion down a city sidewalk.", "Women in blue dress singing on stage.", "A man laughing while at a restaurant.", "Two young musicians play from a stage.", "i love music ; reggae is my favorite genre .", "you must love animals , being a veterinarian !", "that is interesting . favorite animal to work with ?", "what is your favorite video game ?", "do you have a favorite hobby ?"], "speaker_id": "snli:test:3276", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3287", "candidate_text": "well, women look different directions", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3287:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3090:premise", "snli:test:3338:premise", "snli:test:2742:premise", "snli:test:3287:premise", "style:personachat:test:123:self:7:positive:0", "style:personachat:test:123:self:7:positive:1", "style:personachat:test:123:self:7:positive:2", "style:personachat:test:123:self:7:positive:3", "style:personachat:test:123:self:7:positive:4", "style:personachat:test:123:self:7:positive:5", "style:personachat:test:123:self:7:positive:6"], "evidence_texts": ["A band singing and playing electric guitar for a crowd of people.", "The child is hanging upside down with his legs over a pole.", "Two people are sitting on a rock looking off into the distance.", "One man in baseball cap looks out while two men look down.", "pretty good thanks and you ?", "i have lots of dreams but work at mcdonalds", "i run fast that is about it", "i am single . i dropped out of school", "i live in terrible apartment complex", "they stole them from you ?", "yeah i can understand that feeling"], "speaker_id": "snli:test:3287", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3287", "candidate_text": "Honestly, Women look different directions!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3287:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3090:premise", "snli:test:3338:premise", "snli:test:2742:premise", "snli:test:3287:premise", "style:personachat:test:870:self:5:positive:0", "style:personachat:test:870:self:5:positive:1", "style:personachat:test:870:self:5:positive:2", "style:personachat:test:870:self:5:positive:3", "style:personachat:test:870:self:5:positive:4"], "evidence_texts": ["A band singing and playing electric guitar for a crowd of people.", "The child is hanging upside down with his legs over a pole.", "Two people are sitting on a rock looking off into the distance.", "One man in baseball cap looks out while two men look down.", "oh okay . i run and take long walks for exercises . are you dieting ?", "no but i wish i did . i own a dieting business that is why i asked . you sing ?", "get out ! me too ! grace and kelly are my cats . whats your fave car ?", "thank you . john john is my dog . she help me exercise . you act ?", "bbq ! you would be a great voiceover actor . i write cookbooks for dieters ."], "speaker_id": "snli:test:3287", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3289", "candidate_text": "well, people in uniform are working near a fire", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3289:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3271:premise", "snli:test:3289:premise", "snli:test:3315:premise", "snli:test:3244:premise", "style:personachat:test:124:self:1:positive:0"], "evidence_texts": ["Nurses in a medical setting conversing over a plastic cup.", "Police are setting up a barricade due to a fire.", "A woman is walking her dog and using her cellphone.", "Three people are dressed in costumes and playing musical instruments.", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:3289", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3289", "candidate_text": "People in uniform are working near a fire.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3289:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3271:premise", "snli:test:3289:premise", "snli:test:3315:premise", "snli:test:3244:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Nurses in a medical setting conversing over a plastic cup.", "Police are setting up a barricade due to a fire.", "A woman is walking her dog and using her cellphone.", "Three people are dressed in costumes and playing musical instruments.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3289", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3292", "candidate_text": "well, a man waterskiing in a river with a large wall in the background", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3292:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3292:premise", "snli:test:3163:premise", "snli:test:3829:premise", "snli:test:3149:premise", "style:personachat:test:124:self:2:positive:0", "style:personachat:test:124:self:2:positive:1"], "evidence_texts": ["A person waterskiing in a river with a large wall in the background.", "A couple look into the camera as the foreground is filled with balloons.", "A man getting a group of children to move around and have fun.", "A man in a suit cheering in a library in front of children.", "oh okay lol what do you do for work ?", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:3292", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3292", "candidate_text": "I wonder whether a man waterskiing in a river with a large wall in the background?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3292:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3292:premise", "snli:test:3163:premise", "snli:test:3829:premise", "snli:test:3149:premise", "style:personachat:test:481:self:2:positive:0", "style:personachat:test:481:self:2:positive:1"], "evidence_texts": ["A person waterskiing in a river with a large wall in the background.", "A couple look into the camera as the foreground is filled with balloons.", "A man getting a group of children to move around and have fun.", "A man in a suit cheering in a library in front of children.", "do you have any hobbies ? i love to hike .", "do you like to read ? i read a whole ton of poetry ."], "speaker_id": "snli:test:3292", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3302", "candidate_text": "well, the person is a vampire avoiding the sun", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3302:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3247:premise", "snli:test:3310:premise", "snli:test:3302:premise", "snli:test:3035:premise", "style:personachat:test:124:self:5:positive:0", "style:personachat:test:124:self:5:positive:1", "style:personachat:test:124:self:5:positive:2", "style:personachat:test:124:self:5:positive:3", "style:personachat:test:124:self:5:positive:4"], "evidence_texts": ["Everyone is getting ready to play the game.", "A smiling man is holding up a gavel.", "A person obscured in shadow in a gymnasium.", "Cook handling hot tray with food on it.", "that is good where are you from ?", "ca nice i never been hiking before i live in upstate ny", "yes ny has lots of interesting foods what is your favorite type of food ?", "oh okay lol what do you do for work ?", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:3302", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3302", "candidate_text": "Honestly, The person is a vampire avoiding the sun!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3302:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3247:premise", "snli:test:3310:premise", "snli:test:3302:premise", "snli:test:3035:premise", "style:personachat:test:885:self:2:positive:0", "style:personachat:test:885:self:2:positive:1"], "evidence_texts": ["Everyone is getting ready to play the game.", "A smiling man is holding up a gavel.", "A person obscured in shadow in a gymnasium.", "Cook handling hot tray with food on it.", "wow i would love to see that collection ! do you drive them ?", "oh my that is a very early start to a life of crime !"], "speaker_id": "snli:test:3302", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3310", "candidate_text": "well, the man is a judge", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3310:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3302:premise", "snli:test:3344:premise", "snli:test:3310:premise", "snli:test:3247:premise", "style:personachat:test:124:self:6:positive:0", "style:personachat:test:124:self:6:positive:1", "style:personachat:test:124:self:6:positive:2", "style:personachat:test:124:self:6:positive:3", "style:personachat:test:124:self:6:positive:4", "style:personachat:test:124:self:6:positive:5"], "evidence_texts": ["A person obscured in shadow in a gymnasium.", "Two men in polo shirts, reading the paper.", "A smiling man is holding up a gavel.", "Everyone is getting ready to play the game.", "i am okay how are you today ?", "that is good where are you from ?", "ca nice i never been hiking before i live in upstate ny", "yes ny has lots of interesting foods what is your favorite type of food ?", "oh okay lol what do you do for work ?", "i love the scenery the buildings are so tall"], "speaker_id": "snli:test:3310", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3310", "candidate_text": "The man is a judge.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3310:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3302:premise", "snli:test:3344:premise", "snli:test:3310:premise", "snli:test:3247:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A person obscured in shadow in a gymnasium.", "Two men in polo shirts, reading the paper.", "A smiling man is holding up a gavel.", "Everyone is getting ready to play the game.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3310", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3315", "candidate_text": "I wonder whether the woman is using her cellphone?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3315:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3289:premise", "snli:test:3271:premise", "snli:test:3531:premise", "snli:test:3315:premise", "style:personachat:test:125:self:1:positive:0"], "evidence_texts": ["Police are setting up a barricade due to a fire.", "Nurses in a medical setting conversing over a plastic cup.", "A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:3315", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3315", "candidate_text": "The woman is using her cellphone.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3315:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3289:premise", "snli:test:3271:premise", "snli:test:3531:premise", "snli:test:3315:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Police are setting up a barricade due to a fire.", "Nurses in a medical setting conversing over a plastic cup.", "A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3315", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3317", "candidate_text": "I wonder whether the woman is walking her cat?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3317:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3531:premise", "snli:test:3317:premise", "snli:test:3289:premise", "snli:test:3533:premise", "style:personachat:test:125:self:2:positive:0", "style:personachat:test:125:self:2:positive:1"], "evidence_texts": ["A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "Police are setting up a barricade due to a fire.", "A man wearing a welding mask working on a duct.", "wow someone who helps others like me and vegan amazing", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:3317", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3317", "candidate_text": "well, the woman is walking her cat", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3317:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3531:premise", "snli:test:3317:premise", "snli:test:3289:premise", "snli:test:3533:premise", "style:personachat:test:221:self:2:positive:0", "style:personachat:test:221:self:2:positive:1"], "evidence_texts": ["A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "Police are setting up a barricade due to a fire.", "A man wearing a welding mask working on a duct.", "you dislike people that much . do you have children ?", "yes , but they are all grown now ."], "speaker_id": "snli:test:3317", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3324", "candidate_text": "I wonder whether people are gathered on a sunny day at the beach?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3324:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3324:premise", "snli:test:2972:premise", "snli:test:3409:premise", "snli:test:2931:premise", "style:personachat:test:125:self:5:positive:0", "style:personachat:test:125:self:5:positive:1", "style:personachat:test:125:self:5:positive:2", "style:personachat:test:125:self:5:positive:3", "style:personachat:test:125:self:5:positive:4"], "evidence_texts": ["Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "its okay i like to knit hats for babies in nicu", "wow have you discovered any new favorite food i love cooking", "that sounds right up my alley being vegan and all", "wow someone who helps others like me and vegan amazing", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:3324", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3324", "candidate_text": "Honestly, People are gathered on a sunny day at the beach!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3324:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3324:premise", "snli:test:2972:premise", "snli:test:3409:premise", "snli:test:2931:premise", "style:personachat:test:892:self:2:positive:0", "style:personachat:test:892:self:2:positive:1"], "evidence_texts": ["Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A team of men, three in red shirts and two others in baseball jerseys, re-surface a baseball diamond with specialized tools.", "a dog and 2 cute kitties how about you ?", "like the show on nickelodeon !"], "speaker_id": "snli:test:3324", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3333", "candidate_text": "I wonder whether the people are women?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3333:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3236:premise", "snli:test:3333:premise", "snli:test:3178:premise", "snli:test:3442:premise", "style:personachat:test:125:self:6:positive:0", "style:personachat:test:125:self:6:positive:1", "style:personachat:test:125:self:6:positive:2", "style:personachat:test:125:self:6:positive:3", "style:personachat:test:125:self:6:positive:4", "style:personachat:test:125:self:6:positive:5"], "evidence_texts": ["One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "hello i am fine i cannot walk i am disabled", "its okay i like to knit hats for babies in nicu", "wow have you discovered any new favorite food i love cooking", "that sounds right up my alley being vegan and all", "wow someone who helps others like me and vegan amazing", "for me it was the thought of ending a life for greed . . you ?"], "speaker_id": "snli:test:3333", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3333", "candidate_text": "the people are women.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3333:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3236:premise", "snli:test:3333:premise", "snli:test:3178:premise", "snli:test:3442:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A large audience of people seated at banquet tables is watching the happenings on a stage surrounded by blue curtains.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3333", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3335", "candidate_text": "well, the people are running through fire", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3335:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3442:premise", "snli:test:3236:premise", "snli:test:3534:premise", "snli:test:3335:premise", "style:personachat:test:126:self:1:positive:0"], "evidence_texts": ["A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "that sucks . i am currently unemployed so i cannot relate ."], "speaker_id": "snli:test:3335", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3335", "candidate_text": "Honestly, the people are running through fire!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3335:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3442:premise", "snli:test:3236:premise", "snli:test:3534:premise", "snli:test:3335:premise", "style:personachat:test:892:self:7:positive:0", "style:personachat:test:892:self:7:positive:1", "style:personachat:test:892:self:7:positive:2", "style:personachat:test:892:self:7:positive:3", "style:personachat:test:892:self:7:positive:4", "style:personachat:test:892:self:7:positive:5", "style:personachat:test:892:self:7:positive:6"], "evidence_texts": ["A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "One girl sips a soda while another looks on, standing on a street in front of a bunch of bicycles.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "i am not very crafty but i like to write", "wow . i like to work out too . we can be friends", "i have all these yummy healthy recipes . i wrote a cookbook", "i like to read . plays mostly", "it is a good way to unwind .", "a dog and 2 cute kitties how about you ?", "like the show on nickelodeon !"], "speaker_id": "snli:test:3335", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3338", "candidate_text": "well, there is an adult upside down", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3338:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3287:premise", "snli:test:3351:premise", "snli:test:3338:premise", "snli:test:3090:premise", "style:personachat:test:126:self:2:positive:0", "style:personachat:test:126:self:2:positive:1"], "evidence_texts": ["One man in baseball cap looks out while two men look down.", "A man with a bat swings at a ball coming towards him.", "The child is hanging upside down with his legs over a pole.", "A band singing and playing electric guitar for a crowd of people.", "i guess so , are you not allowed to be creative in your work ?", "that sucks . i am currently unemployed so i cannot relate ."], "speaker_id": "snli:test:3338", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3338", "candidate_text": "There is an adult upside down.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3338:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3287:premise", "snli:test:3351:premise", "snli:test:3338:premise", "snli:test:3090:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["One man in baseball cap looks out while two men look down.", "A man with a bat swings at a ball coming towards him.", "The child is hanging upside down with his legs over a pole.", "A band singing and playing electric guitar for a crowd of people.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3338", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3344", "candidate_text": "well, two men read the paper", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3344:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3344:premise", "snli:test:3310:premise", "snli:test:3346:premise", "snli:test:3302:premise", "style:personachat:test:126:self:5:positive:0", "style:personachat:test:126:self:5:positive:1", "style:personachat:test:126:self:5:positive:2", "style:personachat:test:126:self:5:positive:3", "style:personachat:test:126:self:5:positive:4"], "evidence_texts": ["Two men in polo shirts, reading the paper.", "A smiling man is holding up a gavel.", "The 3 dogs are cruising down the street.", "A person obscured in shadow in a gymnasium.", "i am terrible at poker because i do not like to sit still .", "how long have you been a carpenter ?", "why not , it seems like a very reputable thing .", "i guess so , are you not allowed to be creative in your work ?", "that sucks . i am currently unemployed so i cannot relate ."], "speaker_id": "snli:test:3344", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3344", "candidate_text": "I wonder whether two men read the paper?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3344:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3344:premise", "snli:test:3310:premise", "snli:test:3346:premise", "snli:test:3302:premise", "style:personachat:test:485:self:5:positive:0", "style:personachat:test:485:self:5:positive:1", "style:personachat:test:485:self:5:positive:2", "style:personachat:test:485:self:5:positive:3", "style:personachat:test:485:self:5:positive:4"], "evidence_texts": ["Two men in polo shirts, reading the paper.", "A smiling man is holding up a gavel.", "The 3 dogs are cruising down the street.", "A person obscured in shadow in a gymnasium.", "yeah continue it , do you have any siblings ? i have a brother", "what does your mom do ? mines a secretary", "nice , my profession is a doctor , i enjoy it apart from body building", "of course , do you help people ?", "your in school ? wow good for you"], "speaker_id": "snli:test:3344", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3346", "candidate_text": "well, three dogs together", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3346:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3344:premise", "snli:test:3439:premise", "snli:test:3346:premise", "snli:test:3310:premise", "style:personachat:test:126:self:7:positive:0", "style:personachat:test:126:self:7:positive:1", "style:personachat:test:126:self:7:positive:2", "style:personachat:test:126:self:7:positive:3", "style:personachat:test:126:self:7:positive:4", "style:personachat:test:126:self:7:positive:5", "style:personachat:test:126:self:7:positive:6"], "evidence_texts": ["Two men in polo shirts, reading the paper.", "Two little boys are standing in a kitchen.", "The 3 dogs are cruising down the street.", "A smiling man is holding up a gavel.", "that is okay , i am recently divorces so i am lonely .", "i used to drink to get away from my wife , she caused problems .", "i am terrible at poker because i do not like to sit still .", "how long have you been a carpenter ?", "why not , it seems like a very reputable thing .", "i guess so , are you not allowed to be creative in your work ?", "that sucks . i am currently unemployed so i cannot relate ."], "speaker_id": "snli:test:3346", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3346", "candidate_text": "Honestly, three dogs together!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3346:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3344:premise", "snli:test:3439:premise", "snli:test:3346:premise", "snli:test:3310:premise", "style:personachat:test:921:self:5:positive:0", "style:personachat:test:921:self:5:positive:1", "style:personachat:test:921:self:5:positive:2", "style:personachat:test:921:self:5:positive:3", "style:personachat:test:921:self:5:positive:4"], "evidence_texts": ["Two men in polo shirts, reading the paper.", "Two little boys are standing in a kitchen.", "The 3 dogs are cruising down the street.", "A smiling man is holding up a gavel.", "ah . how about pets ? i have 3 one cat and two dogs .", "i love animals . i also love people and caring for them .", "i work in a nursing home . i am a nurses aide .", "steak and mashed potatoes . do you ride a bike ? i love riding mine .", "really ? you are missing out for sure !"], "speaker_id": "snli:test:3346", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3351", "candidate_text": "Honestly, The bat is aluminum!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3351:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3338:premise", "snli:test:3287:premise", "snli:test:3542:premise", "snli:test:3351:premise", "style:personachat:test:127:self:1:positive:0"], "evidence_texts": ["The child is hanging upside down with his legs over a pole.", "One man in baseball cap looks out while two men look down.", "A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:3351", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3351", "candidate_text": "The bat is aluminum.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3351:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3338:premise", "snli:test:3287:premise", "snli:test:3542:premise", "snli:test:3351:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The child is hanging upside down with his legs over a pole.", "One man in baseball cap looks out while two men look down.", "A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3351", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3353", "candidate_text": "Honestly, The man is swinging!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3353:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3542:premise", "snli:test:3353:premise", "snli:test:3338:premise", "snli:test:3712:premise", "style:personachat:test:127:self:2:positive:0", "style:personachat:test:127:self:2:positive:1"], "evidence_texts": ["A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "The child is hanging upside down with his legs over a pole.", "A man in a bright yellow shirt juggles while riding a unicycle.", "i was homeschooled , that is pretty boring too", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:3353", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3353", "candidate_text": "I wonder whether the man is swinging?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3353:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3542:premise", "snli:test:3353:premise", "snli:test:3338:premise", "snli:test:3712:premise", "style:personachat:test:486:self:2:positive:0", "style:personachat:test:486:self:2:positive:1"], "evidence_texts": ["A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "The child is hanging upside down with his legs over a pole.", "A man in a bright yellow shirt juggles while riding a unicycle.", "yes i have . i have dyed my hair many colors .", "i live in america . have you ever been here ?"], "speaker_id": "snli:test:3353", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3355", "candidate_text": "Honestly, A man is wearing a white sport jersey in his cubicle!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3355:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3200:premise", "snli:test:3631:premise", "snli:test:3113:premise", "snli:test:3355:premise", "style:personachat:test:127:self:5:positive:0", "style:personachat:test:127:self:5:positive:1", "style:personachat:test:127:self:5:positive:2", "style:personachat:test:127:self:5:positive:3", "style:personachat:test:127:self:5:positive:4"], "evidence_texts": ["A young man with glasses and his companion are attempting to do laundry in a public place.", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "i live out here in california", "i love the water , it so much fun to go swimming in the ocean", "that is kind of boring , but to each their own !", "i was homeschooled , that is pretty boring too", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:3355", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3355", "candidate_text": "well, a man is wearing a white sport jersey in his cubicle", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3355:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3200:premise", "snli:test:3631:premise", "snli:test:3113:premise", "snli:test:3355:premise", "style:personachat:test:225:self:5:positive:0", "style:personachat:test:225:self:5:positive:1", "style:personachat:test:225:self:5:positive:2", "style:personachat:test:225:self:5:positive:3", "style:personachat:test:225:self:5:positive:4"], "evidence_texts": ["A young man with glasses and his companion are attempting to do laundry in a public place.", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "A young man in a backwards baseball cap balances his skateboard on the edge of the concrete.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "the snow makes it hard to walk the dogs , do you have pets ?", "that is always hard . i love pets . besides my 2 dogs there are 3 cats", "that sounds great . my parents both taught science", "my parents would love you . i think i might try for a teaching degree", "i bet that is not true . who would not want their child to be scientist"], "speaker_id": "snli:test:3355", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3359", "candidate_text": "Honestly, The man in jeans is standing on a bridge working on a outside while listening to music!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3359:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3146:premise", "snli:test:4074:premise", "snli:test:2877:premise", "snli:test:3359:premise", "style:personachat:test:127:self:7:positive:0", "style:personachat:test:127:self:7:positive:1", "style:personachat:test:127:self:7:positive:2", "style:personachat:test:127:self:7:positive:3", "style:personachat:test:127:self:7:positive:4", "style:personachat:test:127:self:7:positive:5", "style:personachat:test:127:self:7:positive:6"], "evidence_texts": ["A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "i am not really into sports . i like robots though !", "that is cool , where do you live ?", "i live out here in california", "i love the water , it so much fun to go swimming in the ocean", "that is kind of boring , but to each their own !", "i was homeschooled , that is pretty boring too", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:3359", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3359", "candidate_text": "The man in jeans is standing on a bridge working on a outside while listening to music.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3359:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3146:premise", "snli:test:4074:premise", "snli:test:2877:premise", "snli:test:3359:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3359", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3367", "candidate_text": "I wonder whether friends wait to get on the bus at night?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3367:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3143:premise", "snli:test:3474:premise", "snli:test:3141:premise", "snli:test:3367:premise", "style:personachat:test:128:self:1:positive:0"], "evidence_texts": ["A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "a different you ? like games ? i like xbox games"], "speaker_id": "snli:test:3367", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3367", "candidate_text": "Friends wait to get on the bus at night.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3367:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3143:premise", "snli:test:3474:premise", "snli:test:3141:premise", "snli:test:3367:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3367", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3369", "candidate_text": "I wonder whether people are in line?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3369:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3264:premise", "snli:test:3369:premise", "snli:test:2993:premise", "snli:test:3373:premise", "style:personachat:test:128:self:2:positive:0", "style:personachat:test:128:self:2:positive:1"], "evidence_texts": ["Two girls in red life jackets lock oars on a river.", "People are waiting in line to board a double decker bus.", "The people await to cross the street while walking the dog.", "An Eastern vendor sits by his stall with a rooster nearby.", "that is cool i stay in florida but from cali", "a different you ? like games ? i like xbox games"], "speaker_id": "snli:test:3369", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3369", "candidate_text": "well, people are in line", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3369:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3264:premise", "snli:test:3369:premise", "snli:test:2993:premise", "snli:test:3373:premise", "style:personachat:test:228:self:2:positive:0", "style:personachat:test:228:self:2:positive:1"], "evidence_texts": ["Two girls in red life jackets lock oars on a river.", "People are waiting in line to board a double decker bus.", "The people await to cross the street while walking the dog.", "An Eastern vendor sits by his stall with a rooster nearby.", "do you like the office ? it is my favorite show .", "you should look it up because it is great ."], "speaker_id": "snli:test:3369", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3371", "candidate_text": "I wonder whether german tourists are waiting in line to board a bus?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3371:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3373:premise", "snli:test:3264:premise", "snli:test:3510:premise", "snli:test:3371:premise", "style:personachat:test:128:self:5:positive:0", "style:personachat:test:128:self:5:positive:1", "style:personachat:test:128:self:5:positive:2", "style:personachat:test:128:self:5:positive:3", "style:personachat:test:128:self:5:positive:4"], "evidence_texts": ["An Eastern vendor sits by his stall with a rooster nearby.", "Two girls in red life jackets lock oars on a river.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "People are waiting in line to board a double decker bus.", "oh i got it now like animals ? or have any ?", "cool , i am a vet , i love live music on the weekend", "really ? i do also an i read", "that is cool i stay in florida but from cali", "a different you ? like games ? i like xbox games"], "speaker_id": "snli:test:3371", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3371", "candidate_text": "Honestly, German tourists are waiting in line to board a bus!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3371:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3373:premise", "snli:test:3264:premise", "snli:test:3510:premise", "snli:test:3371:premise", "style:personachat:test:932:self:2:positive:0", "style:personachat:test:932:self:2:positive:1"], "evidence_texts": ["An Eastern vendor sits by his stall with a rooster nearby.", "Two girls in red life jackets lock oars on a river.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "People are waiting in line to board a double decker bus.", "nice ! i try to cook at home as much as possible !", "well i just had a baby so everything is currently about her !"], "speaker_id": "snli:test:3371", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3373", "candidate_text": "I wonder whether a rooster on the farm?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3373:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3371:premise", "snli:test:3373:premise", "snli:test:3510:premise", "snli:test:3369:premise", "style:personachat:test:128:self:7:positive:0", "style:personachat:test:128:self:7:positive:1", "style:personachat:test:128:self:7:positive:2", "style:personachat:test:128:self:7:positive:3", "style:personachat:test:128:self:7:positive:4", "style:personachat:test:128:self:7:positive:5", "style:personachat:test:128:self:7:positive:6"], "evidence_texts": ["People are waiting in line to board a double decker bus.", "An Eastern vendor sits by his stall with a rooster nearby.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "People are waiting in line to board a double decker bus.", "good evening , i swear i am exhausted", "in the crips ? like books ? my name jordan by the way also", "oh i got it now like animals ? or have any ?", "cool , i am a vet , i love live music on the weekend", "really ? i do also an i read", "that is cool i stay in florida but from cali", "a different you ? like games ? i like xbox games"], "speaker_id": "snli:test:3373", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3373", "candidate_text": "a rooster on the farm.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3373:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3371:premise", "snli:test:3373:premise", "snli:test:3510:premise", "snli:test:3369:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["People are waiting in line to board a double decker bus.", "An Eastern vendor sits by his stall with a rooster nearby.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "People are waiting in line to board a double decker bus.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3373", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3382", "candidate_text": "well, a boy is playing in his new swimming pool", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3382:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3174:premise", "snli:test:3126:premise", "snli:test:3382:premise", "snli:test:3738:premise", "style:personachat:test:129:self:1:positive:0"], "evidence_texts": ["Two men are laying floor tiles in a doorway.", "A group of young people performing on a stage.", "A boy is being splashed in a swimming pool.", "The man is taking a picture of the beach.", "both , i use to want to be a chef"], "speaker_id": "snli:test:3382", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3382", "candidate_text": "Honestly, A boy is playing in his new swimming pool!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3382:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3174:premise", "snli:test:3126:premise", "snli:test:3382:premise", "snli:test:3738:premise", "style:personachat:test:932:self:7:positive:0", "style:personachat:test:932:self:7:positive:1", "style:personachat:test:932:self:7:positive:2", "style:personachat:test:932:self:7:positive:3", "style:personachat:test:932:self:7:positive:4", "style:personachat:test:932:self:7:positive:5", "style:personachat:test:932:self:7:positive:6"], "evidence_texts": ["Two men are laying floor tiles in a doorway.", "A group of young people performing on a stage.", "A boy is being splashed in a swimming pool.", "The man is taking a picture of the beach.", "i love animals too ! i have 1 dog . i also love music . who is your favorite band ?", "not so much . . i am still in school so i have to read for grades", "same here . . do you have any siblings ? i only have a brother", "aww that is too bad ! whats your business called ?", "its hard to keep up . i am 5ft and 110 pounds so i think i am doing okay", "nice ! i try to cook at home as much as possible !", "well i just had a baby so everything is currently about her !"], "speaker_id": "snli:test:3382", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3383", "candidate_text": "well, a boy is splashing others at the beach", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3383:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3738:premise", "snli:test:3174:premise", "snli:test:3740:premise", "snli:test:3383:premise", "style:personachat:test:129:self:2:positive:0", "style:personachat:test:129:self:2:positive:1"], "evidence_texts": ["The man is taking a picture of the beach.", "Two men are laying floor tiles in a doorway.", "The man is taking a picture of the beach.", "A boy is being splashed in a swimming pool.", "one of my favorite passions is food ! if it looks good i will try it at least once", "both , i use to want to be a chef"], "speaker_id": "snli:test:3383", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3383", "candidate_text": "A boy is splashing others at the beach.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3383:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3738:premise", "snli:test:3174:premise", "snli:test:3740:premise", "snli:test:3383:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The man is taking a picture of the beach.", "Two men are laying floor tiles in a doorway.", "The man is taking a picture of the beach.", "A boy is being splashed in a swimming pool.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3383", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3395", "candidate_text": "well, two people are in the same area", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3395:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3122:premise", "snli:test:4578:premise", "snli:test:3119:premise", "snli:test:3395:premise", "style:personachat:test:129:self:5:positive:0", "style:personachat:test:129:self:5:positive:1", "style:personachat:test:129:self:5:positive:2", "style:personachat:test:129:self:5:positive:3", "style:personachat:test:129:self:5:positive:4"], "evidence_texts": ["A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "i bet what are some of your hobbies ?", "i do too , one of my hobbies is to collect odd stickers", "same here ! my sister and i read each other our poetry", "one of my favorite passions is food ! if it looks good i will try it at least once", "both , i use to want to be a chef"], "speaker_id": "snli:test:3395", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3395", "candidate_text": "I wonder whether two people are in the same area?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3395:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3122:premise", "snli:test:4578:premise", "snli:test:3119:premise", "snli:test:3395:premise", "style:personachat:test:496:self:5:positive:0", "style:personachat:test:496:self:5:positive:1", "style:personachat:test:496:self:5:positive:2", "style:personachat:test:496:self:5:positive:3", "style:personachat:test:496:self:5:positive:4"], "evidence_texts": ["A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man in gray and black is playing a black acoustic guitar with a slightly shorter woman who is wearing pink and playing a violin.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "why so tired ? do you have a hobby ? i hunt and love restoring cars , especially classic .", "i understand . do you like seafood ? i do except shellfish , i break out in hives .", "no ! i hate them !", "well because they scare me i guess .", "do you have any pets ?"], "speaker_id": "snli:test:3395", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3399", "candidate_text": "well, the people are standing", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3399:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3254:premise", "snli:test:3518:premise", "snli:test:3252:premise", "snli:test:3399:premise", "style:personachat:test:129:self:7:positive:0", "style:personachat:test:129:self:7:positive:1", "style:personachat:test:129:self:7:positive:2", "style:personachat:test:129:self:7:positive:3", "style:personachat:test:129:self:7:positive:4", "style:personachat:test:129:self:7:positive:5", "style:personachat:test:129:self:7:positive:6"], "evidence_texts": ["A family smiling on the couch.", "A rally in a different country.", "A family smiling on the couch.", "Two people sitting on the sand.", "i am very good thank you , tell me about yourself ?", "thanks cool , i am park ranger i hike in nature for my job", "i bet what are some of your hobbies ?", "i do too , one of my hobbies is to collect odd stickers", "same here ! my sister and i read each other our poetry", "one of my favorite passions is food ! if it looks good i will try it at least once", "both , i use to want to be a chef"], "speaker_id": "snli:test:3399", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3399", "candidate_text": "Honestly, The people are standing!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3399:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3254:premise", "snli:test:3518:premise", "snli:test:3252:premise", "snli:test:3399:premise", "style:personachat:test:933:self:5:positive:0", "style:personachat:test:933:self:5:positive:1", "style:personachat:test:933:self:5:positive:2", "style:personachat:test:933:self:5:positive:3", "style:personachat:test:933:self:5:positive:4"], "evidence_texts": ["A family smiling on the couch.", "A rally in a different country.", "A family smiling on the couch.", "Two people sitting on the sand.", "why are you worried about the world ?", "yeah , humans have been rough on our plant . do you have any other hobbies ?", "do you knit any of your clothes ? i am a knitter .", "i often get funny looks when people see a 40 year old man knitting !", "your mom must be very brave !"], "speaker_id": "snli:test:3399", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3409", "candidate_text": "Honestly, Six men dressed in black and yellow uniforms are standing in a gymnasium!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3409:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3324:premise", "snli:test:3409:premise", "snli:test:3477:premise", "snli:test:2972:premise", "style:personachat:test:130:self:1:positive:0"], "evidence_texts": ["Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:3409", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3409", "candidate_text": "Six men dressed in black and yellow uniforms are standing in a gymnasium.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3409:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3324:premise", "snli:test:3409:premise", "snli:test:3477:premise", "snli:test:2972:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "It is a march of people of different backgrounds all walking and one guy in a green shirt holds a flag.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3409", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3430", "candidate_text": "Honestly, The man is standing!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3430:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3258:premise", "snli:test:3151:premise", "snli:test:3430:premise", "snli:test:3619:premise", "style:personachat:test:130:self:2:positive:0", "style:personachat:test:130:self:2:positive:1"], "evidence_texts": ["A man holds a rope for a boy who's about to climb a wall.", "A girl reaches up to kiss a cat, which is sitting on the counter.", "A man with his finger in his mouth is standing outside of an RV.", "A man in a green shirt and a red kilt is juggling three clubs.", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:3430", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3430", "candidate_text": "I wonder whether the man is standing?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3430:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3258:premise", "snli:test:3151:premise", "snli:test:3430:premise", "snli:test:3619:premise", "style:personachat:test:498:self:2:positive:0", "style:personachat:test:498:self:2:positive:1"], "evidence_texts": ["A man holds a rope for a boy who's about to climb a wall.", "A girl reaches up to kiss a cat, which is sitting on the counter.", "A man with his finger in his mouth is standing outside of an RV.", "A man in a green shirt and a red kilt is juggling three clubs.", "do you have any other hobbies ?", "what do you do for a job ?"], "speaker_id": "snli:test:3430", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3431", "candidate_text": "Honestly, The man is inside his RV!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3431:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3619:premise", "snli:test:3258:premise", "snli:test:3846:premise", "snli:test:3431:premise", "style:personachat:test:130:self:5:positive:0", "style:personachat:test:130:self:5:positive:1", "style:personachat:test:130:self:5:positive:2", "style:personachat:test:130:self:5:positive:3", "style:personachat:test:130:self:5:positive:4"], "evidence_texts": ["A man in a green shirt and a red kilt is juggling three clubs.", "A man holds a rope for a boy who's about to climb a wall.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "ah , so you probably like shopping for clothes like i do ?", "even at my age , i am not married . single is my status .", "that is cool ! my hair is brown .", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:3431", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3431", "candidate_text": "well, the man is inside his rv", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3431:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3619:premise", "snli:test:3258:premise", "snli:test:3846:premise", "snli:test:3431:premise", "style:personachat:test:233:self:5:positive:0", "style:personachat:test:233:self:5:positive:1", "style:personachat:test:233:self:5:positive:2", "style:personachat:test:233:self:5:positive:3", "style:personachat:test:233:self:5:positive:4"], "evidence_texts": ["A man in a green shirt and a red kilt is juggling three clubs.", "A man holds a rope for a boy who's about to climb a wall.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "i have a cat . he likes to watch me play video games .", "yes . i am getting ready to read a book , love a good drama", "interesting . i am a vet , but i enjoy my musician job on the weekends", "neat , i love a good detective show", "that is funny . i used to live in california"], "speaker_id": "snli:test:3431", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3439", "candidate_text": "Honestly, The kitchen is not empty!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3439:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3346:premise", "snli:test:3449:premise", "snli:test:3344:premise", "snli:test:3439:premise", "style:personachat:test:130:self:7:positive:0", "style:personachat:test:130:self:7:positive:1", "style:personachat:test:130:self:7:positive:2", "style:personachat:test:130:self:7:positive:3", "style:personachat:test:130:self:7:positive:4", "style:personachat:test:130:self:7:positive:5", "style:personachat:test:130:self:7:positive:6"], "evidence_texts": ["The 3 dogs are cruising down the street.", "People sitting on benches in a public area.", "Two men in polo shirts, reading the paper.", "Two little boys are standing in a kitchen.", "hi ! no i am not . i actually just finished my education to be a registered nurse .", "no , i will actually be getting my first job as a nurse soon . i cannot wait !", "ah , so you probably like shopping for clothes like i do ?", "even at my age , i am not married . single is my status .", "that is cool ! my hair is brown .", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:3439", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3439", "candidate_text": "The kitchen is not empty.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3439:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3346:premise", "snli:test:3449:premise", "snli:test:3344:premise", "snli:test:3439:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The 3 dogs are cruising down the street.", "People sitting on benches in a public area.", "Two men in polo shirts, reading the paper.", "Two little boys are standing in a kitchen.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3439", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3442", "candidate_text": "well, a child likes the dress", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3442:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3335:premise", "snli:test:3534:premise", "snli:test:3442:premise", "snli:test:3333:premise", "style:personachat:test:131:self:1:positive:0"], "evidence_texts": ["Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "they are good companions . my brother and sister love charlie too ."], "speaker_id": "snli:test:3442", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3442", "candidate_text": "I wonder whether a child likes the dress?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3442:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3335:premise", "snli:test:3534:premise", "snli:test:3442:premise", "snli:test:3333:premise", "style:personachat:test:501:self:1:positive:0"], "evidence_texts": ["Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "no , he does not drive a vw . do you drive to school ?"], "speaker_id": "snli:test:3442", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3446", "candidate_text": "well, a man laughing while at a restaurant eating", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3446:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3276:premise", "snli:test:4113:premise", "snli:test:3446:premise", "snli:test:3106:premise", "style:personachat:test:131:self:2:positive:0", "style:personachat:test:131:self:2:positive:1"], "evidence_texts": ["People in motion down a city sidewalk.", "Children receiving places at an award ceremony.", "A man laughing while at a restaurant.", "Women in blue dress singing on stage.", "charlie is an old boxer . have had him for years .", "they are good companions . my brother and sister love charlie too ."], "speaker_id": "snli:test:3446", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3446", "candidate_text": "Honestly, A man laughing while at a restaurant eating!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3446:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3276:premise", "snli:test:4113:premise", "snli:test:3446:premise", "snli:test:3106:premise", "style:personachat:test:949:self:1:positive:0"], "evidence_texts": ["People in motion down a city sidewalk.", "Children receiving places at an award ceremony.", "A man laughing while at a restaurant.", "Women in blue dress singing on stage.", "you should travel . it is so great !"], "speaker_id": "snli:test:3446", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3449", "candidate_text": "well, there are people sitting on benches outdoors talking to each other", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3449:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3439:premise", "snli:test:3449:premise", "snli:test:3463:premise", "snli:test:3346:premise", "style:personachat:test:131:self:5:positive:0", "style:personachat:test:131:self:5:positive:1", "style:personachat:test:131:self:5:positive:2", "style:personachat:test:131:self:5:positive:3", "style:personachat:test:131:self:5:positive:4"], "evidence_texts": ["Two little boys are standing in a kitchen.", "People sitting on benches in a public area.", "A man touching a hanging strand of lamps.", "The 3 dogs are cruising down the street.", "no , not really . are you ? if i leave some food , my dog charlie will eat it .", "i will take my dog to visit my folks in australia soon . will be interesting .", "sydney . i will go there after the stock markets slow down for winter .", "charlie is an old boxer . have had him for years .", "they are good companions . my brother and sister love charlie too ."], "speaker_id": "snli:test:3449", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3449", "candidate_text": "There are people sitting on benches outdoors talking to each other.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3449:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3439:premise", "snli:test:3449:premise", "snli:test:3463:premise", "snli:test:3346:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two little boys are standing in a kitchen.", "People sitting on benches in a public area.", "A man touching a hanging strand of lamps.", "The 3 dogs are cruising down the street.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3449", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3463", "candidate_text": "well, a man has his hands shoved into his coat pockets", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3463:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3449:premise", "snli:test:3570:premise", "snli:test:3439:premise", "snli:test:3463:premise", "style:personachat:test:131:self:7:positive:0", "style:personachat:test:131:self:7:positive:1", "style:personachat:test:131:self:7:positive:2", "style:personachat:test:131:self:7:positive:3", "style:personachat:test:131:self:7:positive:4", "style:personachat:test:131:self:7:positive:5", "style:personachat:test:131:self:7:positive:6"], "evidence_texts": ["People sitting on benches in a public area.", "A man with his arm around a woman.", "Two little boys are standing in a kitchen.", "A man touching a hanging strand of lamps.", "hey , i am doing good . how about you ? what are you up to ?", "maybe a nice glass of wine while i finish my eggplant parmigiana for dinner .", "no , not really . are you ? if i leave some food , my dog charlie will eat it .", "i will take my dog to visit my folks in australia soon . will be interesting .", "sydney . i will go there after the stock markets slow down for winter .", "charlie is an old boxer . have had him for years .", "they are good companions . my brother and sister love charlie too ."], "speaker_id": "snli:test:3463", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3463", "candidate_text": "I wonder whether a man has his hands shoved into his coat pockets?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3463:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3449:premise", "snli:test:3570:premise", "snli:test:3439:premise", "snli:test:3463:premise", "style:personachat:test:501:self:7:positive:0", "style:personachat:test:501:self:7:positive:1", "style:personachat:test:501:self:7:positive:2", "style:personachat:test:501:self:7:positive:3", "style:personachat:test:501:self:7:positive:4", "style:personachat:test:501:self:7:positive:5", "style:personachat:test:501:self:7:positive:6"], "evidence_texts": ["People sitting on benches in a public area.", "A man with his arm around a woman.", "Two little boys are standing in a kitchen.", "A man touching a hanging strand of lamps.", "oh that is very nice ! what willl you buy there", "i like to go shopping with my two sons", "what do you eat for breakfast ?", "yes . one of them in high school and 1 of them in middle school .", "i enjoy to eat granola because it tastes good", "the oldest is going to san leandro high school", "no , he does not drive a vw . do you drive to school ?"], "speaker_id": "snli:test:3463", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3469", "candidate_text": "well, the woman is the man's wife", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3469:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2815:premise", "snli:test:3469:premise", "snli:test:4812:premise", "snli:test:2500:premise", "style:personachat:test:132:self:1:positive:0"], "evidence_texts": ["an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "a little . i eat sardines every morning ."], "speaker_id": "snli:test:3469", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3469", "candidate_text": "Honestly, The woman is the man's wife!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3469:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:2815:premise", "snli:test:3469:premise", "snli:test:4812:premise", "snli:test:2500:premise", "style:personachat:test:949:self:6:positive:0", "style:personachat:test:949:self:6:positive:1", "style:personachat:test:949:self:6:positive:2", "style:personachat:test:949:self:6:positive:3", "style:personachat:test:949:self:6:positive:4", "style:personachat:test:949:self:6:positive:5"], "evidence_texts": ["an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "A bicycle rider wearing racing gear pedals a yellow bike past the wire fence at the edge of a field, with a stand of trees in the background.", "my dad liked to travel so it has become my passion", "i am 23 in law school you ?", "are you very good at tae kwan do ? i am quite good at law", "i do ! i love adventures of all kinds especially with my girlfriend", "any that involve traveling . we are trying to get to every state", "you should travel . it is so great !"], "speaker_id": "snli:test:3469", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3474", "candidate_text": "well, a person walks a way from a dirt circle", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3474:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3367:premise", "snli:test:3729:premise", "snli:test:3474:premise", "snli:test:3143:premise", "style:personachat:test:132:self:2:positive:0", "style:personachat:test:132:self:2:positive:1"], "evidence_texts": ["Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "there is a lot of rain here currently as well . from the hurricanes probably", "a little . i eat sardines every morning ."], "speaker_id": "snli:test:3474", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3474", "candidate_text": "a person walks a way from a dirt circle.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3474:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3367:premise", "snli:test:3729:premise", "snli:test:3474:premise", "snli:test:3143:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man wearing a white t-shirt and shorts is rock climbing with other people wearing blue hard hats.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3474", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3477", "candidate_text": "well, people are riding bikes in both directions on a road", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3477:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3409:premise", "snli:test:3477:premise", "snli:test:3324:premise", "snli:test:3566:premise", "style:personachat:test:132:self:5:positive:0", "style:personachat:test:132:self:5:positive:1", "style:personachat:test:132:self:5:positive:2", "style:personachat:test:132:self:5:positive:3", "style:personachat:test:132:self:5:positive:4"], "evidence_texts": ["Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "i do not . i spend all of my time playing frisbee", "cats are cool . i have two dogs . pretty big ones as well", "ohio . where are you from ?", "there is a lot of rain here currently as well . from the hurricanes probably", "a little . i eat sardines every morning ."], "speaker_id": "snli:test:3477", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3477", "candidate_text": "I wonder whether people are riding bikes in both directions on a road?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3477:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3409:premise", "snli:test:3477:premise", "snli:test:3324:premise", "snli:test:3566:premise", "style:personachat:test:508:self:5:positive:0", "style:personachat:test:508:self:5:positive:1", "style:personachat:test:508:self:5:positive:2", "style:personachat:test:508:self:5:positive:3", "style:personachat:test:508:self:5:positive:4"], "evidence_texts": ["Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "Women sit on a beach as they watch men bring in a fishing net filled with fish in front of them.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "i am positive my girlfriend would disagree .", "he is max , my girlfriend named him .", "i am in law school , are you still attending ?", "i love to surf as well , max sometimes rides with me .", "does he have a maclaren ? i have always wanted one of those ."], "speaker_id": "snli:test:3477", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3478", "candidate_text": "well, the road is in china", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3478:positive", "evidence_budget": 12, "evidence_ids": ["snli:test:3566:premise", "snli:test:3409:premise", "snli:test:3478:premise", "snli:test:3762:premise", "style:personachat:test:132:self:8:positive:0", "style:personachat:test:132:self:8:positive:1", "style:personachat:test:132:self:8:positive:2", "style:personachat:test:132:self:8:positive:3", "style:personachat:test:132:self:8:positive:4", "style:personachat:test:132:self:8:positive:5", "style:personachat:test:132:self:8:positive:6", "style:personachat:test:132:self:8:positive:7"], "evidence_texts": ["A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "i am good . how are you ?", "that is nice . what do you teach ?", "awesome . i actually never graduated highschool", "i do not . i spend all of my time playing frisbee", "cats are cool . i have two dogs . pretty big ones as well", "ohio . where are you from ?", "there is a lot of rain here currently as well . from the hurricanes probably", "a little . i eat sardines every morning ."], "speaker_id": "snli:test:3478", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3478", "candidate_text": "Honestly, The road is in China!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3478:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3566:premise", "snli:test:3409:premise", "snli:test:3478:premise", "snli:test:3762:premise", "style:personachat:test:964:self:5:positive:0", "style:personachat:test:964:self:5:positive:1", "style:personachat:test:964:self:5:positive:2", "style:personachat:test:964:self:5:positive:3", "style:personachat:test:964:self:5:positive:4"], "evidence_texts": ["A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "Two men stand in a lift as four others, all dressed in black and bright yellow stripes, look up on them.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "i was hoping you knew what i was talking about ! school , no work !", "do i ? maybe a little to ficken", "no like the emphasis on do like duh , who does not love bbq", "homer simpson ? obviously , who else is homer , but explain !", "omg i am ! guess how tall i am"], "speaker_id": "snli:test:3478", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3481", "candidate_text": "Honestly, The man is outdoors!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3481:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9099:premise", "snli:test:3481:premise", "snli:test:7409:premise", "snli:test:4435:premise", "style:personachat:test:133:self:1:positive:0"], "evidence_texts": ["In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:3481", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3481", "candidate_text": "The man is outdoors.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3481:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9099:premise", "snli:test:3481:premise", "snli:test:7409:premise", "snli:test:4435:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3481", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3505", "candidate_text": "Honestly, A group of people wait at a bus stop as a large white and blue bus stops!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3505:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:2059:premise", "snli:test:3505:premise", "snli:test:4177:premise", "snli:test:616:premise", "style:personachat:test:133:self:2:positive:0", "style:personachat:test:133:self:2:positive:1"], "evidence_texts": ["Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:3505", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3505", "candidate_text": "I wonder whether a group of people wait at a bus stop as a large white and blue bus stops?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3505:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:2059:premise", "snli:test:3505:premise", "snli:test:4177:premise", "snli:test:616:premise", "style:personachat:test:510:self:2:positive:0", "style:personachat:test:510:self:2:positive:1"], "evidence_texts": ["Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "i grew up in wisconsin and we used to have a lot of pets with a cold .", "what is your favorite animal ?"], "speaker_id": "snli:test:3505", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3510", "candidate_text": "Honestly, people are swimming in ocean water!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3510:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3373:premise", "snli:test:3371:premise", "snli:test:3510:premise", "snli:test:3560:premise", "style:personachat:test:133:self:5:positive:0", "style:personachat:test:133:self:5:positive:1", "style:personachat:test:133:self:5:positive:2", "style:personachat:test:133:self:5:positive:3", "style:personachat:test:133:self:5:positive:4"], "evidence_texts": ["An Eastern vendor sits by his stall with a rooster nearby.", "People are waiting in line to board a double decker bus.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "A woman with a pink purse walks down a crowded sidewalk.", "do you like to cook ? i make homemade gourmet cat food , and people food , too !", "i love cooking ! and , i am pretty good at it , too ! that explains the extra pounds !", "if i was taller , i might feel better about my body . what do you do ?", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:3510", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3510", "candidate_text": "well, people are swimming in ocean water", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3510:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3373:premise", "snli:test:3371:premise", "snli:test:3510:premise", "snli:test:3560:premise", "style:personachat:test:236:self:5:positive:0", "style:personachat:test:236:self:5:positive:1", "style:personachat:test:236:self:5:positive:2", "style:personachat:test:236:self:5:positive:3", "style:personachat:test:236:self:5:positive:4"], "evidence_texts": ["An Eastern vendor sits by his stall with a rooster nearby.", "People are waiting in line to board a double decker bus.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "A woman with a pink purse walks down a crowded sidewalk.", "not really more into civil war things . too many wives in the past", "that is very nice . what kind of diets ?", "do you eat anything special for it ?", "what are you principles you stick too ?", "not to big on it these days . pretty much just watch what i am eating really"], "speaker_id": "snli:test:3510", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3511", "candidate_text": "Honestly, two people paddle their boat on the ocean!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3511:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3560:premise", "snli:test:3373:premise", "snli:test:3580:premise", "snli:test:3511:premise", "style:personachat:test:133:self:7:positive:0", "style:personachat:test:133:self:7:positive:1", "style:personachat:test:133:self:7:positive:2", "style:personachat:test:133:self:7:positive:3", "style:personachat:test:133:self:7:positive:4", "style:personachat:test:133:self:7:positive:5", "style:personachat:test:133:self:7:positive:6"], "evidence_texts": ["A woman with a pink purse walks down a crowded sidewalk.", "An Eastern vendor sits by his stall with a rooster nearby.", "A brown and black dog rubs his face in the grass.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "no , not anymore . i barely have time for that , with the family !", "i have 3 children , and 2 kitties , that think they are my children as well !", "do you like to cook ? i make homemade gourmet cat food , and people food , too !", "i love cooking ! and , i am pretty good at it , too ! that explains the extra pounds !", "if i was taller , i might feel better about my body . what do you do ?", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:3511", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3511", "candidate_text": "two people paddle their boat on the ocean.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3511:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3560:premise", "snli:test:3373:premise", "snli:test:3580:premise", "snli:test:3511:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman with a pink purse walks down a crowded sidewalk.", "An Eastern vendor sits by his stall with a rooster nearby.", "A brown and black dog rubs his face in the grass.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3511", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3518", "candidate_text": "well, a rally in a seperate country", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3518:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3399:premise", "snli:test:3912:premise", "snli:test:3518:premise", "snli:test:3254:premise", "style:personachat:test:134:self:1:positive:0"], "evidence_texts": ["Two people sitting on the sand.", "Four kids pose on a stage.", "A rally in a different country.", "A family smiling on the couch.", "i do it is simple an easy"], "speaker_id": "snli:test:3518", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3518", "candidate_text": "I wonder whether a rally in a seperate country?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3518:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3399:premise", "snli:test:3912:premise", "snli:test:3518:premise", "snli:test:3254:premise", "style:personachat:test:518:self:1:positive:0"], "evidence_texts": ["Two people sitting on the sand.", "Four kids pose on a stage.", "A rally in a different country.", "A family smiling on the couch.", "where do you live ? are you in the city ?"], "speaker_id": "snli:test:3518", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3531", "candidate_text": "well, the man is kneeling down", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3531:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3317:premise", "snli:test:3315:premise", "snli:test:3537:premise", "snli:test:3531:premise", "style:personachat:test:134:self:2:positive:0", "style:personachat:test:134:self:2:positive:1"], "evidence_texts": ["A woman is walking her dog and using her cellphone.", "A woman is walking her dog and using her cellphone.", "A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "it is we are in a little town so it makes it easy .", "i do it is simple an easy"], "speaker_id": "snli:test:3531", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3531", "candidate_text": "Honestly, The man is kneeling down!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3531:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3317:premise", "snli:test:3315:premise", "snli:test:3537:premise", "snli:test:3531:premise", "style:personachat:test:29:self:1:positive:0"], "evidence_texts": ["A woman is walking her dog and using her cellphone.", "A woman is walking her dog and using her cellphone.", "A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:3531", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3533", "candidate_text": "well, the man is working underneath a vehicle", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3533:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3537:premise", "snli:test:3533:premise", "snli:test:3317:premise", "snli:test:3603:premise", "style:personachat:test:134:self:5:positive:0", "style:personachat:test:134:self:5:positive:1", "style:personachat:test:134:self:5:positive:2", "style:personachat:test:134:self:5:positive:3", "style:personachat:test:134:self:5:positive:4"], "evidence_texts": ["A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "A man in a blue helmet is riding his motorcycle.", "i am not married", "i like to make things with wood like crafts", "i do i am also teaching my kid at home .", "it is we are in a little town so it makes it easy .", "i do it is simple an easy"], "speaker_id": "snli:test:3533", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3533", "candidate_text": "The man is working underneath a vehicle.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3533:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3537:premise", "snli:test:3533:premise", "snli:test:3317:premise", "snli:test:3603:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "A woman is walking her dog and using her cellphone.", "A man in a blue helmet is riding his motorcycle.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3533", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3534", "candidate_text": "well, a man is posing", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3534:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3442:premise", "snli:test:3751:premise", "snli:test:3534:premise", "snli:test:3335:premise", "style:personachat:test:134:self:6:positive:0", "style:personachat:test:134:self:6:positive:1", "style:personachat:test:134:self:6:positive:2", "style:personachat:test:134:self:6:positive:3", "style:personachat:test:134:self:6:positive:4", "style:personachat:test:134:self:6:positive:5"], "evidence_texts": ["A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "i am well an you ?", "i am not married", "i like to make things with wood like crafts", "i do i am also teaching my kid at home .", "it is we are in a little town so it makes it easy .", "i do it is simple an easy"], "speaker_id": "snli:test:3534", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3534", "candidate_text": "I wonder whether a man is posing?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3534:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3442:premise", "snli:test:3751:premise", "snli:test:3534:premise", "snli:test:3335:premise", "style:personachat:test:518:self:7:positive:0", "style:personachat:test:518:self:7:positive:1", "style:personachat:test:518:self:7:positive:2", "style:personachat:test:518:self:7:positive:3", "style:personachat:test:518:self:7:positive:4", "style:personachat:test:518:self:7:positive:5", "style:personachat:test:518:self:7:positive:6"], "evidence_texts": ["A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Many people who appear to be Asian stand in line at a train station waiting patiently for the next train.", "hi , how are you , what type of books do you like ?", "drama is my type of books , on weekends i play instruments", "running is hard on my foot so i play video games", "i play the piano , jordan is my name by the way , do you like animals ?", "can you still afford a nice pair of running shoes ? i have little savings .", "i work as a veterinarian , little money , but i have good running shoes .", "where do you live ? are you in the city ?"], "speaker_id": "snli:test:3534", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3537", "candidate_text": "I wonder whether a man is sitting in a wheelchair in a hospital?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3537:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3533:premise", "snli:test:3537:premise", "snli:test:3603:premise", "snli:test:3531:premise", "style:personachat:test:135:self:1:positive:0"], "evidence_texts": ["A man wearing a welding mask working on a duct.", "A man is playing with his skateboard on a street.", "A man in a blue helmet is riding his motorcycle.", "A man wearing a welding mask working on a duct.", "yeah do you have lots of friends ?"], "speaker_id": "snli:test:3537", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3537", "candidate_text": "well, a man is sitting in a wheelchair in a hospital", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3537:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3533:premise", "snli:test:3537:premise", "snli:test:3603:premise", "snli:test:3531:premise", "style:personachat:test:238:self:1:positive:0"], "evidence_texts": ["A man wearing a welding mask working on a duct.", "A man is playing with his skateboard on a street.", "A man in a blue helmet is riding his motorcycle.", "A man wearing a welding mask working on a duct.", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:3537", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3542", "candidate_text": "I wonder whether a man and woman are getting eloped?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3542:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3353:premise", "snli:test:3712:premise", "snli:test:3542:premise", "snli:test:3351:premise", "style:personachat:test:135:self:2:positive:0", "style:personachat:test:135:self:2:positive:1"], "evidence_texts": ["A man with a bat swings at a ball coming towards him.", "A man in a bright yellow shirt juggles while riding a unicycle.", "A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "impressive i am a 22 year old woman", "yeah do you have lots of friends ?"], "speaker_id": "snli:test:3542", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3542", "candidate_text": "Honestly, A man and woman are getting eloped!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3542:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3353:premise", "snli:test:3712:premise", "snli:test:3542:premise", "snli:test:3351:premise", "style:personachat:test:32:self:1:positive:0"], "evidence_texts": ["A man with a bat swings at a ball coming towards him.", "A man in a bright yellow shirt juggles while riding a unicycle.", "A man and wife stand at the alter as they get married.", "A man with a bat swings at a ball coming towards him.", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:3542", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3560", "candidate_text": "I wonder whether the crowd walks around a woman?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3560:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3560:premise", "snli:test:3511:premise", "snli:test:3580:premise", "snli:test:3510:premise", "style:personachat:test:135:self:5:positive:0", "style:personachat:test:135:self:5:positive:1", "style:personachat:test:135:self:5:positive:2", "style:personachat:test:135:self:5:positive:3", "style:personachat:test:135:self:5:positive:4"], "evidence_texts": ["A woman with a pink purse walks down a crowded sidewalk.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "A brown and black dog rubs his face in the grass.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "what did you do there ?", "just went to my classes really", "college . how old are you ?", "impressive i am a 22 year old woman", "yeah do you have lots of friends ?"], "speaker_id": "snli:test:3560", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3560", "candidate_text": "The crowd walks around a woman.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3560:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3560:premise", "snli:test:3511:premise", "snli:test:3580:premise", "snli:test:3510:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman with a pink purse walks down a crowded sidewalk.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "A brown and black dog rubs his face in the grass.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3560", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3566", "candidate_text": "I wonder whether the man is looking at the camera?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3566:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3478:premise", "snli:test:3762:premise", "snli:test:3566:premise", "snli:test:3477:premise", "style:personachat:test:135:self:7:positive:0", "style:personachat:test:135:self:7:positive:1", "style:personachat:test:135:self:7:positive:2", "style:personachat:test:135:self:7:positive:3", "style:personachat:test:135:self:7:positive:4", "style:personachat:test:135:self:7:positive:5", "style:personachat:test:135:self:7:positive:6"], "evidence_texts": ["A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "pretty good and you tonight ?", "nice what are you doing together ?", "what did you do there ?", "just went to my classes really", "college . how old are you ?", "impressive i am a 22 year old woman", "yeah do you have lots of friends ?"], "speaker_id": "snli:test:3566", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3566", "candidate_text": "well, the man is looking at the camera", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3566:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3478:premise", "snli:test:3762:premise", "snli:test:3566:premise", "snli:test:3477:premise", "style:personachat:test:238:self:6:positive:0", "style:personachat:test:238:self:6:positive:1", "style:personachat:test:238:self:6:positive:2", "style:personachat:test:238:self:6:positive:3", "style:personachat:test:238:self:6:positive:4", "style:personachat:test:238:self:6:positive:5"], "evidence_texts": ["A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "i am well . just finishing up making some barbecue . i love it .", "watching horror movies are one of my favorites and i like to lift weights on weekends", "every weekend i lift weight and watch a horror movie or two . need to go shopping .", "clothes . i am a tall guy and kind of big so shopping is not my favorite", "may i ask what you are studying ?", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:3566", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3570", "candidate_text": "well, the woman and man are smiling", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3570:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3463:premise", "snli:test:3577:premise", "snli:test:3570:premise", "snli:test:3449:premise", "style:personachat:test:136:self:1:positive:0"], "evidence_texts": ["A man touching a hanging strand of lamps.", "A young man holding a microphone and singing.", "A man with his arm around a woman.", "People sitting on benches in a public area.", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:3570", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3570", "candidate_text": "The woman and man are smiling.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3570:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3463:premise", "snli:test:3577:premise", "snli:test:3570:premise", "snli:test:3449:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man touching a hanging strand of lamps.", "A young man holding a microphone and singing.", "A man with his arm around a woman.", "People sitting on benches in a public area.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3570", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3577", "candidate_text": "well, a young man is doing karaoke in a bar", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3577:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3570:premise", "snli:test:3577:premise", "snli:test:3463:premise", "snli:test:3827:premise", "style:personachat:test:136:self:2:positive:0", "style:personachat:test:136:self:2:positive:1"], "evidence_texts": ["A man with his arm around a woman.", "A young man holding a microphone and singing.", "A man touching a hanging strand of lamps.", "A fisherman using a cellphone on a boat.", "cooking and reading are always my relaxers", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:3577", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3577", "candidate_text": "I wonder whether a young man is doing karaoke in a bar?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3577:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3570:premise", "snli:test:3577:premise", "snli:test:3463:premise", "snli:test:3827:premise", "style:personachat:test:525:self:2:positive:0", "style:personachat:test:525:self:2:positive:1"], "evidence_texts": ["A man with his arm around a woman.", "A young man holding a microphone and singing.", "A man touching a hanging strand of lamps.", "A fisherman using a cellphone on a boat.", "awww man ! get some pizza , play some rush ! ! we should hang out sometime .", "what music do you like ?"], "speaker_id": "snli:test:3577", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3578", "candidate_text": "well, a man is performing a song", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3578:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3827:premise", "snli:test:3570:premise", "snli:test:3578:premise", "snli:test:4082:premise", "style:personachat:test:136:self:5:positive:0", "style:personachat:test:136:self:5:positive:1", "style:personachat:test:136:self:5:positive:2", "style:personachat:test:136:self:5:positive:3", "style:personachat:test:136:self:5:positive:4"], "evidence_texts": ["A fisherman using a cellphone on a boat.", "A man with his arm around a woman.", "A young man holding a microphone and singing.", "Some kids are wrestling on an inflatable raft", "i am making dinner while i wait", "i had to grade some papers and do homework with my kids", "my husband works nights so its hard sometimes", "cooking and reading are always my relaxers", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:3578", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3578", "candidate_text": "Honestly, A man is performing a song!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3578:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3827:premise", "snli:test:3570:premise", "snli:test:3578:premise", "snli:test:4082:premise", "style:personachat:test:36:self:2:positive:0", "style:personachat:test:36:self:2:positive:1"], "evidence_texts": ["A fisherman using a cellphone on a boat.", "A man with his arm around a woman.", "A young man holding a microphone and singing.", "Some kids are wrestling on an inflatable raft", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:3578", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3580", "candidate_text": "well, the dog's nose itches", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3580:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3580:premise", "snli:test:3560:premise", "snli:test:3597:premise", "snli:test:3511:premise", "style:personachat:test:136:self:6:positive:0", "style:personachat:test:136:self:6:positive:1", "style:personachat:test:136:self:6:positive:2", "style:personachat:test:136:self:6:positive:3", "style:personachat:test:136:self:6:positive:4", "style:personachat:test:136:self:6:positive:5"], "evidence_texts": ["A brown and black dog rubs his face in the grass.", "A woman with a pink purse walks down a crowded sidewalk.", "Two men have an after dinner discussion at the dinner table.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "i am good . just waiting for the good doctor to come on .", "i am making dinner while i wait", "i had to grade some papers and do homework with my kids", "my husband works nights so its hard sometimes", "cooking and reading are always my relaxers", "yeah i love all the aromas of the food while its cooking"], "speaker_id": "snli:test:3580", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3580", "candidate_text": "The dog's nose itches.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3580:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3580:premise", "snli:test:3560:premise", "snli:test:3597:premise", "snli:test:3511:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A brown and black dog rubs his face in the grass.", "A woman with a pink purse walks down a crowded sidewalk.", "Two men have an after dinner discussion at the dinner table.", "Two silhouetted people paddle a canoe on the ocean during sunset.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3580", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3597", "candidate_text": "Honestly, the men are partners!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3597:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3580:premise", "snli:test:3597:premise", "snli:test:3653:premise", "snli:test:3560:premise", "style:personachat:test:137:self:1:positive:0"], "evidence_texts": ["A brown and black dog rubs his face in the grass.", "Two men have an after dinner discussion at the dinner table.", "Young Asian male child either smoking something or its an instrument.", "A woman with a pink purse walks down a crowded sidewalk.", "yes i would love to hear a secret !"], "speaker_id": "snli:test:3597", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3597", "candidate_text": "I wonder whether the men are partners?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3597:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3580:premise", "snli:test:3597:premise", "snli:test:3653:premise", "snli:test:3560:premise", "style:personachat:test:527:self:1:positive:0"], "evidence_texts": ["A brown and black dog rubs his face in the grass.", "Two men have an after dinner discussion at the dinner table.", "Young Asian male child either smoking something or its an instrument.", "A woman with a pink purse walks down a crowded sidewalk.", "does you mom have heart problems then ?"], "speaker_id": "snli:test:3597", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3603", "candidate_text": "Honestly, The man is sparring in the gym!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3603:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3537:premise", "snli:test:3533:premise", "snli:test:3650:premise", "snli:test:3603:premise", "style:personachat:test:137:self:2:positive:0", "style:personachat:test:137:self:2:positive:1"], "evidence_texts": ["A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:3603", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3603", "candidate_text": "well, the man is sparring in the gym", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3603:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3537:premise", "snli:test:3533:premise", "snli:test:3650:premise", "snli:test:3603:premise", "style:personachat:test:245:self:2:positive:0", "style:personachat:test:245:self:2:positive:1"], "evidence_texts": ["A man is playing with his skateboard on a street.", "A man wearing a welding mask working on a duct.", "Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "haha . i get asked to get stuff off the top self all the time !", "yeah , but it is all good . dynamite comes in small packages ."], "speaker_id": "snli:test:3603", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3605", "candidate_text": "Honestly, The man is driving!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3605:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3650:premise", "snli:test:3605:premise", "snli:test:3537:premise", "snli:test:3732:premise", "style:personachat:test:137:self:5:positive:0", "style:personachat:test:137:self:5:positive:1", "style:personachat:test:137:self:5:positive:2", "style:personachat:test:137:self:5:positive:3", "style:personachat:test:137:self:5:positive:4"], "evidence_texts": ["Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "A man is playing with his skateboard on a street.", "A speaker is talking with a TV in the background.", "right ? ! i have actually been knitting hats for babies in hospitals for last few years", "i am a few years older , whats your favorite music ? mine is country !", "that would be awesome , i want to be a chef since i love cooking", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:3605", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3605", "candidate_text": "The man is driving.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3605:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3650:premise", "snli:test:3605:premise", "snli:test:3537:premise", "snli:test:3732:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "A man is playing with his skateboard on a street.", "A speaker is talking with a TV in the background.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3605", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3619", "candidate_text": "Honestly, A man in a shirt juggles thee clubs and an apple!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3619:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3431:premise", "snli:test:3846:premise", "snli:test:3430:premise", "snli:test:3619:premise", "style:personachat:test:137:self:6:positive:0", "style:personachat:test:137:self:6:positive:1", "style:personachat:test:137:self:6:positive:2", "style:personachat:test:137:self:6:positive:3", "style:personachat:test:137:self:6:positive:4", "style:personachat:test:137:self:6:positive:5"], "evidence_texts": ["A man with his finger in his mouth is standing outside of an RV.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "A man in a green shirt and a red kilt is juggling three clubs.", "i love all cute things ! babies are on the top of my list now", "right ? ! i have actually been knitting hats for babies in hospitals for last few years", "i am a few years older , whats your favorite music ? mine is country !", "that would be awesome , i want to be a chef since i love cooking", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:3619", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3619", "candidate_text": "I wonder whether a man in a shirt juggles thee clubs and an apple?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3619:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3431:premise", "snli:test:3846:premise", "snli:test:3430:premise", "snli:test:3619:premise", "style:personachat:test:527:self:7:positive:0", "style:personachat:test:527:self:7:positive:1", "style:personachat:test:527:self:7:positive:2", "style:personachat:test:527:self:7:positive:3", "style:personachat:test:527:self:7:positive:4", "style:personachat:test:527:self:7:positive:5", "style:personachat:test:527:self:7:positive:6"], "evidence_texts": ["A man with his finger in his mouth is standing outside of an RV.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "A man in a green shirt and a red kilt is juggling three clubs.", "hi ! i am well . just tired after a big run this afternoon .", "i ran 5 miles . how much weight can you lift ?", "that is incredible . what else do you like to do ?", "oh my . i dislike meat and i think it is cruel to eat animals .", "i am a doctor as well . i work with children . what is your specialty ?", "what made you want to work in cardiology ?", "does you mom have heart problems then ?"], "speaker_id": "snli:test:3619", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3621", "candidate_text": "well, a clown rides a tall bike in a parade", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3621:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3081:premise", "snli:test:3621:premise", "snli:test:3774:premise", "snli:test:2712:premise", "style:personachat:test:138:self:1:positive:0"], "evidence_texts": ["Two women competing in a roller derby with several teammates and referee in the background.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "A girl in a black sweater and jeans pours water into an empty soda bottle.", "that is more up to my wife , but maybe ."], "speaker_id": "snli:test:3621", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3621", "candidate_text": "Honestly, A clown rides a tall bike in a parade!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3621:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3081:premise", "snli:test:3621:premise", "snli:test:3774:premise", "snli:test:2712:premise", "style:personachat:test:45:self:7:positive:0", "style:personachat:test:45:self:7:positive:1", "style:personachat:test:45:self:7:positive:2", "style:personachat:test:45:self:7:positive:3", "style:personachat:test:45:self:7:positive:4", "style:personachat:test:45:self:7:positive:5", "style:personachat:test:45:self:7:positive:6"], "evidence_texts": ["Two women competing in a roller derby with several teammates and referee in the background.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "A girl in a black sweater and jeans pours water into an empty soda bottle.", "hi , i am lucy . just getting ready for another day at the office", "i am a secretary . that is neat , what subject do you want to teach", "kindergarten would be fun ! do you play any sports", "i played soccer in college ! we won a cup one season", "that sounds like fun ! i just almost choked on a piece of candy", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:3621", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3631", "candidate_text": "well, a child is playing on christmas", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3631:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3355:premise", "snli:test:3642:premise", "snli:test:3200:premise", "snli:test:3631:premise", "style:personachat:test:138:self:2:positive:0", "style:personachat:test:138:self:2:positive:1"], "evidence_texts": ["A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "A young man with glasses and his companion are attempting to do laundry in a public place.", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "i do not have many friends to travel with . why australia", "that is more up to my wife , but maybe ."], "speaker_id": "snli:test:3631", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3631", "candidate_text": "A child is playing on christmas.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3631:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3355:premise", "snli:test:3642:premise", "snli:test:3200:premise", "snli:test:3631:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "A young man with glasses and his companion are attempting to do laundry in a public place.", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3631", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3642", "candidate_text": "well, a boy is watching a movie in his living room", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3642:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3631:premise", "snli:test:3795:premise", "snli:test:3642:premise", "snli:test:3355:premise", "style:personachat:test:138:self:5:positive:0", "style:personachat:test:138:self:5:positive:1", "style:personachat:test:138:self:5:positive:2", "style:personachat:test:138:self:5:positive:3", "style:personachat:test:138:self:5:positive:4"], "evidence_texts": ["A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "i do , but i prefer carpentry projects or family time", "i have one , but my family hates it . do you", "is three a lot ? do you have kids ?", "i do not have many friends to travel with . why australia", "that is more up to my wife , but maybe ."], "speaker_id": "snli:test:3642", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3642", "candidate_text": "I wonder whether a boy is watching a movie in his living room?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3642:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3631:premise", "snli:test:3795:premise", "snli:test:3642:premise", "snli:test:3355:premise", "style:personachat:test:533:self:5:positive:0", "style:personachat:test:533:self:5:positive:1", "style:personachat:test:533:self:5:positive:2", "style:personachat:test:533:self:5:positive:3", "style:personachat:test:533:self:5:positive:4"], "evidence_texts": ["A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "A man standing in a cubicle with a chair that has a black Ginobili jersey on it.", "a carnie ? not sure what that is . i am at home a lot . even for school", "do you like it ? do you get to dress up ?", "what type of writing ? i like to read comic books .", "i like to dress up like the characters . i dyed my hair too .", "do you ever dress up as one of your characters ?"], "speaker_id": "snli:test:3642", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3650", "candidate_text": "well, he is outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3650:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3605:premise", "snli:test:3732:premise", "snli:test:3650:premise", "snli:test:3603:premise", "style:personachat:test:138:self:7:positive:0", "style:personachat:test:138:self:7:positive:1", "style:personachat:test:138:self:7:positive:2", "style:personachat:test:138:self:7:positive:3", "style:personachat:test:138:self:7:positive:4", "style:personachat:test:138:self:7:positive:5", "style:personachat:test:138:self:7:positive:6"], "evidence_texts": ["A man in a blue helmet is riding his motorcycle.", "A speaker is talking with a TV in the background.", "Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "ouch , why is it painful ?", "ah , what do you do for work ?", "i do , but i prefer carpentry projects or family time", "i have one , but my family hates it . do you", "is three a lot ? do you have kids ?", "i do not have many friends to travel with . why australia", "that is more up to my wife , but maybe ."], "speaker_id": "snli:test:3650", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3650", "candidate_text": "Honestly, He is outside!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3650:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3605:premise", "snli:test:3732:premise", "snli:test:3650:premise", "snli:test:3603:premise", "style:personachat:test:50:self:5:positive:0", "style:personachat:test:50:self:5:positive:1", "style:personachat:test:50:self:5:positive:2", "style:personachat:test:50:self:5:positive:3", "style:personachat:test:50:self:5:positive:4"], "evidence_texts": ["A man in a blue helmet is riding his motorcycle.", "A speaker is talking with a TV in the background.", "Young boy smiles at the camera from the tire swing.", "A man in a blue helmet is riding his motorcycle.", "really ? i went to school at usc . i have a few contacts out there .", "dad tending the farm and mom a strict teacher , anything artsy was always frowned upon !", "where are you from ? i am proudly born and raised in south carolina !", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:3650", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3653", "candidate_text": "I wonder whether a child is asian?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3653:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3597:premise", "snli:test:3653:premise", "snli:test:3870:premise", "snli:test:3580:premise", "style:personachat:test:139:self:1:positive:0"], "evidence_texts": ["Two men have an after dinner discussion at the dinner table.", "Young Asian male child either smoking something or its an instrument.", "A guy taking a nap in the back of a vehicle.", "A brown and black dog rubs his face in the grass.", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:3653", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3653", "candidate_text": "Honestly, A child is Asian!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3653:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3597:premise", "snli:test:3653:premise", "snli:test:3870:premise", "snli:test:3580:premise", "style:personachat:test:50:self:7:positive:0", "style:personachat:test:50:self:7:positive:1", "style:personachat:test:50:self:7:positive:2", "style:personachat:test:50:self:7:positive:3", "style:personachat:test:50:self:7:positive:4", "style:personachat:test:50:self:7:positive:5", "style:personachat:test:50:self:7:positive:6"], "evidence_texts": ["Two men have an after dinner discussion at the dinner table.", "Young Asian male child either smoking something or its an instrument.", "A guy taking a nap in the back of a vehicle.", "A brown and black dog rubs his face in the grass.", "hi , i am joanna . how are you tonight .", "so , i am a singer by trade . what do you do ?", "really ? i went to school at usc . i have a few contacts out there .", "dad tending the farm and mom a strict teacher , anything artsy was always frowned upon !", "where are you from ? i am proudly born and raised in south carolina !", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:3653", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3659", "candidate_text": "I wonder whether a woman has a bright light pink jacket?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3659:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6160:premise", "snli:test:4210:premise", "snli:test:3100:premise", "snli:test:3659:premise", "style:personachat:test:139:self:2:positive:0", "style:personachat:test:139:self:2:positive:1"], "evidence_texts": ["Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "yeah my friend had that removed at his wedding last week when i attended", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:3659", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3659", "candidate_text": "A woman has a bright light pink jacket.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3659:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6160:premise", "snli:test:4210:premise", "snli:test:3100:premise", "snli:test:3659:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3659", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3668", "candidate_text": "I wonder whether boy eats sandwich?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3668:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3668:premise", "snli:test:2614:premise", "snli:test:3690:premise", "snli:test:2168:premise", "style:personachat:test:139:self:5:positive:0", "style:personachat:test:139:self:5:positive:1", "style:personachat:test:139:self:5:positive:2", "style:personachat:test:139:self:5:positive:3", "style:personachat:test:139:self:5:positive:4"], "evidence_texts": ["An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "Right leg in a brace, a young boy lies on the couch while using his cellphone.", "cool beans , do you have any other hobbies then sales ?", "yeah but i do not like it", "i like to eat meat and plants , i eat both", "yeah my friend had that removed at his wedding last week when i attended", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:3668", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3668", "candidate_text": "well, boy eats sandwich", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3668:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3668:premise", "snli:test:2614:premise", "snli:test:3690:premise", "snli:test:2168:premise", "style:personachat:test:247:self:5:positive:0", "style:personachat:test:247:self:5:positive:1", "style:personachat:test:247:self:5:positive:2", "style:personachat:test:247:self:5:positive:3", "style:personachat:test:247:self:5:positive:4"], "evidence_texts": ["An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "Right leg in a brace, a young boy lies on the couch while using his cellphone.", "i would not fight you , instead i would have you over for italian food", "you can bring the family . do they like to sing ?", "lets get a my family against your family softball game going", "you cannot beat home made italian food and a big game of sports", "i know how to make a delicious bowl of captain crunch"], "speaker_id": "snli:test:3668", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3690", "candidate_text": "I wonder whether two dogs are walking throught a gate?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3690:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3668:premise", "snli:test:3943:premise", "snli:test:3690:premise", "snli:test:2614:premise", "style:personachat:test:139:self:7:positive:0", "style:personachat:test:139:self:7:positive:1", "style:personachat:test:139:self:7:positive:2", "style:personachat:test:139:self:7:positive:3", "style:personachat:test:139:self:7:positive:4", "style:personachat:test:139:self:7:positive:5", "style:personachat:test:139:self:7:positive:6"], "evidence_texts": ["An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "A girl is about to hit a tennis ball coming from above her with a racket", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "good , i am in sales , what do you do for your work ?", "i am listening to rush , i love them", "cool beans , do you have any other hobbies then sales ?", "yeah but i do not like it", "i like to eat meat and plants , i eat both", "yeah my friend had that removed at his wedding last week when i attended", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:3690", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3690", "candidate_text": "Honestly, Two dogs are walking throught a gate!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3690:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3668:premise", "snli:test:3943:premise", "snli:test:3690:premise", "snli:test:2614:premise", "style:personachat:test:69:self:5:positive:0", "style:personachat:test:69:self:5:positive:1", "style:personachat:test:69:self:5:positive:2", "style:personachat:test:69:self:5:positive:3", "style:personachat:test:69:self:5:positive:4"], "evidence_texts": ["An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "A girl is about to hit a tennis ball coming from above her with a racket", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "Using a soccer ball for support, a young boy in blue sits atop a park bench.", "i want to visit italy one day .", "do you cook italian food ?", "i understand , i am from the middle east .", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:3690", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3712", "candidate_text": "Honestly, Man throws knives at unsuspecting people!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3712:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3712:premise", "snli:test:3542:premise", "snli:test:3855:premise", "snli:test:3353:premise", "style:personachat:test:140:self:1:positive:0"], "evidence_texts": ["A man in a bright yellow shirt juggles while riding a unicycle.", "A man and wife stand at the alter as they get married.", "A skier goes down a hill with a mountain as a backdrop.", "A man with a bat swings at a ball coming towards him.", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:3712", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3712", "candidate_text": "I wonder whether man throws knives at unsuspecting people?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3712:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3712:premise", "snli:test:3542:premise", "snli:test:3855:premise", "snli:test:3353:premise", "style:personachat:test:546:self:1:positive:0"], "evidence_texts": ["A man in a bright yellow shirt juggles while riding a unicycle.", "A man and wife stand at the alter as they get married.", "A skier goes down a hill with a mountain as a backdrop.", "A man with a bat swings at a ball coming towards him.", "what is your favorite movie ?"], "speaker_id": "snli:test:3712", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3729", "candidate_text": "Honestly, A man prepares to give a speech for a television audience!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3729:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3474:premise", "snli:test:3729:premise", "snli:test:3860:premise", "snli:test:3367:premise", "style:personachat:test:140:self:2:positive:0", "style:personachat:test:140:self:2:positive:1"], "evidence_texts": ["A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:3729", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3729", "candidate_text": "well, a man prepares to give a speech for a television audience", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3729:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3474:premise", "snli:test:3729:premise", "snli:test:3860:premise", "snli:test:3367:premise", "style:personachat:test:248:self:2:positive:0", "style:personachat:test:248:self:2:positive:1"], "evidence_texts": ["A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "Some people are standing on a city sidewalk at night and waiting to get on a double-decker bus.", "oh that is so cool", "how do you usually support him"], "speaker_id": "snli:test:3729", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3732", "candidate_text": "Honestly, There is a live bear in the background!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3732:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3732:premise", "snli:test:3650:premise", "snli:test:3838:premise", "snli:test:3605:premise", "style:personachat:test:140:self:5:positive:0", "style:personachat:test:140:self:5:positive:1", "style:personachat:test:140:self:5:positive:2", "style:personachat:test:140:self:5:positive:3", "style:personachat:test:140:self:5:positive:4"], "evidence_texts": ["A speaker is talking with a TV in the background.", "Young boy smiles at the camera from the tire swing.", "A dog leaps into the air for a tennis ball.", "A man in a blue helmet is riding his motorcycle.", "i am a literature teacher ! nothing like a good book ! nice ! !", "i wish my husband was ! my daughter just loves cooking like me .", "it is fun ! i love granola . what else are you into ?", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:3732", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3732", "candidate_text": "There is a live bear in the background.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3732:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3732:premise", "snli:test:3650:premise", "snli:test:3838:premise", "snli:test:3605:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A speaker is talking with a TV in the background.", "Young boy smiles at the camera from the tire swing.", "A dog leaps into the air for a tennis ball.", "A man in a blue helmet is riding his motorcycle.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3732", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3738", "candidate_text": "Honestly, the man is inside!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3738:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3383:premise", "snli:test:3382:premise", "snli:test:3738:premise", "snli:test:4101:premise", "style:personachat:test:140:self:7:positive:0", "style:personachat:test:140:self:7:positive:1", "style:personachat:test:140:self:7:positive:2", "style:personachat:test:140:self:7:positive:3", "style:personachat:test:140:self:7:positive:4", "style:personachat:test:140:self:7:positive:5", "style:personachat:test:140:self:7:positive:6"], "evidence_texts": ["A boy is being splashed in a swimming pool.", "A boy is being splashed in a swimming pool.", "The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "great ! just grading some papers and throwing some things in the crock pot .", "very cool ! my daughter wants a fish tank !", "i am a literature teacher ! nothing like a good book ! nice ! !", "i wish my husband was ! my daughter just loves cooking like me .", "it is fun ! i love granola . what else are you into ?", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:3738", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3738", "candidate_text": "I wonder whether the man is inside?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3738:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3383:premise", "snli:test:3382:premise", "snli:test:3738:premise", "snli:test:4101:premise", "style:personachat:test:546:self:7:positive:0", "style:personachat:test:546:self:7:positive:1", "style:personachat:test:546:self:7:positive:2", "style:personachat:test:546:self:7:positive:3", "style:personachat:test:546:self:7:positive:4", "style:personachat:test:546:self:7:positive:5", "style:personachat:test:546:self:7:positive:6"], "evidence_texts": ["A boy is being splashed in a swimming pool.", "A boy is being splashed in a swimming pool.", "The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "i am doing well , how are you ?", "i also have a dog named juan . he is my hunting dog . we like to hunt .", "anything but shellfish . i am allergic . mostly we hunt ducks .", "that is a good height for bow hunting . where do you work ?", "i work with a property rental company", "i enjoy it very much . i am somewhat of a handyman in my spare time .", "what is your favorite movie ?"], "speaker_id": "snli:test:3738", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3740", "candidate_text": "well, the man is a photographer", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3740:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3740:premise", "snli:test:4101:premise", "snli:test:3383:premise", "snli:test:4104:premise", "style:personachat:test:141:self:1:positive:0"], "evidence_texts": ["The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "A boy is being splashed in a swimming pool.", "Lab tech looking through a microscope in a lab.", "biofuels seem like the future . at the moment they are double the cost of oil ."], "speaker_id": "snli:test:3740", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3740", "candidate_text": "Honestly, the man is a photographer!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3740:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3740:premise", "snli:test:4101:premise", "snli:test:3383:premise", "snli:test:4104:premise", "style:personachat:test:70:self:7:positive:0", "style:personachat:test:70:self:7:positive:1", "style:personachat:test:70:self:7:positive:2", "style:personachat:test:70:self:7:positive:3", "style:personachat:test:70:self:7:positive:4", "style:personachat:test:70:self:7:positive:5", "style:personachat:test:70:self:7:positive:6"], "evidence_texts": ["The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "A boy is being splashed in a swimming pool.", "Lab tech looking through a microscope in a lab.", "my wife is a model , but we do pretty well . are you married ?", "we adopted because my wife was afraid of that , but i love my 2 kids !", "i like to cook to much to worry about my figure ! do you like food ?", "hmm . . you sound like a liberal .", "i am proud to say i voted for trump . do you like him ?", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:3740", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3751", "candidate_text": "well, a man and his daughter build a doghouse while the dog sits behind them", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3751:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3534:premise", "snli:test:3756:premise", "snli:test:3442:premise", "snli:test:3751:premise", "style:personachat:test:141:self:2:positive:0", "style:personachat:test:141:self:2:positive:1"], "evidence_texts": ["A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "do you ever think about the environment ?", "biofuels seem like the future . at the moment they are double the cost of oil ."], "speaker_id": "snli:test:3751", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3751", "candidate_text": "A man and his daughter build a doghouse while the dog sits behind them.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3751:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3534:premise", "snli:test:3756:premise", "snli:test:3442:premise", "snli:test:3751:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "A child in a jeans jacket and shorts holding up a white dress as another darker-skinned child looks at it.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3751", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3756", "candidate_text": "well, the girl is browsing her phone while another girl watches her", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3756:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3756:premise", "snli:test:3751:premise", "snli:test:3534:premise", "snli:test:4036:premise", "style:personachat:test:141:self:5:positive:0", "style:personachat:test:141:self:5:positive:1", "style:personachat:test:141:self:5:positive:2", "style:personachat:test:141:self:5:positive:3", "style:personachat:test:141:self:5:positive:4"], "evidence_texts": ["Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "cool , i like hanging with my mom as well .", "are they in school yet ?", "my mom was an army nurse , and got deployed once .", "do you ever think about the environment ?", "biofuels seem like the future . at the moment they are double the cost of oil ."], "speaker_id": "snli:test:3756", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3756", "candidate_text": "I wonder whether the girl is browsing her phone while another girl watches her?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3756:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3756:premise", "snli:test:3751:premise", "snli:test:3534:premise", "snli:test:4036:premise", "style:personachat:test:554:self:5:positive:0", "style:personachat:test:554:self:5:positive:1", "style:personachat:test:554:self:5:positive:2", "style:personachat:test:554:self:5:positive:3", "style:personachat:test:554:self:5:positive:4"], "evidence_texts": ["Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "A man in a red muscle shirt poses for a picture beside a stone building whilst other people pass by.", "A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "it is nice to have a hobby like that , i like to sing in a choir .", "if i sing badly , my friends drown my voice out , haha !", "not really . i spend my free time playing softball . and eating !", "it is so fun ! i used to do taekwondo too , since i was very small .", "do you like to go out to eat ?"], "speaker_id": "snli:test:3756", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3758", "candidate_text": "well, the girl is browsing reddit on her phone", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3758:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4036:premise", "snli:test:3751:premise", "snli:test:3758:premise", "snli:test:4293:premise", "style:personachat:test:141:self:6:positive:0", "style:personachat:test:141:self:6:positive:1", "style:personachat:test:141:self:6:positive:2", "style:personachat:test:141:self:6:positive:3", "style:personachat:test:141:self:6:positive:4", "style:personachat:test:141:self:6:positive:5"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "super duper . i am hanging out with my new cat .", "cool , i like hanging with my mom as well .", "are they in school yet ?", "my mom was an army nurse , and got deployed once .", "do you ever think about the environment ?", "biofuels seem like the future . at the moment they are double the cost of oil ."], "speaker_id": "snli:test:3758", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3758", "candidate_text": "Honestly, The girl is browsing reddit on her phone!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3758:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4036:premise", "snli:test:3751:premise", "snli:test:3758:premise", "snli:test:4293:premise", "style:personachat:test:72:self:5:positive:0", "style:personachat:test:72:self:5:positive:1", "style:personachat:test:72:self:5:positive:2", "style:personachat:test:72:self:5:positive:3", "style:personachat:test:72:self:5:positive:4"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Two men holding a Chinese lantern that's yellow and red, one man has a drill and the other is smiling.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "3 kids and a great husband too . we live beside the sea .", "that is a lot ! they would not fit into my mustang convertible .", "do you like to work out ? i am trying to lose some weight .", "i work out about 2 hours a day but it is hard to fit it all in .", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:3758", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3762", "candidate_text": "well, a man is trimming trees", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3762:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3566:premise", "snli:test:3478:premise", "snli:test:3762:premise", "snli:test:4139:premise", "style:personachat:test:142:self:1:positive:0"], "evidence_texts": ["A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "he had to learn how to survive . his parents lived there"], "speaker_id": "snli:test:3762", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3762", "candidate_text": "A man is trimming trees.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3762:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3566:premise", "snli:test:3478:premise", "snli:test:3762:premise", "snli:test:4139:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "A view of bicyclists on a city street, riding in both directions, with red reflective safety cones dividing the two lanes.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3762", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3764", "candidate_text": "well, a woman is working for the phone company", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3764:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3764:premise", "snli:test:4139:premise", "snli:test:3566:premise", "snli:test:4267:premise", "style:personachat:test:142:self:2:positive:0", "style:personachat:test:142:self:2:positive:1"], "evidence_texts": ["A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "a young boy who lost his parents at a young age", "he had to learn how to survive . his parents lived there"], "speaker_id": "snli:test:3764", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3764", "candidate_text": "I wonder whether a woman is working for the phone company?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3764:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3764:premise", "snli:test:4139:premise", "snli:test:3566:premise", "snli:test:4267:premise", "style:personachat:test:555:self:2:positive:0", "style:personachat:test:555:self:2:positive:1"], "evidence_texts": ["A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man is looking at the camera posing for a photo with his arm around a woman who is looking away.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "i love cookies and brownies .", "racquetball and trying new restaurants , you ?"], "speaker_id": "snli:test:3764", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3774", "candidate_text": "well, two girls playing the drums", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3774:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3621:premise", "snli:test:3081:premise", "snli:test:3774:premise", "snli:test:4090:premise", "style:personachat:test:142:self:5:positive:0", "style:personachat:test:142:self:5:positive:1", "style:personachat:test:142:self:5:positive:2", "style:personachat:test:142:self:5:positive:3", "style:personachat:test:142:self:5:positive:4"], "evidence_texts": ["Am man riding a very tall bicycle down the street, past a large brick building.", "Two women competing in a roller derby with several teammates and referee in the background.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "that must be fun . i am a science fiction writer .", "nope still in the writing process", "high in the mountain wilderness", "a young boy who lost his parents at a young age", "he had to learn how to survive . his parents lived there"], "speaker_id": "snli:test:3774", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3774", "candidate_text": "Honestly, Two girls playing the drums!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3774:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3621:premise", "snli:test:3081:premise", "snli:test:3774:premise", "snli:test:4090:premise", "style:personachat:test:74:self:2:positive:0", "style:personachat:test:74:self:2:positive:1"], "evidence_texts": ["Am man riding a very tall bicycle down the street, past a large brick building.", "Two women competing in a roller derby with several teammates and referee in the background.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:3774", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3776", "candidate_text": "well, a woman teaching a boy to sing", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3776:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3776:premise", "snli:test:4090:premise", "snli:test:3621:premise", "snli:test:4251:premise", "style:personachat:test:142:self:7:positive:0", "style:personachat:test:142:self:7:positive:1", "style:personachat:test:142:self:7:positive:2", "style:personachat:test:142:self:7:positive:3", "style:personachat:test:142:self:7:positive:4", "style:personachat:test:142:self:7:positive:5", "style:personachat:test:142:self:7:positive:6"], "evidence_texts": ["A young boy vocalizes in to a microphone held by a blond woman in black.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "lol that is a good science fiction starter", "well if it is a very small house . do you work", "that must be fun . i am a science fiction writer .", "nope still in the writing process", "high in the mountain wilderness", "a young boy who lost his parents at a young age", "he had to learn how to survive . his parents lived there"], "speaker_id": "snli:test:3776", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3776", "candidate_text": "A woman teaching a boy to sing.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3776:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3776:premise", "snli:test:4090:premise", "snli:test:3621:premise", "snli:test:4251:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young boy vocalizes in to a microphone held by a blond woman in black.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "Am man riding a very tall bicycle down the street, past a large brick building.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3776", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3795", "candidate_text": "well, the racetrack is outdoors", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3795:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3642:premise", "snli:test:4020:premise", "snli:test:3631:premise", "snli:test:3795:premise", "style:personachat:test:143:self:1:positive:0"], "evidence_texts": ["A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "it is definitely not for everyone ."], "speaker_id": "snli:test:3795", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3795", "candidate_text": "I wonder whether the racetrack is outdoors?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3795:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3642:premise", "snli:test:4020:premise", "snli:test:3631:premise", "snli:test:3795:premise", "style:personachat:test:557:self:1:positive:0"], "evidence_texts": ["A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "A boy in a red jacket and balck hat sliding on his knees down a snowy hill", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "interesting . so , are you aligned with a specific religion ?"], "speaker_id": "snli:test:3795", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3819", "candidate_text": "well, a woman is blind", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3819:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3086:premise", "snli:test:2989:premise", "snli:test:6790:premise", "snli:test:3819:premise", "style:personachat:test:143:self:2:positive:0", "style:personachat:test:143:self:2:positive:1"], "evidence_texts": ["Two girls playing basketball.", "Two people play horseshoes.", "Woman adjusting her eyeglasses.", "Woman is looking something.", "that sounds cool . i typically just enjoy cleaning .", "it is definitely not for everyone ."], "speaker_id": "snli:test:3819", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3819", "candidate_text": "Honestly, A woman is blind!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3819:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3086:premise", "snli:test:2989:premise", "snli:test:6790:premise", "snli:test:3819:premise", "style:personachat:test:84:self:1:positive:0"], "evidence_texts": ["Two girls playing basketball.", "Two people play horseshoes.", "Woman adjusting her eyeglasses.", "Woman is looking something.", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:3819", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3820", "candidate_text": "well, a woman is looking", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3820:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3820:premise", "snli:test:6790:premise", "snli:test:3086:premise", "snli:test:9479:premise", "style:personachat:test:143:self:5:positive:0", "style:personachat:test:143:self:5:positive:1", "style:personachat:test:143:self:5:positive:2", "style:personachat:test:143:self:5:positive:3", "style:personachat:test:143:self:5:positive:4"], "evidence_texts": ["Woman is looking something.", "Woman adjusting her eyeglasses.", "Two girls playing basketball.", "A boy plays basketball.", "let me see , i have been forgetting ! i am 30 .", "i am single . i have not dated a lot until recently .", "i am the same way . i work a lot to keep myself busy .", "that sounds cool . i typically just enjoy cleaning .", "it is definitely not for everyone ."], "speaker_id": "snli:test:3820", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3820", "candidate_text": "A woman is looking.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3820:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3820:premise", "snli:test:6790:premise", "snli:test:3086:premise", "snli:test:9479:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Woman is looking something.", "Woman adjusting her eyeglasses.", "Two girls playing basketball.", "A boy plays basketball.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3820", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3827", "candidate_text": "well, a fisherman is sleeping on his boat", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3827:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3578:premise", "snli:test:4082:premise", "snli:test:3577:premise", "snli:test:3827:premise", "style:personachat:test:143:self:7:positive:0", "style:personachat:test:143:self:7:positive:1", "style:personachat:test:143:self:7:positive:2", "style:personachat:test:143:self:7:positive:3", "style:personachat:test:143:self:7:positive:4", "style:personachat:test:143:self:7:positive:5", "style:personachat:test:143:self:7:positive:6"], "evidence_texts": ["A young man holding a microphone and singing.", "Some kids are wrestling on an inflatable raft", "A young man holding a microphone and singing.", "A fisherman using a cellphone on a boat.", "i am doing fine . you ?", "good to hear . i am a girl .", "let me see , i have been forgetting ! i am 30 .", "i am single . i have not dated a lot until recently .", "i am the same way . i work a lot to keep myself busy .", "that sounds cool . i typically just enjoy cleaning .", "it is definitely not for everyone ."], "speaker_id": "snli:test:3827", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3827", "candidate_text": "I wonder whether a fisherman is sleeping on his boat?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3827:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3578:premise", "snli:test:4082:premise", "snli:test:3577:premise", "snli:test:3827:premise", "style:personachat:test:557:self:7:positive:0", "style:personachat:test:557:self:7:positive:1", "style:personachat:test:557:self:7:positive:2", "style:personachat:test:557:self:7:positive:3", "style:personachat:test:557:self:7:positive:4", "style:personachat:test:557:self:7:positive:5", "style:personachat:test:557:self:7:positive:6"], "evidence_texts": ["A young man holding a microphone and singing.", "Some kids are wrestling on an inflatable raft", "A young man holding a microphone and singing.", "A fisherman using a cellphone on a boat.", "hi . wow . that is quite an entrance . i say god .", "not as much as i should . i read as much as i can about the arctic .", "we can learn a lot from it . i used to teach but now i explore . you ?", "cool . i take a more active role . i study polar bears and how they hunt .", "did you know that inuits learned how to ice fish by watching the bears ?", "they believe that the earth is like its own god . what would you say to that ?", "interesting . so , are you aligned with a specific religion ?"], "speaker_id": "snli:test:3827", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3829", "candidate_text": "Honestly, A father attempting to encorage his children to run around and have fun outside!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3829:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3292:premise", "snli:test:3829:premise", "snli:test:3865:premise", "snli:test:3163:premise", "style:personachat:test:144:self:1:positive:0"], "evidence_texts": ["A person waterskiing in a river with a large wall in the background.", "A man getting a group of children to move around and have fun.", "A man in a brown shirt and blue shorts walking in Droney Park.", "A couple look into the camera as the foreground is filled with balloons.", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:3829", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3829", "candidate_text": "well, a father attempting to encorage his children to run around and have fun outside", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3829:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3292:premise", "snli:test:3829:premise", "snli:test:3865:premise", "snli:test:3163:premise", "style:personachat:test:253:self:1:positive:0"], "evidence_texts": ["A person waterskiing in a river with a large wall in the background.", "A man getting a group of children to move around and have fun.", "A man in a brown shirt and blue shorts walking in Droney Park.", "A couple look into the camera as the foreground is filled with balloons.", "how old are you i am 33"], "speaker_id": "snli:test:3829", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3834", "candidate_text": "Honestly, Aliens play!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3834:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1010:premise", "snli:test:397:premise", "snli:test:3834:premise", "snli:test:1009:premise", "style:personachat:test:144:self:2:positive:0", "style:personachat:test:144:self:2:positive:1"], "evidence_texts": ["A dog runs.", "A middle eastern marketplace.", "Men play hockey.", "A dog runs.", "i went to college in san diego , but moved out here 2 years ago after i graduated .", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:3834", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3834", "candidate_text": "Aliens play.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3834:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1010:premise", "snli:test:397:premise", "snli:test:3834:premise", "snli:test:1009:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A dog runs.", "A middle eastern marketplace.", "Men play hockey.", "A dog runs.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3834", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3838", "candidate_text": "Honestly, A dog is chasing a frisbee!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3838:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3732:premise", "snli:test:3844:premise", "snli:test:3838:premise", "snli:test:3650:premise", "style:personachat:test:144:self:5:positive:0", "style:personachat:test:144:self:5:positive:1", "style:personachat:test:144:self:5:positive:2", "style:personachat:test:144:self:5:positive:3", "style:personachat:test:144:self:5:positive:4"], "evidence_texts": ["A speaker is talking with a TV in the background.", "An entertainers reaches out to his audience during a performance.", "A dog leaps into the air for a tennis ball.", "Young boy smiles at the camera from the tire swing.", "i teach kindergarten . barely covers the bills ?", "no problem . 7 kids must fry your brain ! lol", "purple ! i live in seattle . how about you ?", "i went to college in san diego , but moved out here 2 years ago after i graduated .", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:3838", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3838", "candidate_text": "I wonder whether a dog is chasing a frisbee?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3838:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3732:premise", "snli:test:3844:premise", "snli:test:3838:premise", "snli:test:3650:premise", "style:personachat:test:562:self:5:positive:0", "style:personachat:test:562:self:5:positive:1", "style:personachat:test:562:self:5:positive:2", "style:personachat:test:562:self:5:positive:3", "style:personachat:test:562:self:5:positive:4"], "evidence_texts": ["A speaker is talking with a TV in the background.", "An entertainers reaches out to his audience during a performance.", "A dog leaps into the air for a tennis ball.", "Young boy smiles at the camera from the tire swing.", "yeah , we work out every day , so staying in shape is no problem .", "did they get to travel much ?", "i am in south korea right now . it is a very tense area politically .", "being in the army helps me take out my frustrations in a positive way .", "are you close to your family ?"], "speaker_id": "snli:test:3838", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3842", "candidate_text": "Honestly, The horse is being ridden by a man!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3842:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:2761:premise", "snli:test:5063:premise", "snli:test:3842:premise", "snli:test:2760:premise", "style:personachat:test:144:self:7:positive:0", "style:personachat:test:144:self:7:positive:1", "style:personachat:test:144:self:7:positive:2", "style:personachat:test:144:self:7:positive:3", "style:personachat:test:144:self:7:positive:4", "style:personachat:test:144:self:7:positive:5", "style:personachat:test:144:self:7:positive:6"], "evidence_texts": ["A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "i am good ! i teach kindergarten , so kids books are my thing !", "i just have 2 roommates and a cat . no kids yet", "i teach kindergarten . barely covers the bills ?", "no problem . 7 kids must fry your brain ! lol", "purple ! i live in seattle . how about you ?", "i went to college in san diego , but moved out here 2 years ago after i graduated .", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:3842", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3842", "candidate_text": "well, the horse is being ridden by a man", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3842:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:2761:premise", "snli:test:5063:premise", "snli:test:3842:premise", "snli:test:2760:premise", "style:personachat:test:253:self:7:positive:0", "style:personachat:test:253:self:7:positive:1", "style:personachat:test:253:self:7:positive:2", "style:personachat:test:253:self:7:positive:3", "style:personachat:test:253:self:7:positive:4", "style:personachat:test:253:self:7:positive:5", "style:personachat:test:253:self:7:positive:6"], "evidence_texts": ["A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "hello how are you doing", "is your name tara by any chance", "where are you from i am from new york", "i am doing ok are you male or female", "i am male and this is a wired hit to do", "i have to brothers and two sisters", "how old are you i am 33"], "speaker_id": "snli:test:3842", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3844", "candidate_text": "well, a man singing to the crowd", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3844:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3844:premise", "snli:test:3838:premise", "snli:test:3899:premise", "snli:test:3732:premise", "style:personachat:test:145:self:1:positive:0"], "evidence_texts": ["An entertainers reaches out to his audience during a performance.", "A dog leaps into the air for a tennis ball.", "Three girls jumping in front of a house's flower garden.", "A speaker is talking with a TV in the background.", "maybe not as exciting , but at least you see snow ."], "speaker_id": "snli:test:3844", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3844", "candidate_text": "A man singing to the crowd.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3844:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3844:premise", "snli:test:3838:premise", "snli:test:3899:premise", "snli:test:3732:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["An entertainers reaches out to his audience during a performance.", "A dog leaps into the air for a tennis ball.", "Three girls jumping in front of a house's flower garden.", "A speaker is talking with a TV in the background.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3844", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3846", "candidate_text": "well, a girl is play fighting", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3846:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3619:premise", "snli:test:3915:premise", "snli:test:3846:premise", "snli:test:3431:premise", "style:personachat:test:145:self:2:positive:0", "style:personachat:test:145:self:2:positive:1"], "evidence_texts": ["A man in a green shirt and a red kilt is juggling three clubs.", "People in bathing suits bend down and reach into the water of a lake.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "are you from cuba ?", "maybe not as exciting , but at least you see snow ."], "speaker_id": "snli:test:3846", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3846", "candidate_text": "I wonder whether a girl is play fighting?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3846:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3619:premise", "snli:test:3915:premise", "snli:test:3846:premise", "snli:test:3431:premise", "style:personachat:test:567:self:2:positive:0", "style:personachat:test:567:self:2:positive:1"], "evidence_texts": ["A man in a green shirt and a red kilt is juggling three clubs.", "People in bathing suits bend down and reach into the water of a lake.", "A girl playfully kicks a guy in the face while another person makes faces.", "A man with his finger in his mouth is standing outside of an RV.", "ok . do you have any hobbies ?", "what kind of music ? i do not sing very well ."], "speaker_id": "snli:test:3846", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3855", "candidate_text": "well, a skier going down the bunny hill", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3855:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3712:premise", "snli:test:3974:premise", "snli:test:3542:premise", "snli:test:3855:premise", "style:personachat:test:145:self:5:positive:0", "style:personachat:test:145:self:5:positive:1", "style:personachat:test:145:self:5:positive:2", "style:personachat:test:145:self:5:positive:3", "style:personachat:test:145:self:5:positive:4"], "evidence_texts": ["A man in a bright yellow shirt juggles while riding a unicycle.", "Two dogs are running next to each other on a grassy hill.", "A man and wife stand at the alter as they get married.", "A skier goes down a hill with a mountain as a backdrop.", "are you a doctor , i work as an engineer in the army .", "that sounds exciting . do you know any languages , i only know english .", "i would like to learn japanese . and visit there someday . anywhere you would like to go ?", "are you from cuba ?", "maybe not as exciting , but at least you see snow ."], "speaker_id": "snli:test:3855", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3855", "candidate_text": "Honestly, A skier going down the bunny hill!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3855:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3712:premise", "snli:test:3974:premise", "snli:test:3542:premise", "snli:test:3855:premise", "style:personachat:test:105:self:2:positive:0", "style:personachat:test:105:self:2:positive:1"], "evidence_texts": ["A man in a bright yellow shirt juggles while riding a unicycle.", "Two dogs are running next to each other on a grassy hill.", "A man and wife stand at the alter as they get married.", "A skier goes down a hill with a mountain as a backdrop.", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:3855", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3860", "candidate_text": "well, the man has been playing the blues", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3860:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3860:premise", "snli:test:3729:premise", "snli:test:4389:premise", "snli:test:3474:premise", "style:personachat:test:145:self:7:positive:0", "style:personachat:test:145:self:7:positive:1", "style:personachat:test:145:self:7:positive:2", "style:personachat:test:145:self:7:positive:3", "style:personachat:test:145:self:7:positive:4", "style:personachat:test:145:self:7:positive:5", "style:personachat:test:145:self:7:positive:6"], "evidence_texts": ["An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "doing well , just listening to some bach . yourself ?", "i thought about going out tonight , but the insects are bad , and i hate them .", "are you a doctor , i work as an engineer in the army .", "that sounds exciting . do you know any languages , i only know english .", "i would like to learn japanese . and visit there someday . anywhere you would like to go ?", "are you from cuba ?", "maybe not as exciting , but at least you see snow ."], "speaker_id": "snli:test:3860", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3860", "candidate_text": "The man has been playing the blues.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3860:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3860:premise", "snli:test:3729:premise", "snli:test:4389:premise", "snli:test:3474:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "A person on blue pants and a white shirt walks away from a dirt circle on multicolored grass.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3860", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3865", "candidate_text": "Honestly, A man walking in Droney Park!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3865:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3829:premise", "snli:test:3865:premise", "snli:test:4062:premise", "snli:test:3292:premise", "style:personachat:test:146:self:1:positive:0"], "evidence_texts": ["A man getting a group of children to move around and have fun.", "A man in a brown shirt and blue shorts walking in Droney Park.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "A person waterskiing in a river with a large wall in the background.", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:3865", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3865", "candidate_text": "I wonder whether a man walking in droney park?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3865:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3829:premise", "snli:test:3865:premise", "snli:test:4062:premise", "snli:test:3292:premise", "style:personachat:test:569:self:1:positive:0"], "evidence_texts": ["A man getting a group of children to move around and have fun.", "A man in a brown shirt and blue shorts walking in Droney Park.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "A person waterskiing in a river with a large wall in the background.", "my name is jamie , yours ?"], "speaker_id": "snli:test:3865", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3867", "candidate_text": "Honestly, This bald man stands naked in protest of fur for fashion!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3867:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3192:premise", "snli:test:3041:premise", "snli:test:3958:premise", "snli:test:3867:premise", "style:personachat:test:146:self:2:positive:0", "style:personachat:test:146:self:2:positive:1"], "evidence_texts": ["A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "so you like the hot weather ? i prefer cold and snow", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:3867", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3867", "candidate_text": "well, this bald man stands naked in protest of fur for fashion", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3867:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3192:premise", "snli:test:3041:premise", "snli:test:3958:premise", "snli:test:3867:premise", "style:personachat:test:256:self:2:positive:0", "style:personachat:test:256:self:2:positive:1"], "evidence_texts": ["A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "Five children, two boys and three girls, with the girls wearing white scarves, sit on the pavement outside in front of a large window.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "i bet they did , that is crazy", "lol . . i have an 88 cutlass that needs to be restored"], "speaker_id": "snli:test:3867", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3868", "candidate_text": "Honestly, This man has an eyesight deficiency!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3868:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3868:premise", "snli:test:3958:premise", "snli:test:3192:premise", "snli:test:4233:premise", "style:personachat:test:146:self:5:positive:0", "style:personachat:test:146:self:5:positive:1", "style:personachat:test:146:self:5:positive:2", "style:personachat:test:146:self:5:positive:3", "style:personachat:test:146:self:5:positive:4"], "evidence_texts": ["A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "no but i have two siblings , both brothers . do you have any ?", "i do not have much time for frisbee , what do you do for a job ?", "i am a managerial consultant originally from wisconsin , where did you grow up ?", "so you like the hot weather ? i prefer cold and snow", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:3868", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3868", "candidate_text": "This man has an eyesight deficiency.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3868:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3868:premise", "snli:test:3958:premise", "snli:test:3192:premise", "snli:test:4233:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A gray rally car is being driven by two adult males while a large white container is tied down on top of the roof.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3868", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3870", "candidate_text": "Honestly, A man is driving a truck blindfolded!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3870:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3653:premise", "snli:test:4118:premise", "snli:test:3870:premise", "snli:test:3597:premise", "style:personachat:test:146:self:6:positive:0", "style:personachat:test:146:self:6:positive:1", "style:personachat:test:146:self:6:positive:2", "style:personachat:test:146:self:6:positive:3", "style:personachat:test:146:self:6:positive:4", "style:personachat:test:146:self:6:positive:5"], "evidence_texts": ["Young Asian male child either smoking something or its an instrument.", "A woman crosses the street with a baby in her arms.", "A guy taking a nap in the back of a vehicle.", "Two men have an after dinner discussion at the dinner table.", "hi , i do not like sardines at any time of the day !", "no but i have two siblings , both brothers . do you have any ?", "i do not have much time for frisbee , what do you do for a job ?", "i am a managerial consultant originally from wisconsin , where did you grow up ?", "so you like the hot weather ? i prefer cold and snow", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:3870", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3870", "candidate_text": "I wonder whether a man is driving a truck blindfolded?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3870:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3653:premise", "snli:test:4118:premise", "snli:test:3870:premise", "snli:test:3597:premise", "style:personachat:test:569:self:7:positive:0", "style:personachat:test:569:self:7:positive:1", "style:personachat:test:569:self:7:positive:2", "style:personachat:test:569:self:7:positive:3", "style:personachat:test:569:self:7:positive:4", "style:personachat:test:569:self:7:positive:5", "style:personachat:test:569:self:7:positive:6"], "evidence_texts": ["Young Asian male child either smoking something or its an instrument.", "A woman crosses the street with a baby in her arms.", "A guy taking a nap in the back of a vehicle.", "Two men have an after dinner discussion at the dinner table.", "great and how are you ?", "yes i fix hair , how about you", "cool . i love christmas , what is your favorite holiday", "awesome ! ! ! i love all the decorations", "everything green , what is yours", "we could totally be friends lol", "my name is jamie , yours ?"], "speaker_id": "snli:test:3870", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3899", "candidate_text": "I wonder whether three girls are jumping?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3899:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3844:premise", "snli:test:3975:premise", "snli:test:3838:premise", "snli:test:3899:premise", "style:personachat:test:147:self:1:positive:0"], "evidence_texts": ["An entertainers reaches out to his audience during a performance.", "Young people are standing around a drum at a party.", "A dog leaps into the air for a tennis ball.", "Three girls jumping in front of a house's flower garden.", "which system are you referring to ?"], "speaker_id": "snli:test:3899", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3899", "candidate_text": "well, three girls are jumping", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3899:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3844:premise", "snli:test:3975:premise", "snli:test:3838:premise", "snli:test:3899:premise", "style:personachat:test:257:self:1:positive:0"], "evidence_texts": ["An entertainers reaches out to his audience during a performance.", "Young people are standing around a drum at a party.", "A dog leaps into the air for a tennis ball.", "Three girls jumping in front of a house's flower garden.", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:3899", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3912", "candidate_text": "I wonder whether four siblings pose for a photo on a stage?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3912:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3912:premise", "snli:test:3518:premise", "snli:test:3997:premise", "snli:test:3399:premise", "style:personachat:test:147:self:2:positive:0", "style:personachat:test:147:self:2:positive:1"], "evidence_texts": ["Four kids pose on a stage.", "A rally in a different country.", "Four people enjoying an art exhibit.", "Two people sitting on the sand.", "i take the kids from the hospital to the zoo weekly", "which system are you referring to ?"], "speaker_id": "snli:test:3912", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3912", "candidate_text": "Honestly, Four siblings pose for a photo on a stage!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3912:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3912:premise", "snli:test:3518:premise", "snli:test:3997:premise", "snli:test:3399:premise", "style:personachat:test:127:self:1:positive:0"], "evidence_texts": ["Four kids pose on a stage.", "A rally in a different country.", "Four people enjoying an art exhibit.", "Two people sitting on the sand.", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:3912", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3915", "candidate_text": "I wonder whether a puppy is sitting?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3915:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3846:premise", "snli:test:3969:premise", "snli:test:3619:premise", "snli:test:3915:premise", "style:personachat:test:147:self:5:positive:0", "style:personachat:test:147:self:5:positive:1", "style:personachat:test:147:self:5:positive:2", "style:personachat:test:147:self:5:positive:3", "style:personachat:test:147:self:5:positive:4"], "evidence_texts": ["A girl playfully kicks a guy in the face while another person makes faces.", "A brown dog and a black and white dog are running over grassy hills.", "A man in a green shirt and a red kilt is juggling three clubs.", "People in bathing suits bend down and reach into the water of a lake.", "i started as a red cross volunteer but i prefer the color green personally", "yea because it lets me visit zoos , which i love", "that can be tough . finding a job to support your rent is hard", "i take the kids from the hospital to the zoo weekly", "which system are you referring to ?"], "speaker_id": "snli:test:3915", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3915", "candidate_text": "A puppy is sitting.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3915:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3846:premise", "snli:test:3969:premise", "snli:test:3619:premise", "snli:test:3915:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A girl playfully kicks a guy in the face while another person makes faces.", "A brown dog and a black and white dog are running over grassy hills.", "A man in a green shirt and a red kilt is juggling three clubs.", "People in bathing suits bend down and reach into the water of a lake.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3915", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3943", "candidate_text": "I wonder whether the girl is swimming?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3943:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3690:premise", "snli:test:4055:premise", "snli:test:3668:premise", "snli:test:3943:premise", "style:personachat:test:147:self:7:positive:0", "style:personachat:test:147:self:7:positive:1", "style:personachat:test:147:self:7:positive:2", "style:personachat:test:147:self:7:positive:3", "style:personachat:test:147:self:7:positive:4", "style:personachat:test:147:self:7:positive:5", "style:personachat:test:147:self:7:positive:6"], "evidence_texts": ["Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "A girl is about to hit a tennis ball coming from above her with a racket", "i am doing great . just hanging out with my teddy bear collection", "i have a very big collection ! since i am a nurse , people give them to me", "i started as a red cross volunteer but i prefer the color green personally", "yea because it lets me visit zoos , which i love", "that can be tough . finding a job to support your rent is hard", "i take the kids from the hospital to the zoo weekly", "which system are you referring to ?"], "speaker_id": "snli:test:3943", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3943", "candidate_text": "well, the girl is swimming", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3943:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3690:premise", "snli:test:4055:premise", "snli:test:3668:premise", "snli:test:3943:premise", "style:personachat:test:257:self:6:positive:0", "style:personachat:test:257:self:6:positive:1", "style:personachat:test:257:self:6:positive:2", "style:personachat:test:257:self:6:positive:3", "style:personachat:test:257:self:6:positive:4", "style:personachat:test:257:self:6:positive:5"], "evidence_texts": ["Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "An athletic woman in a gray top and black shorts, seems very focused on her rollerblading.", "A girl is about to hit a tennis ball coming from above her with a racket", "pretty good . how are you ?", "i just got off work .", "i am looking for another job myself .", "my parents did not want me either .", "what kids are you trying to help ?", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:3943", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3956", "candidate_text": "well, the two men and women are talking to people that they know", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3956:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3956:premise", "snli:test:2946:premise", "snli:test:5367:premise", "snli:test:2571:premise", "style:personachat:test:148:self:1:positive:0"], "evidence_texts": ["Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:3956", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3956", "candidate_text": "The two men and women are talking to people that they know.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3956:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3956:premise", "snli:test:2946:premise", "snli:test:5367:premise", "snli:test:2571:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "A man wearing blue pants, a white helmet, and a green-striped shirt is shown on skis in midair, against a blue sky with a single puffy cloud.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3956", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3958", "candidate_text": "well, a lady is climbing a rope", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3958:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3868:premise", "snli:test:4233:premise", "snli:test:3958:premise", "snli:test:3867:premise", "style:personachat:test:148:self:2:positive:0", "style:personachat:test:148:self:2:positive:1"], "evidence_texts": ["A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "well best of luck to him . i know he must appreciate your support", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:3958", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3958", "candidate_text": "I wonder whether a lady is climbing a rope?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3958:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3868:premise", "snli:test:4233:premise", "snli:test:3958:premise", "snli:test:3867:premise", "style:personachat:test:571:self:2:positive:0", "style:personachat:test:571:self:2:positive:1"], "evidence_texts": ["A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "oh nice . i wonder if my 3 girls would ever go into business with me", "haha . do they like heart songs ? heart is the best band"], "speaker_id": "snli:test:3958", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3969", "candidate_text": "well, two dogs chase a bunny", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3969:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3915:premise", "snli:test:3969:premise", "snli:test:4344:premise", "snli:test:3846:premise", "style:personachat:test:148:self:5:positive:0", "style:personachat:test:148:self:5:positive:1", "style:personachat:test:148:self:5:positive:2", "style:personachat:test:148:self:5:positive:3", "style:personachat:test:148:self:5:positive:4"], "evidence_texts": ["People in bathing suits bend down and reach into the water of a lake.", "A brown dog and a black and white dog are running over grassy hills.", "A little boy in a red shirt and swim trunks plays at the beach.", "A girl playfully kicks a guy in the face while another person makes faces.", "lol that is hilarious . it is a patriotic quilt as a present for my dad", "yes , he is a war vet . i am making it for veterans day", "that is something to be proud of . we should all be proud of our military", "well best of luck to him . i know he must appreciate your support", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:3969", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3969", "candidate_text": "Honestly, two dogs chase a bunny!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3969:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3915:premise", "snli:test:3969:premise", "snli:test:4344:premise", "snli:test:3846:premise", "style:personachat:test:130:self:2:positive:0", "style:personachat:test:130:self:2:positive:1"], "evidence_texts": ["People in bathing suits bend down and reach into the water of a lake.", "A brown dog and a black and white dog are running over grassy hills.", "A little boy in a red shirt and swim trunks plays at the beach.", "A girl playfully kicks a guy in the face while another person makes faces.", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:3969", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3974", "candidate_text": "well, the dogs are eating their food indoors", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3974:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3855:premise", "snli:test:4096:premise", "snli:test:3974:premise", "snli:test:3712:premise", "style:personachat:test:148:self:7:positive:0", "style:personachat:test:148:self:7:positive:1", "style:personachat:test:148:self:7:positive:2", "style:personachat:test:148:self:7:positive:3", "style:personachat:test:148:self:7:positive:4", "style:personachat:test:148:self:7:positive:5", "style:personachat:test:148:self:7:positive:6"], "evidence_texts": ["A skier goes down a hill with a mountain as a backdrop.", "A man is riding up an escalator that is carrying shopping carts.", "Two dogs are running next to each other on a grassy hill.", "A man in a bright yellow shirt juggles while riding a unicycle.", "i am working on my latest quilt , do you have hobbies ?", "wow that is a cool bet . do they have names ?", "lol that is hilarious . it is a patriotic quilt as a present for my dad", "yes , he is a war vet . i am making it for veterans day", "that is something to be proud of . we should all be proud of our military", "well best of luck to him . i know he must appreciate your support", "army . he is from scotland and serving in the army meant a lot to him"], "speaker_id": "snli:test:3974", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3974", "candidate_text": "The dogs are eating their food indoors.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:3974:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3855:premise", "snli:test:4096:premise", "snli:test:3974:premise", "snli:test:3712:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A skier goes down a hill with a mountain as a backdrop.", "A man is riding up an escalator that is carrying shopping carts.", "Two dogs are running next to each other on a grassy hill.", "A man in a bright yellow shirt juggles while riding a unicycle.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3974", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3975", "candidate_text": "I wonder whether people are playing a drum?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3975:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3899:premise", "snli:test:3982:premise", "snli:test:3844:premise", "snli:test:3975:premise", "style:personachat:test:149:self:1:positive:0"], "evidence_texts": ["Three girls jumping in front of a house's flower garden.", "A white man in a red shirt riding a bike.", "An entertainers reaches out to his audience during a performance.", "Young people are standing around a drum at a party.", "very cool how long have you been dating ? i am looking for love in the office"], "speaker_id": "snli:test:3975", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:3975", "candidate_text": "People are playing a drum.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3975:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3899:premise", "snli:test:3982:premise", "snli:test:3844:premise", "snli:test:3975:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Three girls jumping in front of a house's flower garden.", "A white man in a red shirt riding a bike.", "An entertainers reaches out to his audience during a performance.", "Young people are standing around a drum at a party.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:3975", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:3982", "candidate_text": "I wonder whether a guy wears a shirt on a bike?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3982:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3975:premise", "snli:test:4009:premise", "snli:test:3982:premise", "snli:test:3899:premise", "style:personachat:test:149:self:2:positive:0", "style:personachat:test:149:self:2:positive:1"], "evidence_texts": ["Young people are standing around a drum at a party.", "The team swiftly moves their traditional boat down the river.", "A white man in a red shirt riding a bike.", "Three girls jumping in front of a house's flower garden.", "have someone to go polka with ?", "very cool how long have you been dating ? i am looking for love in the office"], "speaker_id": "snli:test:3982", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3982", "candidate_text": "well, a guy wears a shirt on a bike", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3982:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3975:premise", "snli:test:4009:premise", "snli:test:3982:premise", "snli:test:3899:premise", "style:personachat:test:259:self:2:positive:0", "style:personachat:test:259:self:2:positive:1"], "evidence_texts": ["Young people are standing around a drum at a party.", "The team swiftly moves their traditional boat down the river.", "A white man in a red shirt riding a bike.", "Three girls jumping in front of a house's flower garden.", "i just started jujitsu , i may have to start ultimate too", "in the northwest , where i go to college"], "speaker_id": "snli:test:3982", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:3997", "candidate_text": "I wonder whether fours people are looking at art?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3997:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3912:premise", "snli:test:3997:premise", "snli:test:4107:premise", "snli:test:3518:premise", "style:personachat:test:149:self:5:positive:0", "style:personachat:test:149:self:5:positive:1", "style:personachat:test:149:self:5:positive:2", "style:personachat:test:149:self:5:positive:3", "style:personachat:test:149:self:5:positive:4"], "evidence_texts": ["Four kids pose on a stage.", "Four people enjoying an art exhibit.", "Coed choir singing at an event.", "A rally in a different country.", "mine is orange i eat alot of orange candy", "did you enjoy it ? what do you do for fun ?", "yeah ? have a outfit to polka in ?", "have someone to go polka with ?", "very cool how long have you been dating ? i am looking for love in the office"], "speaker_id": "snli:test:3997", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:3997", "candidate_text": "Honestly, Fours people are looking at art!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:3997:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3912:premise", "snli:test:3997:premise", "snli:test:4107:premise", "snli:test:3518:premise", "style:personachat:test:133:self:2:positive:0", "style:personachat:test:133:self:2:positive:1"], "evidence_texts": ["Four kids pose on a stage.", "Four people enjoying an art exhibit.", "Coed choir singing at an event.", "A rally in a different country.", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:3997", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4009", "candidate_text": "I wonder whether the boat has sunk?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4009:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3982:premise", "snli:test:4009:premise", "snli:test:3975:premise", "snli:test:4038:premise", "style:personachat:test:149:self:7:positive:0", "style:personachat:test:149:self:7:positive:1", "style:personachat:test:149:self:7:positive:2", "style:personachat:test:149:self:7:positive:3", "style:personachat:test:149:self:7:positive:4", "style:personachat:test:149:self:7:positive:5", "style:personachat:test:149:self:7:positive:6"], "evidence_texts": ["A white man in a red shirt riding a bike.", "The team swiftly moves their traditional boat down the river.", "Young people are standing around a drum at a party.", "Men in an office with computers and a projector screen.", "doing good just watching some soccer i use to play", "i have not been swimming since i graduated last year from college", "mine is orange i eat alot of orange candy", "did you enjoy it ? what do you do for fun ?", "yeah ? have a outfit to polka in ?", "have someone to go polka with ?", "very cool how long have you been dating ? i am looking for love in the office"], "speaker_id": "snli:test:4009", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4009", "candidate_text": "The boat has sunk.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4009:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3982:premise", "snli:test:4009:premise", "snli:test:3975:premise", "snli:test:4038:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A white man in a red shirt riding a bike.", "The team swiftly moves their traditional boat down the river.", "Young people are standing around a drum at a party.", "Men in an office with computers and a projector screen.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4009", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4010", "candidate_text": "I wonder whether the river is wild?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4010:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4038:premise", "snli:test:3982:premise", "snli:test:4010:premise", "snli:test:4039:premise", "style:personachat:test:150:self:1:positive:0"], "evidence_texts": ["Men in an office with computers and a projector screen.", "A white man in a red shirt riding a bike.", "The team swiftly moves their traditional boat down the river.", "Men in an office with computers and a projector screen.", "so what do you do for fun ?"], "speaker_id": "snli:test:4010", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4010", "candidate_text": "well, the river is wild", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4010:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4038:premise", "snli:test:3982:premise", "snli:test:4010:premise", "snli:test:4039:premise", "style:personachat:test:262:self:1:positive:0"], "evidence_texts": ["Men in an office with computers and a projector screen.", "A white man in a red shirt riding a bike.", "The team swiftly moves their traditional boat down the river.", "Men in an office with computers and a projector screen.", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:4010", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4020", "candidate_text": "I wonder whether the man and women are standing by a river?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4020:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4020:premise", "snli:test:3795:premise", "snli:test:4476:premise", "snli:test:3642:premise", "style:personachat:test:150:self:2:positive:0", "style:personachat:test:150:self:2:positive:1"], "evidence_texts": ["Two people are sitting in a chair a lady reading and a man looking to his phone", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "i am 26 . i only have one kiddo so far . i can not imagine twins .", "so what do you do for fun ?"], "speaker_id": "snli:test:4020", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4020", "candidate_text": "Honestly, The man and women are standing by a river!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4020:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4020:premise", "snli:test:3795:premise", "snli:test:4476:premise", "snli:test:3642:premise", "style:personachat:test:137:self:1:positive:0"], "evidence_texts": ["Two people are sitting in a chair a lady reading and a man looking to his phone", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A guy in a yellow shirt performs a balancing act on a taught chain near a canal.", "yes i would love to hear a secret !"], "speaker_id": "snli:test:4020", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4036", "candidate_text": "I wonder whether a boy puts on some sunglasses?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4036:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4036:premise", "snli:test:3758:premise", "snli:test:4293:premise", "snli:test:3756:premise", "style:personachat:test:150:self:5:positive:0", "style:personachat:test:150:self:5:positive:1", "style:personachat:test:150:self:5:positive:2", "style:personachat:test:150:self:5:positive:3", "style:personachat:test:150:self:5:positive:4"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "so what do you do for a living ?", "i am a kindergarten teacher . i really love kids .", "two in the same grade ! that must be a handful !", "i am 26 . i only have one kiddo so far . i can not imagine twins .", "so what do you do for fun ?"], "speaker_id": "snli:test:4036", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4036", "candidate_text": "A boy puts on some sunglasses.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4036:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4036:premise", "snli:test:3758:premise", "snli:test:4293:premise", "snli:test:3756:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4036", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4038", "candidate_text": "I wonder whether the men are at the gym?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4038:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4010:premise", "snli:test:4009:premise", "snli:test:4038:premise", "snli:test:4042:premise", "style:personachat:test:150:self:6:positive:0", "style:personachat:test:150:self:6:positive:1", "style:personachat:test:150:self:6:positive:2", "style:personachat:test:150:self:6:positive:3", "style:personachat:test:150:self:6:positive:4", "style:personachat:test:150:self:6:positive:5"], "evidence_texts": ["The team swiftly moves their traditional boat down the river.", "The team swiftly moves their traditional boat down the river.", "Men in an office with computers and a projector screen.", "Two kids in black trunks bouncing on a wet trampoline.", "i am fantastic ! how are you ?", "so what do you do for a living ?", "i am a kindergarten teacher . i really love kids .", "two in the same grade ! that must be a handful !", "i am 26 . i only have one kiddo so far . i can not imagine twins .", "so what do you do for fun ?"], "speaker_id": "snli:test:4038", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4038", "candidate_text": "well, the men are at the gym", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4038:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4010:premise", "snli:test:4009:premise", "snli:test:4038:premise", "snli:test:4042:premise", "style:personachat:test:262:self:7:positive:0", "style:personachat:test:262:self:7:positive:1", "style:personachat:test:262:self:7:positive:2", "style:personachat:test:262:self:7:positive:3", "style:personachat:test:262:self:7:positive:4", "style:personachat:test:262:self:7:positive:5", "style:personachat:test:262:self:7:positive:6"], "evidence_texts": ["The team swiftly moves their traditional boat down the river.", "The team swiftly moves their traditional boat down the river.", "Men in an office with computers and a projector screen.", "Two kids in black trunks bouncing on a wet trampoline.", "hi , i am from the south , what about you ?", "i have a mother and five sisters , how about you ?", "you never heard of south carolina", "o yes that in columbia i live a bout 80 miles", "that ok , i really under that", "i am number two next to brothers also", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:4038", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4039", "candidate_text": "well, there are computers and a projector screen in the room", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4039:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4042:premise", "snli:test:4010:premise", "snli:test:4078:premise", "snli:test:4039:premise", "style:personachat:test:151:self:1:positive:0"], "evidence_texts": ["Two kids in black trunks bouncing on a wet trampoline.", "The team swiftly moves their traditional boat down the river.", "A lady is promoting something to the other two girls.", "Men in an office with computers and a projector screen.", "i would highly recommend it ."], "speaker_id": "snli:test:4039", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4039", "candidate_text": "There are computers and a projector screen in the room.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4039:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4042:premise", "snli:test:4010:premise", "snli:test:4078:premise", "snli:test:4039:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two kids in black trunks bouncing on a wet trampoline.", "The team swiftly moves their traditional boat down the river.", "A lady is promoting something to the other two girls.", "Men in an office with computers and a projector screen.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4039", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4042", "candidate_text": "well, two kids are exercising on trampolines", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4042:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4039:premise", "snli:test:4078:premise", "snli:test:4042:premise", "snli:test:4038:premise", "style:personachat:test:151:self:2:positive:0", "style:personachat:test:151:self:2:positive:1"], "evidence_texts": ["Men in an office with computers and a projector screen.", "A lady is promoting something to the other two girls.", "Two kids in black trunks bouncing on a wet trampoline.", "Men in an office with computers and a projector screen.", "i agree . in order to balance life , i live by the sea and swim daily .", "i would highly recommend it ."], "speaker_id": "snli:test:4042", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4042", "candidate_text": "I wonder whether two kids are exercising on trampolines?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4042:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4039:premise", "snli:test:4078:premise", "snli:test:4042:premise", "snli:test:4038:premise", "style:personachat:test:580:self:2:positive:0", "style:personachat:test:580:self:2:positive:1"], "evidence_texts": ["Men in an office with computers and a projector screen.", "A lady is promoting something to the other two girls.", "Two kids in black trunks bouncing on a wet trampoline.", "Men in an office with computers and a projector screen.", "oh ok . well it was nice to chat with you . have a nice evening", "would you like to teach me how to quilt ?"], "speaker_id": "snli:test:4042", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4055", "candidate_text": "well, the boy is asleep in bed", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4055:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3943:premise", "snli:test:4073:premise", "snli:test:3690:premise", "snli:test:4055:premise", "style:personachat:test:151:self:5:positive:0", "style:personachat:test:151:self:5:positive:1", "style:personachat:test:151:self:5:positive:2", "style:personachat:test:151:self:5:positive:3", "style:personachat:test:151:self:5:positive:4"], "evidence_texts": ["A girl is about to hit a tennis ball coming from above her with a racket", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "oh , i like fitness too , but i am a big person . but it does not stop me .", "being married is wonderful . do you have children ? i adore mine .", "that is an important job . my job is a kindergarten teacher , so i can relate .", "i agree . in order to balance life , i live by the sea and swim daily .", "i would highly recommend it ."], "speaker_id": "snli:test:4055", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4055", "candidate_text": "Honestly, The boy is asleep in bed!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4055:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3943:premise", "snli:test:4073:premise", "snli:test:3690:premise", "snli:test:4055:premise", "style:personachat:test:140:self:2:positive:0", "style:personachat:test:140:self:2:positive:1"], "evidence_texts": ["A girl is about to hit a tennis ball coming from above her with a racket", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "Two men and one woman with blond-hair walk through a gate with barbed wire on top.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:4055", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4062", "candidate_text": "well, a dog goes to the vet", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4062:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:3865:premise", "snli:test:4217:premise", "snli:test:4062:premise", "snli:test:3829:premise", "style:personachat:test:151:self:6:positive:0", "style:personachat:test:151:self:6:positive:1", "style:personachat:test:151:self:6:positive:2", "style:personachat:test:151:self:6:positive:3", "style:personachat:test:151:self:6:positive:4", "style:personachat:test:151:self:6:positive:5"], "evidence_texts": ["A man in a brown shirt and blue shorts walking in Droney Park.", "A woman is shaving the hair off of a black and white dog.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "A man getting a group of children to move around and have fun.", "i do , i really love to drive my mustang convertible to an italian restaurant .", "oh , i like fitness too , but i am a big person . but it does not stop me .", "being married is wonderful . do you have children ? i adore mine .", "that is an important job . my job is a kindergarten teacher , so i can relate .", "i agree . in order to balance life , i live by the sea and swim daily .", "i would highly recommend it ."], "speaker_id": "snli:test:4062", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4062", "candidate_text": "A dog goes to the vet.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4062:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3865:premise", "snli:test:4217:premise", "snli:test:4062:premise", "snli:test:3829:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a brown shirt and blue shorts walking in Droney Park.", "A woman is shaving the hair off of a black and white dog.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "A man getting a group of children to move around and have fun.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4062", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4073", "candidate_text": "Honestly, A young man wearing a gray winter jacket walks on a fallen oak tree!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4073:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4055:premise", "snli:test:4073:premise", "snli:test:4195:premise", "snli:test:3943:premise", "style:personachat:test:152:self:1:positive:0"], "evidence_texts": ["A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "A girl is about to hit a tennis ball coming from above her with a racket", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:4073", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4073", "candidate_text": "I wonder whether a young man wearing a gray winter jacket walks on a fallen oak tree?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4073:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4055:premise", "snli:test:4073:premise", "snli:test:4195:premise", "snli:test:3943:premise", "style:personachat:test:582:self:1:positive:0"], "evidence_texts": ["A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "A girl is about to hit a tennis ball coming from above her with a racket", "is it still the same old costumes and skins ?"], "speaker_id": "snli:test:4073", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4074", "candidate_text": "Honestly, A man stands in his room!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4074:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3359:premise", "snli:test:3146:premise", "snli:test:4074:premise", "snli:test:2877:premise", "style:personachat:test:152:self:2:positive:0", "style:personachat:test:152:self:2:positive:1"], "evidence_texts": ["A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "history was my major . i worked in a museum .", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:4074", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4074", "candidate_text": "well, a man stands in his room", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4074:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3359:premise", "snli:test:3146:premise", "snli:test:4074:premise", "snli:test:2877:premise", "style:personachat:test:264:self:2:positive:0", "style:personachat:test:264:self:2:positive:1"], "evidence_texts": ["A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man in a red shirt and a blue hat is sitting on a sidewalk and touching drinking glasses.", "great i bet you do beautiful hair styles yes i sing lovely .", "not really happy but we have to hang in there and do our best ."], "speaker_id": "snli:test:4074", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4075", "candidate_text": "Honestly, A lumberjack works!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4075:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3359:premise", "snli:test:4319:premise", "snli:test:3146:premise", "snli:test:4075:premise", "style:personachat:test:152:self:5:positive:0", "style:personachat:test:152:self:5:positive:1", "style:personachat:test:152:self:5:positive:2", "style:personachat:test:152:self:5:positive:3", "style:personachat:test:152:self:5:positive:4"], "evidence_texts": ["A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "that is great . are you married ?", "i am engaged for a wedding in december .", "i am excited ! especially since we both are now college graduates !", "history was my major . i worked in a museum .", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:4075", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4075", "candidate_text": "A lumberjack works.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4075:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3359:premise", "snli:test:4319:premise", "snli:test:3146:premise", "snli:test:4075:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "A man in a brown shirt and a white baseball cap uses a plastic barrel to water tomato plants.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4075", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4076", "candidate_text": "Honestly, A man walks outdoors!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4076:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4076:premise", "snli:test:4319:premise", "snli:test:4555:premise", "snli:test:3359:premise", "style:personachat:test:152:self:7:positive:0", "style:personachat:test:152:self:7:positive:1", "style:personachat:test:152:self:7:positive:2", "style:personachat:test:152:self:7:positive:3", "style:personachat:test:152:self:7:positive:4", "style:personachat:test:152:self:7:positive:5", "style:personachat:test:152:self:7:positive:6"], "evidence_texts": ["A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "i am fine . just watching some sports .", "i actually enjoy the steelers , do you ?", "that is great . are you married ?", "i am engaged for a wedding in december .", "i am excited ! especially since we both are now college graduates !", "history was my major . i worked in a museum .", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:4076", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4076", "candidate_text": "I wonder whether a man walks outdoors?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4076:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4076:premise", "snli:test:4319:premise", "snli:test:4555:premise", "snli:test:3359:premise", "style:personachat:test:582:self:7:positive:0", "style:personachat:test:582:self:7:positive:1", "style:personachat:test:582:self:7:positive:2", "style:personachat:test:582:self:7:positive:3", "style:personachat:test:582:self:7:positive:4", "style:personachat:test:582:self:7:positive:5", "style:personachat:test:582:self:7:positive:6"], "evidence_texts": ["A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A man wearing a denim jacket and jeans welds on a bridge above water while wearing his safety mask.", "okay , is this a speed dating thing ?", "gotcha . i had to have a tail removed when i was born .", "do you like living that life style ?", "is that still a thing ? wow .", "that really brings back memories .", "i have not been on that in years .", "is it still the same old costumes and skins ?"], "speaker_id": "snli:test:4076", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4078", "candidate_text": "well, a woman is selling jewelry to two girls", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4078:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4042:premise", "snli:test:4133:premise", "snli:test:4078:premise", "snli:test:4039:premise", "style:personachat:test:153:self:1:positive:0"], "evidence_texts": ["Two kids in black trunks bouncing on a wet trampoline.", "A person is walking a bicycle down a city street.", "A lady is promoting something to the other two girls.", "Men in an office with computers and a projector screen.", "i do . i enjoy it ."], "speaker_id": "snli:test:4078", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4078", "candidate_text": "Honestly, A woman is selling jewelry to two girls!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4078:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4042:premise", "snli:test:4133:premise", "snli:test:4078:premise", "snli:test:4039:premise", "style:personachat:test:144:self:7:positive:0", "style:personachat:test:144:self:7:positive:1", "style:personachat:test:144:self:7:positive:2", "style:personachat:test:144:self:7:positive:3", "style:personachat:test:144:self:7:positive:4", "style:personachat:test:144:self:7:positive:5", "style:personachat:test:144:self:7:positive:6"], "evidence_texts": ["Two kids in black trunks bouncing on a wet trampoline.", "A person is walking a bicycle down a city street.", "A lady is promoting something to the other two girls.", "Men in an office with computers and a projector screen.", "i am good ! i teach kindergarten , so kids books are my thing !", "i just have 2 roommates and a cat . no kids yet", "i teach kindergarten . barely covers the bills ?", "no problem . 7 kids must fry your brain ! lol", "purple ! i live in seattle . how about you ?", "i went to college in san diego , but moved out here 2 years ago after i graduated .", "a bit gloomy , but the coffee is great !"], "speaker_id": "snli:test:4078", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4082", "candidate_text": "well, some kids playing with a raft", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4082:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3827:premise", "snli:test:4083:premise", "snli:test:4082:premise", "snli:test:3578:premise", "style:personachat:test:153:self:2:positive:0", "style:personachat:test:153:self:2:positive:1"], "evidence_texts": ["A fisherman using a cellphone on a boat.", "A group of Asian girls are standing together.", "Some kids are wrestling on an inflatable raft", "A young man holding a microphone and singing.", "carrots are great . do you work ?", "i do . i enjoy it ."], "speaker_id": "snli:test:4082", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4082", "candidate_text": "Some kids playing with a raft.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4082:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3827:premise", "snli:test:4083:premise", "snli:test:4082:premise", "snli:test:3578:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A fisherman using a cellphone on a boat.", "A group of Asian girls are standing together.", "Some kids are wrestling on an inflatable raft", "A young man holding a microphone and singing.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4082", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4083", "candidate_text": "well, there are boys standing together", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4083:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4082:premise", "snli:test:4162:premise", "snli:test:3827:premise", "snli:test:4083:premise", "style:personachat:test:153:self:5:positive:0", "style:personachat:test:153:self:5:positive:1", "style:personachat:test:153:self:5:positive:2", "style:personachat:test:153:self:5:positive:3", "style:personachat:test:153:self:5:positive:4"], "evidence_texts": ["Some kids are wrestling on an inflatable raft", "Artists install a new exhibit in a gallery.", "A fisherman using a cellphone on a boat.", "A group of Asian girls are standing together.", "i do , i keep them short though because i skate", "no , but i still enjoy it as a hobby . what do you do ?", "i have never ridden a horse , it sounds fun", "carrots are great . do you work ?", "i do . i enjoy it ."], "speaker_id": "snli:test:4083", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4083", "candidate_text": "I wonder whether there are boys standing together?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4083:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4082:premise", "snli:test:4162:premise", "snli:test:3827:premise", "snli:test:4083:premise", "style:personachat:test:585:self:5:positive:0", "style:personachat:test:585:self:5:positive:1", "style:personachat:test:585:self:5:positive:2", "style:personachat:test:585:self:5:positive:3", "style:personachat:test:585:self:5:positive:4"], "evidence_texts": ["Some kids are wrestling on an inflatable raft", "Artists install a new exhibit in a gallery.", "A fisherman using a cellphone on a boat.", "A group of Asian girls are standing together.", "i love it . it lets me weave in stories about my old job at the carnival .", "not much of a foodie . now , cold scotch on the rocks . bliss . what is yours ?", "yum . that sounds good . how about movies ?", "batman was fabulous in theaters .", "no . i am allergic to everything but gold fish . do you have pets ?"], "speaker_id": "snli:test:4083", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4090", "candidate_text": "well, two girls kiss there dad", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4090:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3776:premise", "snli:test:4251:premise", "snli:test:4090:premise", "snli:test:3774:premise", "style:personachat:test:153:self:7:positive:0", "style:personachat:test:153:self:7:positive:1", "style:personachat:test:153:self:7:positive:2", "style:personachat:test:153:self:7:positive:3", "style:personachat:test:153:self:7:positive:4", "style:personachat:test:153:self:7:positive:5", "style:personachat:test:153:self:7:positive:6"], "evidence_texts": ["A young boy vocalizes in to a microphone held by a blond woman in black.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "i am great , how are you ?", "i am in publishing , heading to nyc next year for my job", "i do , i keep them short though because i skate", "no , but i still enjoy it as a hobby . what do you do ?", "i have never ridden a horse , it sounds fun", "carrots are great . do you work ?", "i do . i enjoy it ."], "speaker_id": "snli:test:4090", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4090", "candidate_text": "Honestly, Two girls kiss there dad!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4090:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3776:premise", "snli:test:4251:premise", "snli:test:4090:premise", "snli:test:3774:premise", "style:personachat:test:146:self:5:positive:0", "style:personachat:test:146:self:5:positive:1", "style:personachat:test:146:self:5:positive:2", "style:personachat:test:146:self:5:positive:3", "style:personachat:test:146:self:5:positive:4"], "evidence_texts": ["A young boy vocalizes in to a microphone held by a blond woman in black.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "no but i have two siblings , both brothers . do you have any ?", "i do not have much time for frisbee , what do you do for a job ?", "i am a managerial consultant originally from wisconsin , where did you grow up ?", "so you like the hot weather ? i prefer cold and snow", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:4090", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4096", "candidate_text": "well, the man is inside a building", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4096:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4096:premise", "snli:test:3974:premise", "snli:test:4119:premise", "snli:test:3855:premise", "style:personachat:test:154:self:1:positive:0"], "evidence_texts": ["A man is riding up an escalator that is carrying shopping carts.", "Two dogs are running next to each other on a grassy hill.", "a man sitting on a motorcycle with a sunset directly behind him", "A skier goes down a hill with a mountain as a backdrop.", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:4096", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4096", "candidate_text": "The man is inside a building.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4096:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4096:premise", "snli:test:3974:premise", "snli:test:4119:premise", "snli:test:3855:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man is riding up an escalator that is carrying shopping carts.", "Two dogs are running next to each other on a grassy hill.", "a man sitting on a motorcycle with a sunset directly behind him", "A skier goes down a hill with a mountain as a backdrop.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4096", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4101", "candidate_text": "well, a woman watches the water", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4101:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3740:premise", "snli:test:4101:premise", "snli:test:4104:premise", "snli:test:3738:premise", "style:personachat:test:154:self:2:positive:0", "style:personachat:test:154:self:2:positive:1"], "evidence_texts": ["The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "Lab tech looking through a microscope in a lab.", "The man is taking a picture of the beach.", "that is very healthy . i like it . do you eat fish ?", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:4101", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4101", "candidate_text": "I wonder whether a woman watches the water?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4101:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3740:premise", "snli:test:4101:premise", "snli:test:4104:premise", "snli:test:3738:premise", "style:personachat:test:586:self:2:positive:0", "style:personachat:test:586:self:2:positive:1"], "evidence_texts": ["The man is taking a picture of the beach.", "A man looking at a woman looking at water.", "Lab tech looking through a microscope in a lab.", "The man is taking a picture of the beach.", "i could use the money for college , i am trying to become a teacher .", "sometimes i wonder , how does horse flesh taste ?"], "speaker_id": "snli:test:4101", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4104", "candidate_text": "well, the lab tech is using the microscope", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4104:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4104:premise", "snli:test:4101:premise", "snli:test:3740:premise", "snli:test:4243:premise", "style:personachat:test:154:self:5:positive:0", "style:personachat:test:154:self:5:positive:1", "style:personachat:test:154:self:5:positive:2", "style:personachat:test:154:self:5:positive:3", "style:personachat:test:154:self:5:positive:4"], "evidence_texts": ["Lab tech looking through a microscope in a lab.", "A man looking at a woman looking at water.", "The man is taking a picture of the beach.", "A runner wearing purple strives for the finish line.", "i love those crafts ! right now i am into my aquarium filled with exotic fish .", "you might like my collection of decorations . they are 50s vintage !", "that is awesome . what do you eat for breakfast ? mine is granola every day .", "that is very healthy . i like it . do you eat fish ?", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:4104", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4104", "candidate_text": "Honestly, The lab tech is using the microscope!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4104:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4104:premise", "snli:test:4101:premise", "snli:test:3740:premise", "snli:test:4243:premise", "style:personachat:test:152:self:2:positive:0", "style:personachat:test:152:self:2:positive:1"], "evidence_texts": ["Lab tech looking through a microscope in a lab.", "A man looking at a woman looking at water.", "The man is taking a picture of the beach.", "A runner wearing purple strives for the finish line.", "history was my major . i worked in a museum .", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:4104", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4105", "candidate_text": "well, the lab tech is measuring chemicals in beakers", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4105:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4243:premise", "snli:test:4105:premise", "snli:test:4101:premise", "snli:test:4281:premise", "style:personachat:test:154:self:6:positive:0", "style:personachat:test:154:self:6:positive:1", "style:personachat:test:154:self:6:positive:2", "style:personachat:test:154:self:6:positive:3", "style:personachat:test:154:self:6:positive:4", "style:personachat:test:154:self:6:positive:5"], "evidence_texts": ["A runner wearing purple strives for the finish line.", "Lab tech looking through a microscope in a lab.", "A man looking at a woman looking at water.", "Children play on a sidewalk with dolls and chalk.", "hi . my day is good . i am hanging out with both of my sons .", "i love those crafts ! right now i am into my aquarium filled with exotic fish .", "you might like my collection of decorations . they are 50s vintage !", "that is awesome . what do you eat for breakfast ? mine is granola every day .", "that is very healthy . i like it . do you eat fish ?", "so do i . i just do not eat it in front of my aquarium ."], "speaker_id": "snli:test:4105", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4105", "candidate_text": "The lab tech is measuring chemicals in beakers.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4105:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4243:premise", "snli:test:4105:premise", "snli:test:4101:premise", "snli:test:4281:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A runner wearing purple strives for the finish line.", "Lab tech looking through a microscope in a lab.", "A man looking at a woman looking at water.", "Children play on a sidewalk with dolls and chalk.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4105", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4107", "candidate_text": "well, a large choir is singing", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4107:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3997:premise", "snli:test:4711:premise", "snli:test:3912:premise", "snli:test:4107:premise", "style:personachat:test:155:self:1:positive:0"], "evidence_texts": ["Four people enjoying an art exhibit.", "Cheerleaders standing on a football field.", "Four kids pose on a stage.", "Coed choir singing at an event.", "so do i and they give me a reason to get out"], "speaker_id": "snli:test:4107", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4107", "candidate_text": "I wonder whether a large choir is singing?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4107:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3997:premise", "snli:test:4711:premise", "snli:test:3912:premise", "snli:test:4107:premise", "style:personachat:test:589:self:1:positive:0"], "evidence_texts": ["Four people enjoying an art exhibit.", "Cheerleaders standing on a football field.", "Four kids pose on a stage.", "Coed choir singing at an event.", "so where exactly are you furry ? like all over ?"], "speaker_id": "snli:test:4107", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4113", "candidate_text": "well, an award ceremony is occurring", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4113:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3446:premise", "snli:test:4113:premise", "snli:test:4408:premise", "snli:test:3276:premise", "style:personachat:test:155:self:2:positive:0", "style:personachat:test:155:self:2:positive:1"], "evidence_texts": ["A man laughing while at a restaurant.", "Children receiving places at an award ceremony.", "Artist on street corner selling his artwork.", "People in motion down a city sidewalk.", "i love walking my dog and two cats as well", "so do i and they give me a reason to get out"], "speaker_id": "snli:test:4113", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4113", "candidate_text": "Honestly, An award ceremony is occurring!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4113:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3446:premise", "snli:test:4113:premise", "snli:test:4408:premise", "snli:test:3276:premise", "style:personachat:test:157:self:1:positive:0"], "evidence_texts": ["A man laughing while at a restaurant.", "Children receiving places at an award ceremony.", "Artist on street corner selling his artwork.", "People in motion down a city sidewalk.", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:4113", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4118", "candidate_text": "well, a woman crosses the street carrying a large sandwich", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4118:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3870:premise", "snli:test:4186:premise", "snli:test:4118:premise", "snli:test:3653:premise", "style:personachat:test:155:self:5:positive:0", "style:personachat:test:155:self:5:positive:1", "style:personachat:test:155:self:5:positive:2", "style:personachat:test:155:self:5:positive:3", "style:personachat:test:155:self:5:positive:4"], "evidence_texts": ["A guy taking a nap in the back of a vehicle.", "Two girls in black dresses enjoying a night on the town.", "A woman crosses the street with a baby in her arms.", "Young Asian male child either smoking something or its an instrument.", "just as a hobby though , i own my own successful dieting business full time", "oh yeah i love exercising for long hours . finding motivation is most important", "eating is also important , i like writing diet cookbooks", "i love walking my dog and two cats as well", "so do i and they give me a reason to get out"], "speaker_id": "snli:test:4118", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4118", "candidate_text": "A woman crosses the street carrying a large sandwich.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4118:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3870:premise", "snli:test:4186:premise", "snli:test:4118:premise", "snli:test:3653:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A guy taking a nap in the back of a vehicle.", "Two girls in black dresses enjoying a night on the town.", "A woman crosses the street with a baby in her arms.", "Young Asian male child either smoking something or its an instrument.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4118", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4119", "candidate_text": "well, a man sitting on his motorcycle with the sunset directly behind him outside", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4119:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4096:premise", "snli:test:4375:premise", "snli:test:3974:premise", "snli:test:4119:premise", "style:personachat:test:155:self:7:positive:0", "style:personachat:test:155:self:7:positive:1", "style:personachat:test:155:self:7:positive:2", "style:personachat:test:155:self:7:positive:3", "style:personachat:test:155:self:7:positive:4", "style:personachat:test:155:self:7:positive:5", "style:personachat:test:155:self:7:positive:6"], "evidence_texts": ["A man is riding up an escalator that is carrying shopping carts.", "a woman with brown hair sitting on a bench outside a cafe.", "Two dogs are running next to each other on a grassy hill.", "a man sitting on a motorcycle with a sunset directly behind him", "i like the beatles how about you ?", "oh i see , well i prefer reading plays mostly in my free time", "just as a hobby though , i own my own successful dieting business full time", "oh yeah i love exercising for long hours . finding motivation is most important", "eating is also important , i like writing diet cookbooks", "i love walking my dog and two cats as well", "so do i and they give me a reason to get out"], "speaker_id": "snli:test:4119", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4119", "candidate_text": "I wonder whether a man sitting on his motorcycle with the sunset directly behind him outside?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4119:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4096:premise", "snli:test:4375:premise", "snli:test:3974:premise", "snli:test:4119:premise", "style:personachat:test:589:self:7:positive:0", "style:personachat:test:589:self:7:positive:1", "style:personachat:test:589:self:7:positive:2", "style:personachat:test:589:self:7:positive:3", "style:personachat:test:589:self:7:positive:4", "style:personachat:test:589:self:7:positive:5", "style:personachat:test:589:self:7:positive:6"], "evidence_texts": ["A man is riding up an escalator that is carrying shopping carts.", "a woman with brown hair sitting on a bench outside a cafe.", "Two dogs are running next to each other on a grassy hill.", "a man sitting on a motorcycle with a sunset directly behind him", "i drive a big rig for work so i am really all over the country", "i hate being at home . probably why i have 7 kids in 7 states", "like i said , 7 kids from 7 different women . spread across the country", "ah age is just a number . what do you do for fun ?", "i read murder mysteries and try not to think about childhood on the farm", "is it blue fur ? that would be cool . its my favorite color", "so where exactly are you furry ? like all over ?"], "speaker_id": "snli:test:4119", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4133", "candidate_text": "I wonder whether a person is walking?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4133:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4078:premise", "snli:test:4133:premise", "snli:test:4169:premise", "snli:test:4042:premise", "style:personachat:test:156:self:1:positive:0"], "evidence_texts": ["A lady is promoting something to the other two girls.", "A person is walking a bicycle down a city street.", "The rhythmic gymnast completes her floor exercise at the competition.", "Two kids in black trunks bouncing on a wet trampoline.", "what do you do for a living ?"], "speaker_id": "snli:test:4133", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4133", "candidate_text": "well, a person is walking", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4133:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4078:premise", "snli:test:4133:premise", "snli:test:4169:premise", "snli:test:4042:premise", "style:personachat:test:273:self:1:positive:0"], "evidence_texts": ["A lady is promoting something to the other two girls.", "A person is walking a bicycle down a city street.", "The rhythmic gymnast completes her floor exercise at the competition.", "Two kids in black trunks bouncing on a wet trampoline.", "i aspire to be a chef someday ."], "speaker_id": "snli:test:4133", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4139", "candidate_text": "I wonder whether the people are at the race?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4139:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3764:premise", "snli:test:4267:premise", "snli:test:3762:premise", "snli:test:4139:premise", "style:personachat:test:156:self:2:positive:0", "style:personachat:test:156:self:2:positive:1"], "evidence_texts": ["A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "lol ! that would be a lot of cleaning up to do .", "what do you do for a living ?"], "speaker_id": "snli:test:4139", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4139", "candidate_text": "Honestly, The people are at the race!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4139:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3764:premise", "snli:test:4267:premise", "snli:test:3762:premise", "snli:test:4139:premise", "style:personachat:test:159:self:1:positive:0"], "evidence_texts": ["A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "i bet your singing is not that bad !"], "speaker_id": "snli:test:4139", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4162", "candidate_text": "I wonder whether people put up new art?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4162:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4083:premise", "snli:test:4082:premise", "snli:test:4162:premise", "snli:test:4290:premise", "style:personachat:test:156:self:5:positive:0", "style:personachat:test:156:self:5:positive:1", "style:personachat:test:156:self:5:positive:2", "style:personachat:test:156:self:5:positive:3", "style:personachat:test:156:self:5:positive:4"], "evidence_texts": ["A group of Asian girls are standing together.", "Some kids are wrestling on an inflatable raft", "Artists install a new exhibit in a gallery.", "group of people sweeping the floor and cleaning", "yes . other than playing guitar , what other hobbies do you have ?", "after retiring from banking , i make quilts for my family . do you have any pets ?", "i own a siamese cat named stormy . what kind of dog do you want ?", "lol ! that would be a lot of cleaning up to do .", "what do you do for a living ?"], "speaker_id": "snli:test:4162", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4162", "candidate_text": "People put up new art.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4162:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4083:premise", "snli:test:4082:premise", "snli:test:4162:premise", "snli:test:4290:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of Asian girls are standing together.", "Some kids are wrestling on an inflatable raft", "Artists install a new exhibit in a gallery.", "group of people sweeping the floor and cleaning", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4162", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4163", "candidate_text": "I wonder whether people put up new art at the towns local gallery?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4163:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4290:premise", "snli:test:4083:premise", "snli:test:4297:premise", "snli:test:4163:premise", "style:personachat:test:156:self:7:positive:0", "style:personachat:test:156:self:7:positive:1", "style:personachat:test:156:self:7:positive:2", "style:personachat:test:156:self:7:positive:3", "style:personachat:test:156:self:7:positive:4", "style:personachat:test:156:self:7:positive:5", "style:personachat:test:156:self:7:positive:6"], "evidence_texts": ["group of people sweeping the floor and cleaning", "A group of Asian girls are standing together.", "Various men in wetsuits and goggles are standing.", "Artists install a new exhibit in a gallery.", "i am good . what are you doing today ?", "wow ! i just got back from walking a mile . i tried to do that everyday", "yes . other than playing guitar , what other hobbies do you have ?", "after retiring from banking , i make quilts for my family . do you have any pets ?", "i own a siamese cat named stormy . what kind of dog do you want ?", "lol ! that would be a lot of cleaning up to do .", "what do you do for a living ?"], "speaker_id": "snli:test:4163", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4163", "candidate_text": "well, people put up new art at the towns local gallery", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4163:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4290:premise", "snli:test:4083:premise", "snli:test:4297:premise", "snli:test:4163:premise", "style:personachat:test:273:self:7:positive:0", "style:personachat:test:273:self:7:positive:1", "style:personachat:test:273:self:7:positive:2", "style:personachat:test:273:self:7:positive:3", "style:personachat:test:273:self:7:positive:4", "style:personachat:test:273:self:7:positive:5", "style:personachat:test:273:self:7:positive:6"], "evidence_texts": ["group of people sweeping the floor and cleaning", "A group of Asian girls are standing together.", "Various men in wetsuits and goggles are standing.", "Artists install a new exhibit in a gallery.", "hi . not too bad . how about you", "glad to hear it . do you like dogs ?", "yeah dogs scare me . i prefer horses too", "hope youre okay . i just turned 32 and am still single . that hurts too haha", "to each their own . i love to read . do you ?", "romance ones are my favorite too !", "i aspire to be a chef someday ."], "speaker_id": "snli:test:4163", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4169", "candidate_text": "Honestly, The gymnast completes her intense exercise!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4169:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4133:premise", "snli:test:4169:premise", "snli:test:4366:premise", "snli:test:4078:premise", "style:personachat:test:157:self:1:positive:0"], "evidence_texts": ["A person is walking a bicycle down a city street.", "The rhythmic gymnast completes her floor exercise at the competition.", "A man jumping a dirt ramp on a bmx bike.", "A lady is promoting something to the other two girls.", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:4169", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4169", "candidate_text": "The gymnast completes her intense exercise.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4169:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4133:premise", "snli:test:4169:premise", "snli:test:4366:premise", "snli:test:4078:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person is walking a bicycle down a city street.", "The rhythmic gymnast completes her floor exercise at the competition.", "A man jumping a dirt ramp on a bmx bike.", "A lady is promoting something to the other two girls.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4169", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4177", "candidate_text": "Honestly, A person is getting ready to snowboard down a hill with one leg!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4177:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3505:premise", "snli:test:4177:premise", "snli:test:4494:premise", "snli:test:2059:premise", "style:personachat:test:157:self:2:positive:0", "style:personachat:test:157:self:2:positive:1"], "evidence_texts": ["A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "i wish i could let you see me , i have to eat dinner with my family though .", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:4177", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4177", "candidate_text": "I wonder whether a person is getting ready to snowboard down a hill with one leg?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4177:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3505:premise", "snli:test:4177:premise", "snli:test:4494:premise", "snli:test:2059:premise", "style:personachat:test:605:self:2:positive:0", "style:personachat:test:605:self:2:positive:1"], "evidence_texts": ["A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "Two hockey players on opposing teams, one in a red and white jersey and one in a blue and black jersey, are standing on the rink.", "thank you ! so tell me more about you", "wow that sounds interesting . do you sell it ?"], "speaker_id": "snli:test:4177", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4186", "candidate_text": "Honestly, Two girls enjoying a night on the town!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4186:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4118:premise", "snli:test:4231:premise", "snli:test:4186:premise", "snli:test:3870:premise", "style:personachat:test:157:self:5:positive:0", "style:personachat:test:157:self:5:positive:1", "style:personachat:test:157:self:5:positive:2", "style:personachat:test:157:self:5:positive:3", "style:personachat:test:157:self:5:positive:4"], "evidence_texts": ["A woman crosses the street with a baby in her arms.", "A white greyhound dog wearing a muzzle runs around a track.", "Two girls in black dresses enjoying a night on the town.", "A guy taking a nap in the back of a vehicle.", "well starfish , jellyfish , squids , fish , they are all my friends !", "that sounds awesome , i have a tail as feet , so i can not work .", "no , i am not , i do have long wavy hair though !", "i wish i could let you see me , i have to eat dinner with my family though .", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:4186", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4186", "candidate_text": "well, two girls enjoying a night on the town", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4186:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4118:premise", "snli:test:4231:premise", "snli:test:4186:premise", "snli:test:3870:premise", "style:personachat:test:274:self:5:positive:0", "style:personachat:test:274:self:5:positive:1", "style:personachat:test:274:self:5:positive:2", "style:personachat:test:274:self:5:positive:3", "style:personachat:test:274:self:5:positive:4"], "evidence_texts": ["A woman crosses the street with a baby in her arms.", "A white greyhound dog wearing a muzzle runs around a track.", "Two girls in black dresses enjoying a night on the town.", "A guy taking a nap in the back of a vehicle.", "i do not really do that kind of stuff . i am only 17 .", "no i do not . i am going to graduate high school in june .", "i do not know who he is ?", "that is cool . i love listening to music in my volkswagen beetle .", "i am going to start auburn university in august ."], "speaker_id": "snli:test:4186", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4195", "candidate_text": "Honestly, A boy is playing in the grass!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4195:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4073:premise", "snli:test:4200:premise", "snli:test:4055:premise", "snli:test:4195:premise", "style:personachat:test:157:self:7:positive:0", "style:personachat:test:157:self:7:positive:1", "style:personachat:test:157:self:7:positive:2", "style:personachat:test:157:self:7:positive:3", "style:personachat:test:157:self:7:positive:4", "style:personachat:test:157:self:7:positive:5", "style:personachat:test:157:self:7:positive:6"], "evidence_texts": ["A man in a gray coat is walking on a fallen tree trunk in a forest.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "i am doing great ! i am just currently exploring with my sea creature friends !", "i like rush as well , i have to see them whenever i can above the sea .", "well starfish , jellyfish , squids , fish , they are all my friends !", "that sounds awesome , i have a tail as feet , so i can not work .", "no , i am not , i do have long wavy hair though !", "i wish i could let you see me , i have to eat dinner with my family though .", "i eat whatever my family makes me , i get to go play with friends after though !"], "speaker_id": "snli:test:4195", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4195", "candidate_text": "A boy is playing in the grass.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4195:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4073:premise", "snli:test:4200:premise", "snli:test:4055:premise", "snli:test:4195:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a gray coat is walking on a fallen tree trunk in a forest.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A boy wearing a shirt with an American flag plays with a basketball on the blacktop.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4195", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4200", "candidate_text": "well, two women are working", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4200:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4200:premise", "snli:test:4195:premise", "snli:test:4073:premise", "snli:test:4240:premise", "style:personachat:test:158:self:1:positive:0"], "evidence_texts": ["Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "yes my birthday is next month"], "speaker_id": "snli:test:4200", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4200", "candidate_text": "I wonder whether two women are working?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4200:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4200:premise", "snli:test:4195:premise", "snli:test:4073:premise", "snli:test:4240:premise", "style:personachat:test:606:self:1:positive:0"], "evidence_texts": ["Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "A man in a gray coat is walking on a fallen tree trunk in a forest.", "A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "thank you . what do you teach in college ?"], "speaker_id": "snli:test:4200", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4202", "candidate_text": "well, two women are running a kitchen", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4202:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4240:premise", "snli:test:4195:premise", "snli:test:4202:premise", "snli:test:4462:premise", "style:personachat:test:158:self:2:positive:0", "style:personachat:test:158:self:2:positive:1"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A cyclist in a red and white uniform is biking across a garage full of people.", "cool i am in my late", "yes my birthday is next month"], "speaker_id": "snli:test:4202", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4202", "candidate_text": "Honestly, Two women are running a kitchen!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4202:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4240:premise", "snli:test:4195:premise", "snli:test:4202:premise", "snli:test:4462:premise", "style:personachat:test:182:self:1:positive:0"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A boy in an all blue uniform kicks a blue soccer ball in a grassy area.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A cyclist in a red and white uniform is biking across a garage full of people.", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:4202", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4210", "candidate_text": "well, two people are paining each others faces", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4210:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3659:premise", "snli:test:8613:premise", "snli:test:4210:premise", "snli:test:6160:premise", "style:personachat:test:158:self:5:positive:0", "style:personachat:test:158:self:5:positive:1", "style:personachat:test:158:self:5:positive:2", "style:personachat:test:158:self:5:positive:3", "style:personachat:test:158:self:5:positive:4"], "evidence_texts": ["A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "cool , do you like it there ?", "i want to visit there so bad", "how old are the bros ?", "cool i am in my late", "yes my birthday is next month"], "speaker_id": "snli:test:4210", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4210", "candidate_text": "Two people are paining each others faces.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4210:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:3659:premise", "snli:test:8613:premise", "snli:test:4210:premise", "snli:test:6160:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4210", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4217", "candidate_text": "well, the woman is holding a some clippers in her hand", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4217:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4062:premise", "snli:test:4217:premise", "snli:test:4623:premise", "snli:test:3865:premise", "style:personachat:test:158:self:7:positive:0", "style:personachat:test:158:self:7:positive:1", "style:personachat:test:158:self:7:positive:2", "style:personachat:test:158:self:7:positive:3", "style:personachat:test:158:self:7:positive:4", "style:personachat:test:158:self:7:positive:5", "style:personachat:test:158:self:7:positive:6"], "evidence_texts": ["A brown dog persues a Frisbee across the grass as the thrower watches.", "A woman is shaving the hair off of a black and white dog.", "Two men and two women in jackets walk down a large city street.", "A man in a brown shirt and blue shorts walking in Droney Park.", "what is that ? like to shop i do which is why i am here lol", "i am almost a nurse lol that is sadly single", "cool , do you like it there ?", "i want to visit there so bad", "how old are the bros ?", "cool i am in my late", "yes my birthday is next month"], "speaker_id": "snli:test:4217", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4217", "candidate_text": "I wonder whether the woman is holding a some clippers in her hand?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4217:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4062:premise", "snli:test:4217:premise", "snli:test:4623:premise", "snli:test:3865:premise", "style:personachat:test:606:self:6:positive:0", "style:personachat:test:606:self:6:positive:1", "style:personachat:test:606:self:6:positive:2", "style:personachat:test:606:self:6:positive:3", "style:personachat:test:606:self:6:positive:4", "style:personachat:test:606:self:6:positive:5"], "evidence_texts": ["A brown dog persues a Frisbee across the grass as the thrower watches.", "A woman is shaving the hair off of a black and white dog.", "Two men and two women in jackets walk down a large city street.", "A man in a brown shirt and blue shorts walking in Droney Park.", "i used to like it , but my job does not get me enough money .", "i work at a fast food place , but want to go to college sometime .", "i am more into heavy rock . fits my life as i lived with my aunt growing up .", "as a kid my parents did not want me so i was not able to join activities .", "i used to run a lot , but find myself working crazy hours to support myself !", "thank you . what do you teach in college ?"], "speaker_id": "snli:test:4217", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4231", "candidate_text": "Honestly, The dog is running!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4231:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4186:premise", "snli:test:4258:premise", "snli:test:4118:premise", "snli:test:4231:premise", "style:personachat:test:159:self:1:positive:0"], "evidence_texts": ["Two girls in black dresses enjoying a night on the town.", "The wind propels a sailing ship on a group of cruisers.", "A woman crosses the street with a baby in her arms.", "A white greyhound dog wearing a muzzle runs around a track.", "i bet your singing is not that bad !"], "speaker_id": "snli:test:4231", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4231", "candidate_text": "well, the dog is running", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4231:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4186:premise", "snli:test:4258:premise", "snli:test:4118:premise", "snli:test:4231:premise", "style:personachat:test:277:self:1:positive:0"], "evidence_texts": ["Two girls in black dresses enjoying a night on the town.", "The wind propels a sailing ship on a group of cruisers.", "A woman crosses the street with a baby in her arms.", "A white greyhound dog wearing a muzzle runs around a track.", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:4231", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4233", "candidate_text": "Honestly, The man the baseball is taking his son to a baseball game today!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4233:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3958:premise", "snli:test:4233:premise", "snli:test:5441:premise", "snli:test:3868:premise", "style:personachat:test:159:self:2:positive:0", "style:personachat:test:159:self:2:positive:1"], "evidence_texts": ["A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "ah . have any pets ? i have 3 . my dogs are my babies .", "i bet your singing is not that bad !"], "speaker_id": "snli:test:4233", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4233", "candidate_text": "The man the baseball is taking his son to a baseball game today.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4233:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3958:premise", "snli:test:4233:premise", "snli:test:5441:premise", "snli:test:3868:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A smiling man with gray hair and glasses, dressed in black, is sitting in a chair with a music stand in front of him.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4233", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4240", "candidate_text": "Honestly, There are multiple people present!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4240:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4240:premise", "snli:test:4202:premise", "snli:test:4462:premise", "snli:test:4200:premise", "style:personachat:test:159:self:5:positive:0", "style:personachat:test:159:self:5:positive:1", "style:personachat:test:159:self:5:positive:2", "style:personachat:test:159:self:5:positive:3", "style:personachat:test:159:self:5:positive:4"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A cyclist in a red and white uniform is biking across a garage full of people.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "watch football on sundays . i love it !", "oh yeah ? it is cold there . i love the cold . christmas is my favorite holiday .", "for sure ! what is your favorite food ? mine is cheese pizza .", "ah . have any pets ? i have 3 . my dogs are my babies .", "i bet your singing is not that bad !"], "speaker_id": "snli:test:4240", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4240", "candidate_text": "I wonder whether there are multiple people present?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4240:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4240:premise", "snli:test:4202:premise", "snli:test:4462:premise", "snli:test:4200:premise", "style:personachat:test:610:self:5:positive:0", "style:personachat:test:610:self:5:positive:1", "style:personachat:test:610:self:5:positive:2", "style:personachat:test:610:self:5:positive:3", "style:personachat:test:610:self:5:positive:4"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "A cyclist in a red and white uniform is biking across a garage full of people.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "ice cream is one of my favourite foods .", "i should work out more , i walk my dogs every day .", "that is cool , i work on the mountains . do you have pets ?", "that is a lot ! i have 3 cats and 2 dogs .", "do you go to school ? my parents are teachers ."], "speaker_id": "snli:test:4240", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4243", "candidate_text": "Honestly, A runner give up before heading to the finish line!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4243:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4105:premise", "snli:test:4281:premise", "snli:test:4104:premise", "snli:test:4243:premise", "style:personachat:test:159:self:7:positive:0", "style:personachat:test:159:self:7:positive:1", "style:personachat:test:159:self:7:positive:2", "style:personachat:test:159:self:7:positive:3", "style:personachat:test:159:self:7:positive:4", "style:personachat:test:159:self:7:positive:5", "style:personachat:test:159:self:7:positive:6"], "evidence_texts": ["Lab tech looking through a microscope in a lab.", "Children play on a sidewalk with dolls and chalk.", "Lab tech looking through a microscope in a lab.", "A runner wearing purple strives for the finish line.", "hello ! very well , and yourself ?", "sounds exciting ! i work as a hair stylist .", "watch football on sundays . i love it !", "oh yeah ? it is cold there . i love the cold . christmas is my favorite holiday .", "for sure ! what is your favorite food ? mine is cheese pizza .", "ah . have any pets ? i have 3 . my dogs are my babies .", "i bet your singing is not that bad !"], "speaker_id": "snli:test:4243", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4243", "candidate_text": "well, a runner give up before heading to the finish line", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4243:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4105:premise", "snli:test:4281:premise", "snli:test:4104:premise", "snli:test:4243:premise", "style:personachat:test:277:self:7:positive:0", "style:personachat:test:277:self:7:positive:1", "style:personachat:test:277:self:7:positive:2", "style:personachat:test:277:self:7:positive:3", "style:personachat:test:277:self:7:positive:4", "style:personachat:test:277:self:7:positive:5", "style:personachat:test:277:self:7:positive:6"], "evidence_texts": ["Lab tech looking through a microscope in a lab.", "Children play on a sidewalk with dolls and chalk.", "Lab tech looking through a microscope in a lab.", "A runner wearing purple strives for the finish line.", "hello , how are you tonight ? i do too and love to cook .", "i would too , i am knitting tiny hats for the babies in the icu", "i am disabled , and not able to work . it makes me feel good to give back .", "i like to listen to music .", "i am vegan , and i love to bake cakes .", "i love dogs as well , and otters . they are like water dogs .", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:4243", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4251", "candidate_text": "well, a man is holding a stone in front of a wall", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4251:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4090:premise", "snli:test:3776:premise", "snli:test:4273:premise", "snli:test:4251:premise", "style:personachat:test:160:self:1:positive:0"], "evidence_texts": ["Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "now that looks cool i would love to"], "speaker_id": "snli:test:4251", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4251", "candidate_text": "A man is holding a stone in front of a wall.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4251:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4090:premise", "snli:test:3776:premise", "snli:test:4273:premise", "snli:test:4251:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A young boy vocalizes in to a microphone held by a blond woman in black.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4251", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4252", "candidate_text": "well, the man is a sculptor", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4252:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4252:premise", "snli:test:4273:premise", "snli:test:4090:premise", "snli:test:4335:premise", "style:personachat:test:160:self:2:positive:0", "style:personachat:test:160:self:2:positive:1"], "evidence_texts": ["A sitting man is holding a partially carved stone in front a partially painted wall.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A man in a blue shirt touches his head while talking to two women outdoors.", "cool i can show you the ropes", "now that looks cool i would love to"], "speaker_id": "snli:test:4252", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4252", "candidate_text": "I wonder whether the man is a sculptor?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4252:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4252:premise", "snli:test:4273:premise", "snli:test:4090:premise", "snli:test:4335:premise", "style:personachat:test:612:self:2:positive:0", "style:personachat:test:612:self:2:positive:1"], "evidence_texts": ["A sitting man is holding a partially carved stone in front a partially painted wall.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "Two girls kissing a man with a black shirt and brown hair on the cheeks.", "A man in a blue shirt touches his head while talking to two women outdoors.", "i am an artist and i also do some coding such as web design .", "what kind of lawyer ? what is your favorite kind of music ?"], "speaker_id": "snli:test:4252", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4258", "candidate_text": "well, the sailing ship is alone on the water", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4258:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4231:premise", "snli:test:4410:premise", "snli:test:4258:premise", "snli:test:4186:premise", "style:personachat:test:160:self:5:positive:0", "style:personachat:test:160:self:5:positive:1", "style:personachat:test:160:self:5:positive:2", "style:personachat:test:160:self:5:positive:3", "style:personachat:test:160:self:5:positive:4"], "evidence_texts": ["A white greyhound dog wearing a muzzle runs around a track.", "This is a very large vegetable stand under a white tent.", "The wind propels a sailing ship on a group of cruisers.", "Two girls in black dresses enjoying a night on the town.", "i absolutely love soccer i wish i still played", "i wish my childhood was as exciting", "yes its a habit i played soccer in hs so i still stick to that regiment", "cool i can show you the ropes", "now that looks cool i would love to"], "speaker_id": "snli:test:4258", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4258", "candidate_text": "Honestly, The sailing ship is alone on the water!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4258:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4231:premise", "snli:test:4410:premise", "snli:test:4258:premise", "snli:test:4186:premise", "style:personachat:test:188:self:2:positive:0", "style:personachat:test:188:self:2:positive:1"], "evidence_texts": ["A white greyhound dog wearing a muzzle runs around a track.", "This is a very large vegetable stand under a white tent.", "The wind propels a sailing ship on a group of cruisers.", "Two girls in black dresses enjoying a night on the town.", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:4258", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4267", "candidate_text": "well, there is a man standing outside having his photo taken by another person with a shadow", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4267:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4139:premise", "snli:test:4379:premise", "snli:test:3764:premise", "snli:test:4267:premise", "style:personachat:test:160:self:7:positive:0", "style:personachat:test:160:self:7:positive:1", "style:personachat:test:160:self:7:positive:2", "style:personachat:test:160:self:7:positive:3", "style:personachat:test:160:self:7:positive:4", "style:personachat:test:160:self:7:positive:5", "style:personachat:test:160:self:7:positive:6"], "evidence_texts": ["A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "i just graduated school . i am gonna get married soon", "when i was in hs i played basketball but now i rather watch it on tv", "i absolutely love soccer i wish i still played", "i wish my childhood was as exciting", "yes its a habit i played soccer in hs so i still stick to that regiment", "cool i can show you the ropes", "now that looks cool i would love to"], "speaker_id": "snli:test:4267", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4267", "candidate_text": "There is a man standing outside having his photo taken by another person with a shadow.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4267:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4139:premise", "snli:test:4379:premise", "snli:test:3764:premise", "snli:test:4267:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "A man in the bucket of a chery-picker cuts down branches using a chainsaw at the end of a long pole.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4267", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4273", "candidate_text": "I wonder whether a human sitting?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4273:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4252:premise", "snli:test:4273:premise", "snli:test:4335:premise", "snli:test:4251:premise", "style:personachat:test:161:self:1:positive:0"], "evidence_texts": ["A sitting man is holding a partially carved stone in front a partially painted wall.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A man in a blue shirt touches his head while talking to two women outdoors.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:4273", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4273", "candidate_text": "A human sitting.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4273:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4252:premise", "snli:test:4273:premise", "snli:test:4335:premise", "snli:test:4251:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A sitting man is holding a partially carved stone in front a partially painted wall.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A man in a blue shirt touches his head while talking to two women outdoors.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4273", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4281", "candidate_text": "I wonder whether kids taking naps inside?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4281:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4243:premise", "snli:test:4281:premise", "snli:test:4459:premise", "snli:test:4105:premise", "style:personachat:test:161:self:2:positive:0", "style:personachat:test:161:self:2:positive:1"], "evidence_texts": ["A runner wearing purple strives for the finish line.", "Children play on a sidewalk with dolls and chalk.", "An orange on the floor next to a dog.", "Lab tech looking through a microscope in a lab.", "i am a writer . since i travel so much , it is the best job for me .", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:4281", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4281", "candidate_text": "well, kids taking naps inside", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4281:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4243:premise", "snli:test:4281:premise", "snli:test:4459:premise", "snli:test:4105:premise", "style:personachat:test:280:self:2:positive:0", "style:personachat:test:280:self:2:positive:1"], "evidence_texts": ["A runner wearing purple strives for the finish line.", "Children play on a sidewalk with dolls and chalk.", "An orange on the floor next to a dog.", "Lab tech looking through a microscope in a lab.", "must be cool having blue eyes .", "being homeschooled is safer these days ."], "speaker_id": "snli:test:4281", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4290", "candidate_text": "I wonder whether there are people working?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4290:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4163:premise", "snli:test:4297:premise", "snli:test:4290:premise", "snli:test:4162:premise", "style:personachat:test:161:self:5:positive:0", "style:personachat:test:161:self:5:positive:1", "style:personachat:test:161:self:5:positive:2", "style:personachat:test:161:self:5:positive:3", "style:personachat:test:161:self:5:positive:4"], "evidence_texts": ["Artists install a new exhibit in a gallery.", "Various men in wetsuits and goggles are standing.", "group of people sweeping the floor and cleaning", "Artists install a new exhibit in a gallery.", "this is the sixth country i have been to . love walking around everywhere .", "well , i have a garden at home . that is very therapeutic . and you ?", "mostly vegetables and herbs . i like nirvana . sad what happened to kurt .", "i am a writer . since i travel so much , it is the best job for me .", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:4290", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4290", "candidate_text": "Honestly, There are people working!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4290:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4163:premise", "snli:test:4297:premise", "snli:test:4290:premise", "snli:test:4162:premise", "style:personachat:test:200:self:2:positive:0", "style:personachat:test:200:self:2:positive:1"], "evidence_texts": ["Artists install a new exhibit in a gallery.", "Various men in wetsuits and goggles are standing.", "group of people sweeping the floor and cleaning", "Artists install a new exhibit in a gallery.", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:4290", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4293", "candidate_text": "I wonder whether six people have an absolutely normal poker game?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4293:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4036:premise", "snli:test:4293:premise", "snli:test:3758:premise", "snli:test:4313:premise", "style:personachat:test:161:self:6:positive:0", "style:personachat:test:161:self:6:positive:1", "style:personachat:test:161:self:6:positive:2", "style:personachat:test:161:self:6:positive:3", "style:personachat:test:161:self:6:positive:4", "style:personachat:test:161:self:6:positive:5"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "backpacking through europe . love it ! i am in italy now .", "this is the sixth country i have been to . love walking around everywhere .", "well , i have a garden at home . that is very therapeutic . and you ?", "mostly vegetables and herbs . i like nirvana . sad what happened to kurt .", "i am a writer . since i travel so much , it is the best job for me .", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:4293", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4293", "candidate_text": "Six people have an absolutely normal poker game.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4293:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4036:premise", "snli:test:4293:premise", "snli:test:3758:premise", "snli:test:4313:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "Two girls are sitting on a park bench one is looking at her phone and the other is looking on.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4293", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4295", "candidate_text": "well, five men are kneeling, while another man puts his hands into the air", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4295:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4313:premise", "snli:test:4036:premise", "snli:test:5074:premise", "snli:test:4295:premise", "style:personachat:test:162:self:1:positive:0"], "evidence_texts": ["A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "take care and tell marty where is mc fly"], "speaker_id": "snli:test:4295", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4295", "candidate_text": "Honestly, Five men are kneeling, while another man puts his hands into the air!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4295:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4313:premise", "snli:test:4036:premise", "snli:test:5074:premise", "snli:test:4295:premise", "style:personachat:test:200:self:7:positive:0", "style:personachat:test:200:self:7:positive:1", "style:personachat:test:200:self:7:positive:2", "style:personachat:test:200:self:7:positive:3", "style:personachat:test:200:self:7:positive:4", "style:personachat:test:200:self:7:positive:5", "style:personachat:test:200:self:7:positive:6"], "evidence_texts": ["A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "A little boy in a blue shirt is putting some sunglasses up to a window while a lady watches him.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "judy . what do you like about sundays ? football watching is what i like !", "what is your favorite food ? cheese pizza is mine .", "do you have any pets ? i have three . all dogs . i love them !", "wow expensive . i have a yorkie , maltese and a mix breed .", "christmas , joy and jingle . i love christmas ! my favorite holiday . how about you ?", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:4295", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4297", "candidate_text": "well, men are standing up", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4297:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4290:premise", "snli:test:4297:premise", "snli:test:4349:premise", "snli:test:4163:premise", "style:personachat:test:162:self:2:positive:0", "style:personachat:test:162:self:2:positive:1"], "evidence_texts": ["group of people sweeping the floor and cleaning", "Various men in wetsuits and goggles are standing.", "people are enjoying food at a crowded restaurant.", "Artists install a new exhibit in a gallery.", "oh i see , well have a great evening", "take care and tell marty where is mc fly"], "speaker_id": "snli:test:4297", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4297", "candidate_text": "Men are standing up.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4297:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4290:premise", "snli:test:4297:premise", "snli:test:4349:premise", "snli:test:4163:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["group of people sweeping the floor and cleaning", "Various men in wetsuits and goggles are standing.", "people are enjoying food at a crowded restaurant.", "Artists install a new exhibit in a gallery.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4297", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4313", "candidate_text": "well, a man walks down the street with other people around", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4313:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4295:premise", "snli:test:4313:premise", "snli:test:5074:premise", "snli:test:4293:premise", "style:personachat:test:162:self:5:positive:0", "style:personachat:test:162:self:5:positive:1", "style:personachat:test:162:self:5:positive:2", "style:personachat:test:162:self:5:positive:3", "style:personachat:test:162:self:5:positive:4"], "evidence_texts": ["Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "oh , i do not have siblings so i am not sure what that is like", "are you in a hospital treating folks", "who is marty relation to you", "oh i see , well have a great evening", "take care and tell marty where is mc fly"], "speaker_id": "snli:test:4313", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4313", "candidate_text": "I wonder whether a man walks down the street with other people around?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4313:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4295:premise", "snli:test:4313:premise", "snli:test:5074:premise", "snli:test:4293:premise", "style:personachat:test:623:self:5:positive:0", "style:personachat:test:623:self:5:positive:1", "style:personachat:test:623:self:5:positive:2", "style:personachat:test:623:self:5:positive:3", "style:personachat:test:623:self:5:positive:4"], "evidence_texts": ["Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "i enjoy walking but my favorite hobby is participating in board games .", "i am terrible at seeing but enjoy perusing biographies .", "my brother , who just said i do , loves monopoly !", "i also raise cats for a hobby and have one who we call arizona .", "are you an investment advisor ?"], "speaker_id": "snli:test:4313", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4319", "candidate_text": "well, a old man in a plaid shirt buys apples at the market", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4319:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4076:premise", "snli:test:4555:premise", "snli:test:4075:premise", "snli:test:4319:premise", "style:personachat:test:162:self:6:positive:0", "style:personachat:test:162:self:6:positive:1", "style:personachat:test:162:self:6:positive:2", "style:personachat:test:162:self:6:positive:3", "style:personachat:test:162:self:6:positive:4", "style:personachat:test:162:self:6:positive:5"], "evidence_texts": ["A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "great , i am also a student for 2 more years", "oh , i do not have siblings so i am not sure what that is like", "are you in a hospital treating folks", "who is marty relation to you", "oh i see , well have a great evening", "take care and tell marty where is mc fly"], "speaker_id": "snli:test:4319", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4319", "candidate_text": "Honestly, A old man in a plaid shirt buys apples at the market!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4319:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4076:premise", "snli:test:4555:premise", "snli:test:4075:premise", "snli:test:4319:premise", "style:personachat:test:209:self:5:positive:0", "style:personachat:test:209:self:5:positive:1", "style:personachat:test:209:self:5:positive:2", "style:personachat:test:209:self:5:positive:3", "style:personachat:test:209:self:5:positive:4"], "evidence_texts": ["A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "no feet , i am sorry ! i am tall english speaking adult !", "that is fun ! do you work in the sea too ?", "yes , as an engineer . do you have pets ?", "i want one , but i think their food will bring in insects .", "i am terrified of them ! !"], "speaker_id": "snli:test:4319", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4335", "candidate_text": "well, three people on work break", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4335:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4273:premise", "snli:test:4486:premise", "snli:test:4252:premise", "snli:test:4335:premise", "style:personachat:test:163:self:1:positive:0"], "evidence_texts": ["An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "A man in a blue shirt touches his head while talking to two women outdoors.", "i lay here and look at the paint peal"], "speaker_id": "snli:test:4335", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4335", "candidate_text": "three people on work break.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4335:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4273:premise", "snli:test:4486:premise", "snli:test:4252:premise", "snli:test:4335:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["An old woman wearing a ski hat and glasses is sitting on a blue chair.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A sitting man is holding a partially carved stone in front a partially painted wall.", "A man in a blue shirt touches his head while talking to two women outdoors.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4335", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4344", "candidate_text": "well, the girl is playing at the beach", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4344:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4344:premise", "snli:test:3969:premise", "snli:test:4394:premise", "snli:test:3915:premise", "style:personachat:test:163:self:2:positive:0", "style:personachat:test:163:self:2:positive:1"], "evidence_texts": ["A little boy in a red shirt and swim trunks plays at the beach.", "A brown dog and a black and white dog are running over grassy hills.", "A cement worker is working on a new sidewalk outside of a clothing store.", "People in bathing suits bend down and reach into the water of a lake.", "no i have not , it is hard to even breath", "i lay here and look at the paint peal"], "speaker_id": "snli:test:4344", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4344", "candidate_text": "I wonder whether the girl is playing at the beach?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4344:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4344:premise", "snli:test:3969:premise", "snli:test:4394:premise", "snli:test:3915:premise", "style:personachat:test:625:self:2:positive:0", "style:personachat:test:625:self:2:positive:1"], "evidence_texts": ["A little boy in a red shirt and swim trunks plays at the beach.", "A brown dog and a black and white dog are running over grassy hills.", "A cement worker is working on a new sidewalk outside of a clothing store.", "People in bathing suits bend down and reach into the water of a lake.", "i have never seen one myself .", "is there anything unusual about you ?"], "speaker_id": "snli:test:4344", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4349", "candidate_text": "well, the people are celebrating their father's birthday at a local restaurant", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4349:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4297:premise", "snli:test:4349:premise", "snli:test:4404:premise", "snli:test:4290:premise", "style:personachat:test:163:self:5:positive:0", "style:personachat:test:163:self:5:positive:1", "style:personachat:test:163:self:5:positive:2", "style:personachat:test:163:self:5:positive:3", "style:personachat:test:163:self:5:positive:4"], "evidence_texts": ["Various men in wetsuits and goggles are standing.", "people are enjoying food at a crowded restaurant.", "A gas station and a car on fire.", "group of people sweeping the floor and cleaning", "oh you have a boss , i can not work due to disability", "that must make work fun", "well at least you have a job , i wish i could get out of bed", "no i have not , it is hard to even breath", "i lay here and look at the paint peal"], "speaker_id": "snli:test:4349", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4349", "candidate_text": "Honestly, The people are celebrating their father's birthday at a local restaurant!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4349:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4297:premise", "snli:test:4349:premise", "snli:test:4404:premise", "snli:test:4290:premise", "style:personachat:test:212:self:2:positive:0", "style:personachat:test:212:self:2:positive:1"], "evidence_texts": ["Various men in wetsuits and goggles are standing.", "people are enjoying food at a crowded restaurant.", "A gas station and a car on fire.", "group of people sweeping the floor and cleaning", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:4349", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4366", "candidate_text": "well, a man crashes his bike", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4366:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4169:premise", "snli:test:4419:premise", "snli:test:4366:premise", "snli:test:4133:premise", "style:personachat:test:163:self:6:positive:0", "style:personachat:test:163:self:6:positive:1", "style:personachat:test:163:self:6:positive:2", "style:personachat:test:163:self:6:positive:3", "style:personachat:test:163:self:6:positive:4", "style:personachat:test:163:self:6:positive:5"], "evidence_texts": ["The rhythmic gymnast completes her floor exercise at the competition.", "Two people are running in the grass by a hill.", "A man jumping a dirt ramp on a bmx bike.", "A person is walking a bicycle down a city street.", "my name is sam short for sammie", "oh you have a boss , i can not work due to disability", "that must make work fun", "well at least you have a job , i wish i could get out of bed", "no i have not , it is hard to even breath", "i lay here and look at the paint peal"], "speaker_id": "snli:test:4366", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4366", "candidate_text": "A man crashes his bike.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4366:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4169:premise", "snli:test:4419:premise", "snli:test:4366:premise", "snli:test:4133:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The rhythmic gymnast completes her floor exercise at the competition.", "Two people are running in the grass by a hill.", "A man jumping a dirt ramp on a bmx bike.", "A person is walking a bicycle down a city street.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4366", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4375", "candidate_text": "I wonder whether woman waiting for a friend at a cafe?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4375:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4119:premise", "snli:test:4382:premise", "snli:test:4096:premise", "snli:test:4375:premise", "style:personachat:test:164:self:1:positive:0"], "evidence_texts": ["a man sitting on a motorcycle with a sunset directly behind him", "children are sitting in chairs with a music stand in the background.", "A man is riding up an escalator that is carrying shopping carts.", "a woman with brown hair sitting on a bench outside a cafe.", "what do you do in your spare time ?"], "speaker_id": "snli:test:4375", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4375", "candidate_text": "Woman waiting for a friend at a cafe.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4375:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4119:premise", "snli:test:4382:premise", "snli:test:4096:premise", "snli:test:4375:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["a man sitting on a motorcycle with a sunset directly behind him", "children are sitting in chairs with a music stand in the background.", "A man is riding up an escalator that is carrying shopping carts.", "a woman with brown hair sitting on a bench outside a cafe.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4375", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4379", "candidate_text": "I wonder whether the man is standing?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4379:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4267:premise", "snli:test:4656:premise", "snli:test:4139:premise", "snli:test:4379:premise", "style:personachat:test:164:self:2:positive:0", "style:personachat:test:164:self:2:positive:1"], "evidence_texts": ["a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "lately i have been exercising . my weight has become a problem this past year .", "what do you do in your spare time ?"], "speaker_id": "snli:test:4379", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4379", "candidate_text": "well, the man is standing", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4379:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4267:premise", "snli:test:4656:premise", "snli:test:4139:premise", "snli:test:4379:premise", "style:personachat:test:284:self:2:positive:0", "style:personachat:test:284:self:2:positive:1"], "evidence_texts": ["a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A crowd gathered on either side of a soap box derby, with a single soap box car on the road track.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "makes sense . i never had any direction", "that is why i am trying to pursue a career in fashion"], "speaker_id": "snli:test:4379", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4382", "candidate_text": "I wonder whether there is a group of people sitting?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4382:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4375:premise", "snli:test:4633:premise", "snli:test:4382:premise", "snli:test:4119:premise", "style:personachat:test:164:self:5:positive:0", "style:personachat:test:164:self:5:positive:1", "style:personachat:test:164:self:5:positive:2", "style:personachat:test:164:self:5:positive:3", "style:personachat:test:164:self:5:positive:4"], "evidence_texts": ["a woman with brown hair sitting on a bench outside a cafe.", "A person in a black and green outfit is riding a bicycle.", "children are sitting in chairs with a music stand in the background.", "a man sitting on a motorcycle with a sunset directly behind him", "that is nice . i would go back to school but my husband works nights .", "it is hard to pay bills here too . we work different shifts to avoid daycare bills .", "i have a cat , too ! i just love them .", "lately i have been exercising . my weight has become a problem this past year .", "what do you do in your spare time ?"], "speaker_id": "snli:test:4382", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4382", "candidate_text": "Honestly, There is a group of people sitting!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4382:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4375:premise", "snli:test:4633:premise", "snli:test:4382:premise", "snli:test:4119:premise", "style:personachat:test:216:self:2:positive:0", "style:personachat:test:216:self:2:positive:1"], "evidence_texts": ["a woman with brown hair sitting on a bench outside a cafe.", "A person in a black and green outfit is riding a bicycle.", "children are sitting in chairs with a music stand in the background.", "a man sitting on a motorcycle with a sunset directly behind him", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:4382", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4389", "candidate_text": "I wonder whether a band is playing music?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4389:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3860:premise", "snli:test:4389:premise", "snli:test:4502:premise", "snli:test:3729:premise", "style:personachat:test:164:self:7:positive:0", "style:personachat:test:164:self:7:positive:1", "style:personachat:test:164:self:7:positive:2", "style:personachat:test:164:self:7:positive:3", "style:personachat:test:164:self:7:positive:4", "style:personachat:test:164:self:7:positive:5", "style:personachat:test:164:self:7:positive:6"], "evidence_texts": ["An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "hi . i am so tired from work . how are you ?", "are you a teacher ? my youngest is in kindergarten .", "that is nice . i would go back to school but my husband works nights .", "it is hard to pay bills here too . we work different shifts to avoid daycare bills .", "i have a cat , too ! i just love them .", "lately i have been exercising . my weight has become a problem this past year .", "what do you do in your spare time ?"], "speaker_id": "snli:test:4389", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4389", "candidate_text": "A band is playing music.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4389:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3860:premise", "snli:test:4389:premise", "snli:test:4502:premise", "snli:test:3729:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "A man in a button up shirt with a black shirt underneath presents in front of a television.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4389", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4394", "candidate_text": "I wonder whether a worker is working in a restuarant?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4394:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4344:premise", "snli:test:4428:premise", "snli:test:4394:premise", "snli:test:3969:premise", "style:personachat:test:165:self:1:positive:0"], "evidence_texts": ["A little boy in a red shirt and swim trunks plays at the beach.", "a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A cement worker is working on a new sidewalk outside of a clothing store.", "A brown dog and a black and white dog are running over grassy hills.", "what is your favorite one ?"], "speaker_id": "snli:test:4394", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4394", "candidate_text": "well, a worker is working in a restuarant", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4394:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4344:premise", "snli:test:4428:premise", "snli:test:4394:premise", "snli:test:3969:premise", "style:personachat:test:286:self:1:positive:0"], "evidence_texts": ["A little boy in a red shirt and swim trunks plays at the beach.", "a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A cement worker is working on a new sidewalk outside of a clothing store.", "A brown dog and a black and white dog are running over grassy hills.", "i do not sing , sadly ."], "speaker_id": "snli:test:4394", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4404", "candidate_text": "I wonder whether a car sets a gas station on fire?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4404:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4404:premise", "snli:test:4349:premise", "snli:test:4439:premise", "snli:test:4297:premise", "style:personachat:test:165:self:2:positive:0", "style:personachat:test:165:self:2:positive:1"], "evidence_texts": ["A gas station and a car on fire.", "people are enjoying food at a crowded restaurant.", "A man a woman posing for a picture.", "Various men in wetsuits and goggles are standing.", "my diet is extreme , i want to be healthy though eventually", "what is your favorite one ?"], "speaker_id": "snli:test:4404", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4404", "candidate_text": "Honestly, A car sets a gas station on fire!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4404:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4404:premise", "snli:test:4349:premise", "snli:test:4439:premise", "snli:test:4297:premise", "style:personachat:test:222:self:1:positive:0"], "evidence_texts": ["A gas station and a car on fire.", "people are enjoying food at a crowded restaurant.", "A man a woman posing for a picture.", "Various men in wetsuits and goggles are standing.", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:4404", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4408", "candidate_text": "I wonder whether someone is selling artwork?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4408:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4408:premise", "snli:test:4113:premise", "snli:test:3446:premise", "snli:test:4515:premise", "style:personachat:test:165:self:5:positive:0", "style:personachat:test:165:self:5:positive:1", "style:personachat:test:165:self:5:positive:2", "style:personachat:test:165:self:5:positive:3", "style:personachat:test:165:self:5:positive:4"], "evidence_texts": ["Artist on street corner selling his artwork.", "Children receiving places at an award ceremony.", "A man laughing while at a restaurant.", "Three kids perform a skit about camping.", "i like those animal , but i love horses", "oh can you swim or are you afraid of it ?", "i do like it , but i am dieting now so i am not strong enough", "my diet is extreme , i want to be healthy though eventually", "what is your favorite one ?"], "speaker_id": "snli:test:4408", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4408", "candidate_text": "Someone is selling artwork.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4408:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4408:premise", "snli:test:4113:premise", "snli:test:3446:premise", "snli:test:4515:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Artist on street corner selling his artwork.", "Children receiving places at an award ceremony.", "A man laughing while at a restaurant.", "Three kids perform a skit about camping.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4408", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4409", "candidate_text": "I wonder whether someone is begging for money?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4409:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4515:premise", "snli:test:4409:premise", "snli:test:4113:premise", "snli:test:4681:premise", "style:personachat:test:165:self:7:positive:0", "style:personachat:test:165:self:7:positive:1", "style:personachat:test:165:self:7:positive:2", "style:personachat:test:165:self:7:positive:3", "style:personachat:test:165:self:7:positive:4", "style:personachat:test:165:self:7:positive:5", "style:personachat:test:165:self:7:positive:6"], "evidence_texts": ["Three kids perform a skit about camping.", "Artist on street corner selling his artwork.", "Children receiving places at an award ceremony.", "Three people are walking on a beach.", "yeah a little i just had a manicure . i like gardens", "oh my , what is your name ?", "i like those animal , but i love horses", "oh can you swim or are you afraid of it ?", "i do like it , but i am dieting now so i am not strong enough", "my diet is extreme , i want to be healthy though eventually", "what is your favorite one ?"], "speaker_id": "snli:test:4409", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4409", "candidate_text": "well, someone is begging for money", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4409:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4515:premise", "snli:test:4409:premise", "snli:test:4113:premise", "snli:test:4681:premise", "style:personachat:test:286:self:6:positive:0", "style:personachat:test:286:self:6:positive:1", "style:personachat:test:286:self:6:positive:2", "style:personachat:test:286:self:6:positive:3", "style:personachat:test:286:self:6:positive:4", "style:personachat:test:286:self:6:positive:5"], "evidence_texts": ["Three kids perform a skit about camping.", "Artist on street corner selling his artwork.", "Children receiving places at an award ceremony.", "Three people are walking on a beach.", "oh , my 3 kids are annoying me chasing the cat around .", "my cat . she is a siamese .", "i do love cooking for my kids tho .", "pork chops . sadly i cook alot and not work out as i should .", "do you enjoy that kind of work ?", "i do not sing , sadly ."], "speaker_id": "snli:test:4409", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4410", "candidate_text": "I wonder whether there is a fresh food market?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4410:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4258:premise", "snli:test:4231:premise", "snli:test:4410:premise", "snli:test:4503:premise", "style:personachat:test:166:self:1:positive:0"], "evidence_texts": ["The wind propels a sailing ship on a group of cruisers.", "A white greyhound dog wearing a muzzle runs around a track.", "This is a very large vegetable stand under a white tent.", "A man is pushing many cartons of bread down the street.", "yes , did you get injuries playing soccer ?"], "speaker_id": "snli:test:4410", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4410", "candidate_text": "Honestly, There is a fresh food market!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4410:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4258:premise", "snli:test:4231:premise", "snli:test:4410:premise", "snli:test:4503:premise", "style:personachat:test:222:self:6:positive:0", "style:personachat:test:222:self:6:positive:1", "style:personachat:test:222:self:6:positive:2", "style:personachat:test:222:self:6:positive:3", "style:personachat:test:222:self:6:positive:4", "style:personachat:test:222:self:6:positive:5"], "evidence_texts": ["The wind propels a sailing ship on a group of cruisers.", "A white greyhound dog wearing a muzzle runs around a track.", "This is a very large vegetable stand under a white tent.", "A man is pushing many cartons of bread down the street.", "i am good . i prefer artistic endeavors to scholarly ones . you ?", "i earn a paycheck at a research facility cleaning cages . what would you study ?", "i understand . luckily i decided to create my art from trash so it is free !", "cool ! what do you like to do in your free time ?", "that sounds fun . i spend my free time on art .", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:4410", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4411", "candidate_text": "I wonder whether vegetables are being sold under a white tent?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4411:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4503:premise", "snli:test:4258:premise", "snli:test:4505:premise", "snli:test:4411:premise", "style:personachat:test:166:self:2:positive:0", "style:personachat:test:166:self:2:positive:1"], "evidence_texts": ["A man is pushing many cartons of bread down the street.", "The wind propels a sailing ship on a group of cruisers.", "A man is pushing many cartons of bread down the street.", "This is a very large vegetable stand under a white tent.", "yes , unfortunately i fell from my horse .", "yes , did you get injuries playing soccer ?"], "speaker_id": "snli:test:4411", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4411", "candidate_text": "Vegetables are being sold under a white tent.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4411:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4503:premise", "snli:test:4258:premise", "snli:test:4505:premise", "snli:test:4411:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man is pushing many cartons of bread down the street.", "The wind propels a sailing ship on a group of cruisers.", "A man is pushing many cartons of bread down the street.", "This is a very large vegetable stand under a white tent.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4411", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4419", "candidate_text": "I wonder whether a man and a woman are running as part of their training for a marathon?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4419:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4366:premise", "snli:test:4443:premise", "snli:test:4169:premise", "snli:test:4419:premise", "style:personachat:test:166:self:5:positive:0", "style:personachat:test:166:self:5:positive:1", "style:personachat:test:166:self:5:positive:2", "style:personachat:test:166:self:5:positive:3", "style:personachat:test:166:self:5:positive:4"], "evidence_texts": ["A man jumping a dirt ramp on a bmx bike.", "A man and a woman are dancing in the crowd.", "The rhythmic gymnast completes her floor exercise at the competition.", "Two people are running in the grass by a hill.", "i have a leg injury and i am in pain every day", "seems like you really loved college . i am about to retire", "no , unless horseback riding is a sport to you", "yes , unfortunately i fell from my horse .", "yes , did you get injuries playing soccer ?"], "speaker_id": "snli:test:4419", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4419", "candidate_text": "well, a man and a woman are running as part of their training for a marathon", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4419:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4366:premise", "snli:test:4443:premise", "snli:test:4169:premise", "snli:test:4419:premise", "style:personachat:test:291:self:5:positive:0", "style:personachat:test:291:self:5:positive:1", "style:personachat:test:291:self:5:positive:2", "style:personachat:test:291:self:5:positive:3", "style:personachat:test:291:self:5:positive:4"], "evidence_texts": ["A man jumping a dirt ramp on a bmx bike.", "A man and a woman are dancing in the crowd.", "The rhythmic gymnast completes her floor exercise at the competition.", "Two people are running in the grass by a hill.", "oh . i train dogs . i get to work indoors , a bit warmer .", "boy , you really like hunting and fishing . not me , sweets i love .", "you really like cold weather . take pictures of yourself while exploring .", "oh , i would not say i am a narcissists , just like sharing my adventures .", "cool . i take a lot of photos of me dancing at the local night clubs ."], "speaker_id": "snli:test:4419", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4428", "candidate_text": "I wonder whether the woman are enjoying a nice date out at the zoo?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4428:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4428:premise", "snli:test:4394:premise", "snli:test:4473:premise", "snli:test:4344:premise", "style:personachat:test:166:self:7:positive:0", "style:personachat:test:166:self:7:positive:1", "style:personachat:test:166:self:7:positive:2", "style:personachat:test:166:self:7:positive:3", "style:personachat:test:166:self:7:positive:4", "style:personachat:test:166:self:7:positive:5", "style:personachat:test:166:self:7:positive:6"], "evidence_texts": ["a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A cement worker is working on a new sidewalk outside of a clothing store.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A little boy in a red shirt and swim trunks plays at the beach.", "hi , i am doing well , you ? just have a bum leg .", "that sounds cool . do you have pets ? i own horses", "i have a leg injury and i am in pain every day", "seems like you really loved college . i am about to retire", "no , unless horseback riding is a sport to you", "yes , unfortunately i fell from my horse .", "yes , did you get injuries playing soccer ?"], "speaker_id": "snli:test:4428", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4428", "candidate_text": "Honestly, The woman are enjoying a nice date out at the zoo!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4428:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4428:premise", "snli:test:4394:premise", "snli:test:4473:premise", "snli:test:4344:premise", "style:personachat:test:226:self:5:positive:0", "style:personachat:test:226:self:5:positive:1", "style:personachat:test:226:self:5:positive:2", "style:personachat:test:226:self:5:positive:3", "style:personachat:test:226:self:5:positive:4"], "evidence_texts": ["a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A cement worker is working on a new sidewalk outside of a clothing store.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A little boy in a red shirt and swim trunks plays at the beach.", "i work in advertising . how about you ?", "awww , how fulfilling ! so are you enjoying your pizza all alone or with friends ?", "just the significant other , or do you have kids of your own ?", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:4428", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4435", "candidate_text": "well, the man is at home sleeping", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4435:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:2891:premise", "snli:test:9099:premise", "snli:test:6938:premise", "snli:test:4435:premise", "style:personachat:test:167:self:1:positive:0"], "evidence_texts": ["A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "so you understand the appeal"], "speaker_id": "snli:test:4435", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4435", "candidate_text": "I wonder whether the man is at home sleeping?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4435:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:2891:premise", "snli:test:9099:premise", "snli:test:6938:premise", "snli:test:4435:premise", "style:personachat:test:650:self:1:positive:0"], "evidence_texts": ["A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "are you in drama theatre in school ?"], "speaker_id": "snli:test:4435", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4439", "candidate_text": "well, a happy young couple are posing for a portrait", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4439:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4404:premise", "snli:test:4591:premise", "snli:test:4349:premise", "snli:test:4439:premise", "style:personachat:test:167:self:2:positive:0", "style:personachat:test:167:self:2:positive:1"], "evidence_texts": ["A gas station and a car on fire.", "A person in an orange shirt reaching up.", "people are enjoying food at a crowded restaurant.", "A man a woman posing for a picture.", "that sounds like a cool hobby . i enjoy restoring classic cars", "so you understand the appeal"], "speaker_id": "snli:test:4439", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4439", "candidate_text": "Honestly, A happy young couple are posing for a portrait!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4439:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4404:premise", "snli:test:4591:premise", "snli:test:4349:premise", "snli:test:4439:premise", "style:personachat:test:231:self:1:positive:0"], "evidence_texts": ["A gas station and a car on fire.", "A person in an orange shirt reaching up.", "people are enjoying food at a crowded restaurant.", "A man a woman posing for a picture.", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:4439", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4443", "candidate_text": "well, a man and woman dance alone", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4443:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4419:premise", "snli:test:4366:premise", "snli:test:4169:premise", "snli:test:4443:premise", "style:personachat:test:167:self:5:positive:0", "style:personachat:test:167:self:5:positive:1", "style:personachat:test:167:self:5:positive:2", "style:personachat:test:167:self:5:positive:3", "style:personachat:test:167:self:5:positive:4"], "evidence_texts": ["Two people are running in the grass by a hill.", "A man jumping a dirt ramp on a bmx bike.", "The rhythmic gymnast completes her floor exercise at the competition.", "A man and a woman are dancing in the crowd.", "that is a great degree to have nowadays .", "are you glad you went for it , or would have rather pursued something else ?", "i am glad . do you have any hobbies ?", "that sounds like a cool hobby . i enjoy restoring classic cars", "so you understand the appeal"], "speaker_id": "snli:test:4443", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4443", "candidate_text": "a man and woman dance alone.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4443:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4419:premise", "snli:test:4366:premise", "snli:test:4169:premise", "snli:test:4443:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two people are running in the grass by a hill.", "A man jumping a dirt ramp on a bmx bike.", "The rhythmic gymnast completes her floor exercise at the competition.", "A man and a woman are dancing in the crowd.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4443", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4444", "candidate_text": "well, a couple dances among other people", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4444:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4444:premise", "snli:test:4419:premise", "snli:test:4464:premise", "snli:test:4366:premise", "style:personachat:test:167:self:7:positive:0", "style:personachat:test:167:self:7:positive:1", "style:personachat:test:167:self:7:positive:2", "style:personachat:test:167:self:7:positive:3", "style:personachat:test:167:self:7:positive:4", "style:personachat:test:167:self:7:positive:5", "style:personachat:test:167:self:7:positive:6"], "evidence_texts": ["A man and a woman are dancing in the crowd.", "Two people are running in the grass by a hill.", "A golfer shoots his ball out of a sand trap.", "A man jumping a dirt ramp on a bmx bike.", "i am pretty good ! glad work is over . how are you ?", "my company rents out properties . what do you do ?", "that is a great degree to have nowadays .", "are you glad you went for it , or would have rather pursued something else ?", "i am glad . do you have any hobbies ?", "that sounds like a cool hobby . i enjoy restoring classic cars", "so you understand the appeal"], "speaker_id": "snli:test:4444", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4444", "candidate_text": "I wonder whether a couple dances among other people?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4444:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4444:premise", "snli:test:4419:premise", "snli:test:4464:premise", "snli:test:4366:premise", "style:personachat:test:650:self:6:positive:0", "style:personachat:test:650:self:6:positive:1", "style:personachat:test:650:self:6:positive:2", "style:personachat:test:650:self:6:positive:3", "style:personachat:test:650:self:6:positive:4", "style:personachat:test:650:self:6:positive:5"], "evidence_texts": ["A man and a woman are dancing in the crowd.", "Two people are running in the grass by a hill.", "A golfer shoots his ball out of a sand trap.", "A man jumping a dirt ramp on a bmx bike.", "i am doing well . glad that monday is done and over with . how are you ?", "i really like pop and rock music . how about you ?", "i am sure there are kids your age who likes pink floyd", "oh . do you currently work , or in school ?", "yes , i am in college . i am looking to be a fashion designer . how about you ?", "are you in drama theatre in school ?"], "speaker_id": "snli:test:4444", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4445", "candidate_text": "well, a couple dance together at their wedding reception", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4445:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4464:premise", "snli:test:4445:premise", "snli:test:4560:premise", "snli:test:4419:premise", "style:personachat:test:168:self:1:positive:0"], "evidence_texts": ["A golfer shoots his ball out of a sand trap.", "A man and a woman are dancing in the crowd.", "The man in the dark maroon shirt is kicking something.", "Two people are running in the grass by a hill.", "my dream is to have kids one day , but i need to get it together ."], "speaker_id": "snli:test:4445", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4445", "candidate_text": "Honestly, a couple dance together at their wedding reception!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4445:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4464:premise", "snli:test:4445:premise", "snli:test:4560:premise", "snli:test:4419:premise", "style:personachat:test:231:self:6:positive:0", "style:personachat:test:231:self:6:positive:1", "style:personachat:test:231:self:6:positive:2", "style:personachat:test:231:self:6:positive:3", "style:personachat:test:231:self:6:positive:4", "style:personachat:test:231:self:6:positive:5"], "evidence_texts": ["A golfer shoots his ball out of a sand trap.", "A man and a woman are dancing in the crowd.", "The man in the dark maroon shirt is kicking something.", "Two people are running in the grass by a hill.", "i enjoy spending time with my 3 girls not my ex though", "i grew up in the last was . not sure what that means but it is true", "brr ! what do your parents do ? my dad was a firefighter and mom was a teacher", "wow ! i am not much into teaching . too torn up over my divorce .", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:4445", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4450", "candidate_text": "well, a dog on a red leash walking away", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4450:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:1784:premise", "snli:test:5577:premise", "snli:test:4450:premise", "snli:test:9840:premise", "style:personachat:test:168:self:2:positive:0", "style:personachat:test:168:self:2:positive:1"], "evidence_texts": ["A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "yes , i was born and raise in northern california . you ?", "my dream is to have kids one day , but i need to get it together ."], "speaker_id": "snli:test:4450", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4450", "candidate_text": "A dog on a red leash walking away.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4450:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1784:premise", "snli:test:5577:premise", "snli:test:4450:premise", "snli:test:9840:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4450", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4459", "candidate_text": "well, there's a dog and an orange on the floor", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4459:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4281:premise", "snli:test:4243:premise", "snli:test:4470:premise", "snli:test:4459:premise", "style:personachat:test:168:self:5:positive:0", "style:personachat:test:168:self:5:positive:1", "style:personachat:test:168:self:5:positive:2", "style:personachat:test:168:self:5:positive:3", "style:personachat:test:168:self:5:positive:4"], "evidence_texts": ["Children play on a sidewalk with dolls and chalk.", "A runner wearing purple strives for the finish line.", "A group of mountain climbers rests at the summit.", "An orange on the floor next to a dog.", "oh . i always wanted to do fashion , but i had a rough life growing up .", "cool . stay focus . my parents were never around . i started stealing and got arrested .", "yeah . when i was in high school , my best friend stole my boyfriend . rough !", "yes , i was born and raise in northern california . you ?", "my dream is to have kids one day , but i need to get it together ."], "speaker_id": "snli:test:4459", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4459", "candidate_text": "I wonder whether there's a dog and an orange on the floor?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4459:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4281:premise", "snli:test:4243:premise", "snli:test:4470:premise", "snli:test:4459:premise", "style:personachat:test:651:self:5:positive:0", "style:personachat:test:651:self:5:positive:1", "style:personachat:test:651:self:5:positive:2", "style:personachat:test:651:self:5:positive:3", "style:personachat:test:651:self:5:positive:4"], "evidence_texts": ["Children play on a sidewalk with dolls and chalk.", "A runner wearing purple strives for the finish line.", "A group of mountain climbers rests at the summit.", "An orange on the floor next to a dog.", "that might be chaos ! i used to quilt on my grandpa s farm though .", "honduras , really ? we had a small dairy farm in wisconsin . did you have cows ?", "i could not farm . i drive trucks for a living . nothing will grow for me .", "it is not a bad job . little lonely sometimes . i read mysteries to distract me .", "what kind of novel are you writing ? i m a mystery fan ."], "speaker_id": "snli:test:4459", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4460", "candidate_text": "well, the fruit next to the dog is an apple", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4460:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4460:premise", "snli:test:4470:premise", "snli:test:4281:premise", "snli:test:4481:premise", "style:personachat:test:168:self:6:positive:0", "style:personachat:test:168:self:6:positive:1", "style:personachat:test:168:self:6:positive:2", "style:personachat:test:168:self:6:positive:3", "style:personachat:test:168:self:6:positive:4", "style:personachat:test:168:self:6:positive:5"], "evidence_texts": ["An orange on the floor next to a dog.", "A group of mountain climbers rests at the summit.", "Children play on a sidewalk with dolls and chalk.", "A woman holds a newspaper that says \"Real change\".", "i am good . what is your occupation ?", "oh . i always wanted to do fashion , but i had a rough life growing up .", "cool . stay focus . my parents were never around . i started stealing and got arrested .", "yeah . when i was in high school , my best friend stole my boyfriend . rough !", "yes , i was born and raise in northern california . you ?", "my dream is to have kids one day , but i need to get it together ."], "speaker_id": "snli:test:4460", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4460", "candidate_text": "Honestly, The fruit next to the dog is an apple!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4460:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4460:premise", "snli:test:4470:premise", "snli:test:4281:premise", "snli:test:4481:premise", "style:personachat:test:232:self:5:positive:0", "style:personachat:test:232:self:5:positive:1", "style:personachat:test:232:self:5:positive:2", "style:personachat:test:232:self:5:positive:3", "style:personachat:test:232:self:5:positive:4"], "evidence_texts": ["An orange on the floor next to a dog.", "A group of mountain climbers rests at the summit.", "Children play on a sidewalk with dolls and chalk.", "A woman holds a newspaper that says \"Real change\".", "jerry . i used to live in arizona . now i am living in new york .", "yes , i love to sing on my modeling jobs .", "what is your dream job ? mine is becoming an actress .", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:4460", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4462", "candidate_text": "I wonder whether the man is wearing a pijama?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4462:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4240:premise", "snli:test:4571:premise", "snli:test:4462:premise", "snli:test:4202:premise", "style:personachat:test:169:self:1:positive:0"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "A cyclist in a red and white uniform is biking across a garage full of people.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:4462", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4462", "candidate_text": "Honestly, The man is wearing a pijama!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4462:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4240:premise", "snli:test:4571:premise", "snli:test:4462:premise", "snli:test:4202:premise", "style:personachat:test:232:self:6:positive:0", "style:personachat:test:232:self:6:positive:1", "style:personachat:test:232:self:6:positive:2", "style:personachat:test:232:self:6:positive:3", "style:personachat:test:232:self:6:positive:4", "style:personachat:test:232:self:6:positive:5"], "evidence_texts": ["A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "A cyclist in a red and white uniform is biking across a garage full of people.", "Two women wearing aprons and hairnets look at each other while they reach into metal canisters.", "i have a dog . it is perfect for my single life", "jerry . i used to live in arizona . now i am living in new york .", "yes , i love to sing on my modeling jobs .", "what is your dream job ? mine is becoming an actress .", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:4462", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4464", "candidate_text": "I wonder whether a duffer hits to the green using a sand wedge?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4464:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4464:premise", "snli:test:4445:premise", "snli:test:4560:premise", "snli:test:4444:premise", "style:personachat:test:169:self:2:positive:0", "style:personachat:test:169:self:2:positive:1"], "evidence_texts": ["A golfer shoots his ball out of a sand trap.", "A man and a woman are dancing in the crowd.", "The man in the dark maroon shirt is kicking something.", "A man and a woman are dancing in the crowd.", "lucky me . we cook duck where i live . you ever tasted duck ?", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:4464", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4464", "candidate_text": "A duffer hits to the green using a sand wedge.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4464:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4464:premise", "snli:test:4445:premise", "snli:test:4560:premise", "snli:test:4444:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A golfer shoots his ball out of a sand trap.", "A man and a woman are dancing in the crowd.", "The man in the dark maroon shirt is kicking something.", "A man and a woman are dancing in the crowd.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4464", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4470", "candidate_text": "I wonder whether a group of climbers takes a break?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4470:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4460:premise", "snli:test:4481:premise", "snli:test:4470:premise", "snli:test:4459:premise", "style:personachat:test:169:self:5:positive:0", "style:personachat:test:169:self:5:positive:1", "style:personachat:test:169:self:5:positive:2", "style:personachat:test:169:self:5:positive:3", "style:personachat:test:169:self:5:positive:4"], "evidence_texts": ["An orange on the floor next to a dog.", "A woman holds a newspaper that says \"Real change\".", "A group of mountain climbers rests at the summit.", "An orange on the floor next to a dog.", "good for you ! i am a rn . catholic and love one tree hill .", "great ! i see many things at the hospital i work . real estate business good ?", "hmmm . my uniform is purple . i like chicken sausage pizza . you ?", "lucky me . we cook duck where i live . you ever tasted duck ?", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:4470", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4470", "candidate_text": "well, a group of climbers takes a break", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4470:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4460:premise", "snli:test:4481:premise", "snli:test:4470:premise", "snli:test:4459:premise", "style:personachat:test:296:self:5:positive:0", "style:personachat:test:296:self:5:positive:1", "style:personachat:test:296:self:5:positive:2", "style:personachat:test:296:self:5:positive:3", "style:personachat:test:296:self:5:positive:4"], "evidence_texts": ["An orange on the floor next to a dog.", "A woman holds a newspaper that says \"Real change\".", "A group of mountain climbers rests at the summit.", "An orange on the floor next to a dog.", "i do not but i love facial hair", "i ride my horses it is nice", "nice are you retired yet ?", "yeah remember the horses ? i want to retire though", "i am just tired and want to retire soon"], "speaker_id": "snli:test:4470", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4473", "candidate_text": "I wonder whether a boy throwing a stone?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4473:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4428:premise", "snli:test:4473:premise", "snli:test:4590:premise", "snli:test:4394:premise", "style:personachat:test:169:self:6:positive:0", "style:personachat:test:169:self:6:positive:1", "style:personachat:test:169:self:6:positive:2", "style:personachat:test:169:self:6:positive:3", "style:personachat:test:169:self:6:positive:4", "style:personachat:test:169:self:6:positive:5"], "evidence_texts": ["a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "two teen boys in school clothes are walking with something in a garbage bag.", "A cement worker is working on a new sidewalk outside of a clothing store.", "hi , you like purple ? i love it . tracy here and i love pizza . you ?", "good for you ! i am a rn . catholic and love one tree hill .", "great ! i see many things at the hospital i work . real estate business good ?", "hmmm . my uniform is purple . i like chicken sausage pizza . you ?", "lucky me . we cook duck where i live . you ever tasted duck ?", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:4473", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4473", "candidate_text": "Honestly, A boy throwing a stone!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4473:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4428:premise", "snli:test:4473:premise", "snli:test:4590:premise", "snli:test:4394:premise", "style:personachat:test:240:self:5:positive:0", "style:personachat:test:240:self:5:positive:1", "style:personachat:test:240:self:5:positive:2", "style:personachat:test:240:self:5:positive:3", "style:personachat:test:240:self:5:positive:4"], "evidence_texts": ["a woman in jeans and a woman in shorts are petting a baby kangaroo.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "two teen boys in school clothes are walking with something in a garbage bag.", "A cement worker is working on a new sidewalk outside of a clothing store.", "how about paint a picture of me eating onion rings ? i love onions . . . lol", "my dad is single . he is a doctor and i am his nurse in his practice .", "he might . which store do you work at ?", "he loves to eat . he will need that discount .", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:4473", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4476", "candidate_text": "well, a man walking around in nature", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4476:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4476:premise", "snli:test:4020:premise", "snli:test:4492:premise", "snli:test:3795:premise", "style:personachat:test:170:self:1:positive:0"], "evidence_texts": ["A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "i see blue but it is really green ."], "speaker_id": "snli:test:4476", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4476", "candidate_text": "I wonder whether a man walking around in nature?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4476:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4476:premise", "snli:test:4020:premise", "snli:test:4492:premise", "snli:test:3795:premise", "style:personachat:test:658:self:1:positive:0"], "evidence_texts": ["A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "Four cyclists are racing on an indoor racetrack all wearing the same red, white, and blue uniforms.", "do you have any books published ?"], "speaker_id": "snli:test:4476", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4481", "candidate_text": "well, a woman with a newspaper that says \"real change\" on the ground next to her", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4481:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4470:premise", "snli:test:4481:premise", "snli:test:4605:premise", "snli:test:4460:premise", "style:personachat:test:170:self:2:positive:0", "style:personachat:test:170:self:2:positive:1"], "evidence_texts": ["A group of mountain climbers rests at the summit.", "A woman holds a newspaper that says \"Real change\".", "A large crowd is standing around the start line.", "An orange on the floor next to a dog.", "it really relaxes me and i love to see it come to life !", "i see blue but it is really green ."], "speaker_id": "snli:test:4481", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4481", "candidate_text": "Honestly, a woman with a newspaper that says \"real change\" on the ground next to her!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4481:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4470:premise", "snli:test:4481:premise", "snli:test:4605:premise", "snli:test:4460:premise", "style:personachat:test:249:self:1:positive:0"], "evidence_texts": ["A group of mountain climbers rests at the summit.", "A woman holds a newspaper that says \"Real change\".", "A large crowd is standing around the start line.", "An orange on the floor next to a dog.", "i met my wife at a wedding !"], "speaker_id": "snli:test:4481", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4486", "candidate_text": "well, a single person stands on a heavily-eroded cliff-side spaceship", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4486:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4335:premise", "snli:test:4968:premise", "snli:test:4486:premise", "snli:test:4273:premise", "style:personachat:test:170:self:5:positive:0", "style:personachat:test:170:self:5:positive:1", "style:personachat:test:170:self:5:positive:2", "style:personachat:test:170:self:5:positive:3", "style:personachat:test:170:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt touches his head while talking to two women outdoors.", "A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "i am in the army . i joined right after high school .", "i like to go to the karaoke bars", "i like to draw and create art", "it really relaxes me and i love to see it come to life !", "i see blue but it is really green ."], "speaker_id": "snli:test:4486", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4486", "candidate_text": "A single person stands on a heavily-eroded cliff-side spaceship.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4486:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4335:premise", "snli:test:4968:premise", "snli:test:4486:premise", "snli:test:4273:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt touches his head while talking to two women outdoors.", "A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "An old woman wearing a ski hat and glasses is sitting on a blue chair.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4486", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4492", "candidate_text": "well, two people are going for a stroll in the woods", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4492:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4492:premise", "snli:test:4476:premise", "snli:test:4523:premise", "snli:test:4020:premise", "style:personachat:test:170:self:7:positive:0", "style:personachat:test:170:self:7:positive:1", "style:personachat:test:170:self:7:positive:2", "style:personachat:test:170:self:7:positive:3", "style:personachat:test:170:self:7:positive:4", "style:personachat:test:170:self:7:positive:5", "style:personachat:test:170:self:7:positive:6"], "evidence_texts": ["Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A bunch of people are standing all together in a street with a building in the background.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "are you allergic ? dogs give me bad allergies .", "my mother use to have an english accent she left when i was 5", "i am in the army . i joined right after high school .", "i like to go to the karaoke bars", "i like to draw and create art", "it really relaxes me and i love to see it come to life !", "i see blue but it is really green ."], "speaker_id": "snli:test:4492", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4492", "candidate_text": "I wonder whether two people are going for a stroll in the woods?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4492:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4492:premise", "snli:test:4476:premise", "snli:test:4523:premise", "snli:test:4020:premise", "style:personachat:test:658:self:6:positive:0", "style:personachat:test:658:self:6:positive:1", "style:personachat:test:658:self:6:positive:2", "style:personachat:test:658:self:6:positive:3", "style:personachat:test:658:self:6:positive:4", "style:personachat:test:658:self:6:positive:5"], "evidence_texts": ["Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A bunch of people are standing all together in a street with a building in the background.", "Two people are sitting in a chair a lady reading and a man looking to his phone", "i am well how are you", "its nice here i am on the porch cross stitching", "yes quite beautiful today . i love doing my crafts outside", "i enjoy quilting", "i sure do since i retired and what do you do ?", "do you have any books published ?"], "speaker_id": "snli:test:4492", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4494", "candidate_text": "well, a boy has a white t-shirt and pink sweater", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4494:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4177:premise", "snli:test:5828:premise", "snli:test:4494:premise", "snli:test:3505:premise", "style:personachat:test:171:self:1:positive:0"], "evidence_texts": ["A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "you are missing out on some great music"], "speaker_id": "snli:test:4494", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4494", "candidate_text": "Honestly, A boy has a white t-shirt and pink sweater!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4494:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4177:premise", "snli:test:5828:premise", "snli:test:4494:premise", "snli:test:3505:premise", "style:personachat:test:249:self:7:positive:0", "style:personachat:test:249:self:7:positive:1", "style:personachat:test:249:self:7:positive:2", "style:personachat:test:249:self:7:positive:3", "style:personachat:test:249:self:7:positive:4", "style:personachat:test:249:self:7:positive:5", "style:personachat:test:249:self:7:positive:6"], "evidence_texts": ["A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A group of people, one of which is a woman with a large white and colorful purse, wait to board a large white and blue bus.", "i just got home from a nice nature hike . do you like the outdoors ?", "hiking gives me inspiration for my drawing class . do you like art ?", "yes ! what kind of work do you do indoors ?", "i would put wierd stickers all over the walls ! do you have any hobbies ?", "i collect stickers , i try all kinds of strange food . can you recommend any strange food ?", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:4494", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4502", "candidate_text": "well, the man is outdoors", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4502:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4389:premise", "snli:test:4535:premise", "snli:test:4502:premise", "snli:test:3860:premise", "style:personachat:test:171:self:2:positive:0", "style:personachat:test:171:self:2:positive:1"], "evidence_texts": ["A group of men playing music, the man holding a guitar is talking to a man with drums.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "simple plan band is what energize me", "you are missing out on some great music"], "speaker_id": "snli:test:4502", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4502", "candidate_text": "The man is outdoors.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4502:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4389:premise", "snli:test:4535:premise", "snli:test:4502:premise", "snli:test:3860:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of men playing music, the man holding a guitar is talking to a man with drums.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "An old black man sitting on a couple of grocery crates playing the string guitar during the day.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4502", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4503", "candidate_text": "well, a vendor is selling loaves of bread to pedestrians", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4503:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4411:premise", "snli:test:4410:premise", "snli:test:4704:premise", "snli:test:4503:premise", "style:personachat:test:171:self:5:positive:0", "style:personachat:test:171:self:5:positive:1", "style:personachat:test:171:self:5:positive:2", "style:personachat:test:171:self:5:positive:3", "style:personachat:test:171:self:5:positive:4"], "evidence_texts": ["This is a very large vegetable stand under a white tent.", "This is a very large vegetable stand under a white tent.", "Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "i actually help out at my animal shelter today so that is a bonus", "do you like listening to music in your free time ?", "that is not a good thing , i would say", "simple plan band is what energize me", "you are missing out on some great music"], "speaker_id": "snli:test:4503", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4503", "candidate_text": "I wonder whether a vendor is selling loaves of bread to pedestrians?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4503:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4411:premise", "snli:test:4410:premise", "snli:test:4704:premise", "snli:test:4503:premise", "style:personachat:test:660:self:5:positive:0", "style:personachat:test:660:self:5:positive:1", "style:personachat:test:660:self:5:positive:2", "style:personachat:test:660:self:5:positive:3", "style:personachat:test:660:self:5:positive:4"], "evidence_texts": ["This is a very large vegetable stand under a white tent.", "This is a very large vegetable stand under a white tent.", "Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "wow ! you probably would think i am boring . i am an accountant .", "yes , it is hard . but i like to hike in my free time so that helps .", "do you do anything for fun ?", "ha , i remember those party days . but now i have two kids to take care of .", "awe , how old are they ?"], "speaker_id": "snli:test:4503", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4505", "candidate_text": "well, a woman is baking bread in the oven", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4505:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4704:premise", "snli:test:4505:premise", "snli:test:4411:premise", "snli:test:4722:premise", "style:personachat:test:171:self:7:positive:0", "style:personachat:test:171:self:7:positive:1", "style:personachat:test:171:self:7:positive:2", "style:personachat:test:171:self:7:positive:3", "style:personachat:test:171:self:7:positive:4", "style:personachat:test:171:self:7:positive:5", "style:personachat:test:171:self:7:positive:6"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "This is a very large vegetable stand under a white tent.", "a young woman siting on a park bench in the sun.", "hello , yes i am here", "just my usual routine , working , coming home . what about you ?", "i actually help out at my animal shelter today so that is a bonus", "do you like listening to music in your free time ?", "that is not a good thing , i would say", "simple plan band is what energize me", "you are missing out on some great music"], "speaker_id": "snli:test:4505", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4505", "candidate_text": "Honestly, A woman is baking bread in the oven!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4505:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4704:premise", "snli:test:4505:premise", "snli:test:4411:premise", "snli:test:4722:premise", "style:personachat:test:261:self:5:positive:0", "style:personachat:test:261:self:5:positive:1", "style:personachat:test:261:self:5:positive:2", "style:personachat:test:261:self:5:positive:3", "style:personachat:test:261:self:5:positive:4"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "This is a very large vegetable stand under a white tent.", "a young woman siting on a park bench in the sun.", "i listen to pop music", "my dad owns a car dealership across town", "no but i get a discount", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:4505", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4515", "candidate_text": "I wonder whether three kids performed a camping skit?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4515:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4409:premise", "snli:test:4681:premise", "snli:test:4408:premise", "snli:test:4515:premise", "style:personachat:test:172:self:1:positive:0"], "evidence_texts": ["Artist on street corner selling his artwork.", "Three people are walking on a beach.", "Artist on street corner selling his artwork.", "Three kids perform a skit about camping.", "do you have any hobbies ?"], "speaker_id": "snli:test:4515", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4515", "candidate_text": "Honestly, Three kids performed a camping skit!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4515:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4409:premise", "snli:test:4681:premise", "snli:test:4408:premise", "snli:test:4515:premise", "style:personachat:test:261:self:7:positive:0", "style:personachat:test:261:self:7:positive:1", "style:personachat:test:261:self:7:positive:2", "style:personachat:test:261:self:7:positive:3", "style:personachat:test:261:self:7:positive:4", "style:personachat:test:261:self:7:positive:5", "style:personachat:test:261:self:7:positive:6"], "evidence_texts": ["Artist on street corner selling his artwork.", "Three people are walking on a beach.", "Artist on street corner selling his artwork.", "Three kids perform a skit about camping.", "i love music ! i also love horses", "i do not . i would get sweaty", "i listen to pop music", "my dad owns a car dealership across town", "no but i get a discount", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:4515", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4523", "candidate_text": "I wonder whether a group of people are standing?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4523:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4492:premise", "snli:test:4789:premise", "snli:test:4476:premise", "snli:test:4523:premise", "style:personachat:test:172:self:2:positive:0", "style:personachat:test:172:self:2:positive:1"], "evidence_texts": ["Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A bunch of people are standing all together in a street with a building in the background.", "no i wish , got to keep my head clear for trades .", "do you have any hobbies ?"], "speaker_id": "snli:test:4523", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4523", "candidate_text": "A group of people are standing.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4523:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4492:premise", "snli:test:4789:premise", "snli:test:4476:premise", "snli:test:4523:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A man in dress pants and a blue dress shirt is walking in the shadows of nature.", "A bunch of people are standing all together in a street with a building in the background.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4523", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4535", "candidate_text": "I wonder whether two men are reading books?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4535:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4502:premise", "snli:test:4723:premise", "snli:test:4389:premise", "snli:test:4535:premise", "style:personachat:test:172:self:5:positive:0", "style:personachat:test:172:self:5:positive:1", "style:personachat:test:172:self:5:positive:2", "style:personachat:test:172:self:5:positive:3", "style:personachat:test:172:self:5:positive:4"], "evidence_texts": ["A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "cool do you have any brothers or sisters ? i have both .", "do you live with your parents as well ?", "i am in the stocks myself .", "no i wish , got to keep my head clear for trades .", "do you have any hobbies ?"], "speaker_id": "snli:test:4535", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4535", "candidate_text": "well, two men are reading books", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4535:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4502:premise", "snli:test:4723:premise", "snli:test:4389:premise", "snli:test:4535:premise", "style:personachat:test:301:self:5:positive:0", "style:personachat:test:301:self:5:positive:1", "style:personachat:test:301:self:5:positive:2", "style:personachat:test:301:self:5:positive:3", "style:personachat:test:301:self:5:positive:4"], "evidence_texts": ["A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "A group of men playing music, the man holding a guitar is talking to a man with drums.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "running is good . do you have hobbies ? i sing in a community choir .", "i love playing cards , especially euchre .", "euchre is a a card game where you only use ace through 9 cards . lots of fun .", "4 people , deal out 5 cards each . if spades are trump , jack of spades is highest card", "win by taking the most tricks . you have to take at least 3 tricks or you lose ."], "speaker_id": "snli:test:4535", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4555", "candidate_text": "I wonder whether a man is signaling the start of a race?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4555:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4319:premise", "snli:test:4650:premise", "snli:test:4076:premise", "snli:test:4555:premise", "style:personachat:test:172:self:7:positive:0", "style:personachat:test:172:self:7:positive:1", "style:personachat:test:172:self:7:positive:2", "style:personachat:test:172:self:7:positive:3", "style:personachat:test:172:self:7:positive:4", "style:personachat:test:172:self:7:positive:5", "style:personachat:test:172:self:7:positive:6"], "evidence_texts": ["An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "no but i am eating eggplant parmigiana !", "i love it the best ! even my dog charlie loves it !", "cool do you have any brothers or sisters ? i have both .", "do you live with your parents as well ?", "i am in the stocks myself .", "no i wish , got to keep my head clear for trades .", "do you have any hobbies ?"], "speaker_id": "snli:test:4555", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4555", "candidate_text": "Honestly, A man is signaling the start of a race!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4555:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4319:premise", "snli:test:4650:premise", "snli:test:4076:premise", "snli:test:4555:premise", "style:personachat:test:269:self:5:positive:0", "style:personachat:test:269:self:5:positive:1", "style:personachat:test:269:self:5:positive:2", "style:personachat:test:269:self:5:positive:3", "style:personachat:test:269:self:5:positive:4"], "evidence_texts": ["An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "A man in a jacket wearing black boots walking on the truck of a fallen tree over a ravine.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "i have two small dogs , max and marsha and a cat named ginger .", "we do , though mostly they ignore us . what do you do for work john ?", "nice ! i work with the elderly helping them care for their daily needs .", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:4555", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4560", "candidate_text": "I wonder whether the guy kicked something?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4560:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4560:premise", "snli:test:4464:premise", "snli:test:4602:premise", "snli:test:4445:premise", "style:personachat:test:173:self:1:positive:0"], "evidence_texts": ["The man in the dark maroon shirt is kicking something.", "A golfer shoots his ball out of a sand trap.", "Several people wearing green shirts walk on the beach together.", "A man and a woman are dancing in the crowd.", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:4560", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4560", "candidate_text": "the guy kicked something.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4560:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4560:premise", "snli:test:4464:premise", "snli:test:4602:premise", "snli:test:4445:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The man in the dark maroon shirt is kicking something.", "A golfer shoots his ball out of a sand trap.", "Several people wearing green shirts walk on the beach together.", "A man and a woman are dancing in the crowd.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4560", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4571", "candidate_text": "I wonder whether a topless man is working outside?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4571:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4462:premise", "snli:test:4586:premise", "snli:test:4240:premise", "snli:test:4571:premise", "style:personachat:test:173:self:2:positive:0", "style:personachat:test:173:self:2:positive:1"], "evidence_texts": ["A cyclist in a red and white uniform is biking across a garage full of people.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "oh nice ! my mother works as a secretary . and , no .", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:4571", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4571", "candidate_text": "well, a topless man is working outside", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4571:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4462:premise", "snli:test:4586:premise", "snli:test:4240:premise", "snli:test:4571:premise", "style:personachat:test:302:self:2:positive:0", "style:personachat:test:302:self:2:positive:1"], "evidence_texts": ["A cyclist in a red and white uniform is biking across a garage full of people.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "A man wearing shorts looks at his cellphone while a woman walks outside a small building.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "my hair color is dirty blonde .", "i do not sing and i was not homeschooled ."], "speaker_id": "snli:test:4571", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4578", "candidate_text": "I wonder whether three people are leading a parade?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4578:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3395:premise", "snli:test:5007:premise", "snli:test:4578:premise", "snli:test:3122:premise", "style:personachat:test:173:self:5:positive:0", "style:personachat:test:173:self:5:positive:1", "style:personachat:test:173:self:5:positive:2", "style:personachat:test:173:self:5:positive:3", "style:personachat:test:173:self:5:positive:4"], "evidence_texts": ["A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "that is fun ! whose wedding did you go to ?", "not much . i work at a hospital so i do not have much time .", "what did you graduate for ?", "oh nice ! my mother works as a secretary . and , no .", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:4578", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4578", "candidate_text": "Honestly, Three people are leading a parade!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4578:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3395:premise", "snli:test:5007:premise", "snli:test:4578:premise", "snli:test:3122:premise", "style:personachat:test:275:self:2:positive:0", "style:personachat:test:275:self:2:positive:1"], "evidence_texts": ["A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man wearing a stocking hat and has a beard is looking at the camera with two women sitting on the couch next to him.", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:4578", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4586", "candidate_text": "I wonder whether a child sleeps in a crib?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4586:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4571:premise", "snli:test:4613:premise", "snli:test:4586:premise", "snli:test:4462:premise", "style:personachat:test:173:self:6:positive:0", "style:personachat:test:173:self:6:positive:1", "style:personachat:test:173:self:6:positive:2", "style:personachat:test:173:self:6:positive:3", "style:personachat:test:173:self:6:positive:4", "style:personachat:test:173:self:6:positive:5"], "evidence_texts": ["A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "A cyclist in a red and white uniform is biking across a garage full of people.", "not too much . i just got done working out at the gym .", "that is fun ! whose wedding did you go to ?", "not much . i work at a hospital so i do not have much time .", "what did you graduate for ?", "oh nice ! my mother works as a secretary . and , no .", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:4586", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4586", "candidate_text": "A child sleeps in a crib.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4586:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4571:premise", "snli:test:4613:premise", "snli:test:4586:premise", "snli:test:4462:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "A cyclist in a red and white uniform is biking across a garage full of people.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4586", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4590", "candidate_text": "Honestly, Two boys are sleeping beside the garbage bag!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4590:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4473:premise", "snli:test:4595:premise", "snli:test:4590:premise", "snli:test:4428:premise", "style:personachat:test:174:self:1:positive:0"], "evidence_texts": ["A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A young couple riding on a bus with the boy's arm around the girl.", "two teen boys in school clothes are walking with something in a garbage bag.", "a woman in jeans and a woman in shorts are petting a baby kangaroo.", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:4590", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4590", "candidate_text": "well, two boys are sleeping beside the garbage bag", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4590:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4473:premise", "snli:test:4595:premise", "snli:test:4590:premise", "snli:test:4428:premise", "style:personachat:test:303:self:1:positive:0"], "evidence_texts": ["A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A young couple riding on a bus with the boy's arm around the girl.", "two teen boys in school clothes are walking with something in a garbage bag.", "a woman in jeans and a woman in shorts are petting a baby kangaroo.", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:4590", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4591", "candidate_text": "Honestly, A person is reaching up!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4591:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4439:premise", "snli:test:5087:premise", "snli:test:4404:premise", "snli:test:4591:premise", "style:personachat:test:174:self:2:positive:0", "style:personachat:test:174:self:2:positive:1"], "evidence_texts": ["A man a woman posing for a picture.", "Two boys in front of a soda machine.", "A gas station and a car on fire.", "A person in an orange shirt reaching up.", "it is fun . but modeling pays more ! you have hobbies ?", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:4591", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4591", "candidate_text": "A person is reaching up.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4591:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4439:premise", "snli:test:5087:premise", "snli:test:4404:premise", "snli:test:4591:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man a woman posing for a picture.", "Two boys in front of a soda machine.", "A gas station and a car on fire.", "A person in an orange shirt reaching up.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4591", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4595", "candidate_text": "Honestly, The couple are on a bus!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4595:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4590:premise", "snli:test:4473:premise", "snli:test:4922:premise", "snli:test:4595:premise", "style:personachat:test:174:self:5:positive:0", "style:personachat:test:174:self:5:positive:1", "style:personachat:test:174:self:5:positive:2", "style:personachat:test:174:self:5:positive:3", "style:personachat:test:174:self:5:positive:4"], "evidence_texts": ["two teen boys in school clothes are walking with something in a garbage bag.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A bald man with a white shirt and watch eating food and drinking wine.", "A young couple riding on a bus with the boy's arm around the girl.", "that is why i do not ever want kids . where are you from ?", "new york . a great place to be single ! married yourself ?", "came here to be a singer . why did you leave ?", "it is fun . but modeling pays more ! you have hobbies ?", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:4595", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4595", "candidate_text": "I wonder whether the couple are on a bus?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4595:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4590:premise", "snli:test:4473:premise", "snli:test:4922:premise", "snli:test:4595:premise", "style:personachat:test:667:self:5:positive:0", "style:personachat:test:667:self:5:positive:1", "style:personachat:test:667:self:5:positive:2", "style:personachat:test:667:self:5:positive:3", "style:personachat:test:667:self:5:positive:4"], "evidence_texts": ["two teen boys in school clothes are walking with something in a garbage bag.", "A boy wearing jeans, standing on rocks in a river and throwing a stone.", "A bald man with a white shirt and watch eating food and drinking wine.", "A young couple riding on a bus with the boy's arm around the girl.", "my brother is in the army one of them anyway . you like cats ?", "well that is an interesting hobby . do you listen to music ?", "i like rock and heavy metal . metallica is my favorite band though .", "i grew up listening to eighties rock with my dad and that started my obsession", "i am not sure i would have to ask him . how long have you listened ?"], "speaker_id": "snli:test:4595", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4596", "candidate_text": "Honestly, The couple are in their parent's car!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4596:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4596:premise", "snli:test:4922:premise", "snli:test:4590:premise", "snli:test:4980:premise", "style:personachat:test:174:self:7:positive:0", "style:personachat:test:174:self:7:positive:1", "style:personachat:test:174:self:7:positive:2", "style:personachat:test:174:self:7:positive:3", "style:personachat:test:174:self:7:positive:4", "style:personachat:test:174:self:7:positive:5", "style:personachat:test:174:self:7:positive:6"], "evidence_texts": ["A young couple riding on a bus with the boy's arm around the girl.", "A bald man with a white shirt and watch eating food and drinking wine.", "two teen boys in school clothes are walking with something in a garbage bag.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "wonderful . just finished work . how are you ?", "right now i am modeling . it is why i moved to this city . what do you do ?", "that is why i do not ever want kids . where are you from ?", "new york . a great place to be single ! married yourself ?", "came here to be a singer . why did you leave ?", "it is fun . but modeling pays more ! you have hobbies ?", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:4596", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4596", "candidate_text": "well, the couple are in their parent's car", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4596:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4596:premise", "snli:test:4922:premise", "snli:test:4590:premise", "snli:test:4980:premise", "style:personachat:test:303:self:7:positive:0", "style:personachat:test:303:self:7:positive:1", "style:personachat:test:303:self:7:positive:2", "style:personachat:test:303:self:7:positive:3", "style:personachat:test:303:self:7:positive:4", "style:personachat:test:303:self:7:positive:5", "style:personachat:test:303:self:7:positive:6"], "evidence_texts": ["A young couple riding on a bus with the boy's arm around the girl.", "A bald man with a white shirt and watch eating food and drinking wine.", "two teen boys in school clothes are walking with something in a garbage bag.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "hello nice to meet you !", "no , imam not into that . . i like to travel . . visited 6 countries already", "good choice ! what do you do ? i , a biology teacher at college", "interesting ! i guess you are into nature ! i have a big garden and i love it !", "nice ! do you have any hobbies ? i love walking", "i a , a biology teacher", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:4596", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4602", "candidate_text": "well, a man in a red shirt is reading at the park", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4602:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4560:premise", "snli:test:4886:premise", "snli:test:4602:premise", "snli:test:4464:premise", "style:personachat:test:175:self:1:positive:0"], "evidence_texts": ["The man in the dark maroon shirt is kicking something.", "Family gathered together in a house enjoying each other company.", "Several people wearing green shirts walk on the beach together.", "A golfer shoots his ball out of a sand trap.", "i may only be an accountant but . . . pretty sure dogs should not eat pizza"], "speaker_id": "snli:test:4602", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4602", "candidate_text": "A man in a red shirt is reading at the park.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4602:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4560:premise", "snli:test:4886:premise", "snli:test:4602:premise", "snli:test:4464:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The man in the dark maroon shirt is kicking something.", "Family gathered together in a house enjoying each other company.", "Several people wearing green shirts walk on the beach together.", "A golfer shoots his ball out of a sand trap.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4602", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4605", "candidate_text": "well, people are getting ready to run", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4605:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4481:premise", "snli:test:4605:premise", "snli:test:4638:premise", "snli:test:4470:premise", "style:personachat:test:175:self:2:positive:0", "style:personachat:test:175:self:2:positive:1"], "evidence_texts": ["A woman holds a newspaper that says \"Real change\".", "A large crowd is standing around the start line.", "Families with strollers waiting in front of a carousel.", "A group of mountain climbers rests at the summit.", "i would love some ! but make it two . . . for each boy !", "i may only be an accountant but . . . pretty sure dogs should not eat pizza"], "speaker_id": "snli:test:4605", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4605", "candidate_text": "I wonder whether people are getting ready to run?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4605:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4481:premise", "snli:test:4605:premise", "snli:test:4638:premise", "snli:test:4470:premise", "style:personachat:test:668:self:2:positive:0", "style:personachat:test:668:self:2:positive:1"], "evidence_texts": ["A woman holds a newspaper that says \"Real change\".", "A large crowd is standing around the start line.", "Families with strollers waiting in front of a carousel.", "A group of mountain climbers rests at the summit.", "that sounds like lots of fun", "not particularly fond of them , do you ?"], "speaker_id": "snli:test:4605", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4613", "candidate_text": "well, gymnasts wearing matching leotards practicing", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4613:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4586:premise", "snli:test:4613:premise", "snli:test:4770:premise", "snli:test:4571:premise", "style:personachat:test:175:self:5:positive:0", "style:personachat:test:175:self:5:positive:1", "style:personachat:test:175:self:5:positive:2", "style:personachat:test:175:self:5:positive:3", "style:personachat:test:175:self:5:positive:4"], "evidence_texts": ["A small child wearing a helmet is in a baby carrier that is on the floor.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "oh you are . luckily my job is mostly paperwork . . . what do you do for a living ?", "maybe you could do my hair sometime !", "that is okay , i spend my weekends hiking , usually .", "i would love some ! but make it two . . . for each boy !", "i may only be an accountant but . . . pretty sure dogs should not eat pizza"], "speaker_id": "snli:test:4613", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4613", "candidate_text": "Honestly, Gymnasts wearing matching leotards practicing!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4613:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4586:premise", "snli:test:4613:premise", "snli:test:4770:premise", "snli:test:4571:premise", "style:personachat:test:290:self:2:positive:0", "style:personachat:test:290:self:2:positive:1"], "evidence_texts": ["A small child wearing a helmet is in a baby carrier that is on the floor.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A man wearing no shirt is lifting a shovel by a wheelbarrow as he works outside.", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:4613", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4623", "candidate_text": "well, people eating lunch together", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4623:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4217:premise", "snli:test:4647:premise", "snli:test:4062:premise", "snli:test:4623:premise", "style:personachat:test:175:self:7:positive:0", "style:personachat:test:175:self:7:positive:1", "style:personachat:test:175:self:7:positive:2", "style:personachat:test:175:self:7:positive:3", "style:personachat:test:175:self:7:positive:4", "style:personachat:test:175:self:7:positive:5", "style:personachat:test:175:self:7:positive:6"], "evidence_texts": ["A woman is shaving the hair off of a black and white dog.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "Two men and two women in jackets walk down a large city street.", "hehe , i am actually a miss . it is going well , what about yourself ?", "what are their names ? i do not have any pets , my two boys are enough !", "oh you are . luckily my job is mostly paperwork . . . what do you do for a living ?", "maybe you could do my hair sometime !", "that is okay , i spend my weekends hiking , usually .", "i would love some ! but make it two . . . for each boy !", "i may only be an accountant but . . . pretty sure dogs should not eat pizza"], "speaker_id": "snli:test:4623", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4623", "candidate_text": "People eating lunch together.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4623:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4217:premise", "snli:test:4647:premise", "snli:test:4062:premise", "snli:test:4623:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman is shaving the hair off of a black and white dog.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "A brown dog persues a Frisbee across the grass as the thrower watches.", "Two men and two women in jackets walk down a large city street.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4623", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4633", "candidate_text": "I wonder whether a boy rides his bike?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4633:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4382:premise", "snli:test:4633:premise", "snli:test:4375:premise", "snli:test:4854:premise", "style:personachat:test:176:self:1:positive:0"], "evidence_texts": ["children are sitting in chairs with a music stand in the background.", "A person in a black and green outfit is riding a bicycle.", "a woman with brown hair sitting on a bench outside a cafe.", "Three guys in red uniforms celebrating a goal in a soccer game.", "what do you do now ?"], "speaker_id": "snli:test:4633", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4633", "candidate_text": "a boy rides his bike.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4633:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4382:premise", "snli:test:4633:premise", "snli:test:4375:premise", "snli:test:4854:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["children are sitting in chairs with a music stand in the background.", "A person in a black and green outfit is riding a bicycle.", "a woman with brown hair sitting on a bench outside a cafe.", "Three guys in red uniforms celebrating a goal in a soccer game.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4633", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4634", "candidate_text": "I wonder whether a girl runs down the street?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4634:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4854:premise", "snli:test:4382:premise", "snli:test:4634:premise", "snli:test:4907:premise", "style:personachat:test:176:self:2:positive:0", "style:personachat:test:176:self:2:positive:1"], "evidence_texts": ["Three guys in red uniforms celebrating a goal in a soccer game.", "children are sitting in chairs with a music stand in the background.", "A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "i am thinking about being a nurse . my aunt works in a hospital , and likes it .", "what do you do now ?"], "speaker_id": "snli:test:4634", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4634", "candidate_text": "well, a girl runs down the street", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4634:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4854:premise", "snli:test:4382:premise", "snli:test:4634:premise", "snli:test:4907:premise", "style:personachat:test:305:self:2:positive:0", "style:personachat:test:305:self:2:positive:1"], "evidence_texts": ["Three guys in red uniforms celebrating a goal in a soccer game.", "children are sitting in chairs with a music stand in the background.", "A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "not really , i do not like sports", "yes just about all genres"], "speaker_id": "snli:test:4634", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4638", "candidate_text": "I wonder whether families have some dogs in front of a carousel?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4638:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4605:premise", "snli:test:4671:premise", "snli:test:4638:premise", "snli:test:4481:premise", "style:personachat:test:176:self:5:positive:0", "style:personachat:test:176:self:5:positive:1", "style:personachat:test:176:self:5:positive:2", "style:personachat:test:176:self:5:positive:3", "style:personachat:test:176:self:5:positive:4"], "evidence_texts": ["A large crowd is standing around the start line.", "A boy works on a project in shop class.", "Families with strollers waiting in front of a carousel.", "A woman holds a newspaper that says \"Real change\".", "i sell hamburgers and fries .", "i am full time . i am thinking about finishing school .", "i thought it would help me get a better job .", "i am thinking about being a nurse . my aunt works in a hospital , and likes it .", "what do you do now ?"], "speaker_id": "snli:test:4638", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4638", "candidate_text": "Honestly, Families have some dogs in front of a carousel!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4638:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4605:premise", "snli:test:4671:premise", "snli:test:4638:premise", "snli:test:4481:premise", "style:personachat:test:295:self:2:positive:0", "style:personachat:test:295:self:2:positive:1"], "evidence_texts": ["A large crowd is standing around the start line.", "A boy works on a project in shop class.", "Families with strollers waiting in front of a carousel.", "A woman holds a newspaper that says \"Real change\".", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:4638", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4647", "candidate_text": "I wonder whether a youth skateboards by a mural?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4647:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4623:premise", "snli:test:4787:premise", "snli:test:4217:premise", "snli:test:4647:premise", "style:personachat:test:176:self:6:positive:0", "style:personachat:test:176:self:6:positive:1", "style:personachat:test:176:self:6:positive:2", "style:personachat:test:176:self:6:positive:3", "style:personachat:test:176:self:6:positive:4", "style:personachat:test:176:self:6:positive:5"], "evidence_texts": ["Two men and two women in jackets walk down a large city street.", "A man is ducking down between the red tulips in a flower field.", "A woman is shaving the hair off of a black and white dog.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "i am fine . i am on a break at work .", "i sell hamburgers and fries .", "i am full time . i am thinking about finishing school .", "i thought it would help me get a better job .", "i am thinking about being a nurse . my aunt works in a hospital , and likes it .", "what do you do now ?"], "speaker_id": "snli:test:4647", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4647", "candidate_text": "A youth skateboards by a mural.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4647:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4623:premise", "snli:test:4787:premise", "snli:test:4217:premise", "snli:test:4647:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men and two women in jackets walk down a large city street.", "A man is ducking down between the red tulips in a flower field.", "A woman is shaving the hair off of a black and white dog.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4647", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4650", "candidate_text": "well, runners see something", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4650:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4555:premise", "snli:test:4927:premise", "snli:test:4650:premise", "snli:test:4319:premise", "style:personachat:test:177:self:1:positive:0"], "evidence_texts": ["A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "i can see why . heights can be a little terrifying sometimes"], "speaker_id": "snli:test:4650", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4650", "candidate_text": "Honestly, Runners see something!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4650:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4555:premise", "snli:test:4927:premise", "snli:test:4650:premise", "snli:test:4319:premise", "style:personachat:test:295:self:7:positive:0", "style:personachat:test:295:self:7:positive:1", "style:personachat:test:295:self:7:positive:2", "style:personachat:test:295:self:7:positive:3", "style:personachat:test:295:self:7:positive:4", "style:personachat:test:295:self:7:positive:5", "style:personachat:test:295:self:7:positive:6"], "evidence_texts": ["A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "An old man in a plaid shirt and blue pants is looking in a telescope on a sunny day.", "i just got home from work , i do graphic design .", "nope , my real job is wheelchair sales and design is my hobby , i freelance .", "i can hook you up . do you like music ? my fav band is rush ? awesome .", "big question , what does an omnivore do at a vegans wedding ? my challenge last weekend !", "i like to pimp out my clients wheelchairs , what would your brother like ?", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:4650", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4656", "candidate_text": "well, a man is about to throw a snowball at someone", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4656:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4656:premise", "snli:test:4379:premise", "snli:test:4924:premise", "snli:test:4267:premise", "style:personachat:test:177:self:2:positive:0", "style:personachat:test:177:self:2:positive:1"], "evidence_texts": ["A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "depends . being an avid runner , i would say you should make some time for it ;", "i can see why . heights can be a little terrifying sometimes"], "speaker_id": "snli:test:4656", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4656", "candidate_text": "A man is about to throw a snowball at someone.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4656:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4656:premise", "snli:test:4379:premise", "snli:test:4924:premise", "snli:test:4267:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "a man standing against a building, in the sunlight, as you can see the shadow of the person taking his picture", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4656", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4671", "candidate_text": "well, the boy peeks through a doorway at people working on their shop class project", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4671:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4638:premise", "snli:test:4785:premise", "snli:test:4605:premise", "snli:test:4671:premise", "style:personachat:test:177:self:5:positive:0", "style:personachat:test:177:self:5:positive:1", "style:personachat:test:177:self:5:positive:2", "style:personachat:test:177:self:5:positive:3", "style:personachat:test:177:self:5:positive:4"], "evidence_texts": ["Families with strollers waiting in front of a carousel.", "A woman is taking a picture of a man.", "A large crowd is standing around the start line.", "A boy works on a project in shop class.", "i am not an animal fan myself . low income apartments do not allow them anyway", "i live in the us . you ?", "oh wow that is cool . you like to run ? that is what keeps me sane", "depends . being an avid runner , i would say you should make some time for it ;", "i can see why . heights can be a little terrifying sometimes"], "speaker_id": "snli:test:4671", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4671", "candidate_text": "I wonder whether the boy peeks through a doorway at people working on their shop class project?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4671:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4638:premise", "snli:test:4785:premise", "snli:test:4605:premise", "snli:test:4671:premise", "style:personachat:test:674:self:5:positive:0", "style:personachat:test:674:self:5:positive:1", "style:personachat:test:674:self:5:positive:2", "style:personachat:test:674:self:5:positive:3", "style:personachat:test:674:self:5:positive:4"], "evidence_texts": ["Families with strollers waiting in front of a carousel.", "A woman is taking a picture of a man.", "A large crowd is standing around the start line.", "A boy works on a project in shop class.", "do you like horses ? i love working with horses .", "what is your favorite musician ? mine is john denver .", "one of my hobbies is working out every day . you ?", "i like lifting weights and doing squats .", "oh i misunderstood , i am a doctor . you ?"], "speaker_id": "snli:test:4671", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4681", "candidate_text": "well, there are people at the beach", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4681:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4515:premise", "snli:test:4681:premise", "snli:test:4735:premise", "snli:test:4409:premise", "style:personachat:test:177:self:7:positive:0", "style:personachat:test:177:self:7:positive:1", "style:personachat:test:177:self:7:positive:2", "style:personachat:test:177:self:7:positive:3", "style:personachat:test:177:self:7:positive:4", "style:personachat:test:177:self:7:positive:5", "style:personachat:test:177:self:7:positive:6"], "evidence_texts": ["Three kids perform a skit about camping.", "Three people are walking on a beach.", "A brown dog races through a field.", "Artist on street corner selling his artwork.", "okay i suppose . just got in from work at a fast food joint", "bet that is a nice income", "i am not an animal fan myself . low income apartments do not allow them anyway", "i live in the us . you ?", "oh wow that is cool . you like to run ? that is what keeps me sane", "depends . being an avid runner , i would say you should make some time for it ;", "i can see why . heights can be a little terrifying sometimes"], "speaker_id": "snli:test:4681", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4681", "candidate_text": "Honestly, There are people at the beach!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4681:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4515:premise", "snli:test:4681:premise", "snli:test:4735:premise", "snli:test:4409:premise", "style:personachat:test:332:self:5:positive:0", "style:personachat:test:332:self:5:positive:1", "style:personachat:test:332:self:5:positive:2", "style:personachat:test:332:self:5:positive:3", "style:personachat:test:332:self:5:positive:4"], "evidence_texts": ["Three kids perform a skit about camping.", "Three people are walking on a beach.", "A brown dog races through a field.", "Artist on street corner selling his artwork.", "i take my obedience dogs hiking ! i have mountains behind my new house .", "sorry ! was taking pictures of myself in my new club dress ! yes ! ! lets do it .", "only if i can hang it in my obedience school !", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:4681", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4704", "candidate_text": "I wonder whether the people are outside?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4704:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4704:premise", "snli:test:4505:premise", "snli:test:4722:premise", "snli:test:4503:premise", "style:personachat:test:178:self:1:positive:0"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "a young woman siting on a park bench in the sun.", "A man is pushing many cartons of bread down the street.", "ooh what kind ? any specific genre ?"], "speaker_id": "snli:test:4704", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4704", "candidate_text": "Honestly, The people are outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4704:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4704:premise", "snli:test:4505:premise", "snli:test:4722:premise", "snli:test:4503:premise", "style:personachat:test:332:self:7:positive:0", "style:personachat:test:332:self:7:positive:1", "style:personachat:test:332:self:7:positive:2", "style:personachat:test:332:self:7:positive:3", "style:personachat:test:332:self:7:positive:4", "style:personachat:test:332:self:7:positive:5", "style:personachat:test:332:self:7:positive:6"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "A man is pushing many cartons of bread down the street.", "a young woman siting on a park bench in the sun.", "A man is pushing many cartons of bread down the street.", "no stickers . . but i have candy ! i always have candy", "i love candy ! ! ! it is the best ! and brownies .", "i take my obedience dogs hiking ! i have mountains behind my new house .", "sorry ! was taking pictures of myself in my new club dress ! yes ! ! lets do it .", "only if i can hang it in my obedience school !", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:4704", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4711", "candidate_text": "I wonder whether cheerleaders are standing on a bridge?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4711:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4107:premise", "snli:test:3997:premise", "snli:test:4824:premise", "snli:test:4711:premise", "style:personachat:test:178:self:2:positive:0", "style:personachat:test:178:self:2:positive:1"], "evidence_texts": ["Coed choir singing at an event.", "Four people enjoying an art exhibit.", "Xylophonists play in a dark setting.", "Cheerleaders standing on a football field.", "yes ! my mom was an army nurse so i read a lot of military history", "ooh what kind ? any specific genre ?"], "speaker_id": "snli:test:4711", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4711", "candidate_text": "Cheerleaders are standing on a bridge.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4711:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4107:premise", "snli:test:3997:premise", "snli:test:4824:premise", "snli:test:4711:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Coed choir singing at an event.", "Four people enjoying an art exhibit.", "Xylophonists play in a dark setting.", "Cheerleaders standing on a football field.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4711", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4712", "candidate_text": "I wonder whether cheerleaders are wearing outside?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4712:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4712:premise", "snli:test:4824:premise", "snli:test:4107:premise", "snli:test:4867:premise", "style:personachat:test:178:self:5:positive:0", "style:personachat:test:178:self:5:positive:1", "style:personachat:test:178:self:5:positive:2", "style:personachat:test:178:self:5:positive:3", "style:personachat:test:178:self:5:positive:4"], "evidence_texts": ["Cheerleaders standing on a football field.", "Xylophonists play in a dark setting.", "Coed choir singing at an event.", "People shopping at an outside market", "well i did just adopt a cat from the street . . . i could eat it for my diet", "i do not have a problem with eating animals", "no , but i do sing folk songs . i could sing one about dieting for you ?", "yes ! my mom was an army nurse so i read a lot of military history", "ooh what kind ? any specific genre ?"], "speaker_id": "snli:test:4712", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4712", "candidate_text": "well, cheerleaders are wearing outside", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4712:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4712:premise", "snli:test:4824:premise", "snli:test:4107:premise", "snli:test:4867:premise", "style:personachat:test:307:self:5:positive:0", "style:personachat:test:307:self:5:positive:1", "style:personachat:test:307:self:5:positive:2", "style:personachat:test:307:self:5:positive:3", "style:personachat:test:307:self:5:positive:4"], "evidence_texts": ["Cheerleaders standing on a football field.", "Xylophonists play in a dark setting.", "Coed choir singing at an event.", "People shopping at an outside market", "that is nice , i was born and raised in the united states have you painted there ?", "awesome , we should chat more often . well i am very busy with my study .", "yeah , i play her music to scare whales in deep ocean trenches . .", "one whale died because of us . and we as a class spread lies that it was china .", "can you come to our university for an interview , to help our study ."], "speaker_id": "snli:test:4712", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4722", "candidate_text": "I wonder whether the young woman is sitting on a bench in the rain?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4722:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4704:premise", "snli:test:4890:premise", "snli:test:4722:premise", "snli:test:4505:premise", "style:personachat:test:178:self:7:positive:0", "style:personachat:test:178:self:7:positive:1", "style:personachat:test:178:self:7:positive:2", "style:personachat:test:178:self:7:positive:3", "style:personachat:test:178:self:7:positive:4", "style:personachat:test:178:self:7:positive:5", "style:personachat:test:178:self:7:positive:6"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "The blond boy is riding roller-coaster with his mouth wide open.", "a young woman siting on a park bench in the sun.", "A man is pushing many cartons of bread down the street.", "hey there ! i am freaking out about nature and the environment !", "nice ! i also make my own clothes , it helps as well .", "well i did just adopt a cat from the street . . . i could eat it for my diet", "i do not have a problem with eating animals", "no , but i do sing folk songs . i could sing one about dieting for you ?", "yes ! my mom was an army nurse so i read a lot of military history", "ooh what kind ? any specific genre ?"], "speaker_id": "snli:test:4722", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4722", "candidate_text": "Honestly, The young woman is sitting on a bench in the rain!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4722:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4704:premise", "snli:test:4890:premise", "snli:test:4722:premise", "snli:test:4505:premise", "style:personachat:test:339:self:5:positive:0", "style:personachat:test:339:self:5:positive:1", "style:personachat:test:339:self:5:positive:2", "style:personachat:test:339:self:5:positive:3", "style:personachat:test:339:self:5:positive:4"], "evidence_texts": ["Group of people standing and sitting in a field of grass.", "The blond boy is riding roller-coaster with his mouth wide open.", "a young woman siting on a park bench in the sun.", "A man is pushing many cartons of bread down the street.", "music is my bread and butter , i like surrounding myself with music . what do you teach ?", "wow , that is cool , you must be very smart . i work at petsmart right now", "what is your favorite place to visit in europe ?", "my father was chief of police in france , i have been there a lot !", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:4722", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4723", "candidate_text": "well, a blonde woman sits on a bench outside in a park", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4723:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4535:premise", "snli:test:4502:premise", "snli:test:4920:premise", "snli:test:4723:premise", "style:personachat:test:179:self:1:positive:0"], "evidence_texts": ["Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "they dress up like superheroes and the lord of the rings character is there too"], "speaker_id": "snli:test:4723", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4723", "candidate_text": "I wonder whether a blonde woman sits on a bench outside in a park?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4723:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4535:premise", "snli:test:4502:premise", "snli:test:4920:premise", "snli:test:4723:premise", "style:personachat:test:679:self:1:positive:0"], "evidence_texts": ["Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A man wearing red ski pants, a black jacket, and a white helmet is skiing down a mountain.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "yes cats suck . gosh , did i mention i am transexual ?"], "speaker_id": "snli:test:4723", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4724", "candidate_text": "well, a brunette woman sits on a sofa", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4724:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4724:premise", "snli:test:4920:premise", "snli:test:4535:premise", "snli:test:5220:premise", "style:personachat:test:179:self:2:positive:0", "style:personachat:test:179:self:2:positive:1"], "evidence_texts": ["A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "it is a huge festival that last for days", "they dress up like superheroes and the lord of the rings character is there too"], "speaker_id": "snli:test:4724", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4724", "candidate_text": "Honestly, A brunette woman sits on a sofa!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4724:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4724:premise", "snli:test:4920:premise", "snli:test:4535:premise", "snli:test:5220:premise", "style:personachat:test:342:self:1:positive:0"], "evidence_texts": ["A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "Two men cooking, one stirring a pot while making a funny face while the other points at him.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:4724", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4735", "candidate_text": "well, a dog is barking out of the window", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4735:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4681:premise", "snli:test:4515:premise", "snli:test:4749:premise", "snli:test:4735:premise", "style:personachat:test:179:self:5:positive:0", "style:personachat:test:179:self:5:positive:1", "style:personachat:test:179:self:5:positive:2", "style:personachat:test:179:self:5:positive:3", "style:personachat:test:179:self:5:positive:4"], "evidence_texts": ["Three people are walking on a beach.", "Three kids perform a skit about camping.", "Two girls feeding a horse an apple.", "A brown dog races through a field.", "i want to just stay home and write .", "i like to role play and play fantasy games . have you been to com icon ?", "have you ever heard of the lord of the rings ?", "it is a huge festival that last for days", "they dress up like superheroes and the lord of the rings character is there too"], "speaker_id": "snli:test:4735", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4735", "candidate_text": "A dog is barking out of the window.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4735:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4681:premise", "snli:test:4515:premise", "snli:test:4749:premise", "snli:test:4735:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Three people are walking on a beach.", "Three kids perform a skit about camping.", "Two girls feeding a horse an apple.", "A brown dog races through a field.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4735", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4736", "candidate_text": "well, a brown dog is racing another dog", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4736:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4736:premise", "snli:test:4749:premise", "snli:test:4681:premise", "snli:test:4882:premise", "style:personachat:test:179:self:6:positive:0", "style:personachat:test:179:self:6:positive:1", "style:personachat:test:179:self:6:positive:2", "style:personachat:test:179:self:6:positive:3", "style:personachat:test:179:self:6:positive:4", "style:personachat:test:179:self:6:positive:5"], "evidence_texts": ["A brown dog races through a field.", "Two girls feeding a horse an apple.", "Three people are walking on a beach.", "A girl in black pants doing exercises.", "i am so tired ! i have to work a double !", "i want to just stay home and write .", "i like to role play and play fantasy games . have you been to com icon ?", "have you ever heard of the lord of the rings ?", "it is a huge festival that last for days", "they dress up like superheroes and the lord of the rings character is there too"], "speaker_id": "snli:test:4736", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4736", "candidate_text": "I wonder whether a brown dog is racing another dog?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4736:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4736:premise", "snli:test:4749:premise", "snli:test:4681:premise", "snli:test:4882:premise", "style:personachat:test:679:self:7:positive:0", "style:personachat:test:679:self:7:positive:1", "style:personachat:test:679:self:7:positive:2", "style:personachat:test:679:self:7:positive:3", "style:personachat:test:679:self:7:positive:4", "style:personachat:test:679:self:7:positive:5", "style:personachat:test:679:self:7:positive:6"], "evidence_texts": ["A brown dog races through a field.", "Two girls feeding a horse an apple.", "Three people are walking on a beach.", "A girl in black pants doing exercises.", "what do you sell ? i would like to be an actor", "that is neat , what do you design", "that sounds neat ! do you have a favorite band", "pink floyd ! they rock . i am only 14 years old", "ha ha . do you have any dogs ?", "yes i have a ton , i love them . i cannot stand cats though !", "yes cats suck . gosh , did i mention i am transexual ?"], "speaker_id": "snli:test:4736", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4749", "candidate_text": "well, two girls are near an animal", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4749:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4736:premise", "snli:test:4749:premise", "snli:test:4882:premise", "snli:test:4735:premise", "style:personachat:test:180:self:1:positive:0"], "evidence_texts": ["A brown dog races through a field.", "Two girls feeding a horse an apple.", "A girl in black pants doing exercises.", "A brown dog races through a field.", "not a fan of star wars ."], "speaker_id": "snli:test:4749", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4749", "candidate_text": "Honestly, Two girls are near an animal!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4749:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4736:premise", "snli:test:4749:premise", "snli:test:4882:premise", "snli:test:4735:premise", "style:personachat:test:342:self:7:positive:0", "style:personachat:test:342:self:7:positive:1", "style:personachat:test:342:self:7:positive:2", "style:personachat:test:342:self:7:positive:3", "style:personachat:test:342:self:7:positive:4", "style:personachat:test:342:self:7:positive:5", "style:personachat:test:342:self:7:positive:6"], "evidence_texts": ["A brown dog races through a field.", "Two girls feeding a horse an apple.", "A girl in black pants doing exercises.", "A brown dog races through a field.", "i am good , thanks ! and you ?", "you performing there or going as audience member ?", "aww , all the best to you ! i just finished baking a cake for my kids and hubby .", "sending you a slice ! i do not like either . i enjoy reading .", "you could always read really fast ! !", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:4749", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4770", "candidate_text": "well, a woman carrying a girl, with a bag behind her", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4770:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4613:premise", "snli:test:5198:premise", "snli:test:4770:premise", "snli:test:4586:premise", "style:personachat:test:180:self:2:positive:0", "style:personachat:test:180:self:2:positive:1"], "evidence_texts": ["Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "those are also very good usually .", "not a fan of star wars ."], "speaker_id": "snli:test:4770", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4770", "candidate_text": "A woman carrying a girl, with a bag behind her.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4770:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4613:premise", "snli:test:5198:premise", "snli:test:4770:premise", "snli:test:4586:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A small child wearing a helmet is in a baby carrier that is on the floor.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4770", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4785", "candidate_text": "well, the wife is taking a photo of her husband", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4785:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4671:premise", "snli:test:4785:premise", "snli:test:4972:premise", "snli:test:4638:premise", "style:personachat:test:180:self:5:positive:0", "style:personachat:test:180:self:5:positive:1", "style:personachat:test:180:self:5:positive:2", "style:personachat:test:180:self:5:positive:3", "style:personachat:test:180:self:5:positive:4"], "evidence_texts": ["A boy works on a project in shop class.", "A woman is taking a picture of a man.", "The boy in pajama pants jumps off the sofa.", "Families with strollers waiting in front of a carousel.", "i have a goat named galadriel .", "i love animals but have no others .", "fantasy movie scores are great .", "those are also very good usually .", "not a fan of star wars ."], "speaker_id": "snli:test:4785", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4785", "candidate_text": "I wonder whether the wife is taking a photo of her husband?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4785:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4671:premise", "snli:test:4785:premise", "snli:test:4972:premise", "snli:test:4638:premise", "style:personachat:test:683:self:5:positive:0", "style:personachat:test:683:self:5:positive:1", "style:personachat:test:683:self:5:positive:2", "style:personachat:test:683:self:5:positive:3", "style:personachat:test:683:self:5:positive:4"], "evidence_texts": ["A boy works on a project in shop class.", "A woman is taking a picture of a man.", "The boy in pajama pants jumps off the sofa.", "Families with strollers waiting in front of a carousel.", "sorry to heard that , i am single just to let you know", "you kinda old , do you like to read books ? i read romance novels", "you still like the army ? i love watching movies and you ?", "okay , i hope to be a chef , i love to cook for others .", "what is your favorite station ? you listen to music or the news ?"], "speaker_id": "snli:test:4785", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4787", "candidate_text": "well, the man is inside", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4787:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4647:premise", "snli:test:4623:premise", "snli:test:4809:premise", "snli:test:4787:premise", "style:personachat:test:180:self:7:positive:0", "style:personachat:test:180:self:7:positive:1", "style:personachat:test:180:self:7:positive:2", "style:personachat:test:180:self:7:positive:3", "style:personachat:test:180:self:7:positive:4", "style:personachat:test:180:self:7:positive:5", "style:personachat:test:180:self:7:positive:6"], "evidence_texts": ["A youth skateboards by a mural of a griffin wielding a flaming sword.", "Two men and two women in jackets walk down a large city street.", "A bald person in green clothing stands in front of a brick wall.", "A man is ducking down between the red tulips in a flower field.", "i am great . i am watching lord of the rings .", "awesome . i like movies , especially fantasy .", "i have a goat named galadriel .", "i love animals but have no others .", "fantasy movie scores are great .", "those are also very good usually .", "not a fan of star wars ."], "speaker_id": "snli:test:4787", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4787", "candidate_text": "Honestly, The man is inside!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4787:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4647:premise", "snli:test:4623:premise", "snli:test:4809:premise", "snli:test:4787:premise", "style:personachat:test:347:self:5:positive:0", "style:personachat:test:347:self:5:positive:1", "style:personachat:test:347:self:5:positive:2", "style:personachat:test:347:self:5:positive:3", "style:personachat:test:347:self:5:positive:4"], "evidence_texts": ["A youth skateboards by a mural of a griffin wielding a flaming sword.", "Two men and two women in jackets walk down a large city street.", "A bald person in green clothing stands in front of a brick wall.", "A man is ducking down between the red tulips in a flower field.", "it is indeed what i do ! i like it , it is good pay !", "yes , it does ! also , i was just hanging with my wife and 3 girls", "those 3 are all i need ! i do not have very many friends", "i quit drinking years ago", "well , i did not allow me to think !"], "speaker_id": "snli:test:4787", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4788", "candidate_text": "well, the man is outside", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4788:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4788:premise", "snli:test:4809:premise", "snli:test:4647:premise", "snli:test:4857:premise", "style:personachat:test:181:self:1:positive:0"], "evidence_texts": ["A man is ducking down between the red tulips in a flower field.", "A bald person in green clothing stands in front of a brick wall.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "A young boy and an adult man looking at ducks in a pond.", "yeah it is tough they disowned me at age 12 and now i am 20"], "speaker_id": "snli:test:4788", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4788", "candidate_text": "The man is outside.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4788:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4788:premise", "snli:test:4809:premise", "snli:test:4647:premise", "snli:test:4857:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man is ducking down between the red tulips in a flower field.", "A bald person in green clothing stands in front of a brick wall.", "A youth skateboards by a mural of a griffin wielding a flaming sword.", "A young boy and an adult man looking at ducks in a pond.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4788", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4789", "candidate_text": "well, a lady with an axe is standing in the middle of a crowded mall", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4789:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4523:premise", "snli:test:4789:premise", "snli:test:4492:premise", "snli:test:5274:premise", "style:personachat:test:181:self:2:positive:0", "style:personachat:test:181:self:2:positive:1"], "evidence_texts": ["A bunch of people are standing all together in a street with a building in the background.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "my aunt and uncle but my parents do not want anything to do with me .", "yeah it is tough they disowned me at age 12 and now i am 20"], "speaker_id": "snli:test:4789", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4789", "candidate_text": "I wonder whether a lady with an axe is standing in the middle of a crowded mall?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4789:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4523:premise", "snli:test:4789:premise", "snli:test:4492:premise", "snli:test:5274:premise", "style:personachat:test:684:self:2:positive:0", "style:personachat:test:684:self:2:positive:1"], "evidence_texts": ["A bunch of people are standing all together in a street with a building in the background.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "Two people walking outside in the woods in the winter time wearing jackets with hoods on them.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "maybe but i am not good around others . i tend to keep to myself", "i try . what do you enjoy doing with your girls ?"], "speaker_id": "snli:test:4789", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4790", "candidate_text": "well, a lady with an axe is posing in the middle of the woods", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4790:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5274:premise", "snli:test:4523:premise", "snli:test:4790:premise", "snli:test:5419:premise", "style:personachat:test:181:self:5:positive:0", "style:personachat:test:181:self:5:positive:1", "style:personachat:test:181:self:5:positive:2", "style:personachat:test:181:self:5:positive:3", "style:personachat:test:181:self:5:positive:4"], "evidence_texts": ["A small child in a red jacket is sitting on a man's shoulders and holding his head.", "A bunch of people are standing all together in a street with a building in the background.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "yeah doing what ? hard to live on your on with no support from parents .", "i bet that keeps you busy . i am trying to find a new job but its tough .", "would be easier if i had more support from family .", "my aunt and uncle but my parents do not want anything to do with me .", "yeah it is tough they disowned me at age 12 and now i am 20"], "speaker_id": "snli:test:4790", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4790", "candidate_text": "Honestly, A lady with an axe is posing in the middle of the woods!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4790:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5274:premise", "snli:test:4523:premise", "snli:test:4790:premise", "snli:test:5419:premise", "style:personachat:test:348:self:2:positive:0", "style:personachat:test:348:self:2:positive:1"], "evidence_texts": ["A small child in a red jacket is sitting on a man's shoulders and holding his head.", "A bunch of people are standing all together in a street with a building in the background.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:4790", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4809", "candidate_text": "well, a hairy person naps", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4809:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4788:premise", "snli:test:4809:premise", "snli:test:4857:premise", "snli:test:4787:premise", "style:personachat:test:181:self:7:positive:0", "style:personachat:test:181:self:7:positive:1", "style:personachat:test:181:self:7:positive:2", "style:personachat:test:181:self:7:positive:3", "style:personachat:test:181:self:7:positive:4", "style:personachat:test:181:self:7:positive:5", "style:personachat:test:181:self:7:positive:6"], "evidence_texts": ["A man is ducking down between the red tulips in a flower field.", "A bald person in green clothing stands in front of a brick wall.", "A young boy and an adult man looking at ducks in a pond.", "A man is ducking down between the red tulips in a flower field.", "hey ! bored waiting to go to my boring job . how are you ?", "you like that movie ? that sounds way better than going to mcdonalds for 8 hours .", "yeah doing what ? hard to live on your on with no support from parents .", "i bet that keeps you busy . i am trying to find a new job but its tough .", "would be easier if i had more support from family .", "my aunt and uncle but my parents do not want anything to do with me .", "yeah it is tough they disowned me at age 12 and now i am 20"], "speaker_id": "snli:test:4809", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4809", "candidate_text": "A hairy person naps.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4809:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4788:premise", "snli:test:4809:premise", "snli:test:4857:premise", "snli:test:4787:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man is ducking down between the red tulips in a flower field.", "A bald person in green clothing stands in front of a brick wall.", "A young boy and an adult man looking at ducks in a pond.", "A man is ducking down between the red tulips in a flower field.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4809", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4812", "candidate_text": "Honestly, The woman is a secret agent on mission to murder the man!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4812:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4812:premise", "snli:test:3469:premise", "snli:test:6681:premise", "snli:test:2815:premise", "style:personachat:test:182:self:1:positive:0"], "evidence_texts": ["Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:4812", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4812", "candidate_text": "I wonder whether the woman is a secret agent on mission to murder the man?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4812:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4812:premise", "snli:test:3469:premise", "snli:test:6681:premise", "snli:test:2815:premise", "style:personachat:test:688:self:1:positive:0"], "evidence_texts": ["Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "an Asian woman dances with long ribbons, she is mid jump with her legs pointed to the right and the red ribbons flowing on either side of her", "how about scary movies ? those are my favorite"], "speaker_id": "snli:test:4812", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4824", "candidate_text": "Honestly, xylophone is played by someone!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4824:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4824:premise", "snli:test:4712:premise", "snli:test:4867:premise", "snli:test:4711:premise", "style:personachat:test:182:self:2:positive:0", "style:personachat:test:182:self:2:positive:1"], "evidence_texts": ["Xylophonists play in a dark setting.", "Cheerleaders standing on a football field.", "People shopping at an outside market", "Cheerleaders standing on a football field.", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:4824", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4824", "candidate_text": "well, xylophone is played by someone", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4824:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4824:premise", "snli:test:4712:premise", "snli:test:4867:premise", "snli:test:4711:premise", "style:personachat:test:313:self:2:positive:0", "style:personachat:test:313:self:2:positive:1"], "evidence_texts": ["Xylophonists play in a dark setting.", "Cheerleaders standing on a football field.", "People shopping at an outside market", "Cheerleaders standing on a football field.", "i work as a nurses aid .", "not really , i am a home body mostly , sometimes i go clubbing"], "speaker_id": "snli:test:4824", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4854", "candidate_text": "Honestly, Three men in red uniforms celebrating a goal at a soccer game outside!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4854:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4634:premise", "snli:test:4907:premise", "snli:test:4854:premise", "snli:test:4633:premise", "style:personachat:test:182:self:5:positive:0", "style:personachat:test:182:self:5:positive:1", "style:personachat:test:182:self:5:positive:2", "style:personachat:test:182:self:5:positive:3", "style:personachat:test:182:self:5:positive:4"], "evidence_texts": ["A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "Three guys in red uniforms celebrating a goal in a soccer game.", "A person in a black and green outfit is riding a bicycle.", "you should ! i moved to the states to become a stock broker but i miss it .", "one of my sisters is in fashion . i have a brother too but he is a doctor .", "are you close to your siblings ?", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:4854", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4854", "candidate_text": "Three men in red uniforms celebrating a goal at a soccer game outside.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4854:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4634:premise", "snli:test:4907:premise", "snli:test:4854:premise", "snli:test:4633:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "Three guys in red uniforms celebrating a goal in a soccer game.", "A person in a black and green outfit is riding a bicycle.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4854", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4857", "candidate_text": "Honestly, The man and his son stand at the edge of a pond!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4857:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4809:premise", "snli:test:4857:premise", "snli:test:5137:premise", "snli:test:4788:premise", "style:personachat:test:182:self:7:positive:0", "style:personachat:test:182:self:7:positive:1", "style:personachat:test:182:self:7:positive:2", "style:personachat:test:182:self:7:positive:3", "style:personachat:test:182:self:7:positive:4", "style:personachat:test:182:self:7:positive:5", "style:personachat:test:182:self:7:positive:6"], "evidence_texts": ["A bald person in green clothing stands in front of a brick wall.", "A young boy and an adult man looking at ducks in a pond.", "While two students are jumping, others in a classroom are sitting or standing.", "A man is ducking down between the red tulips in a flower field.", "hi there . i do not any children but i do have a dog named charlie .", "neat ! where are you from ? i am from australia . my parents still live there .", "you should ! i moved to the states to become a stock broker but i miss it .", "one of my sisters is in fashion . i have a brother too but he is a doctor .", "are you close to your siblings ?", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:4857", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4857", "candidate_text": "I wonder whether the man and his son stand at the edge of a pond?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4857:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4809:premise", "snli:test:4857:premise", "snli:test:5137:premise", "snli:test:4788:premise", "style:personachat:test:688:self:7:positive:0", "style:personachat:test:688:self:7:positive:1", "style:personachat:test:688:self:7:positive:2", "style:personachat:test:688:self:7:positive:3", "style:personachat:test:688:self:7:positive:4", "style:personachat:test:688:self:7:positive:5", "style:personachat:test:688:self:7:positive:6"], "evidence_texts": ["A bald person in green clothing stands in front of a brick wall.", "A young boy and an adult man looking at ducks in a pond.", "While two students are jumping, others in a classroom are sitting or standing.", "A man is ducking down between the red tulips in a flower field.", "great . how are you doing", "how many do you have", "a boy and a girl . 12 and 9 . angelic monsters lol", "who you telling . haha . my girl is a hundred times more stressful .", "kudos to her . i am exhausted half way through the day .", "well good for her . do you like barbecue . i am a pro at it", "how about scary movies ? those are my favorite"], "speaker_id": "snli:test:4857", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4867", "candidate_text": "well, people are outside", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4867:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4824:premise", "snli:test:4914:premise", "snli:test:4712:premise", "snli:test:4867:premise", "style:personachat:test:183:self:1:positive:0"], "evidence_texts": ["Xylophonists play in a dark setting.", "Two boys in running clothes stretching.", "Cheerleaders standing on a football field.", "People shopping at an outside market", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:4867", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4867", "candidate_text": "Honestly, People are outside!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4867:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4824:premise", "snli:test:4914:premise", "snli:test:4712:premise", "snli:test:4867:premise", "style:personachat:test:363:self:7:positive:0", "style:personachat:test:363:self:7:positive:1", "style:personachat:test:363:self:7:positive:2", "style:personachat:test:363:self:7:positive:3", "style:personachat:test:363:self:7:positive:4", "style:personachat:test:363:self:7:positive:5", "style:personachat:test:363:self:7:positive:6"], "evidence_texts": ["Xylophonists play in a dark setting.", "Two boys in running clothes stretching.", "Cheerleaders standing on a football field.", "People shopping at an outside market", "i am 12 , so i probably drive my mom crazy , too ! rollercoasters are awesome !", "i am going to see taylor swift in concert in california next week !", "cool ! i think i am more excited to leave my state for the first time .", "where is the coolest place you have been ?", "i know tae kwan do ! are you into martial arts ?", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:4867", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4882", "candidate_text": "well, a girl moving her muscles to get exercise in", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4882:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4749:premise", "snli:test:4736:premise", "snli:test:4882:premise", "snli:test:4905:premise", "style:personachat:test:183:self:2:positive:0", "style:personachat:test:183:self:2:positive:1"], "evidence_texts": ["Two girls feeding a horse an apple.", "A brown dog races through a field.", "A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "hopefully you have a loving family to help , what do you do for fun ?", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:4882", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4882", "candidate_text": "A girl moving her muscles to get exercise in.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4882:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4749:premise", "snli:test:4736:premise", "snli:test:4882:premise", "snli:test:4905:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two girls feeding a horse an apple.", "A brown dog races through a field.", "A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4882", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4884", "candidate_text": "well, two guys lifting oranges to build up muscles", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4884:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4884:premise", "snli:test:4905:premise", "snli:test:4749:premise", "snli:test:4932:premise", "style:personachat:test:183:self:5:positive:0", "style:personachat:test:183:self:5:positive:1", "style:personachat:test:183:self:5:positive:2", "style:personachat:test:183:self:5:positive:3", "style:personachat:test:183:self:5:positive:4"], "evidence_texts": ["A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "Two girls feeding a horse an apple.", "Expert snowboard is trying a new trick.", "yeah very cool , i quit to become a skater , i get paid to skate", "not to me but other people say yes , what kind of actor ?", "i would watch that show i am open to everything", "hopefully you have a loving family to help , what do you do for fun ?", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:4884", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4884", "candidate_text": "I wonder whether two guys lifting oranges to build up muscles?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4884:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4884:premise", "snli:test:4905:premise", "snli:test:4749:premise", "snli:test:4932:premise", "style:personachat:test:691:self:5:positive:0", "style:personachat:test:691:self:5:positive:1", "style:personachat:test:691:self:5:positive:2", "style:personachat:test:691:self:5:positive:3", "style:personachat:test:691:self:5:positive:4"], "evidence_texts": ["A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "Two girls feeding a horse an apple.", "Expert snowboard is trying a new trick.", "what kind of music do you listen to ?", "i do sometimes . i also enjoy taking long walks . do you ?", "i love to travel . do you vacation much ?", "the office is funny . do you read much ?", "do you ever read books on traveling to europe ?"], "speaker_id": "snli:test:4884", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4886", "candidate_text": "well, a family is sitting together inside a house", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4886:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4602:premise", "snli:test:4897:premise", "snli:test:4886:premise", "snli:test:4560:premise", "style:personachat:test:183:self:7:positive:0", "style:personachat:test:183:self:7:positive:1", "style:personachat:test:183:self:7:positive:2", "style:personachat:test:183:self:7:positive:3", "style:personachat:test:183:self:7:positive:4", "style:personachat:test:183:self:7:positive:5", "style:personachat:test:183:self:7:positive:6"], "evidence_texts": ["Several people wearing green shirts walk on the beach together.", "Two men help each other cut a sheet of plastic.", "Family gathered together in a house enjoying each other company.", "The man in the dark maroon shirt is kicking something.", "how are you doing tonight , i am missing my tail it was removed", "oh no do you like hollister i worked there back in high school ?", "yeah very cool , i quit to become a skater , i get paid to skate", "not to me but other people say yes , what kind of actor ?", "i would watch that show i am open to everything", "hopefully you have a loving family to help , what do you do for fun ?", "oh i do not like those at all i avoid chips"], "speaker_id": "snli:test:4886", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4886", "candidate_text": "Honestly, A family is sitting together inside a house!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4886:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4602:premise", "snli:test:4897:premise", "snli:test:4886:premise", "snli:test:4560:premise", "style:personachat:test:374:self:5:positive:0", "style:personachat:test:374:self:5:positive:1", "style:personachat:test:374:self:5:positive:2", "style:personachat:test:374:self:5:positive:3", "style:personachat:test:374:self:5:positive:4"], "evidence_texts": ["Several people wearing green shirts walk on the beach together.", "Two men help each other cut a sheet of plastic.", "Family gathered together in a house enjoying each other company.", "The man in the dark maroon shirt is kicking something.", "i write as well since i hate exercise or physical stuff .", "oh lord you work two jobs . wish i had your energy .", "do you have a supportive family ? my husband is my world .", "yes , do not ever stop writing . it really calms the soul or awakens it .", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:4886", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4890", "candidate_text": "well, a little boy is riding a roller-coaster", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4890:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4722:premise", "snli:test:5034:premise", "snli:test:4890:premise", "snli:test:4704:premise", "style:personachat:test:184:self:1:positive:0"], "evidence_texts": ["a young woman siting on a park bench in the sun.", "Many people sit, some stand outside of a large brick building.", "The blond boy is riding roller-coaster with his mouth wide open.", "Group of people standing and sitting in a field of grass.", "no they are all in places that can be covered up ."], "speaker_id": "snli:test:4890", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4890", "candidate_text": "a little boy is riding a roller-coaster.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4890:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4722:premise", "snli:test:5034:premise", "snli:test:4890:premise", "snli:test:4704:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["a young woman siting on a park bench in the sun.", "Many people sit, some stand outside of a large brick building.", "The blond boy is riding roller-coaster with his mouth wide open.", "Group of people standing and sitting in a field of grass.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4890", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4897", "candidate_text": "well, two men cut some plastic", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4897:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4886:premise", "snli:test:4897:premise", "snli:test:5170:premise", "snli:test:4602:premise", "style:personachat:test:184:self:2:positive:0", "style:personachat:test:184:self:2:positive:1"], "evidence_texts": ["Family gathered together in a house enjoying each other company.", "Two men help each other cut a sheet of plastic.", "Several bicyclers are carrying their bikes up a grassy hill.", "Several people wearing green shirts walk on the beach together.", "that it does . i am an admin assistant .", "no they are all in places that can be covered up ."], "speaker_id": "snli:test:4897", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4897", "candidate_text": "I wonder whether two men cut some plastic?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4897:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4886:premise", "snli:test:4897:premise", "snli:test:5170:premise", "snli:test:4602:premise", "style:personachat:test:703:self:2:positive:0", "style:personachat:test:703:self:2:positive:1"], "evidence_texts": ["Family gathered together in a house enjoying each other company.", "Two men help each other cut a sheet of plastic.", "Several bicyclers are carrying their bikes up a grassy hill.", "Several people wearing green shirts walk on the beach together.", "hmmm . titanic is something to see for the eleven times .", "hmmm . really ? i met alot of foreign exchanged student studying abroad ."], "speaker_id": "snli:test:4897", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4905", "candidate_text": "well, two girls play jump rope in the street", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4905:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4884:premise", "snli:test:4905:premise", "snli:test:4932:premise", "snli:test:4882:premise", "style:personachat:test:184:self:5:positive:0", "style:personachat:test:184:self:5:positive:1", "style:personachat:test:184:self:5:positive:2", "style:personachat:test:184:self:5:positive:3", "style:personachat:test:184:self:5:positive:4"], "evidence_texts": ["A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "Expert snowboard is trying a new trick.", "A girl in black pants doing exercises.", "i have them , my favorite is of a barbecue pit .", "i play racquetball , maybe i should get one of that .", "i have an oldies jukebox on my back !", "that it does . i am an admin assistant .", "no they are all in places that can be covered up ."], "speaker_id": "snli:test:4905", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4905", "candidate_text": "Honestly, Two girls play jump rope in the street!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4905:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4884:premise", "snli:test:4905:premise", "snli:test:4932:premise", "snli:test:4882:premise", "style:personachat:test:375:self:2:positive:0", "style:personachat:test:375:self:2:positive:1"], "evidence_texts": ["A girl in black pants doing exercises.", "Two girls aboard an amusement park ride.", "Expert snowboard is trying a new trick.", "A girl in black pants doing exercises.", "no . it is my favorite color so i am on the hunt for one !", "that would be cool ! what is your favorite movie ?"], "speaker_id": "snli:test:4905", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4907", "candidate_text": "well, a woman older than thirty is outside in winter", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4907:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4854:premise", "snli:test:4915:premise", "snli:test:4634:premise", "snli:test:4907:premise", "style:personachat:test:184:self:6:positive:0", "style:personachat:test:184:self:6:positive:1", "style:personachat:test:184:self:6:positive:2", "style:personachat:test:184:self:6:positive:3", "style:personachat:test:184:self:6:positive:4", "style:personachat:test:184:self:6:positive:5"], "evidence_texts": ["Three guys in red uniforms celebrating a goal in a soccer game.", "Three women sit at a table, taking notes from three identical magazines.", "A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "that is awesome ! i am retiring in five years .", "i have them , my favorite is of a barbecue pit .", "i play racquetball , maybe i should get one of that .", "i have an oldies jukebox on my back !", "that it does . i am an admin assistant .", "no they are all in places that can be covered up ."], "speaker_id": "snli:test:4907", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4907", "candidate_text": "A woman older than thirty is outside in winter.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4907:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4854:premise", "snli:test:4915:premise", "snli:test:4634:premise", "snli:test:4907:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Three guys in red uniforms celebrating a goal in a soccer game.", "Three women sit at a table, taking notes from three identical magazines.", "A person in a black and green outfit is riding a bicycle.", "A woman in a long, dark coat looks through trashcans in Winter.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4907", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4914", "candidate_text": "Honestly, The boys are sleeping!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4914:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4867:premise", "snli:test:5604:premise", "snli:test:4914:premise", "snli:test:4824:premise", "style:personachat:test:185:self:1:positive:0"], "evidence_texts": ["People shopping at an outside market", "A young girl running on grass", "Two boys in running clothes stretching.", "Xylophonists play in a dark setting.", "i do sing , just not well ! lol"], "speaker_id": "snli:test:4914", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4914", "candidate_text": "I wonder whether the boys are sleeping?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4914:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4867:premise", "snli:test:5604:premise", "snli:test:4914:premise", "snli:test:4824:premise", "style:personachat:test:704:self:1:positive:0"], "evidence_texts": ["People shopping at an outside market", "A young girl running on grass", "Two boys in running clothes stretching.", "Xylophonists play in a dark setting.", "i do . how old are you ?"], "speaker_id": "snli:test:4914", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4915", "candidate_text": "Honestly, Three college women are studing!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4915:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4907:premise", "snli:test:4854:premise", "snli:test:4941:premise", "snli:test:4915:premise", "style:personachat:test:185:self:2:positive:0", "style:personachat:test:185:self:2:positive:1"], "evidence_texts": ["A woman in a long, dark coat looks through trashcans in Winter.", "Three guys in red uniforms celebrating a goal in a soccer game.", "Two young girls show prints of a line drawing of an animal.", "Three women sit at a table, taking notes from three identical magazines.", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:4915", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4915", "candidate_text": "well, three college women are studing", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4915:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4907:premise", "snli:test:4854:premise", "snli:test:4941:premise", "snli:test:4915:premise", "style:personachat:test:317:self:2:positive:0", "style:personachat:test:317:self:2:positive:1"], "evidence_texts": ["A woman in a long, dark coat looks through trashcans in Winter.", "Three guys in red uniforms celebrating a goal in a soccer game.", "Two young girls show prints of a line drawing of an animal.", "Three women sit at a table, taking notes from three identical magazines.", "i am sorry to hear that , pets are the best .", "no , i ran a rabbit rescue blog . i would not be able to eat it ."], "speaker_id": "snli:test:4915", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4916", "candidate_text": "Honestly, Three woman are watching TV!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4916:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4916:premise", "snli:test:4941:premise", "snli:test:4907:premise", "snli:test:5001:premise", "style:personachat:test:185:self:5:positive:0", "style:personachat:test:185:self:5:positive:1", "style:personachat:test:185:self:5:positive:2", "style:personachat:test:185:self:5:positive:3", "style:personachat:test:185:self:5:positive:4"], "evidence_texts": ["Three women sit at a table, taking notes from three identical magazines.", "Two young girls show prints of a line drawing of an animal.", "A woman in a long, dark coat looks through trashcans in Winter.", "A woman in a black hooded sweatshirt walking with a large dog.", "that is sick ! i want to act when i am an adult", "sick as in cool ! i am too shy to start now . i just came out as trans", "good luck to you too ! do you like animals ?", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:4916", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4916", "candidate_text": "Three woman are watching TV.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4916:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4916:premise", "snli:test:4941:premise", "snli:test:4907:premise", "snli:test:5001:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Three women sit at a table, taking notes from three identical magazines.", "Two young girls show prints of a line drawing of an animal.", "A woman in a long, dark coat looks through trashcans in Winter.", "A woman in a black hooded sweatshirt walking with a large dog.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4916", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4920", "candidate_text": "Honestly, There is a field with a parking lot!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4920:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4920:premise", "snli:test:4724:premise", "snli:test:5220:premise", "snli:test:4723:premise", "style:personachat:test:185:self:6:positive:0", "style:personachat:test:185:self:6:positive:1", "style:personachat:test:185:self:6:positive:2", "style:personachat:test:185:self:6:positive:3", "style:personachat:test:185:self:6:positive:4", "style:personachat:test:185:self:6:positive:5"], "evidence_texts": ["a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "i am a middle schooler who absolutely loves pink floyd !", "that is sick ! i want to act when i am an adult", "sick as in cool ! i am too shy to start now . i just came out as trans", "good luck to you too ! do you like animals ?", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:4920", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4920", "candidate_text": "I wonder whether there is a field with a parking lot?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4920:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4920:premise", "snli:test:4724:premise", "snli:test:5220:premise", "snli:test:4723:premise", "style:personachat:test:704:self:6:positive:0", "style:personachat:test:704:self:6:positive:1", "style:personachat:test:704:self:6:positive:2", "style:personachat:test:704:self:6:positive:3", "style:personachat:test:704:self:6:positive:4", "style:personachat:test:704:self:6:positive:5"], "evidence_texts": ["a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "hi i am a high school drop out with three sisters", "no i am lead singer in a band", "yes i am are you a parent ?", "i home schooled while my dad drove a semi", "no i am in a punk grunge band . what do you do ?", "i do . how old are you ?"], "speaker_id": "snli:test:4920", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4922", "candidate_text": "well, there is a bald man", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4922:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4596:premise", "snli:test:4980:premise", "snli:test:4922:premise", "snli:test:4595:premise", "style:personachat:test:186:self:1:positive:0"], "evidence_texts": ["A young couple riding on a bus with the boy's arm around the girl.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A bald man with a white shirt and watch eating food and drinking wine.", "A young couple riding on a bus with the boy's arm around the girl.", "in my spare time environmentalism is my passion ."], "speaker_id": "snli:test:4922", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4922", "candidate_text": "Honestly, There is a bald man!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4922:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4596:premise", "snli:test:4980:premise", "snli:test:4922:premise", "snli:test:4595:premise", "style:personachat:test:377:self:7:positive:0", "style:personachat:test:377:self:7:positive:1", "style:personachat:test:377:self:7:positive:2", "style:personachat:test:377:self:7:positive:3", "style:personachat:test:377:self:7:positive:4", "style:personachat:test:377:self:7:positive:5", "style:personachat:test:377:self:7:positive:6"], "evidence_texts": ["A young couple riding on a bus with the boy's arm around the girl.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A bald man with a white shirt and watch eating food and drinking wine.", "A young couple riding on a bus with the boy's arm around the girl.", "i am well , what are your plans today ?", "that sounds very relaxing . i think i am going to wash my car . do you study ?", "awe , how sweet . i start college next year , i am studying to be a doctor .", "wow ! i bet he came accross a lot of interesting patients . that is admiring .", "well that is amazing . do you like to shop ?", "yes , i love it ! i love fashion and clothes .", "oh i bet there is ! i am an expert when it comes to shopping ."], "speaker_id": "snli:test:4922", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4924", "candidate_text": "well, the children are less than 20 yrs old", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4924:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4924:premise", "snli:test:4656:premise", "snli:test:4379:premise", "snli:test:4951:premise", "style:personachat:test:186:self:2:positive:0", "style:personachat:test:186:self:2:positive:1"], "evidence_texts": ["Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "oh , you are not that old .", "in my spare time environmentalism is my passion ."], "speaker_id": "snli:test:4924", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4924", "candidate_text": "The children are less than 20 yrs old.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4924:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4924:premise", "snli:test:4656:premise", "snli:test:4379:premise", "snli:test:4951:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "A man in reflective sunglasses, a skull T-shirt, and long red shorts sits out of the sun in a folding chair.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4924", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4926", "candidate_text": "well, the three children are related to each other", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4926:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4951:premise", "snli:test:4656:premise", "snli:test:4926:premise", "snli:test:4953:premise", "style:personachat:test:186:self:5:positive:0", "style:personachat:test:186:self:5:positive:1", "style:personachat:test:186:self:5:positive:2", "style:personachat:test:186:self:5:positive:3", "style:personachat:test:186:self:5:positive:4"], "evidence_texts": ["Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "right on . i went to u . miss . where did you go ?", "lol . i have been saving my pennies for a touch screen .", "cool . i am 34 . how old are you ?", "oh , you are not that old .", "in my spare time environmentalism is my passion ."], "speaker_id": "snli:test:4926", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4926", "candidate_text": "I wonder whether the three children are related to each other?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4926:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4951:premise", "snli:test:4656:premise", "snli:test:4926:premise", "snli:test:4953:premise", "style:personachat:test:705:self:5:positive:0", "style:personachat:test:705:self:5:positive:1", "style:personachat:test:705:self:5:positive:2", "style:personachat:test:705:self:5:positive:3", "style:personachat:test:705:self:5:positive:4"], "evidence_texts": ["Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "A man in a winter jacket and knit cap stand with his arms up at his side in a snow landscape.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "no , i am really into nutrition and fitness though .", "not these days , i am mostly too busy with my dieting business to play many sports .", "i am not sure where that is .", "is ralph a pet of yours ?", "crazy guess , what kind of dog is he ?"], "speaker_id": "snli:test:4926", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4927", "candidate_text": "well, a man is sound checking his guitar in the back room before a gig", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4927:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4650:premise", "snli:test:5080:premise", "snli:test:4555:premise", "snli:test:4927:premise", "style:personachat:test:186:self:7:positive:0", "style:personachat:test:186:self:7:positive:1", "style:personachat:test:186:self:7:positive:2", "style:personachat:test:186:self:7:positive:3", "style:personachat:test:186:self:7:positive:4", "style:personachat:test:186:self:7:positive:5", "style:personachat:test:186:self:7:positive:6"], "evidence_texts": ["Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "doing well . just got my 2 kids into bed . they are in kindergarten .", "we did too but working in real estate for 12 years sucked up a lot of time .", "right on . i went to u . miss . where did you go ?", "lol . i have been saving my pennies for a touch screen .", "cool . i am 34 . how old are you ?", "oh , you are not that old .", "in my spare time environmentalism is my passion ."], "speaker_id": "snli:test:4927", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4927", "candidate_text": "Honestly, A man is sound checking his guitar in the back room before a gig!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4927:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4650:premise", "snli:test:5080:premise", "snli:test:4555:premise", "snli:test:4927:premise", "style:personachat:test:379:self:5:positive:0", "style:personachat:test:379:self:5:positive:1", "style:personachat:test:379:self:5:positive:2", "style:personachat:test:379:self:5:positive:3", "style:personachat:test:379:self:5:positive:4"], "evidence_texts": ["Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "A man with a light blue t-shirt outside in the open wearing red ear protection while firing a gun.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "you kind of sounds like your single , i am too and love it !", "really ? ! ? ! i love water ! in fact that is the only liquid i will drink !", "that is funny ! i love swimming at the pool in my gym , it is a great workout", "gotcha , i have some of those as well which is why i am single and alone", "it really is ! it was nice to have met you jane !"], "speaker_id": "snli:test:4927", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4932", "candidate_text": "well, the person is snowboarding", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4932:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4932:premise", "snli:test:4905:premise", "snli:test:5021:premise", "snli:test:4884:premise", "style:personachat:test:187:self:1:positive:0"], "evidence_texts": ["Expert snowboard is trying a new trick.", "Two girls aboard an amusement park ride.", "The cyclist rides on a wooded path.", "A girl in black pants doing exercises.", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:4932", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4932", "candidate_text": "The person is snowboarding.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4932:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4932:premise", "snli:test:4905:premise", "snli:test:5021:premise", "snli:test:4884:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Expert snowboard is trying a new trick.", "Two girls aboard an amusement park ride.", "The cyclist rides on a wooded path.", "A girl in black pants doing exercises.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4932", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4941", "candidate_text": "well, there are young girls showing animal drawings", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4941:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4916:premise", "snli:test:4941:premise", "snli:test:5001:premise", "snli:test:4915:premise", "style:personachat:test:187:self:2:positive:0", "style:personachat:test:187:self:2:positive:1"], "evidence_texts": ["Three women sit at a table, taking notes from three identical magazines.", "Two young girls show prints of a line drawing of an animal.", "A woman in a black hooded sweatshirt walking with a large dog.", "Three women sit at a table, taking notes from three identical magazines.", "i am on the same page . what music do you ike ?", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:4941", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4941", "candidate_text": "I wonder whether there are young girls showing animal drawings?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4941:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4916:premise", "snli:test:4941:premise", "snli:test:5001:premise", "snli:test:4915:premise", "style:personachat:test:706:self:2:positive:0", "style:personachat:test:706:self:2:positive:1"], "evidence_texts": ["Three women sit at a table, taking notes from three identical magazines.", "Two young girls show prints of a line drawing of an animal.", "A woman in a black hooded sweatshirt walking with a large dog.", "Three women sit at a table, taking notes from three identical magazines.", "i want to go everywhere , but do not have that kind of money .", "i have never been in there , do they make pets smart ?"], "speaker_id": "snli:test:4941", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4951", "candidate_text": "well, two men pose in suits", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4951:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4926:premise", "snli:test:4924:premise", "snli:test:5205:premise", "snli:test:4951:premise", "style:personachat:test:187:self:5:positive:0", "style:personachat:test:187:self:5:positive:1", "style:personachat:test:187:self:5:positive:2", "style:personachat:test:187:self:5:positive:3", "style:personachat:test:187:self:5:positive:4"], "evidence_texts": ["Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "i did a lot of those when i was in school learning to be an engineer", "that is a good subject to get into . do you speak any other languages ?", "that is the only language i speak as well .", "i am on the same page . what music do you ike ?", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:4951", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4951", "candidate_text": "Honestly, two men pose in suits!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4951:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4926:premise", "snli:test:4924:premise", "snli:test:5205:premise", "snli:test:4951:premise", "style:personachat:test:384:self:2:positive:0", "style:personachat:test:384:self:2:positive:1"], "evidence_texts": ["Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "awesome ! i am originally from wisconsin .", "that is cool ! do you visit them often ?"], "speaker_id": "snli:test:4951", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4953", "candidate_text": "well, a girl with a helmet wears a dog tag", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4953:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5205:premise", "snli:test:4953:premise", "snli:test:4926:premise", "snli:test:5744:premise", "style:personachat:test:187:self:6:positive:0", "style:personachat:test:187:self:6:positive:1", "style:personachat:test:187:self:6:positive:2", "style:personachat:test:187:self:6:positive:3", "style:personachat:test:187:self:6:positive:4", "style:personachat:test:187:self:6:positive:5"], "evidence_texts": ["Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "i am doing great . just having a few slices of bacon for breakfast . you ?", "i did a lot of those when i was in school learning to be an engineer", "that is a good subject to get into . do you speak any other languages ?", "that is the only language i speak as well .", "i am on the same page . what music do you ike ?", "i really love classical music . i really love bach ."], "speaker_id": "snli:test:4953", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4953", "candidate_text": "a girl with a helmet wears a dog tag.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4953:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5205:premise", "snli:test:4953:premise", "snli:test:4926:premise", "snli:test:5744:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "Three children, two boys and a girl, are playing with the swings on a playground with residential houses in the background.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4953", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4968", "candidate_text": "Honestly, The child wearing sneakers skips down a forest path!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4968:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4968:premise", "snli:test:4486:premise", "snli:test:5243:premise", "snli:test:4335:premise", "style:personachat:test:188:self:1:positive:0"], "evidence_texts": ["A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "A man in a blue shirt touches his head while talking to two women outdoors.", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:4968", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4968", "candidate_text": "I wonder whether the child wearing sneakers skips down a forest path?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:4968:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4968:premise", "snli:test:4486:premise", "snli:test:5243:premise", "snli:test:4335:premise", "style:personachat:test:707:self:1:positive:0"], "evidence_texts": ["A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "A man in a blue shirt touches his head while talking to two women outdoors.", "that is cool , do they do cosplay ?"], "speaker_id": "snli:test:4968", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4972", "candidate_text": "Honestly, The boy is jumping up!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4972:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4972:premise", "snli:test:4785:premise", "snli:test:4976:premise", "snli:test:4671:premise", "style:personachat:test:188:self:2:positive:0", "style:personachat:test:188:self:2:positive:1"], "evidence_texts": ["The boy in pajama pants jumps off the sofa.", "A woman is taking a picture of a man.", "Two kids running past a dinosaur in the woods.", "A boy works on a project in shop class.", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:4972", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:4972", "candidate_text": "well, the boy is jumping up", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4972:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4972:premise", "snli:test:4785:premise", "snli:test:4976:premise", "snli:test:4671:premise", "style:personachat:test:320:self:2:positive:0", "style:personachat:test:320:self:2:positive:1"], "evidence_texts": ["The boy in pajama pants jumps off the sofa.", "A woman is taking a picture of a man.", "Two kids running past a dinosaur in the woods.", "A boy works on a project in shop class.", "i think we can agree there lol", "me as well . never bothered to learn another"], "speaker_id": "snli:test:4972", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:4976", "candidate_text": "Honestly, Children are being chased by a velociraptor!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4976:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4976:premise", "snli:test:4972:premise", "snli:test:4987:premise", "snli:test:4785:premise", "style:personachat:test:188:self:5:positive:0", "style:personachat:test:188:self:5:positive:1", "style:personachat:test:188:self:5:positive:2", "style:personachat:test:188:self:5:positive:3", "style:personachat:test:188:self:5:positive:4"], "evidence_texts": ["Two kids running past a dinosaur in the woods.", "The boy in pajama pants jumps off the sofa.", "Five boys in blue shirts drawing at a table.", "A woman is taking a picture of a man.", "where do you get the tattoos ?", "i live in seattle with two derelicts . they drink all the time .", "do you have a southern accent ?", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:4976", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4976", "candidate_text": "Children are being chased by a velociraptor.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4976:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4976:premise", "snli:test:4972:premise", "snli:test:4987:premise", "snli:test:4785:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two kids running past a dinosaur in the woods.", "The boy in pajama pants jumps off the sofa.", "Five boys in blue shirts drawing at a table.", "A woman is taking a picture of a man.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4976", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4980", "candidate_text": "Honestly, A drummer practicing for a concert!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4980:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4980:premise", "snli:test:4922:premise", "snli:test:4984:premise", "snli:test:4596:premise", "style:personachat:test:188:self:7:positive:0", "style:personachat:test:188:self:7:positive:1", "style:personachat:test:188:self:7:positive:2", "style:personachat:test:188:self:7:positive:3", "style:personachat:test:188:self:7:positive:4", "style:personachat:test:188:self:7:positive:5", "style:personachat:test:188:self:7:positive:6"], "evidence_texts": ["A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A bald man with a white shirt and watch eating food and drinking wine.", "Two identical brown dogs side by side holding something white in both their mouths.", "A young couple riding on a bus with the boy's arm around the girl.", "no , i can barely pay my monthly bills .", "i teach kindergarten , but i hate children .", "where do you get the tattoos ?", "i live in seattle with two derelicts . they drink all the time .", "do you have a southern accent ?", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:4980", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4980", "candidate_text": "I wonder whether a drummer practicing for a concert?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4980:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4980:premise", "snli:test:4922:premise", "snli:test:4984:premise", "snli:test:4596:premise", "style:personachat:test:707:self:7:positive:0", "style:personachat:test:707:self:7:positive:1", "style:personachat:test:707:self:7:positive:2", "style:personachat:test:707:self:7:positive:3", "style:personachat:test:707:self:7:positive:4", "style:personachat:test:707:self:7:positive:5", "style:personachat:test:707:self:7:positive:6"], "evidence_texts": ["A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A bald man with a white shirt and watch eating food and drinking wine.", "Two identical brown dogs side by side holding something white in both their mouths.", "A young couple riding on a bus with the boy's arm around the girl.", "i am enjoying a nice scotch to relax", "trying to come up with stories all day stresses me out .", "yea , i write for a living , what do you do at your job ?", "cool , but i am afraid of snakes , along with dogs and clowns .", "what do you like to do on your time off , i enjoy superhero movies like batman", "what gencon are there dogs there ?", "that is cool , do they do cosplay ?"], "speaker_id": "snli:test:4980", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4984", "candidate_text": "well, two twin pups holding onto the same bone", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4984:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4984:premise", "snli:test:4980:premise", "snli:test:5247:premise", "snli:test:4922:premise", "style:personachat:test:189:self:1:positive:0"], "evidence_texts": ["Two identical brown dogs side by side holding something white in both their mouths.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "A bald man with a white shirt and watch eating food and drinking wine.", "yes for the 4th time . my first three husbands passed away ."], "speaker_id": "snli:test:4984", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4984", "candidate_text": "Honestly, Two twin pups holding onto the same bone!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4984:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4984:premise", "snli:test:4980:premise", "snli:test:5247:premise", "snli:test:4922:premise", "style:personachat:test:389:self:7:positive:0", "style:personachat:test:389:self:7:positive:1", "style:personachat:test:389:self:7:positive:2", "style:personachat:test:389:self:7:positive:3", "style:personachat:test:389:self:7:positive:4", "style:personachat:test:389:self:7:positive:5", "style:personachat:test:389:self:7:positive:6"], "evidence_texts": ["Two identical brown dogs side by side holding something white in both their mouths.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "A bald man with a white shirt and watch eating food and drinking wine.", "hello i am doing good thanks ! i write books .", "thank you for your service .", "i tend to write about romance novels .", "bacon is always a good cure .", "anger is nothing to play around with , keep it under control .", "my two cats think they own the island .", "you have a great mother !"], "speaker_id": "snli:test:4984", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:4987", "candidate_text": "well, the boys are working on an art project", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4987:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4976:premise", "snli:test:5194:premise", "snli:test:4972:premise", "snli:test:4987:premise", "style:personachat:test:189:self:2:positive:0", "style:personachat:test:189:self:2:positive:1"], "evidence_texts": ["Two kids running past a dinosaur in the woods.", "Women playing volleyball in the sand near the ocean.", "The boy in pajama pants jumps off the sofa.", "Five boys in blue shirts drawing at a table.", "that would be super cool . my 3rd husband loved cars .", "yes for the 4th time . my first three husbands passed away ."], "speaker_id": "snli:test:4987", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:4987", "candidate_text": "The boys are working on an art project.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:4987:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4976:premise", "snli:test:5194:premise", "snli:test:4972:premise", "snli:test:4987:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two kids running past a dinosaur in the woods.", "Women playing volleyball in the sand near the ocean.", "The boy in pajama pants jumps off the sofa.", "Five boys in blue shirts drawing at a table.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:4987", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5001", "candidate_text": "well, a woman walking her dog before leaving to work", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5001:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4941:premise", "snli:test:5001:premise", "snli:test:5110:premise", "snli:test:4916:premise", "style:personachat:test:189:self:5:positive:0", "style:personachat:test:189:self:5:positive:1", "style:personachat:test:189:self:5:positive:2", "style:personachat:test:189:self:5:positive:3", "style:personachat:test:189:self:5:positive:4"], "evidence_texts": ["Two young girls show prints of a line drawing of an animal.", "A woman in a black hooded sweatshirt walking with a large dog.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "Three women sit at a table, taking notes from three identical magazines.", "that is great that you have a job that you can do that !", "i love civil war movies . my hobby is collecting civil war antiques .", "that is a great movie . that was my 2nd husbands favorite", "that would be super cool . my 3rd husband loved cars .", "yes for the 4th time . my first three husbands passed away ."], "speaker_id": "snli:test:5001", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5001", "candidate_text": "I wonder whether a woman walking her dog before leaving to work?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5001:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4941:premise", "snli:test:5001:premise", "snli:test:5110:premise", "snli:test:4916:premise", "style:personachat:test:709:self:5:positive:0", "style:personachat:test:709:self:5:positive:1", "style:personachat:test:709:self:5:positive:2", "style:personachat:test:709:self:5:positive:3", "style:personachat:test:709:self:5:positive:4"], "evidence_texts": ["Two young girls show prints of a line drawing of an animal.", "A woman in a black hooded sweatshirt walking with a large dog.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "Three women sit at a table, taking notes from three identical magazines.", "the best part , is the feast of a wild boar .", "yea , amongst other things , i installed a loud stereo in my truck .", "at my moms healthcare job , they as well have an adopted cat .", "did she like the healthcare field ?", "would it interfere with my love for camping and hiking ?"], "speaker_id": "snli:test:5001", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5007", "candidate_text": "well, a young girl is by a waterfall", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5007:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4578:premise", "snli:test:5058:premise", "snli:test:3395:premise", "snli:test:5007:premise", "style:personachat:test:189:self:7:positive:0", "style:personachat:test:189:self:7:positive:1", "style:personachat:test:189:self:7:positive:2", "style:personachat:test:189:self:7:positive:3", "style:personachat:test:189:self:7:positive:4", "style:personachat:test:189:self:7:positive:5", "style:personachat:test:189:self:7:positive:6"], "evidence_texts": ["Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "hi ! i am doing well how are you ?", "congratulations ! have you had to take time off work ?", "that is great that you have a job that you can do that !", "i love civil war movies . my hobby is collecting civil war antiques .", "that is a great movie . that was my 2nd husbands favorite", "that would be super cool . my 3rd husband loved cars .", "yes for the 4th time . my first three husbands passed away ."], "speaker_id": "snli:test:5007", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5007", "candidate_text": "Honestly, A young girl is by a waterfall!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5007:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4578:premise", "snli:test:5058:premise", "snli:test:3395:premise", "snli:test:5007:premise", "style:personachat:test:399:self:5:positive:0", "style:personachat:test:399:self:5:positive:1", "style:personachat:test:399:self:5:positive:2", "style:personachat:test:399:self:5:positive:3", "style:personachat:test:399:self:5:positive:4"], "evidence_texts": ["Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "A young man wearing goggles is putting some liquid in a beaker while a young girl wearing blue gloves looks down while holding a pen.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "hmmm . interesting . i model gloves and sing sappy jingles . i surf too . you surf ?", "wow ! i am leaving virginia to move to ny next year . i am from california . you ?", "okay . you would be a great voice over actor . what color are the fish ?", "great ! i wish i could sing better . you sing or model ? i was born in california . you ?", "get out ! a singing fisherman . ever heard of cate mesa ?"], "speaker_id": "snli:test:5007", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5021", "candidate_text": "well, the cyclist rides alone on a wooded path", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5021:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4932:premise", "snli:test:5021:premise", "snli:test:5167:premise", "snli:test:4905:premise", "style:personachat:test:190:self:1:positive:0"], "evidence_texts": ["Expert snowboard is trying a new trick.", "The cyclist rides on a wooded path.", "An old man holds up a sign.", "Two girls aboard an amusement park ride.", "bless her soul . i am a biology professor"], "speaker_id": "snli:test:5021", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5021", "candidate_text": "The cyclist rides alone on a wooded path.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5021:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4932:premise", "snli:test:5021:premise", "snli:test:5167:premise", "snli:test:4905:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Expert snowboard is trying a new trick.", "The cyclist rides on a wooded path.", "An old man holds up a sign.", "Two girls aboard an amusement park ride.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5021", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5023", "candidate_text": "well, the brazilian woman is walking through the desert", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5023:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3217:premise", "snli:test:2221:premise", "snli:test:5098:premise", "snli:test:5023:premise", "style:personachat:test:190:self:2:positive:0", "style:personachat:test:190:self:2:positive:1"], "evidence_texts": ["Four African boys playing soccer.", "several people are bike riding.", "A child hugs a birdhouse.", "African woman walking through field.", "i would love to meet him , i am a big fan of strippers .", "bless her soul . i am a biology professor"], "speaker_id": "snli:test:5023", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5023", "candidate_text": "I wonder whether the brazilian woman is walking through the desert?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5023:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3217:premise", "snli:test:2221:premise", "snli:test:5098:premise", "snli:test:5023:premise", "style:personachat:test:710:self:2:positive:0", "style:personachat:test:710:self:2:positive:1"], "evidence_texts": ["Four African boys playing soccer.", "several people are bike riding.", "A child hugs a birdhouse.", "African woman walking through field.", "i love ed sheeran ! he is very dreamy", "i like to play poker . what about you ?"], "speaker_id": "snli:test:5023", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5024", "candidate_text": "well, there is women navigating through the fields", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5024:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5024:premise", "snli:test:5098:premise", "snli:test:3217:premise", "snli:test:5099:premise", "style:personachat:test:190:self:5:positive:0", "style:personachat:test:190:self:5:positive:1", "style:personachat:test:190:self:5:positive:2", "style:personachat:test:190:self:5:positive:3", "style:personachat:test:190:self:5:positive:4"], "evidence_texts": ["African woman walking through field.", "A child hugs a birdhouse.", "Four African boys playing soccer.", "A child hugs a birdhouse.", "yes , definitely . i have been to 6 countries actually .", "that sounds so harsh , but i like meat as well . i guess i get it", "do you live out in the country ?", "i would love to meet him , i am a big fan of strippers .", "bless her soul . i am a biology professor"], "speaker_id": "snli:test:5024", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5024", "candidate_text": "Honestly, THere is women navigating through the fields!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5024:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5024:premise", "snli:test:5098:premise", "snli:test:3217:premise", "snli:test:5099:premise", "style:personachat:test:405:self:2:positive:0", "style:personachat:test:405:self:2:positive:1"], "evidence_texts": ["African woman walking through field.", "A child hugs a birdhouse.", "Four African boys playing soccer.", "A child hugs a birdhouse.", "and beans . but my favorite is sushi . vegan sushi .", "cool ! i am going to go running now and listen to rap ."], "speaker_id": "snli:test:5024", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5034", "candidate_text": "well, people are throwing rocks at each other", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5034:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4890:premise", "snli:test:5048:premise", "snli:test:5034:premise", "snli:test:4722:premise", "style:personachat:test:190:self:7:positive:0", "style:personachat:test:190:self:7:positive:1", "style:personachat:test:190:self:7:positive:2", "style:personachat:test:190:self:7:positive:3", "style:personachat:test:190:self:7:positive:4", "style:personachat:test:190:self:7:positive:5", "style:personachat:test:190:self:7:positive:6"], "evidence_texts": ["The blond boy is riding roller-coaster with his mouth wide open.", "A person is sitting on the edge of a rock cliff.", "Many people sit, some stand outside of a large brick building.", "a young woman siting on a park bench in the sun.", "good , just gardening in my large garden !", "meat rabbits ? do you like traveling ? i love it", "yes , definitely . i have been to 6 countries actually .", "that sounds so harsh , but i like meat as well . i guess i get it", "do you live out in the country ?", "i would love to meet him , i am a big fan of strippers .", "bless her soul . i am a biology professor"], "speaker_id": "snli:test:5034", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5034", "candidate_text": "people are throwing rocks at each other.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5034:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4890:premise", "snli:test:5048:premise", "snli:test:5034:premise", "snli:test:4722:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The blond boy is riding roller-coaster with his mouth wide open.", "A person is sitting on the edge of a rock cliff.", "Many people sit, some stand outside of a large brick building.", "a young woman siting on a park bench in the sun.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5034", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5048", "candidate_text": "I wonder whether a person sitting on a rock cliff?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5048:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5048:premise", "snli:test:5034:premise", "snli:test:5184:premise", "snli:test:4890:premise", "style:personachat:test:191:self:1:positive:0"], "evidence_texts": ["A person is sitting on the edge of a rock cliff.", "Many people sit, some stand outside of a large brick building.", "A helmeted male airborne on a bike on a dirt road.", "The blond boy is riding roller-coaster with his mouth wide open.", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:5048", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5048", "candidate_text": "A person sitting on a rock cliff.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5048:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5048:premise", "snli:test:5034:premise", "snli:test:5184:premise", "snli:test:4890:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person is sitting on the edge of a rock cliff.", "Many people sit, some stand outside of a large brick building.", "A helmeted male airborne on a bike on a dirt road.", "The blond boy is riding roller-coaster with his mouth wide open.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5048", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5058", "candidate_text": "I wonder whether it is food collection room as a donation?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5058:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5007:premise", "snli:test:6309:premise", "snli:test:5058:premise", "snli:test:4578:premise", "style:personachat:test:191:self:2:positive:0", "style:personachat:test:191:self:2:positive:1"], "evidence_texts": ["A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "do you like to read ?", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:5058", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5058", "candidate_text": "well, it is food collection room as a donation", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5058:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5007:premise", "snli:test:6309:premise", "snli:test:5058:premise", "snli:test:4578:premise", "style:personachat:test:325:self:2:positive:0", "style:personachat:test:325:self:2:positive:1"], "evidence_texts": ["A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "Two men on stilts and a woman twirling a ribbon, lead a festive parade along the streets, while a woman looks on from her window.", "do you have a favorite tv show ?", "oh i really like blindspot its really action packed"], "speaker_id": "snli:test:5058", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5063", "candidate_text": "I wonder whether a worker pulls up his long boots before climbing onto the flotation device?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5063:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3842:premise", "snli:test:5152:premise", "snli:test:2761:premise", "snli:test:5063:premise", "style:personachat:test:191:self:5:positive:0", "style:personachat:test:191:self:5:positive:1", "style:personachat:test:191:self:5:positive:2", "style:personachat:test:191:self:5:positive:3", "style:personachat:test:191:self:5:positive:4"], "evidence_texts": ["A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "a very honorable profession . what do you do for fun ?", "i love beetles ! is it a convertible ?", "i am looking forward to a day of xbox tomorrow on my day off .", "do you like to read ?", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:5063", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5063", "candidate_text": "Honestly, A worker pulls up his long boots before climbing onto the flotation device!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5063:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3842:premise", "snli:test:5152:premise", "snli:test:2761:premise", "snli:test:5063:premise", "style:personachat:test:413:self:2:positive:0", "style:personachat:test:413:self:2:positive:1"], "evidence_texts": ["A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man with a paddle stands next to a woman with a bikini top; and her arms are in a dance pose.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "yes . time is supposed to heal all wounds .", "that takes talent ! i am not very coordinated ."], "speaker_id": "snli:test:5063", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5074", "candidate_text": "I wonder whether young women are surfing in the ocean?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5074:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4313:premise", "snli:test:5277:premise", "snli:test:5074:premise", "snli:test:4295:premise", "style:personachat:test:191:self:7:positive:0", "style:personachat:test:191:self:7:positive:1", "style:personachat:test:191:self:7:positive:2", "style:personachat:test:191:self:7:positive:3", "style:personachat:test:191:self:7:positive:4", "style:personachat:test:191:self:7:positive:5", "style:personachat:test:191:self:7:positive:6"], "evidence_texts": ["A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "nice to meet you too ! what is your name ?", "full time veterinarian . what will you be majoring in ?", "a very honorable profession . what do you do for fun ?", "i love beetles ! is it a convertible ?", "i am looking forward to a day of xbox tomorrow on my day off .", "do you like to read ?", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:5074", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5074", "candidate_text": "Young women are surfing in the ocean.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5074:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4313:premise", "snli:test:5277:premise", "snli:test:5074:premise", "snli:test:4295:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Five People are kneeling between two park cars, and one man is approaching with his arm raised into the air.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5074", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5080", "candidate_text": "well, the people are dying of old age", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5080:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5080:premise", "snli:test:4927:premise", "snli:test:5173:premise", "snli:test:4650:premise", "style:personachat:test:192:self:1:positive:0"], "evidence_texts": ["A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "thank you , it changes a persons life ."], "speaker_id": "snli:test:5080", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5080", "candidate_text": "Honestly, the people are dying of old age!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5080:negative", "evidence_budget": 12, "evidence_ids": ["snli:test:5080:premise", "snli:test:4927:premise", "snli:test:5173:premise", "snli:test:4650:premise", "style:personachat:test:413:self:8:positive:0", "style:personachat:test:413:self:8:positive:1", "style:personachat:test:413:self:8:positive:2", "style:personachat:test:413:self:8:positive:3", "style:personachat:test:413:self:8:positive:4", "style:personachat:test:413:self:8:positive:5", "style:personachat:test:413:self:8:positive:6", "style:personachat:test:413:self:8:positive:7"], "evidence_texts": ["A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "Runners in various athletic garb, as well as one individual in a spotted-green dragon costume, run to the right.", "hi . i am doing pretty well . listening to heart , the band . you like music ?", "nice . my three daughters all like bluegrass .", "whoa . crazy stuff . my dad was a firefighter . he told me lots of crazy stories .", "no worries . i am patient . i get that from my mom . she was a teacher .", "really ? do they make you sick ? my ex makes me sick , lol .", "yeah . it is hard to forget him even though we are divorced .", "yes . time is supposed to heal all wounds .", "that takes talent ! i am not very coordinated ."], "speaker_id": "snli:test:5080", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5087", "candidate_text": "well, two boys are going to purchase sodas", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5087:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4591:premise", "snli:test:5147:premise", "snli:test:4439:premise", "snli:test:5087:premise", "style:personachat:test:192:self:2:positive:0", "style:personachat:test:192:self:2:positive:1"], "evidence_texts": ["A person in an orange shirt reaching up.", "Businessman walking as two arrows point to him.", "A man a woman posing for a picture.", "Two boys in front of a soda machine.", "my dad died of cancer so i wanted to help others .", "thank you , it changes a persons life ."], "speaker_id": "snli:test:5087", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5087", "candidate_text": "Two boys are going to purchase sodas.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5087:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4591:premise", "snli:test:5147:premise", "snli:test:4439:premise", "snli:test:5087:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A person in an orange shirt reaching up.", "Businessman walking as two arrows point to him.", "A man a woman posing for a picture.", "Two boys in front of a soda machine.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5087", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5098", "candidate_text": "well, an insane child hugs a birdhouse", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5098:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5024:premise", "snli:test:5023:premise", "snli:test:5098:premise", "snli:test:5490:premise", "style:personachat:test:192:self:5:positive:0", "style:personachat:test:192:self:5:positive:1", "style:personachat:test:192:self:5:positive:2", "style:personachat:test:192:self:5:positive:3", "style:personachat:test:192:self:5:positive:4"], "evidence_texts": ["African woman walking through field.", "African woman walking through field.", "A child hugs a birdhouse.", "A couple walking their dog.", "muffins are my favorite food .", "i am just watching a movie and eating muffins .", "i am in the healthcare field so i am just chilling tonight .", "my dad died of cancer so i wanted to help others .", "thank you , it changes a persons life ."], "speaker_id": "snli:test:5098", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5098", "candidate_text": "I wonder whether an insane child hugs a birdhouse?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5098:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5024:premise", "snli:test:5023:premise", "snli:test:5098:premise", "snli:test:5490:premise", "style:personachat:test:722:self:5:positive:0", "style:personachat:test:722:self:5:positive:1", "style:personachat:test:722:self:5:positive:2", "style:personachat:test:722:self:5:positive:3", "style:personachat:test:722:self:5:positive:4"], "evidence_texts": ["African woman walking through field.", "African woman walking through field.", "A child hugs a birdhouse.", "A couple walking their dog.", "i do not like stickers , what is the best bear", "i do know how to draw i love to draw eggplants", "it is easy just draw everything with purple , it is my favorite color", "what are those , are they gardening related", "what is your favorite football team ?"], "speaker_id": "snli:test:5098", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5099", "candidate_text": "well, there is a child and a birdhouse", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5099:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5490:premise", "snli:test:5024:premise", "snli:test:5610:premise", "snli:test:5099:premise", "style:personachat:test:192:self:7:positive:0", "style:personachat:test:192:self:7:positive:1", "style:personachat:test:192:self:7:positive:2", "style:personachat:test:192:self:7:positive:3", "style:personachat:test:192:self:7:positive:4", "style:personachat:test:192:self:7:positive:5", "style:personachat:test:192:self:7:positive:6"], "evidence_texts": ["A couple walking their dog.", "African woman walking through field.", "pedestrian walking on the street", "A child hugs a birdhouse.", "good evening how are you tonight ?", "i do not like bugs either but bacon is good .", "muffins are my favorite food .", "i am just watching a movie and eating muffins .", "i am in the healthcare field so i am just chilling tonight .", "my dad died of cancer so i wanted to help others .", "thank you , it changes a persons life ."], "speaker_id": "snli:test:5099", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5099", "candidate_text": "Honestly, There is a child and a birdhouse!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5099:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5490:premise", "snli:test:5024:premise", "snli:test:5610:premise", "snli:test:5099:premise", "style:personachat:test:425:self:5:positive:0", "style:personachat:test:425:self:5:positive:1", "style:personachat:test:425:self:5:positive:2", "style:personachat:test:425:self:5:positive:3", "style:personachat:test:425:self:5:positive:4"], "evidence_texts": ["A couple walking their dog.", "African woman walking through field.", "pedestrian walking on the street", "A child hugs a birdhouse.", "what shift do you work in maternity ? could be some great fantasy come out that department !", "books better than movies but a superb job here . how long are your shifts ?", "anything crazy happen there today ? baby ork or anything ?", "just got off work a day ago ? or are we in a time loop ?", "crazy ! nurses are like elves bringing in new life into the world to fight evil"], "speaker_id": "snli:test:5099", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5110", "candidate_text": "well, a man is at a concert playing for a large crowd", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5110:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5001:premise", "snli:test:5188:premise", "snli:test:5110:premise", "snli:test:4941:premise", "style:personachat:test:193:self:1:positive:0"], "evidence_texts": ["A woman in a black hooded sweatshirt walking with a large dog.", "A man walks past an arch in an old gothic style building.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "Two young girls show prints of a line drawing of an animal.", "no i really do not but night"], "speaker_id": "snli:test:5110", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5110", "candidate_text": "A man is at a concert playing for a large crowd.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5110:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5001:premise", "snli:test:5188:premise", "snli:test:5110:premise", "snli:test:4941:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A woman in a black hooded sweatshirt walking with a large dog.", "A man walks past an arch in an old gothic style building.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "Two young girls show prints of a line drawing of an animal.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5110", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5137", "candidate_text": "well, two students jumping in a class", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5137:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4857:premise", "snli:test:5137:premise", "snli:test:5200:premise", "snli:test:4809:premise", "style:personachat:test:193:self:2:positive:0", "style:personachat:test:193:self:2:positive:1"], "evidence_texts": ["A young boy and an adult man looking at ducks in a pond.", "While two students are jumping, others in a classroom are sitting or standing.", "A gray-haired man rides a bike in front of a bus advertising Ironman.", "A bald person in green clothing stands in front of a brick wall.", "gotcha my dad has awesome cars there", "no i really do not but night"], "speaker_id": "snli:test:5137", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5137", "candidate_text": "I wonder whether two students jumping in a class?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5137:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4857:premise", "snli:test:5137:premise", "snli:test:5200:premise", "snli:test:4809:premise", "style:personachat:test:723:self:2:positive:0", "style:personachat:test:723:self:2:positive:1"], "evidence_texts": ["A young boy and an adult man looking at ducks in a pond.", "While two students are jumping, others in a classroom are sitting or standing.", "A gray-haired man rides a bike in front of a bus advertising Ironman.", "A bald person in green clothing stands in front of a brick wall.", "i love sweet tea but then again i like anything with sugar", "so what hobbies are you into ?"], "speaker_id": "snli:test:5137", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5142", "candidate_text": "well, the woman in the two paneled photo is the other woman's relative", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5142:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3226:premise", "snli:test:1832:premise", "snli:test:5142:premise", "snli:test:3225:premise", "style:personachat:test:193:self:5:positive:0", "style:personachat:test:193:self:5:positive:1", "style:personachat:test:193:self:5:positive:2", "style:personachat:test:193:self:5:positive:3", "style:personachat:test:193:self:5:positive:4"], "evidence_texts": ["A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "really ? what is your fav animal ? mine are horses", "lol or so ? i love the ocean an beach", "really ? wow , need a new ride ?", "gotcha my dad has awesome cars there", "no i really do not but night"], "speaker_id": "snli:test:5142", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5142", "candidate_text": "Honestly, The woman in the two paneled photo is the other woman's relative!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5142:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3226:premise", "snli:test:1832:premise", "snli:test:5142:premise", "snli:test:3225:premise", "style:personachat:test:429:self:2:positive:0", "style:personachat:test:429:self:2:positive:1"], "evidence_texts": ["A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A man dressed in a surfing shirt and blue and white shorts skillfully guides his yellow board with its multicolored sail over the crest of an ocean wave, flipping over in the process.", "A photo of a woman in a black coat standing in front of a large two-paneled photo depicting a woman in a gray dress looking down with her hand in a bag.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "thank you ! life will be so awesome after retirement !", "oh , definitely ! i can enjoy life to the fullest !"], "speaker_id": "snli:test:5142", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5147", "candidate_text": "well, there is a businessman is walking", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5147:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5087:premise", "snli:test:4591:premise", "snli:test:5253:premise", "snli:test:5147:premise", "style:personachat:test:193:self:7:positive:0", "style:personachat:test:193:self:7:positive:1", "style:personachat:test:193:self:7:positive:2", "style:personachat:test:193:self:7:positive:3", "style:personachat:test:193:self:7:positive:4", "style:personachat:test:193:self:7:positive:5", "style:personachat:test:193:self:7:positive:6"], "evidence_texts": ["Two boys in front of a soda machine.", "A person in an orange shirt reaching up.", "A fireman searching for something using a flashlight.", "Businessman walking as two arrows point to him.", "hey there how are you today ?", "i am just getting to it an i just got my nails finished", "really ? what is your fav animal ? mine are horses", "lol or so ? i love the ocean an beach", "really ? wow , need a new ride ?", "gotcha my dad has awesome cars there", "no i really do not but night"], "speaker_id": "snli:test:5147", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5147", "candidate_text": "There is a businessman is walking.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5147:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5087:premise", "snli:test:4591:premise", "snli:test:5253:premise", "snli:test:5147:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two boys in front of a soda machine.", "A person in an orange shirt reaching up.", "A fireman searching for something using a flashlight.", "Businessman walking as two arrows point to him.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5147", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5148", "candidate_text": "well, a guy is jogging in a sweatsuit", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5148:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5148:premise", "snli:test:5253:premise", "snli:test:5087:premise", "snli:test:5336:premise", "style:personachat:test:194:self:1:positive:0"], "evidence_texts": ["Businessman walking as two arrows point to him.", "A fireman searching for something using a flashlight.", "Two boys in front of a soda machine.", "This woman is skiing down a snowy hill.", "yes i have you must love it"], "speaker_id": "snli:test:5148", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5148", "candidate_text": "I wonder whether a guy is jogging in a sweatsuit?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5148:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5148:premise", "snli:test:5253:premise", "snli:test:5087:premise", "snli:test:5336:premise", "style:personachat:test:728:self:1:positive:0"], "evidence_texts": ["Businessman walking as two arrows point to him.", "A fireman searching for something using a flashlight.", "Two boys in front of a soda machine.", "This woman is skiing down a snowy hill.", "italian dishes mostly what about you ?"], "speaker_id": "snli:test:5148", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5152", "candidate_text": "well, a woman prepares to shoot a free throw at the staples center", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5152:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5152:premise", "snli:test:5063:premise", "snli:test:3842:premise", "snli:test:5338:premise", "style:personachat:test:194:self:2:positive:0", "style:personachat:test:194:self:2:positive:1"], "evidence_texts": ["A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "what are those i am lost", "yes i have you must love it"], "speaker_id": "snli:test:5152", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5152", "candidate_text": "Honestly, A woman prepares to shoot a free throw at the Staples Center!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5152:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5152:premise", "snli:test:5063:premise", "snli:test:3842:premise", "snli:test:5338:premise", "style:personachat:test:435:self:1:positive:0"], "evidence_texts": ["A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A woman with a blond ponytail and a white hat is riding a white horse, inside a fence with a horned cow.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "ohhhh nooo i will send you the snow !"], "speaker_id": "snli:test:5152", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5153", "candidate_text": "well, a woman in a white and red basketball uniform", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5153:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5338:premise", "snli:test:5153:premise", "snli:test:5063:premise", "snli:test:5339:premise", "style:personachat:test:194:self:5:positive:0", "style:personachat:test:194:self:5:positive:1", "style:personachat:test:194:self:5:positive:2", "style:personachat:test:194:self:5:positive:3", "style:personachat:test:194:self:5:positive:4"], "evidence_texts": ["A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "i understand that i have 1 cat 2 dogs", "i get that i do , like bikes ?", "yes i am do you ride ? i do", "what are those i am lost", "yes i have you must love it"], "speaker_id": "snli:test:5153", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5153", "candidate_text": "A woman in a white and red basketball uniform.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5153:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5338:premise", "snli:test:5153:premise", "snli:test:5063:premise", "snli:test:5339:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man wearing a blue apron and long rubber boots is dragging a flotation device from a long row of flotation devices.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5153", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5167", "candidate_text": "well, a man is working", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5167:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5021:premise", "snli:test:5594:premise", "snli:test:4932:premise", "snli:test:5167:premise", "style:personachat:test:194:self:7:positive:0", "style:personachat:test:194:self:7:positive:1", "style:personachat:test:194:self:7:positive:2", "style:personachat:test:194:self:7:positive:3", "style:personachat:test:194:self:7:positive:4", "style:personachat:test:194:self:7:positive:5", "style:personachat:test:194:self:7:positive:6"], "evidence_texts": ["The cyclist rides on a wooded path.", "A man in glasses in a basement.", "Expert snowboard is trying a new trick.", "An old man holds up a sign.", "i do lol do you like to be cared for ?", "like animals ? i have several of them", "i understand that i have 1 cat 2 dogs", "i get that i do , like bikes ?", "yes i am do you ride ? i do", "what are those i am lost", "yes i have you must love it"], "speaker_id": "snli:test:5167", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5167", "candidate_text": "I wonder whether a man is working?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5167:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5021:premise", "snli:test:5594:premise", "snli:test:4932:premise", "snli:test:5167:premise", "style:personachat:test:728:self:7:positive:0", "style:personachat:test:728:self:7:positive:1", "style:personachat:test:728:self:7:positive:2", "style:personachat:test:728:self:7:positive:3", "style:personachat:test:728:self:7:positive:4", "style:personachat:test:728:self:7:positive:5", "style:personachat:test:728:self:7:positive:6"], "evidence_texts": ["The cyclist rides on a wooded path.", "A man in glasses in a basement.", "Expert snowboard is trying a new trick.", "An old man holds up a sign.", "i am wonderful thank you . what do you like ?", "watching tv , music , anything that involves home lol", "i like ice cream and married", "no we are trying for kids tho", "that is nice . you have a busy day planned ?", "housework it is never done and dinner", "italian dishes mostly what about you ?"], "speaker_id": "snli:test:5167", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5170", "candidate_text": "well, people carrying bikes", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5170:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4897:premise", "snli:test:5308:premise", "snli:test:5170:premise", "snli:test:4886:premise", "style:personachat:test:195:self:1:positive:0"], "evidence_texts": ["Two men help each other cut a sheet of plastic.", "The small child is wearing winter clothes and a hat.", "Several bicyclers are carrying their bikes up a grassy hill.", "Family gathered together in a house enjoying each other company.", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:5170", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5170", "candidate_text": "Honestly, People carrying bikes!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5170:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4897:premise", "snli:test:5308:premise", "snli:test:5170:premise", "snli:test:4886:premise", "style:personachat:test:435:self:7:positive:0", "style:personachat:test:435:self:7:positive:1", "style:personachat:test:435:self:7:positive:2", "style:personachat:test:435:self:7:positive:3", "style:personachat:test:435:self:7:positive:4", "style:personachat:test:435:self:7:positive:5", "style:personachat:test:435:self:7:positive:6"], "evidence_texts": ["Two men help each other cut a sheet of plastic.", "The small child is wearing winter clothes and a hat.", "Several bicyclers are carrying their bikes up a grassy hill.", "Family gathered together in a house enjoying each other company.", "hello well i love ice tea , country music , and love traveling hbu ?", "that is cool , my friends and i love to relax to bluegrass", "what do you compete in ? i compete in swimming", "that is cool , do you live in the states ?", "i live in toronto canada", "super cold ! cali sounds nice about right now !", "ohhhh nooo i will send you the snow !"], "speaker_id": "snli:test:5170", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5173", "candidate_text": "well, a man in a purple suit is dumping a bucket of water over his head", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5173:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5080:premise", "snli:test:5173:premise", "snli:test:5560:premise", "snli:test:4927:premise", "style:personachat:test:195:self:2:positive:0", "style:personachat:test:195:self:2:positive:1"], "evidence_texts": ["A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "A group of three people are digging a large trench in a yard while another person walks close by.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "it can be fun . just not when my sweet tooth kicks in lol", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:5173", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5173", "candidate_text": "A man in a purple suit is dumping a bucket of water over his head.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5173:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5080:premise", "snli:test:5173:premise", "snli:test:5560:premise", "snli:test:4927:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "A group of three people are digging a large trench in a yard while another person walks close by.", "A rocker standing with one foot on the stage holding the microphone attached to the stand over his head.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5173", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5184", "candidate_text": "well, the man fell off his bike", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5184:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5184:premise", "snli:test:5048:premise", "snli:test:5323:premise", "snli:test:5034:premise", "style:personachat:test:195:self:5:positive:0", "style:personachat:test:195:self:5:positive:1", "style:personachat:test:195:self:5:positive:2", "style:personachat:test:195:self:5:positive:3", "style:personachat:test:195:self:5:positive:4"], "evidence_texts": ["A helmeted male airborne on a bike on a dirt road.", "A person is sitting on the edge of a rock cliff.", "A group of people sitting around a small round table eating.", "Many people sit, some stand outside of a large brick building.", "yeah would help if i stay out of the clubs will not find him there", "would you say obedience school for dogs is one ?", "that is a good one kinda play that with my dogs i teach", "it can be fun . just not when my sweet tooth kicks in lol", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:5184", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5184", "candidate_text": "I wonder whether the man fell off his bike?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5184:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5184:premise", "snli:test:5048:premise", "snli:test:5323:premise", "snli:test:5034:premise", "style:personachat:test:731:self:5:positive:0", "style:personachat:test:731:self:5:positive:1", "style:personachat:test:731:self:5:positive:2", "style:personachat:test:731:self:5:positive:3", "style:personachat:test:731:self:5:positive:4"], "evidence_texts": ["A helmeted male airborne on a bike on a dirt road.", "A person is sitting on the edge of a rock cliff.", "A group of people sitting around a small round table eating.", "Many people sit, some stand outside of a large brick building.", "yes but i am mad . becky stole my boyfriend .", "what would you be if you could be anything ?", "i would be a fashion designer but i have a criminal record .", "life is half full for you .", "but should you be breaking into all of these houses ?"], "speaker_id": "snli:test:5184", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5188", "candidate_text": "well, the man is in a submarine", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5188:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5188:premise", "snli:test:5110:premise", "snli:test:5233:premise", "snli:test:5001:premise", "style:personachat:test:195:self:7:positive:0", "style:personachat:test:195:self:7:positive:1", "style:personachat:test:195:self:7:positive:2", "style:personachat:test:195:self:7:positive:3", "style:personachat:test:195:self:7:positive:4", "style:personachat:test:195:self:7:positive:5", "style:personachat:test:195:self:7:positive:6"], "evidence_texts": ["A man walks past an arch in an old gothic style building.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "A red hot air balloon lands in a field among a crowd.", "A woman in a black hooded sweatshirt walking with a large dog.", "i am good how about you", "oh how exciting that is . still looking for mister right", "yeah would help if i stay out of the clubs will not find him there", "would you say obedience school for dogs is one ?", "that is a good one kinda play that with my dogs i teach", "it can be fun . just not when my sweet tooth kicks in lol", "right now mostly movies and clubbing . still young so wanna get it out of my system now"], "speaker_id": "snli:test:5188", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5188", "candidate_text": "Honestly, The man is in a submarine!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5188:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5188:premise", "snli:test:5110:premise", "snli:test:5233:premise", "snli:test:5001:premise", "style:personachat:test:438:self:5:positive:0", "style:personachat:test:438:self:5:positive:1", "style:personachat:test:438:self:5:positive:2", "style:personachat:test:438:self:5:positive:3", "style:personachat:test:438:self:5:positive:4"], "evidence_texts": ["A man walks past an arch in an old gothic style building.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "A red hot air balloon lands in a field among a crowd.", "A woman in a black hooded sweatshirt walking with a large dog.", "i like checking out the sea creatures .", "i like cats . but i like my tail as feet", "i have long wavy hair what about you ?", "bills cost money . after dinner i usually go out and play with my friends under the sea", "yes ! i must be . i love being me . board games are difficult to play under the sea"], "speaker_id": "snli:test:5188", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5194", "candidate_text": "I wonder whether women cooking dinner?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5194:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4987:premise", "snli:test:4976:premise", "snli:test:5194:premise", "snli:test:5264:premise", "style:personachat:test:196:self:1:positive:0"], "evidence_texts": ["Five boys in blue shirts drawing at a table.", "Two kids running past a dinosaur in the woods.", "Women playing volleyball in the sand near the ocean.", "A man is sitting at a restaurant table outdoors.", "what kind of music you like ?"], "speaker_id": "snli:test:5194", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5194", "candidate_text": "Honestly, Women cooking dinner!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5194:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:4987:premise", "snli:test:4976:premise", "snli:test:5194:premise", "snli:test:5264:premise", "style:personachat:test:438:self:7:positive:0", "style:personachat:test:438:self:7:positive:1", "style:personachat:test:438:self:7:positive:2", "style:personachat:test:438:self:7:positive:3", "style:personachat:test:438:self:7:positive:4", "style:personachat:test:438:self:7:positive:5", "style:personachat:test:438:self:7:positive:6"], "evidence_texts": ["Five boys in blue shirts drawing at a table.", "Two kids running past a dinosaur in the woods.", "Women playing volleyball in the sand near the ocean.", "A man is sitting at a restaurant table outdoors.", "ok for living in the sea", "not seattle but ocean . i spend all my days at school .", "i like checking out the sea creatures .", "i like cats . but i like my tail as feet", "i have long wavy hair what about you ?", "bills cost money . after dinner i usually go out and play with my friends under the sea", "yes ! i must be . i love being me . board games are difficult to play under the sea"], "speaker_id": "snli:test:5194", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5196", "candidate_text": "I wonder whether women playing a sport?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5196:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5196:premise", "snli:test:5264:premise", "snli:test:4987:premise", "snli:test:5320:premise", "style:personachat:test:196:self:2:positive:0", "style:personachat:test:196:self:2:positive:1"], "evidence_texts": ["Women playing volleyball in the sand near the ocean.", "A man is sitting at a restaurant table outdoors.", "Five boys in blue shirts drawing at a table.", "A boy wearing a helmet is jumping his skateboard.", "i bet that can be interesting", "what kind of music you like ?"], "speaker_id": "snli:test:5196", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5196", "candidate_text": "Women playing a sport.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5196:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5196:premise", "snli:test:5264:premise", "snli:test:4987:premise", "snli:test:5320:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Women playing volleyball in the sand near the ocean.", "A man is sitting at a restaurant table outdoors.", "Five boys in blue shirts drawing at a table.", "A boy wearing a helmet is jumping his skateboard.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5196", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5198", "candidate_text": "I wonder whether the kids are playing soccer in the field with their dog?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5198:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4770:premise", "snli:test:5459:premise", "snli:test:5198:premise", "snli:test:4613:premise", "style:personachat:test:196:self:5:positive:0", "style:personachat:test:196:self:5:positive:1", "style:personachat:test:196:self:5:positive:2", "style:personachat:test:196:self:5:positive:3", "style:personachat:test:196:self:5:positive:4"], "evidence_texts": ["A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "i wanna write fantasy but for now i will just was lord of the rings", "i tend too where i get ideas from", "what are some of yours", "i bet that can be interesting", "what kind of music you like ?"], "speaker_id": "snli:test:5198", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5198", "candidate_text": "well, the kids are playing soccer in the field with their dog", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5198:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4770:premise", "snli:test:5459:premise", "snli:test:5198:premise", "snli:test:4613:premise", "style:personachat:test:330:self:5:positive:0", "style:personachat:test:330:self:5:positive:1", "style:personachat:test:330:self:5:positive:2", "style:personachat:test:330:self:5:positive:3", "style:personachat:test:330:self:5:positive:4"], "evidence_texts": ["A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "Four young female gymnasts dressed in black and blue leotards are preparing for their next routine.", "that is awesome i really need to adapt to that type of lifestyle", "no not really i use to but its usually too painful", "i have some nerve issues arm and back pains badly", "probably so lol my medical situation is a lot", "i have my boyfriend but he can only help so much since he works"], "speaker_id": "snli:test:5198", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5200", "candidate_text": "I wonder whether an older man rides a bike in front of a bus with a movie poster on it?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5200:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5200:premise", "snli:test:5137:premise", "snli:test:5535:premise", "snli:test:4857:premise", "style:personachat:test:196:self:7:positive:0", "style:personachat:test:196:self:7:positive:1", "style:personachat:test:196:self:7:positive:2", "style:personachat:test:196:self:7:positive:3", "style:personachat:test:196:self:7:positive:4", "style:personachat:test:196:self:7:positive:5", "style:personachat:test:196:self:7:positive:6"], "evidence_texts": ["A gray-haired man rides a bike in front of a bus advertising Ironman.", "While two students are jumping, others in a classroom are sitting or standing.", "Two young teenage girls hiking through the snow on a cold winter day.", "A young boy and an adult man looking at ducks in a pond.", "i am good busy with 2 part time jobs", "yeah i wanna write book someday", "i wanna write fantasy but for now i will just was lord of the rings", "i tend too where i get ideas from", "what are some of yours", "i bet that can be interesting", "what kind of music you like ?"], "speaker_id": "snli:test:5200", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5200", "candidate_text": "Honestly, An older man rides a bike in front of a bus with a movie poster on it!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5200:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5200:premise", "snli:test:5137:premise", "snli:test:5535:premise", "snli:test:4857:premise", "style:personachat:test:445:self:5:positive:0", "style:personachat:test:445:self:5:positive:1", "style:personachat:test:445:self:5:positive:2", "style:personachat:test:445:self:5:positive:3", "style:personachat:test:445:self:5:positive:4"], "evidence_texts": ["A gray-haired man rides a bike in front of a bus advertising Ironman.", "While two students are jumping, others in a classroom are sitting or standing.", "Two young teenage girls hiking through the snow on a cold winter day.", "A young boy and an adult man looking at ducks in a pond.", "what ? my dad is awesome , i dropped out of school to travel with him .", "love nyc ! party all the time , my dad drove a semi , i helped .", "my sister does too ! ! i have 31 do you have siblings ?", "3 not 31 . . that would be scary ! sad", "totally , one just like me ! ! lol"], "speaker_id": "snli:test:5200", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5205", "candidate_text": "well, the women are hot", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5205:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:4953:premise", "snli:test:5205:premise", "snli:test:5744:premise", "snli:test:4951:premise", "style:personachat:test:197:self:1:positive:0"], "evidence_texts": ["Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "that is the way to do it"], "speaker_id": "snli:test:5205", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5205", "candidate_text": "I wonder whether the women are hot?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5205:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:4953:premise", "snli:test:5205:premise", "snli:test:5744:premise", "snli:test:4951:premise", "style:personachat:test:739:self:1:positive:0"], "evidence_texts": ["Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "lol i never heard of that . is it an actual bug ?"], "speaker_id": "snli:test:5205", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5220", "candidate_text": "well, a woman is sitting near food", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5220:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5220:premise", "snli:test:4920:premise", "snli:test:5756:premise", "snli:test:4724:premise", "style:personachat:test:197:self:2:positive:0", "style:personachat:test:197:self:2:positive:1"], "evidence_texts": ["A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "that sounds cool i would like that for my collection", "that is the way to do it"], "speaker_id": "snli:test:5220", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5220", "candidate_text": "Honestly, A woman is sitting near food!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5220:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5220:premise", "snli:test:4920:premise", "snli:test:5756:premise", "snli:test:4724:premise", "style:personachat:test:446:self:1:positive:0"], "evidence_texts": ["A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A blond woman sits on a park bench while wearing a brown tank top and olive green shorts.", "i remember those ! my mother had one in the army"], "speaker_id": "snli:test:5220", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5233", "candidate_text": "well, a hot air balloon is crashing down to earth", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5233:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5188:premise", "snli:test:5233:premise", "snli:test:5110:premise", "snli:test:5240:premise", "style:personachat:test:197:self:5:positive:0", "style:personachat:test:197:self:5:positive:1", "style:personachat:test:197:self:5:positive:2", "style:personachat:test:197:self:5:positive:3", "style:personachat:test:197:self:5:positive:4"], "evidence_texts": ["A man walks past an arch in an old gothic style building.", "A red hot air balloon lands in a field among a crowd.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "A young boy wearing a red shirt runs along a gravel road.", "i love zoos visit them every chance i get", "zoos are always green and that is my favorite color", "i have a teddy bear collection i hope to turn in to some green", "that sounds cool i would like that for my collection", "that is the way to do it"], "speaker_id": "snli:test:5233", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5233", "candidate_text": "A hot air balloon is crashing down to earth.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5233:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5188:premise", "snli:test:5233:premise", "snli:test:5110:premise", "snli:test:5240:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man walks past an arch in an old gothic style building.", "A red hot air balloon lands in a field among a crowd.", "Man in plaid shirt sings into microphone while playing an acoustic guitar.", "A young boy wearing a red shirt runs along a gravel road.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5233", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5234", "candidate_text": "well, a hot air balloon is rising in to the sky", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5234:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5240:premise", "snli:test:5188:premise", "snli:test:5234:premise", "snli:test:5288:premise", "style:personachat:test:197:self:6:positive:0", "style:personachat:test:197:self:6:positive:1", "style:personachat:test:197:self:6:positive:2", "style:personachat:test:197:self:6:positive:3", "style:personachat:test:197:self:6:positive:4", "style:personachat:test:197:self:6:positive:5"], "evidence_texts": ["A young boy wearing a red shirt runs along a gravel road.", "A man walks past an arch in an old gothic style building.", "A red hot air balloon lands in a field among a crowd.", "People waiting to cross the street on the side of a sidewalk.", "yes i am i am even a volunteer with red cross", "i love zoos visit them every chance i get", "zoos are always green and that is my favorite color", "i have a teddy bear collection i hope to turn in to some green", "that sounds cool i would like that for my collection", "that is the way to do it"], "speaker_id": "snli:test:5234", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5234", "candidate_text": "I wonder whether a hot air balloon is rising in to the sky?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5234:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5240:premise", "snli:test:5188:premise", "snli:test:5234:premise", "snli:test:5288:premise", "style:personachat:test:739:self:6:positive:0", "style:personachat:test:739:self:6:positive:1", "style:personachat:test:739:self:6:positive:2", "style:personachat:test:739:self:6:positive:3", "style:personachat:test:739:self:6:positive:4", "style:personachat:test:739:self:6:positive:5"], "evidence_texts": ["A young boy wearing a red shirt runs along a gravel road.", "A man walks past an arch in an old gothic style building.", "A red hot air balloon lands in a field among a crowd.", "People waiting to cross the street on the side of a sidewalk.", "i am doing good . so tell me where you are from ?", "i love new york . i stay in texas . what are your hobbies ?", "fashion is cool . i am an avid gamer playing second life .", "it is an online game where i portray a furry .", "i do not drive since i work at home and online so much .", "lol i never heard of that . is it an actual bug ?"], "speaker_id": "snli:test:5234", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5240", "candidate_text": "well, a boy is outdoors", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5240:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5240:premise", "snli:test:5234:premise", "snli:test:5288:premise", "snli:test:5233:premise", "style:personachat:test:198:self:1:positive:0"], "evidence_texts": ["A young boy wearing a red shirt runs along a gravel road.", "A red hot air balloon lands in a field among a crowd.", "People waiting to cross the street on the side of a sidewalk.", "A red hot air balloon lands in a field among a crowd.", "i studied abroad so it is the only movie i had in my room ."], "speaker_id": "snli:test:5240", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5240", "candidate_text": "Honestly, A boy is outdoors!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5240:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5240:premise", "snli:test:5234:premise", "snli:test:5288:premise", "snli:test:5233:premise", "style:personachat:test:446:self:6:positive:0", "style:personachat:test:446:self:6:positive:1", "style:personachat:test:446:self:6:positive:2", "style:personachat:test:446:self:6:positive:3", "style:personachat:test:446:self:6:positive:4", "style:personachat:test:446:self:6:positive:5"], "evidence_texts": ["A young boy wearing a red shirt runs along a gravel road.", "A red hot air balloon lands in a field among a crowd.", "People waiting to cross the street on the side of a sidewalk.", "A red hot air balloon lands in a field among a crowd.", "that is cool so did my husband . . i am a hippy and make my clothes", "nothing like the good ol us of a i love singing folk songs", "that is neat ! i just found a kitty we named her bubbles", "me too ! we have to keep our environment clean to help save them all", "very true . its advancing so quickly !", "i remember those ! my mother had one in the army"], "speaker_id": "snli:test:5240", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5243", "candidate_text": "well, a wife taking groceries home to her family", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5243:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4968:premise", "snli:test:5254:premise", "snli:test:4486:premise", "snli:test:5243:premise", "style:personachat:test:198:self:2:positive:0", "style:personachat:test:198:self:2:positive:1"], "evidence_texts": ["A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "i learned to love that movie since that is all we had to watch at school .", "i studied abroad so it is the only movie i had in my room ."], "speaker_id": "snli:test:5243", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5243", "candidate_text": "a wife taking groceries home to her family.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5243:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4968:premise", "snli:test:5254:premise", "snli:test:4486:premise", "snli:test:5243:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "A single person stands on a heavily-eroded cliff-side rock while holding a bare branch aloft.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5243", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5247", "candidate_text": "well, a woman is walking away from trader joe's with a full shopping cart", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5247:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4984:premise", "snli:test:5433:premise", "snli:test:4980:premise", "snli:test:5247:premise", "style:personachat:test:198:self:5:positive:0", "style:personachat:test:198:self:5:positive:1", "style:personachat:test:198:self:5:positive:2", "style:personachat:test:198:self:5:positive:3", "style:personachat:test:198:self:5:positive:4"], "evidence_texts": ["Two identical brown dogs side by side holding something white in both their mouths.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "football is awesome . what is your favorite team", "i love patriots . i have a dog too , her name is ginger .", "that is nice of you . do you like movies ? my favorite is titanic .", "i learned to love that movie since that is all we had to watch at school .", "i studied abroad so it is the only movie i had in my room ."], "speaker_id": "snli:test:5247", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5247", "candidate_text": "I wonder whether a woman is walking away from trader joe's with a full shopping cart?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5247:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4984:premise", "snli:test:5433:premise", "snli:test:4980:premise", "snli:test:5247:premise", "style:personachat:test:741:self:5:positive:0", "style:personachat:test:741:self:5:positive:1", "style:personachat:test:741:self:5:positive:2", "style:personachat:test:741:self:5:positive:3", "style:personachat:test:741:self:5:positive:4"], "evidence_texts": ["Two identical brown dogs side by side holding something white in both their mouths.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A drummer wearing a plaid shirt, jeans, and a purple had bangs his drum.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "do you like it ?", "i like to play the game snakes and ladders but nothing else about snakes is good", "i read a biography once about a snake charmer . interesting", "i play monopoly or risk or any board game really", "i never really got into them but i like reading too . do you read ?"], "speaker_id": "snli:test:5247", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5253", "candidate_text": "well, a fireman is in the dark", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5253:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5148:premise", "snli:test:5253:premise", "snli:test:5336:premise", "snli:test:5147:premise", "style:personachat:test:198:self:7:positive:0", "style:personachat:test:198:self:7:positive:1", "style:personachat:test:198:self:7:positive:2", "style:personachat:test:198:self:7:positive:3", "style:personachat:test:198:self:7:positive:4", "style:personachat:test:198:self:7:positive:5", "style:personachat:test:198:self:7:positive:6"], "evidence_texts": ["Businessman walking as two arrows point to him.", "A fireman searching for something using a flashlight.", "This woman is skiing down a snowy hill.", "Businessman walking as two arrows point to him.", "i study but my hobby is modeling cars . you ?", "that sounds interesting . i just became a father . do you have a family ?", "football is awesome . what is your favorite team", "i love patriots . i have a dog too , her name is ginger .", "that is nice of you . do you like movies ? my favorite is titanic .", "i learned to love that movie since that is all we had to watch at school .", "i studied abroad so it is the only movie i had in my room ."], "speaker_id": "snli:test:5253", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5253", "candidate_text": "Honestly, A fireman is in the dark!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5253:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5148:premise", "snli:test:5253:premise", "snli:test:5336:premise", "snli:test:5147:premise", "style:personachat:test:456:self:5:positive:0", "style:personachat:test:456:self:5:positive:1", "style:personachat:test:456:self:5:positive:2", "style:personachat:test:456:self:5:positive:3", "style:personachat:test:456:self:5:positive:4"], "evidence_texts": ["Businessman walking as two arrows point to him.", "A fireman searching for something using a flashlight.", "This woman is skiing down a snowy hill.", "Businessman walking as two arrows point to him.", "i do not want to share that information . my parents would get mad", "my three sisters watch us play sometimes , and then we party", "everywhere . my dad drives a semi and i travel with him when i am not playing", "sometimes , yeah ! we love meat", "we are going on another trip soon , i will !"], "speaker_id": "snli:test:5253", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5254", "candidate_text": "I wonder whether a fireman with an axe stands in a burning house?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5254:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5243:premise", "snli:test:5278:premise", "snli:test:5254:premise", "snli:test:4968:premise", "style:personachat:test:199:self:1:positive:0"], "evidence_texts": ["A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:5254", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5254", "candidate_text": "Honestly, A fireman with an axe stands in a burning house!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5254:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5243:premise", "snli:test:5278:premise", "snli:test:5254:premise", "snli:test:4968:premise", "style:personachat:test:456:self:7:positive:0", "style:personachat:test:456:self:7:positive:1", "style:personachat:test:456:self:7:positive:2", "style:personachat:test:456:self:7:positive:3", "style:personachat:test:456:self:7:positive:4", "style:personachat:test:456:self:7:positive:5", "style:personachat:test:456:self:7:positive:6"], "evidence_texts": ["A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "A young barefoot girl wearing a helmet walks down a mossy covered fallen down tree.", "i am getting ready for bed , you ?", "i am a lead singer in a band myself", "i do not want to share that information . my parents would get mad", "my three sisters watch us play sometimes , and then we party", "everywhere . my dad drives a semi and i travel with him when i am not playing", "sometimes , yeah ! we love meat", "we are going on another trip soon , i will !"], "speaker_id": "snli:test:5254", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5264", "candidate_text": "I wonder whether the man is being served food under an umbrella?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5264:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5264:premise", "snli:test:5196:premise", "snli:test:5320:premise", "snli:test:5194:premise", "style:personachat:test:199:self:2:positive:0", "style:personachat:test:199:self:2:positive:1"], "evidence_texts": ["A man is sitting at a restaurant table outdoors.", "Women playing volleyball in the sand near the ocean.", "A boy wearing a helmet is jumping his skateboard.", "Women playing volleyball in the sand near the ocean.", "new york ! a big slice of cheese pizza is the best !", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:5264", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5264", "candidate_text": "The man is being served food under an umbrella.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5264:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5264:premise", "snli:test:5196:premise", "snli:test:5320:premise", "snli:test:5194:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man is sitting at a restaurant table outdoors.", "Women playing volleyball in the sand near the ocean.", "A boy wearing a helmet is jumping his skateboard.", "Women playing volleyball in the sand near the ocean.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5264", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5274", "candidate_text": "I wonder whether th small child is watching a performance?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5274:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:4790:premise", "snli:test:5419:premise", "snli:test:5274:premise", "snli:test:4789:premise", "style:personachat:test:199:self:5:positive:0", "style:personachat:test:199:self:5:positive:1", "style:personachat:test:199:self:5:positive:2", "style:personachat:test:199:self:5:positive:3", "style:personachat:test:199:self:5:positive:4"], "evidence_texts": ["A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "i am in my twenties . not going to say exactly . what do you do for fun ?", "cosplay ? i have never been into that . unless you count dressing as santa claus ! i love christmas !", "i went through a blue phase . hair stylists do that .", "new york ! a big slice of cheese pizza is the best !", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:5274", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5274", "candidate_text": "well, th small child is watching a performance", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5274:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:4790:premise", "snli:test:5419:premise", "snli:test:5274:premise", "snli:test:4789:premise", "style:personachat:test:336:self:5:positive:0", "style:personachat:test:336:self:5:positive:1", "style:personachat:test:336:self:5:positive:2", "style:personachat:test:336:self:5:positive:3", "style:personachat:test:336:self:5:positive:4"], "evidence_texts": ["A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "that is great . i like to hunt in my spare time .", "i am not a big music fan . i prefer the silence . you ?", "i just prefer to work on restoring cars . i am also a handyman", "what do you do for work ?", "i do rent out properties . it is a job ."], "speaker_id": "snli:test:5274", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5277", "candidate_text": "I wonder whether a group of kids slide down a water slide as their parents watch them?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5277:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5074:premise", "snli:test:5277:premise", "snli:test:5652:premise", "snli:test:4313:premise", "style:personachat:test:199:self:6:positive:0", "style:personachat:test:199:self:6:positive:1", "style:personachat:test:199:self:6:positive:2", "style:personachat:test:199:self:6:positive:3", "style:personachat:test:199:self:6:positive:4", "style:personachat:test:199:self:6:positive:5"], "evidence_texts": ["A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "hey there ! i am well . how are you ?", "i am in my twenties . not going to say exactly . what do you do for fun ?", "cosplay ? i have never been into that . unless you count dressing as santa claus ! i love christmas !", "i went through a blue phase . hair stylists do that .", "new york ! a big slice of cheese pizza is the best !", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:5277", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5277", "candidate_text": "Honestly, A group of kids slide down a water slide as their parents watch them!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5277:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5074:premise", "snli:test:5277:premise", "snli:test:5652:premise", "snli:test:4313:premise", "style:personachat:test:466:self:5:positive:0", "style:personachat:test:466:self:5:positive:1", "style:personachat:test:466:self:5:positive:2", "style:personachat:test:466:self:5:positive:3", "style:personachat:test:466:self:5:positive:4"], "evidence_texts": ["A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "A man in tribal dress is walking down the street while spectators are standing behind red and white barrier tape.", "only as old as you feel . and no , just me and my oldies music", "hm , not my expertise , only hobby of mine is trying barbeque restaurants", "as long as its oldies i will dance to anything ! aha you ?", "my father and grandfather were both army", "i have never been there ! and formally married"], "speaker_id": "snli:test:5277", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5278", "candidate_text": "Honestly, A young man is standing on both feet barefooted!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5278:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5254:premise", "snli:test:5429:premise", "snli:test:5278:premise", "snli:test:5243:premise", "style:personachat:test:200:self:1:positive:0"], "evidence_texts": ["A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:5278", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5278", "candidate_text": "I wonder whether a young man is standing on both feet barefooted?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5278:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5254:premise", "snli:test:5429:premise", "snli:test:5278:premise", "snli:test:5243:premise", "style:personachat:test:745:self:1:positive:0"], "evidence_texts": ["A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "A woman in a burgundy coat exits Trader Joe's with a cart full of groceries.", "do you enjoy rap music ? it is my favorite ?"], "speaker_id": "snli:test:5278", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5288", "candidate_text": "Honestly, People are crossing the street during a parade!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5288:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5288:premise", "snli:test:5240:premise", "snli:test:5371:premise", "snli:test:5234:premise", "style:personachat:test:200:self:2:positive:0", "style:personachat:test:200:self:2:positive:1"], "evidence_texts": ["People waiting to cross the street on the side of a sidewalk.", "A young boy wearing a red shirt runs along a gravel road.", "The red motocross bike and its rider are airborne above the trees.", "A red hot air balloon lands in a field among a crowd.", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:5288", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5288", "candidate_text": "well, people are crossing the street during a parade", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5288:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5288:premise", "snli:test:5240:premise", "snli:test:5371:premise", "snli:test:5234:premise", "style:personachat:test:337:self:2:positive:0", "style:personachat:test:337:self:2:positive:1"], "evidence_texts": ["People waiting to cross the street on the side of a sidewalk.", "A young boy wearing a red shirt runs along a gravel road.", "The red motocross bike and its rider are airborne above the trees.", "A red hot air balloon lands in a field among a crowd.", "life is the thing that is the thing and its dressed like the thing", "just being random to pass the time trying to get my 30 cents"], "speaker_id": "snli:test:5288", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5308", "candidate_text": "Honestly, The small child is dressed in tank top and shorts!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5308:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5308:premise", "snli:test:5170:premise", "snli:test:4897:premise", "snli:test:5401:premise", "style:personachat:test:200:self:5:positive:0", "style:personachat:test:200:self:5:positive:1", "style:personachat:test:200:self:5:positive:2", "style:personachat:test:200:self:5:positive:3", "style:personachat:test:200:self:5:positive:4"], "evidence_texts": ["The small child is wearing winter clothes and a hat.", "Several bicyclers are carrying their bikes up a grassy hill.", "Two men help each other cut a sheet of plastic.", "A man performing auto repair in front of a building.", "do you have any pets ? i have three . all dogs . i love them !", "wow expensive . i have a yorkie , maltese and a mix breed .", "christmas , joy and jingle . i love christmas ! my favorite holiday . how about you ?", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:5308", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5308", "candidate_text": "The small child is dressed in tank top and shorts.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5308:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5308:premise", "snli:test:5170:premise", "snli:test:4897:premise", "snli:test:5401:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The small child is wearing winter clothes and a hat.", "Several bicyclers are carrying their bikes up a grassy hill.", "Two men help each other cut a sheet of plastic.", "A man performing auto repair in front of a building.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5308", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5310", "candidate_text": "Honestly, The small child is dressed in a parka!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5310:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5401:premise", "snli:test:5170:premise", "snli:test:5310:premise", "snli:test:5530:premise", "style:personachat:test:200:self:7:positive:0", "style:personachat:test:200:self:7:positive:1", "style:personachat:test:200:self:7:positive:2", "style:personachat:test:200:self:7:positive:3", "style:personachat:test:200:self:7:positive:4", "style:personachat:test:200:self:7:positive:5", "style:personachat:test:200:self:7:positive:6"], "evidence_texts": ["A man performing auto repair in front of a building.", "Several bicyclers are carrying their bikes up a grassy hill.", "The small child is wearing winter clothes and a hat.", "Two girls wearing hats are running through snow drifts outside.", "judy . what do you like about sundays ? football watching is what i like !", "what is your favorite food ? cheese pizza is mine .", "do you have any pets ? i have three . all dogs . i love them !", "wow expensive . i have a yorkie , maltese and a mix breed .", "christmas , joy and jingle . i love christmas ! my favorite holiday . how about you ?", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:5310", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5310", "candidate_text": "I wonder whether the small child is dressed in a parka?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5310:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5401:premise", "snli:test:5170:premise", "snli:test:5310:premise", "snli:test:5530:premise", "style:personachat:test:745:self:6:positive:0", "style:personachat:test:745:self:6:positive:1", "style:personachat:test:745:self:6:positive:2", "style:personachat:test:745:self:6:positive:3", "style:personachat:test:745:self:6:positive:4", "style:personachat:test:745:self:6:positive:5"], "evidence_texts": ["A man performing auto repair in front of a building.", "Several bicyclers are carrying their bikes up a grassy hill.", "The small child is wearing winter clothes and a hat.", "Two girls wearing hats are running through snow drifts outside.", "oh really are you from there ?", "that is amazing , i spend my time working out and listening to rap . how about you ?", "that sounds nice . do you have any family ?", "that must be rough , i have a brother and a father", "i do go to church , we go out for vegan sushi afterwards !", "do you enjoy rap music ? it is my favorite ?"], "speaker_id": "snli:test:5310", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5320", "candidate_text": "I wonder whether a boy is jumping his skateboard while his friends watch?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5320:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5320:premise", "snli:test:5264:premise", "snli:test:5196:premise", "snli:test:5409:premise", "style:personachat:test:201:self:1:positive:0"], "evidence_texts": ["A boy wearing a helmet is jumping his skateboard.", "A man is sitting at a restaurant table outdoors.", "Women playing volleyball in the sand near the ocean.", "A man holding a yellow surfboard facing the ocean.", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:5320", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5320", "candidate_text": "well, a boy is jumping his skateboard while his friends watch", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5320:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5320:premise", "snli:test:5264:premise", "snli:test:5196:premise", "snli:test:5409:premise", "style:personachat:test:338:self:1:positive:0"], "evidence_texts": ["A boy wearing a helmet is jumping his skateboard.", "A man is sitting at a restaurant table outdoors.", "Women playing volleyball in the sand near the ocean.", "A man holding a yellow surfboard facing the ocean.", "no really i learned everything from those movies ."], "speaker_id": "snli:test:5320", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5321", "candidate_text": "I wonder whether a girl is picks up her skateboard?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5321:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5409:premise", "snli:test:5321:premise", "snli:test:5264:premise", "snli:test:5412:premise", "style:personachat:test:201:self:2:positive:0", "style:personachat:test:201:self:2:positive:1"], "evidence_texts": ["A man holding a yellow surfboard facing the ocean.", "A boy wearing a helmet is jumping his skateboard.", "A man is sitting at a restaurant table outdoors.", "A water scene with a sunset in the background.", "that sounds nice , do you eat dinner as a family ?", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:5321", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5321", "candidate_text": "Honestly, A girl is picks up her skateboard!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5321:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5409:premise", "snli:test:5321:premise", "snli:test:5264:premise", "snli:test:5412:premise", "style:personachat:test:479:self:1:positive:0"], "evidence_texts": ["A man holding a yellow surfboard facing the ocean.", "A boy wearing a helmet is jumping his skateboard.", "A man is sitting at a restaurant table outdoors.", "A water scene with a sunset in the background.", "yeah , scared of heights . i take my risks in card games both feet on the ground !"], "speaker_id": "snli:test:5321", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5323", "candidate_text": "I wonder whether the people are enjoying a meal?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5323:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5184:premise", "snli:test:5362:premise", "snli:test:5048:premise", "snli:test:5323:premise", "style:personachat:test:201:self:5:positive:0", "style:personachat:test:201:self:5:positive:1", "style:personachat:test:201:self:5:positive:2", "style:personachat:test:201:self:5:positive:3", "style:personachat:test:201:self:5:positive:4"], "evidence_texts": ["A helmeted male airborne on a bike on a dirt road.", "A boy making an \"oh!\" face in front of a bay.", "A person is sitting on the edge of a rock cliff.", "A group of people sitting around a small round table eating.", "kind of . i have a tail as feet and wavy hair . are you ?", "you do not live in the sea like me ? where do you live ?", "what do you do there ? i explore with my friends after dinner", "that sounds nice , do you eat dinner as a family ?", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:5323", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5323", "candidate_text": "The people are enjoying a meal.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5323:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5184:premise", "snli:test:5362:premise", "snli:test:5048:premise", "snli:test:5323:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A helmeted male airborne on a bike on a dirt road.", "A boy making an \"oh!\" face in front of a bay.", "A person is sitting on the edge of a rock cliff.", "A group of people sitting around a small round table eating.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5323", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5336", "candidate_text": "I wonder whether this woman is skiing down a grassy hill?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5336:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5336:premise", "snli:test:5253:premise", "snli:test:6048:premise", "snli:test:5148:premise", "style:personachat:test:201:self:7:positive:0", "style:personachat:test:201:self:7:positive:1", "style:personachat:test:201:self:7:positive:2", "style:personachat:test:201:self:7:positive:3", "style:personachat:test:201:self:7:positive:4", "style:personachat:test:201:self:7:positive:5", "style:personachat:test:201:self:7:positive:6"], "evidence_texts": ["This woman is skiing down a snowy hill.", "A fireman searching for something using a flashlight.", "A young couple holding hands in the park.", "Businessman walking as two arrows point to him.", "nice , is that what you always do after dinner ?", "i spend all most all day in a school with my sea creature friends .", "kind of . i have a tail as feet and wavy hair . are you ?", "you do not live in the sea like me ? where do you live ?", "what do you do there ? i explore with my friends after dinner", "that sounds nice , do you eat dinner as a family ?", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:5336", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5336", "candidate_text": "well, this woman is skiing down a grassy hill", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5336:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5336:premise", "snli:test:5253:premise", "snli:test:6048:premise", "snli:test:5148:premise", "style:personachat:test:338:self:7:positive:0", "style:personachat:test:338:self:7:positive:1", "style:personachat:test:338:self:7:positive:2", "style:personachat:test:338:self:7:positive:3", "style:personachat:test:338:self:7:positive:4", "style:personachat:test:338:self:7:positive:5", "style:personachat:test:338:self:7:positive:6"], "evidence_texts": ["This woman is skiing down a snowy hill.", "A fireman searching for something using a flashlight.", "A young couple holding hands in the park.", "Businessman walking as two arrows point to him.", "i am doing great today and you ?", "going to walk my dog soon . do you have one ?", "my wife and i just had a baby boy . do you have kids ?", "first time dad over here . team no sleep .", "yes , i played a little company softball", "cool , i watch bruce lee films since i was little . we share the same skills", "no really i learned everything from those movies ."], "speaker_id": "snli:test:5336", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5338", "candidate_text": "I wonder whether a woman and two kids next to each other?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5338:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5153:premise", "snli:test:5152:premise", "snli:test:5338:premise", "snli:test:5360:premise", "style:personachat:test:202:self:1:positive:0"], "evidence_texts": ["A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:5338", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5338", "candidate_text": "Honestly, A woman and two kids next to each other!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5338:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5153:premise", "snli:test:5152:premise", "snli:test:5338:premise", "snli:test:5360:premise", "style:personachat:test:479:self:7:positive:0", "style:personachat:test:479:self:7:positive:1", "style:personachat:test:479:self:7:positive:2", "style:personachat:test:479:self:7:positive:3", "style:personachat:test:479:self:7:positive:4", "style:personachat:test:479:self:7:positive:5", "style:personachat:test:479:self:7:positive:6"], "evidence_texts": ["A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "i am well . how about you ?", "nice ! i just signed up for a jujitsu class ! so far it seems fun .", "wow ! that is amazing ! do you do anything else to help support your tae kwan do ?", "ok , i run to help keep my mind clear .", "at least 5 times a week . do you have any other hobbies ? work ?", "oh i hate them . xd", "yeah , scared of heights . i take my risks in card games both feet on the ground !"], "speaker_id": "snli:test:5338", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5339", "candidate_text": "I wonder whether a mother is waiting for the bus with her kids?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5339:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5360:premise", "snli:test:5153:premise", "snli:test:6184:premise", "snli:test:5339:premise", "style:personachat:test:202:self:2:positive:0", "style:personachat:test:202:self:2:positive:1"], "evidence_texts": ["A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "sounds great . yes it can be . do you have any pets ? i have frogs .", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:5339", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5339", "candidate_text": "A mother is waiting for the bus with her kids.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5339:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5360:premise", "snli:test:5153:premise", "snli:test:6184:premise", "snli:test:5339:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A woman in a white and red basketball uniform with the number fifteen on her chest prepares to shoot a free throw.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5339", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5351", "candidate_text": "I wonder whether the man is going to turn left at the green arrow?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5351:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1634:premise", "snli:test:5732:premise", "snli:test:1633:premise", "snli:test:5351:premise", "style:personachat:test:202:self:5:positive:0", "style:personachat:test:202:self:5:positive:1", "style:personachat:test:202:self:5:positive:2", "style:personachat:test:202:self:5:positive:3", "style:personachat:test:202:self:5:positive:4"], "evidence_texts": ["A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "that is great ! i was just watching some tv , sports , i love them .", "lol , i have been playing sports all through elementary and high school .", "i have to say i favor soccer . what do you enjoy doing ?", "sounds great . yes it can be . do you have any pets ? i have frogs .", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:5351", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5351", "candidate_text": "well, the man is going to turn left at the green arrow", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5351:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:1634:premise", "snli:test:5732:premise", "snli:test:1633:premise", "snli:test:5351:premise", "style:personachat:test:340:self:5:positive:0", "style:personachat:test:340:self:5:positive:1", "style:personachat:test:340:self:5:positive:2", "style:personachat:test:340:self:5:positive:3", "style:personachat:test:340:self:5:positive:4"], "evidence_texts": ["A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "i go to the gym a lot", "walking can be good exercise", "oh spending time with your granddaughter must be great . how old is she ?", "i am sure , but she must be a lot of fun .", "quilting is a great hobby very artistic"], "speaker_id": "snli:test:5351", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5360", "candidate_text": "I wonder whether the woman is wearing a full sleeved jacket?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5360:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5360:premise", "snli:test:5339:premise", "snli:test:6184:premise", "snli:test:5338:premise", "style:personachat:test:202:self:7:positive:0", "style:personachat:test:202:self:7:positive:1", "style:personachat:test:202:self:7:positive:2", "style:personachat:test:202:self:7:positive:3", "style:personachat:test:202:self:7:positive:4", "style:personachat:test:202:self:7:positive:5", "style:personachat:test:202:self:7:positive:6"], "evidence_texts": ["A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "good evening lucy , i am rayne . i hope you are doing well .", "i am a wedding planner , i just graduated from college .", "that is great ! i was just watching some tv , sports , i love them .", "lol , i have been playing sports all through elementary and high school .", "i have to say i favor soccer . what do you enjoy doing ?", "sounds great . yes it can be . do you have any pets ? i have frogs .", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:5360", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5360", "candidate_text": "Honestly, The woman is wearing a full sleeved jacket!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5360:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5360:premise", "snli:test:5339:premise", "snli:test:6184:premise", "snli:test:5338:premise", "style:personachat:test:500:self:5:positive:0", "style:personachat:test:500:self:5:positive:1", "style:personachat:test:500:self:5:positive:2", "style:personachat:test:500:self:5:positive:3", "style:personachat:test:500:self:5:positive:4"], "evidence_texts": ["A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "what do you teach ?", "i should teach history because i travel a lot .", "no children yet . i have traveled to 6 countries .", "traveling is so wonderful plus i love to walk so it works .", "good night , have a wonderful night !"], "speaker_id": "snli:test:5360", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5362", "candidate_text": "I wonder whether a boy is swimming in the bay?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5362:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5323:premise", "snli:test:5524:premise", "snli:test:5362:premise", "snli:test:5184:premise", "style:personachat:test:203:self:1:positive:0"], "evidence_texts": ["A group of people sitting around a small round table eating.", "A child enjoying themselves on a yellow slide at a park.", "A boy making an \"oh!\" face in front of a bay.", "A helmeted male airborne on a bike on a dirt road.", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:5362", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5362", "candidate_text": "A boy is swimming in the bay.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5362:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5323:premise", "snli:test:5524:premise", "snli:test:5362:premise", "snli:test:5184:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A group of people sitting around a small round table eating.", "A child enjoying themselves on a yellow slide at a park.", "A boy making an \"oh!\" face in front of a bay.", "A helmeted male airborne on a bike on a dirt road.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5362", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5367", "candidate_text": "I wonder whether three men adn a woman look at produce?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5367:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:3956:premise", "snli:test:5427:premise", "snli:test:2946:premise", "snli:test:5367:premise", "style:personachat:test:203:self:2:positive:0", "style:personachat:test:203:self:2:positive:1"], "evidence_texts": ["Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "can anyone be jewish or is it exclusive to a certain group ?", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:5367", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5367", "candidate_text": "well, three men adn a woman look at produce", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5367:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:3956:premise", "snli:test:5427:premise", "snli:test:2946:premise", "snli:test:5367:premise", "style:personachat:test:343:self:2:positive:0", "style:personachat:test:343:self:2:positive:1"], "evidence_texts": ["Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "Children at daycare playing on rings, and poles, one parent standing and holding her child and one teacher is sitting on the bench watching the others play.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "yes , he was on tv", "you must really like cars"], "speaker_id": "snli:test:5367", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5371", "candidate_text": "I wonder whether a red motocross bike is being taken over a jump by a rider, while performing a trick?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5371:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5288:premise", "snli:test:5377:premise", "snli:test:5240:premise", "snli:test:5371:premise", "style:personachat:test:203:self:5:positive:0", "style:personachat:test:203:self:5:positive:1", "style:personachat:test:203:self:5:positive:2", "style:personachat:test:203:self:5:positive:3", "style:personachat:test:203:self:5:positive:4"], "evidence_texts": ["People waiting to cross the street on the side of a sidewalk.", "A crowd of people fill the street in front of a McDonald's.", "A young boy wearing a red shirt runs along a gravel road.", "The red motocross bike and its rider are airborne above the trees.", "what do you like to do in your spare moments ?", "i am getting married soon , hope she starts to enjoy sports like i do", "where did the jews come from ?", "can anyone be jewish or is it exclusive to a certain group ?", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:5371", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5371", "candidate_text": "Honestly, A red motocross bike is being taken over a jump by a rider, while performing a trick!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5371:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5288:premise", "snli:test:5377:premise", "snli:test:5240:premise", "snli:test:5371:premise", "style:personachat:test:505:self:2:positive:0", "style:personachat:test:505:self:2:positive:1"], "evidence_texts": ["People waiting to cross the street on the side of a sidewalk.", "A crowd of people fill the street in front of a McDonald's.", "A young boy wearing a red shirt runs along a gravel road.", "The red motocross bike and its rider are airborne above the trees.", "well have a good time . it is good to be with people", "well have a great time tonight !"], "speaker_id": "snli:test:5371", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5377", "candidate_text": "I wonder whether the people are hungry?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5377:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5371:premise", "snli:test:5377:premise", "snli:test:5288:premise", "snli:test:5384:premise", "style:personachat:test:203:self:7:positive:0", "style:personachat:test:203:self:7:positive:1", "style:personachat:test:203:self:7:positive:2", "style:personachat:test:203:self:7:positive:3", "style:personachat:test:203:self:7:positive:4", "style:personachat:test:203:self:7:positive:5", "style:personachat:test:203:self:7:positive:6"], "evidence_texts": ["The red motocross bike and its rider are airborne above the trees.", "A crowd of people fill the street in front of a McDonald's.", "People waiting to cross the street on the side of a sidewalk.", "A young, well-dressed black man seated behind a drum set and smiling.", "doing great watching a little sports on tv", "i enjoy watching soccer a lot . i played some sports in high school .", "what do you like to do in your spare moments ?", "i am getting married soon , hope she starts to enjoy sports like i do", "where did the jews come from ?", "can anyone be jewish or is it exclusive to a certain group ?", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:5377", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5377", "candidate_text": "The people are hungry.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5377:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5371:premise", "snli:test:5377:premise", "snli:test:5288:premise", "snli:test:5384:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The red motocross bike and its rider are airborne above the trees.", "A crowd of people fill the street in front of a McDonald's.", "People waiting to cross the street on the side of a sidewalk.", "A young, well-dressed black man seated behind a drum set and smiling.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5377", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5379", "candidate_text": "well, there is a crowd outside the restaurant", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5379:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5384:premise", "snli:test:5371:premise", "snli:test:5393:premise", "snli:test:5379:premise", "style:personachat:test:204:self:1:positive:0"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "The red motocross bike and its rider are airborne above the trees.", "A person, in a striped blue shirt and pants, is running along.", "A crowd of people fill the street in front of a McDonald's.", "i am very new , but i am learning . i also run"], "speaker_id": "snli:test:5379", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5379", "candidate_text": "Honestly, There is a crowd outside the restaurant!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5379:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5384:premise", "snli:test:5371:premise", "snli:test:5393:premise", "snli:test:5379:premise", "style:personachat:test:505:self:7:positive:0", "style:personachat:test:505:self:7:positive:1", "style:personachat:test:505:self:7:positive:2", "style:personachat:test:505:self:7:positive:3", "style:personachat:test:505:self:7:positive:4", "style:personachat:test:505:self:7:positive:5", "style:personachat:test:505:self:7:positive:6"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "The red motocross bike and its rider are airborne above the trees.", "A person, in a striped blue shirt and pants, is running along.", "A crowd of people fill the street in front of a McDonald's.", "well i am a nurse for the elderly , what do you do ?", "what a noble profession . good for you . my son wants to teach", "much older than that ! i have 6 grandkids !", "you might be surprised . all these kids keep be young", "i have a bicycle . but i am not out playing football", "well have a good time . it is good to be with people", "well have a great time tonight !"], "speaker_id": "snli:test:5379", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5384", "candidate_text": "well, a man is smiling", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5384:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5384:premise", "snli:test:5379:premise", "snli:test:5393:premise", "snli:test:5377:premise", "style:personachat:test:204:self:2:positive:0", "style:personachat:test:204:self:2:positive:1"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "A crowd of people fill the street in front of a McDonald's.", "A person, in a striped blue shirt and pants, is running along.", "A crowd of people fill the street in front of a McDonald's.", "i take jujitsu classes for fitness", "i am very new , but i am learning . i also run"], "speaker_id": "snli:test:5384", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5384", "candidate_text": "a man is smiling.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5384:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5384:premise", "snli:test:5379:premise", "snli:test:5393:premise", "snli:test:5377:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "A crowd of people fill the street in front of a McDonald's.", "A person, in a striped blue shirt and pants, is running along.", "A crowd of people fill the street in front of a McDonald's.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5384", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5393", "candidate_text": "well, the person is running", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5393:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5384:premise", "snli:test:5393:premise", "snli:test:5589:premise", "snli:test:5379:premise", "style:personachat:test:204:self:5:positive:0", "style:personachat:test:204:self:5:positive:1", "style:personachat:test:204:self:5:positive:2", "style:personachat:test:204:self:5:positive:3", "style:personachat:test:204:self:5:positive:4"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "A person, in a striped blue shirt and pants, is running along.", "A surfer is riding a wave in a large body of water.", "A crowd of people fill the street in front of a McDonald's.", "do you like that ? i am in the teaching program", "do you like music ? i love ed sheeran", "well that is no reason not to take care of your body", "i take jujitsu classes for fitness", "i am very new , but i am learning . i also run"], "speaker_id": "snli:test:5393", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5393", "candidate_text": "I wonder whether the person is running?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5393:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5384:premise", "snli:test:5393:premise", "snli:test:5589:premise", "snli:test:5379:premise", "style:personachat:test:757:self:5:positive:0", "style:personachat:test:757:self:5:positive:1", "style:personachat:test:757:self:5:positive:2", "style:personachat:test:757:self:5:positive:3", "style:personachat:test:757:self:5:positive:4"], "evidence_texts": ["A young, well-dressed black man seated behind a drum set and smiling.", "A person, in a striped blue shirt and pants, is running along.", "A surfer is riding a wave in a large body of water.", "A crowd of people fill the street in front of a McDonald's.", "oo i enlisted when i was still in high school", "oo , so you have to wait until youre 18", "i like cold weather and the snow", "i am from wisconsin but live in texas", "definitely texas , do you have siblings ?"], "speaker_id": "snli:test:5393", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5401", "candidate_text": "well, the man is fixing a car", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5401:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5310:premise", "snli:test:5401:premise", "snli:test:5530:premise", "snli:test:5308:premise", "style:personachat:test:204:self:7:positive:0", "style:personachat:test:204:self:7:positive:1", "style:personachat:test:204:self:7:positive:2", "style:personachat:test:204:self:7:positive:3", "style:personachat:test:204:self:7:positive:4", "style:personachat:test:204:self:7:positive:5", "style:personachat:test:204:self:7:positive:6"], "evidence_texts": ["The small child is wearing winter clothes and a hat.", "A man performing auto repair in front of a building.", "Two girls wearing hats are running through snow drifts outside.", "The small child is wearing winter clothes and a hat.", "just finished a poker night with my friends , you ?", "that sounds nice . when i finish college i hope to start a family", "do you like that ? i am in the teaching program", "do you like music ? i love ed sheeran", "well that is no reason not to take care of your body", "i take jujitsu classes for fitness", "i am very new , but i am learning . i also run"], "speaker_id": "snli:test:5401", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5401", "candidate_text": "Honestly, The man is fixing a car!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5401:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5310:premise", "snli:test:5401:premise", "snli:test:5530:premise", "snli:test:5308:premise", "style:personachat:test:512:self:5:positive:0", "style:personachat:test:512:self:5:positive:1", "style:personachat:test:512:self:5:positive:2", "style:personachat:test:512:self:5:positive:3", "style:personachat:test:512:self:5:positive:4"], "evidence_texts": ["The small child is wearing winter clothes and a hat.", "A man performing auto repair in front of a building.", "Two girls wearing hats are running through snow drifts outside.", "The small child is wearing winter clothes and a hat.", "what kind of practice do you do ?", "very cool . i am a real estate agent .", "yes , lots . i have two kids in kindergarten so i am busy !", "do you have any kids ?", "ha ! well then good . i am 34 ."], "speaker_id": "snli:test:5401", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5409", "candidate_text": "well, a man is holding a surfboard", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5409:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5321:premise", "snli:test:5409:premise", "snli:test:5412:premise", "snli:test:5320:premise", "style:personachat:test:205:self:1:positive:0"], "evidence_texts": ["A boy wearing a helmet is jumping his skateboard.", "A man holding a yellow surfboard facing the ocean.", "A water scene with a sunset in the background.", "A boy wearing a helmet is jumping his skateboard.", "yeah , my mom works a lot and i always thought i should balance more"], "speaker_id": "snli:test:5409", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5409", "candidate_text": "A man is holding a surfboard.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5409:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5321:premise", "snli:test:5409:premise", "snli:test:5412:premise", "snli:test:5320:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A boy wearing a helmet is jumping his skateboard.", "A man holding a yellow surfboard facing the ocean.", "A water scene with a sunset in the background.", "A boy wearing a helmet is jumping his skateboard.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5409", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5412", "candidate_text": "well, there is a water scene with the sunset in the back", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5412:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5412:premise", "snli:test:5409:premise", "snli:test:5516:premise", "snli:test:5321:premise", "style:personachat:test:205:self:2:positive:0", "style:personachat:test:205:self:2:positive:1"], "evidence_texts": ["A water scene with a sunset in the background.", "A man holding a yellow surfboard facing the ocean.", "A person standing next to an old dark building.", "A boy wearing a helmet is jumping his skateboard.", "oh that is pretty sweet , i watch a lot of sports do you ?", "yeah , my mom works a lot and i always thought i should balance more"], "speaker_id": "snli:test:5412", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5412", "candidate_text": "I wonder whether there is a water scene with the sunset in the back?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5412:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5412:premise", "snli:test:5409:premise", "snli:test:5516:premise", "snli:test:5321:premise", "style:personachat:test:760:self:2:positive:0", "style:personachat:test:760:self:2:positive:1"], "evidence_texts": ["A water scene with a sunset in the background.", "A man holding a yellow surfboard facing the ocean.", "A person standing next to an old dark building.", "A boy wearing a helmet is jumping his skateboard.", "so sorry to hear that , but it sounds like your uncle is a good support .", "that is wonderful . do you need help looking for a job ?"], "speaker_id": "snli:test:5412", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5418", "candidate_text": "well, a person rides on a wakeboard along the beach", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5418:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:1009:premise", "snli:test:1010:premise", "snli:test:5418:premise", "snli:test:3834:premise", "style:personachat:test:205:self:5:positive:0", "style:personachat:test:205:self:5:positive:1", "style:personachat:test:205:self:5:positive:2", "style:personachat:test:205:self:5:positive:3", "style:personachat:test:205:self:5:positive:4"], "evidence_texts": ["A dog runs.", "A dog runs.", "A wakeboarder.", "Men play hockey.", "that is cool , i like playing video games a lot do you play ?", "that is cool , i like them but they can be weird . any hobbies ?", "oh that is cool . do you work fulltime ?", "oh that is pretty sweet , i watch a lot of sports do you ?", "yeah , my mom works a lot and i always thought i should balance more"], "speaker_id": "snli:test:5418", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5418", "candidate_text": "Honestly, A person rides on a wakeboard along the beach!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5418:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:1009:premise", "snli:test:1010:premise", "snli:test:5418:premise", "snli:test:3834:premise", "style:personachat:test:531:self:2:positive:0", "style:personachat:test:531:self:2:positive:1"], "evidence_texts": ["A dog runs.", "A dog runs.", "A wakeboarder.", "Men play hockey.", "i hear you . yes , i am happy to be single . my best friend has 5 children !", "me too ! my parents had 8 . i am eldest daughter , number 4 from top"], "speaker_id": "snli:test:5418", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5419", "candidate_text": "well, two people are going to kiss", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5419:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5274:premise", "snli:test:5448:premise", "snli:test:4790:premise", "snli:test:5419:premise", "style:personachat:test:205:self:7:positive:0", "style:personachat:test:205:self:7:positive:1", "style:personachat:test:205:self:7:positive:2", "style:personachat:test:205:self:7:positive:3", "style:personachat:test:205:self:7:positive:4", "style:personachat:test:205:self:7:positive:5", "style:personachat:test:205:self:7:positive:6"], "evidence_texts": ["A small child in a red jacket is sitting on a man's shoulders and holding his head.", "Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "i am good , how are you ?", "yes i work from home , do you work ?", "that is cool , i like playing video games a lot do you play ?", "that is cool , i like them but they can be weird . any hobbies ?", "oh that is cool . do you work fulltime ?", "oh that is pretty sweet , i watch a lot of sports do you ?", "yeah , my mom works a lot and i always thought i should balance more"], "speaker_id": "snli:test:5419", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5419", "candidate_text": "Two people are going to kiss.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5419:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5274:premise", "snli:test:5448:premise", "snli:test:4790:premise", "snli:test:5419:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A small child in a red jacket is sitting on a man's shoulders and holding his head.", "Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A lady, dressed in a costume, is posing with an axe in the middle of the woods.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5419", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5427", "candidate_text": "I wonder whether a leprechaun is skateboarding?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5427:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5367:premise", "snli:test:9688:premise", "snli:test:3956:premise", "snli:test:5427:premise", "style:personachat:test:206:self:1:positive:0"], "evidence_texts": ["Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "i see . does your dad like riding rollercoasters too ?"], "speaker_id": "snli:test:5427", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5427", "candidate_text": "A leprechaun is skateboarding.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5427:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5367:premise", "snli:test:9688:premise", "snli:test:3956:premise", "snli:test:5427:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "Two women, standing at a fence, are each talking on their cellphones, while two men sitting on the opposite side of the fence are having a conversation.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5427", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5429", "candidate_text": "I wonder whether people excercise on court?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5429:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5278:premise", "snli:test:5429:premise", "snli:test:5722:premise", "snli:test:5254:premise", "style:personachat:test:206:self:2:positive:0", "style:personachat:test:206:self:2:positive:1"], "evidence_texts": ["A young man standing on one foot in colorful bowling shoes after throwing the ball.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "hmmm . what does your dad do ?", "i see . does your dad like riding rollercoasters too ?"], "speaker_id": "snli:test:5429", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5429", "candidate_text": "well, people excercise on court", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5429:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5278:premise", "snli:test:5429:premise", "snli:test:5722:premise", "snli:test:5254:premise", "style:personachat:test:346:self:2:positive:0", "style:personachat:test:346:self:2:positive:1"], "evidence_texts": ["A young man standing on one foot in colorful bowling shoes after throwing the ball.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A fireman with a neon helmet and a pickaxe inspects some debris with a flashlight.", "i feel so sorry . you should get it soon .", "god help you and make you rich one day ."], "speaker_id": "snli:test:5429", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5433", "candidate_text": "I wonder whether humans in orange?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5433:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5247:premise", "snli:test:5433:premise", "snli:test:5767:premise", "snli:test:4984:premise", "style:personachat:test:206:self:5:positive:0", "style:personachat:test:206:self:5:positive:1", "style:personachat:test:206:self:5:positive:2", "style:personachat:test:206:self:5:positive:3", "style:personachat:test:206:self:5:positive:4"], "evidence_texts": ["A woman is walking with a full shopping cart away from a Trader Joe's.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A dog with a purple leash is held by a woman wearing white shoes.", "Two identical brown dogs side by side holding something white in both their mouths.", "great ! you ever left your state . i have not .", "really ? i do rollercoasters at amusement parks . you ?", "wow ! i am a japanese boxing champion . you box ?", "hmmm . what does your dad do ?", "i see . does your dad like riding rollercoasters too ?"], "speaker_id": "snli:test:5433", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5433", "candidate_text": "Honestly, Humans in orange!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5433:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5247:premise", "snli:test:5433:premise", "snli:test:5767:premise", "snli:test:4984:premise", "style:personachat:test:549:self:2:positive:0", "style:personachat:test:549:self:2:positive:1"], "evidence_texts": ["A woman is walking with a full shopping cart away from a Trader Joe's.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A dog with a purple leash is held by a woman wearing white shoes.", "Two identical brown dogs side by side holding something white in both their mouths.", "its no big deal . its in the past . it was high school .", "i am too ! i went through a bad phase then . . . i was even arrested"], "speaker_id": "snli:test:5433", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5441", "candidate_text": "I wonder whether a man covered in paint after a paintball fight takes a photo?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5441:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4233:premise", "snli:test:5441:premise", "snli:test:5597:premise", "snli:test:3958:premise", "style:personachat:test:206:self:7:positive:0", "style:personachat:test:206:self:7:positive:1", "style:personachat:test:206:self:7:positive:2", "style:personachat:test:206:self:7:positive:3", "style:personachat:test:206:self:7:positive:4", "style:personachat:test:206:self:7:positive:5", "style:personachat:test:206:self:7:positive:6"], "evidence_texts": ["A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "hi , my dad box and teaches boxing ? you do anything ?", "i see but can you sing and dance ?", "great ! you ever left your state . i have not .", "really ? i do rollercoasters at amusement parks . you ?", "wow ! i am a japanese boxing champion . you box ?", "hmmm . what does your dad do ?", "i see . does your dad like riding rollercoasters too ?"], "speaker_id": "snli:test:5441", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5441", "candidate_text": "A man covered in paint after a paintball fight takes a photo.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5441:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4233:premise", "snli:test:5441:premise", "snli:test:5597:premise", "snli:test:3958:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "A lady in the kitchen cleaning her feet on a chopping board with a red towel, or red apron, with her husband standing there.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5441", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5448", "candidate_text": "well, two girls with short blonde hair playing a game", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5448:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5448:premise", "snli:test:5419:premise", "snli:test:5669:premise", "snli:test:5274:premise", "style:personachat:test:207:self:1:positive:0"], "evidence_texts": ["Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "A group of people taking pictures on a walkway in front of a large body of water.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "fresh is better . microwaves are for losers . and now you have me hungry ."], "speaker_id": "snli:test:5448", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5448", "candidate_text": "Honestly, Two girls with short blonde hair playing a game!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5448:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5448:premise", "snli:test:5419:premise", "snli:test:5669:premise", "snli:test:5274:premise", "style:personachat:test:549:self:7:positive:0", "style:personachat:test:549:self:7:positive:1", "style:personachat:test:549:self:7:positive:2", "style:personachat:test:549:self:7:positive:3", "style:personachat:test:549:self:7:positive:4", "style:personachat:test:549:self:7:positive:5", "style:personachat:test:549:self:7:positive:6"], "evidence_texts": ["Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "A group of people taking pictures on a walkway in front of a large body of water.", "A small child in a red jacket is sitting on a man's shoulders and holding his head.", "hey ! i am great , thanks for asking . how are you ?", "oh that does sound rough , i want to do fashion .", "that sounds awesome ! that is what i want to do . fashion design .", "i have never listened to them . i prefered pop growing up .", "ah . i see . my best friend stole my boyfriend . . .", "its no big deal . its in the past . it was high school .", "i am too ! i went through a bad phase then . . . i was even arrested"], "speaker_id": "snli:test:5448", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5459", "candidate_text": "well, a little girl is standing outside", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5459:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5198:premise", "snli:test:5495:premise", "snli:test:4770:premise", "snli:test:5459:premise", "style:personachat:test:207:self:2:positive:0", "style:personachat:test:207:self:2:positive:1"], "evidence_texts": ["There are two kids, a soccer ball, a dog and a goat outside in a field.", "A small child with a backpack walks down a narrow road next to a painted wall.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "i will starve to death by then !", "fresh is better . microwaves are for losers . and now you have me hungry ."], "speaker_id": "snli:test:5459", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5459", "candidate_text": "A little girl is standing outside.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5459:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5198:premise", "snli:test:5495:premise", "snli:test:4770:premise", "snli:test:5459:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["There are two kids, a soccer ball, a dog and a goat outside in a field.", "A small child with a backpack walks down a narrow road next to a painted wall.", "A person walks outside carrying a bag, while a young girl in pink walks behind her.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5459", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5490", "candidate_text": "well, a dog is going for a walk", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5490:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5099:premise", "snli:test:5610:premise", "snli:test:5490:premise", "snli:test:5098:premise", "style:personachat:test:207:self:5:positive:0", "style:personachat:test:207:self:5:positive:1", "style:personachat:test:207:self:5:positive:2", "style:personachat:test:207:self:5:positive:3", "style:personachat:test:207:self:5:positive:4"], "evidence_texts": ["A child hugs a birdhouse.", "pedestrian walking on the street", "A couple walking their dog.", "A child hugs a birdhouse.", "i love country music the best . i am a petite little thing only five feet tall .", "oh i could go for some angel hair alfredo .", "i love sauce . the more the better . come cook for me !", "i will starve to death by then !", "fresh is better . microwaves are for losers . and now you have me hungry ."], "speaker_id": "snli:test:5490", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5490", "candidate_text": "I wonder whether a dog is going for a walk?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5490:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5099:premise", "snli:test:5610:premise", "snli:test:5490:premise", "snli:test:5098:premise", "style:personachat:test:773:self:5:positive:0", "style:personachat:test:773:self:5:positive:1", "style:personachat:test:773:self:5:positive:2", "style:personachat:test:773:self:5:positive:3", "style:personachat:test:773:self:5:positive:4"], "evidence_texts": ["A child hugs a birdhouse.", "pedestrian walking on the street", "A couple walking their dog.", "A child hugs a birdhouse.", "i tore up my knee playing frisbee with my 2 dogs . where are you driving to ?", "its going well so far . an excuse to lay around . you drive truck ?", "i am currently looking . i did not finish high school so its not easy finding something worth it .", "wow that is a lot . i do not have any kids . how old are yours ?", "i would imagine so . where are you driving from to chicago ?"], "speaker_id": "snli:test:5490", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5495", "candidate_text": "well, a small child is walking on a road", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5495:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5459:premise", "snli:test:5929:premise", "snli:test:5198:premise", "snli:test:5495:premise", "style:personachat:test:207:self:7:positive:0", "style:personachat:test:207:self:7:positive:1", "style:personachat:test:207:self:7:positive:2", "style:personachat:test:207:self:7:positive:3", "style:personachat:test:207:self:7:positive:4", "style:personachat:test:207:self:7:positive:5", "style:personachat:test:207:self:7:positive:6"], "evidence_texts": ["A young girl with a purple shirt and a pink hair band standing near the woods.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "A small child with a backpack walks down a narrow road next to a painted wall.", "i am good . just sitting here with my sister and mom and dad . the 4 of us .", "i love music so i am studying that in school . i hope to be a music teacher .", "i love country music the best . i am a petite little thing only five feet tall .", "oh i could go for some angel hair alfredo .", "i love sauce . the more the better . come cook for me !", "i will starve to death by then !", "fresh is better . microwaves are for losers . and now you have me hungry ."], "speaker_id": "snli:test:5495", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5495", "candidate_text": "Honestly, A small child is walking on a road!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5495:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5459:premise", "snli:test:5929:premise", "snli:test:5198:premise", "snli:test:5495:premise", "style:personachat:test:553:self:5:positive:0", "style:personachat:test:553:self:5:positive:1", "style:personachat:test:553:self:5:positive:2", "style:personachat:test:553:self:5:positive:3", "style:personachat:test:553:self:5:positive:4"], "evidence_texts": ["A young girl with a purple shirt and a pink hair band standing near the woods.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "There are two kids, a soccer ball, a dog and a goat outside in a field.", "A small child with a backpack walks down a narrow road next to a painted wall.", "i am a secretary at an office . nothing too exciting", "that is awesome . i love animals", "that is a beautiful name for an animal", "i do think . i went to grace college last year", "haha yes i am . what a small world we live in !"], "speaker_id": "snli:test:5495", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5516", "candidate_text": "well, a man standing next to a run down building", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5516:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5516:premise", "snli:test:5412:premise", "snli:test:5665:premise", "snli:test:5409:premise", "style:personachat:test:208:self:1:positive:0"], "evidence_texts": ["A person standing next to an old dark building.", "A water scene with a sunset in the background.", "One female and two male musicians holding musical equipment.", "A man holding a yellow surfboard facing the ocean.", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:5516", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5516", "candidate_text": "A man standing next to a run down building.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5516:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5516:premise", "snli:test:5412:premise", "snli:test:5665:premise", "snli:test:5409:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person standing next to an old dark building.", "A water scene with a sunset in the background.", "One female and two male musicians holding musical equipment.", "A man holding a yellow surfboard facing the ocean.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5516", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5524", "candidate_text": "well, children enjoy the last few moments in a park before a rainstorm", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5524:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5524:premise", "snli:test:5362:premise", "snli:test:5527:premise", "snli:test:5323:premise", "style:personachat:test:208:self:2:positive:0", "style:personachat:test:208:self:2:positive:1"], "evidence_texts": ["A child enjoying themselves on a yellow slide at a park.", "A boy making an \"oh!\" face in front of a bay.", "A person stands between two giant statues that look like faces.", "A group of people sitting around a small round table eating.", "i have never heard of vegan sushi .", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:5524", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5524", "candidate_text": "I wonder whether children enjoy the last few moments in a park before a rainstorm?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5524:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5524:premise", "snli:test:5362:premise", "snli:test:5527:premise", "snli:test:5323:premise", "style:personachat:test:778:self:2:positive:0", "style:personachat:test:778:self:2:positive:1"], "evidence_texts": ["A child enjoying themselves on a yellow slide at a park.", "A boy making an \"oh!\" face in front of a bay.", "A person stands between two giant statues that look like faces.", "A group of people sitting around a small round table eating.", "not true . they are specifically all different inside .", "why would anyone be called beesly ?"], "speaker_id": "snli:test:5524", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5527", "candidate_text": "well, a human is standing between two large objects", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5527:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5524:premise", "snli:test:5648:premise", "snli:test:5362:premise", "snli:test:5527:premise", "style:personachat:test:208:self:5:positive:0", "style:personachat:test:208:self:5:positive:1", "style:personachat:test:208:self:5:positive:2", "style:personachat:test:208:self:5:positive:3", "style:personachat:test:208:self:5:positive:4"], "evidence_texts": ["A child enjoying themselves on a yellow slide at a park.", "A four piece band plays a song on a medium-size stage.", "A boy making an \"oh!\" face in front of a bay.", "A person stands between two giant statues that look like faces.", "that sounds interesting , do you not get sick ?", "you are a vegan and eat sushi ? how does that work ?", "those usually have cream cheese in them , so they are not vegan .", "i have never heard of vegan sushi .", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:5527", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5527", "candidate_text": "Honestly, A human is standing between two large objects!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5527:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5524:premise", "snli:test:5648:premise", "snli:test:5362:premise", "snli:test:5527:premise", "style:personachat:test:565:self:2:positive:0", "style:personachat:test:565:self:2:positive:1"], "evidence_texts": ["A child enjoying themselves on a yellow slide at a park.", "A four piece band plays a song on a medium-size stage.", "A boy making an \"oh!\" face in front of a bay.", "A person stands between two giant statues that look like faces.", "can she remake jack . i have seen it 10 times", "no ! jack from titanic . i want to use the puppy money to remake it"], "speaker_id": "snli:test:5527", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5530", "candidate_text": "well, one girl running", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5530:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5401:premise", "snli:test:5553:premise", "snli:test:5530:premise", "snli:test:5310:premise", "style:personachat:test:208:self:7:positive:0", "style:personachat:test:208:self:7:positive:1", "style:personachat:test:208:self:7:positive:2", "style:personachat:test:208:self:7:positive:3", "style:personachat:test:208:self:7:positive:4", "style:personachat:test:208:self:7:positive:5", "style:personachat:test:208:self:7:positive:6"], "evidence_texts": ["A man performing auto repair in front of a building.", "Two men scrimmage in soccer, as a referee looks on.", "Two girls wearing hats are running through snow drifts outside.", "The small child is wearing winter clothes and a hat.", "hello , how are you tonight ? i am in a band , actually", "what do you do for a living ?", "that sounds interesting , do you not get sick ?", "you are a vegan and eat sushi ? how does that work ?", "those usually have cream cheese in them , so they are not vegan .", "i have never heard of vegan sushi .", "that is okay , i do not care one way or another"], "speaker_id": "snli:test:5530", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5530", "candidate_text": "one girl running.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5530:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5401:premise", "snli:test:5553:premise", "snli:test:5530:premise", "snli:test:5310:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man performing auto repair in front of a building.", "Two men scrimmage in soccer, as a referee looks on.", "Two girls wearing hats are running through snow drifts outside.", "The small child is wearing winter clothes and a hat.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5530", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5535", "candidate_text": "Honestly, Teenagers are hiking in the cold!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5535:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5200:premise", "snli:test:5817:premise", "snli:test:5137:premise", "snli:test:5535:premise", "style:personachat:test:209:self:1:positive:0"], "evidence_texts": ["A gray-haired man rides a bike in front of a bus advertising Ironman.", "A photographer is using a tripod to take a picture of a sunset.", "While two students are jumping, others in a classroom are sitting or standing.", "Two young teenage girls hiking through the snow on a cold winter day.", "i am terrified of them ! !"], "speaker_id": "snli:test:5535", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5535", "candidate_text": "I wonder whether teenagers are hiking in the cold?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5535:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5200:premise", "snli:test:5817:premise", "snli:test:5137:premise", "snli:test:5535:premise", "style:personachat:test:788:self:1:positive:0"], "evidence_texts": ["A gray-haired man rides a bike in front of a bus advertising Ironman.", "A photographer is using a tripod to take a picture of a sunset.", "While two students are jumping, others in a classroom are sitting or standing.", "Two young teenage girls hiking through the snow on a cold winter day.", "that sounds interesting . what are your hobbies ?"], "speaker_id": "snli:test:5535", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5553", "candidate_text": "Honestly, the ref watches the two men in the scrimmage game!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5553:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5530:premise", "snli:test:5553:premise", "snli:test:5614:premise", "snli:test:5401:premise", "style:personachat:test:209:self:2:positive:0", "style:personachat:test:209:self:2:positive:1"], "evidence_texts": ["Two girls wearing hats are running through snow drifts outside.", "Two men scrimmage in soccer, as a referee looks on.", "People ordering and picking up food at a restaurant window.", "A man performing auto repair in front of a building.", "i want one , but i think their food will bring in insects .", "i am terrified of them ! !"], "speaker_id": "snli:test:5553", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5553", "candidate_text": "well, the ref watches the two men in the scrimmage game", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5553:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5530:premise", "snli:test:5553:premise", "snli:test:5614:premise", "snli:test:5401:premise", "style:personachat:test:351:self:2:positive:0", "style:personachat:test:351:self:2:positive:1"], "evidence_texts": ["Two girls wearing hats are running through snow drifts outside.", "Two men scrimmage in soccer, as a referee looks on.", "People ordering and picking up food at a restaurant window.", "A man performing auto repair in front of a building.", "that must be warm in texas . what color is your fur ?", "no , but my pets do , have you always lived in texas"], "speaker_id": "snli:test:5553", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5556", "candidate_text": "Honestly, Workers preparing for a chili cook-off!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5556:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5556:premise", "snli:test:1694:premise", "snli:test:6441:premise", "snli:test:1692:premise", "style:personachat:test:209:self:5:positive:0", "style:personachat:test:209:self:5:positive:1", "style:personachat:test:209:self:5:positive:2", "style:personachat:test:209:self:5:positive:3", "style:personachat:test:209:self:5:positive:4"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "no feet , i am sorry ! i am tall english speaking adult !", "that is fun ! do you work in the sea too ?", "yes , as an engineer . do you have pets ?", "i want one , but i think their food will bring in insects .", "i am terrified of them ! !"], "speaker_id": "snli:test:5556", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5556", "candidate_text": "Workers preparing for a chili cook-off.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5556:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5556:premise", "snli:test:1694:premise", "snli:test:6441:premise", "snli:test:1692:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5556", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5558", "candidate_text": "Honestly, A man is laying down on a park bench as people walk by!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5558:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3104:premise", "snli:test:6031:premise", "snli:test:5558:premise", "snli:test:3103:premise", "style:personachat:test:209:self:7:positive:0", "style:personachat:test:209:self:7:positive:1", "style:personachat:test:209:self:7:positive:2", "style:personachat:test:209:self:7:positive:3", "style:personachat:test:209:self:7:positive:4", "style:personachat:test:209:self:7:positive:5", "style:personachat:test:209:self:7:positive:6"], "evidence_texts": ["A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "i love to listen to bach and compose songs in english . you ?", "very nice ! i love bacon but i am so afraid it will draw in insects .", "no feet , i am sorry ! i am tall english speaking adult !", "that is fun ! do you work in the sea too ?", "yes , as an engineer . do you have pets ?", "i want one , but i think their food will bring in insects .", "i am terrified of them ! !"], "speaker_id": "snli:test:5558", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5558", "candidate_text": "I wonder whether a man is laying down on a park bench as people walk by?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5558:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:3104:premise", "snli:test:6031:premise", "snli:test:5558:premise", "snli:test:3103:premise", "style:personachat:test:788:self:7:positive:0", "style:personachat:test:788:self:7:positive:1", "style:personachat:test:788:self:7:positive:2", "style:personachat:test:788:self:7:positive:3", "style:personachat:test:788:self:7:positive:4", "style:personachat:test:788:self:7:positive:5", "style:personachat:test:788:self:7:positive:6"], "evidence_texts": ["A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "i just finished hiking and camping , how about you ?", "i have a lot of meat to grill up this weekend . any plans ?", "no but i love music as my truck has a great sound system", "i should get my father a pet since he is retired now .", "i will think about it as my mother is busy working in healthcare .", "yes it would . what do you do ?", "that sounds interesting . what are your hobbies ?"], "speaker_id": "snli:test:5558", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5560", "candidate_text": "well, there is treasure in the trench", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5560:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5560:premise", "snli:test:5173:premise", "snli:test:5789:premise", "snli:test:5080:premise", "style:personachat:test:210:self:1:positive:0"], "evidence_texts": ["A group of three people are digging a large trench in a yard while another person walks close by.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "army beats that . i am a wedding singer"], "speaker_id": "snli:test:5560", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5560", "candidate_text": "Honestly, There is treasure in the trench!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5560:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5560:premise", "snli:test:5173:premise", "snli:test:5789:premise", "snli:test:5080:premise", "style:personachat:test:566:self:6:positive:0", "style:personachat:test:566:self:6:positive:1", "style:personachat:test:566:self:6:positive:2", "style:personachat:test:566:self:6:positive:3", "style:personachat:test:566:self:6:positive:4", "style:personachat:test:566:self:6:positive:5"], "evidence_texts": ["A group of three people are digging a large trench in a yard while another person walks close by.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A young boy wearing a suit and top hat is holding hands with a girl in a green dress.", "me too ! where are you from ?", "not me ! ! i like to be alone !", "that is cool ! i am a writer ! where do you live ?", "i write stories for work ! romance usually !", "i do ! but i live on an island so there are not many stores around !", "yep ! i am single too ! wish i had a husband though ."], "speaker_id": "snli:test:5560", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5577", "candidate_text": "well, a man and a woman are jumping in the snow", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5577:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4450:premise", "snli:test:5577:premise", "snli:test:6016:premise", "snli:test:1784:premise", "style:personachat:test:210:self:2:positive:0", "style:personachat:test:210:self:2:positive:1"], "evidence_texts": ["The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "haha . youre funny . are you in the army ?", "army beats that . i am a wedding singer"], "speaker_id": "snli:test:5577", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5577", "candidate_text": "A man and a woman are jumping in the snow.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5577:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4450:premise", "snli:test:5577:premise", "snli:test:6016:premise", "snli:test:1784:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5577", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5589", "candidate_text": "well, the surfer is skateboarding", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5589:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5393:premise", "snli:test:5589:premise", "snli:test:5830:premise", "snli:test:5384:premise", "style:personachat:test:210:self:5:positive:0", "style:personachat:test:210:self:5:positive:1", "style:personachat:test:210:self:5:positive:2", "style:personachat:test:210:self:5:positive:3", "style:personachat:test:210:self:5:positive:4"], "evidence_texts": ["A person, in a striped blue shirt and pants, is running along.", "A surfer is riding a wave in a large body of water.", "A shirtless man with red paint on his torso holding a stick.", "A young, well-dressed black man seated behind a drum set and smiling.", "wow where did that happen ? things never happen like that in south carolina", "are you in trouble ?", "what caused this ? i seen a sick bar fight before i graduated college", "haha . youre funny . are you in the army ?", "army beats that . i am a wedding singer"], "speaker_id": "snli:test:5589", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5589", "candidate_text": "I wonder whether the surfer is skateboarding?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5589:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5393:premise", "snli:test:5589:premise", "snli:test:5830:premise", "snli:test:5384:premise", "style:personachat:test:789:self:5:positive:0", "style:personachat:test:789:self:5:positive:1", "style:personachat:test:789:self:5:positive:2", "style:personachat:test:789:self:5:positive:3", "style:personachat:test:789:self:5:positive:4"], "evidence_texts": ["A person, in a striped blue shirt and pants, is running along.", "A surfer is riding a wave in a large body of water.", "A shirtless man with red paint on his torso holding a stick.", "A young, well-dressed black man seated behind a drum set and smiling.", "enjoying a much needed night off with my pup . you ?", "cool beans . where are you working ?", "er nurse , and still not good enough for my dad lol .", "not really . no time with my schedule . you ?", "oh yeah . soccer is pretty fun though . do you play in a club ?"], "speaker_id": "snli:test:5589", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5594", "candidate_text": "well, the man is sitting in his living room", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5594:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5167:premise", "snli:test:5716:premise", "snli:test:5594:premise", "snli:test:5021:premise", "style:personachat:test:210:self:7:positive:0", "style:personachat:test:210:self:7:positive:1", "style:personachat:test:210:self:7:positive:2", "style:personachat:test:210:self:7:positive:3", "style:personachat:test:210:self:7:positive:4", "style:personachat:test:210:self:7:positive:5", "style:personachat:test:210:self:7:positive:6"], "evidence_texts": ["An old man holds up a sign.", "A football layer wearing a red shirt.", "A man in glasses in a basement.", "The cyclist rides on a wooded path.", "its going well . just got done with a singing gig . you ?", "so sorry to hear that . reminds me of a horror movie i just watched", "wow where did that happen ? things never happen like that in south carolina", "are you in trouble ?", "what caused this ? i seen a sick bar fight before i graduated college", "haha . youre funny . are you in the army ?", "army beats that . i am a wedding singer"], "speaker_id": "snli:test:5594", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5594", "candidate_text": "Honestly, The man is sitting in his living room!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5594:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5167:premise", "snli:test:5716:premise", "snli:test:5594:premise", "snli:test:5021:premise", "style:personachat:test:584:self:5:positive:0", "style:personachat:test:584:self:5:positive:1", "style:personachat:test:584:self:5:positive:2", "style:personachat:test:584:self:5:positive:3", "style:personachat:test:584:self:5:positive:4"], "evidence_texts": ["An old man holds up a sign.", "A football layer wearing a red shirt.", "A man in glasses in a basement.", "The cyclist rides on a wooded path.", "i am religious , but i am christian .", "that is sweet . do you play sports ?", "i play racquetball and listen to the oldies .", "i do like movies , and i hope to watch many when i retire .", "i love to eat ! candy is delicious and chicken !"], "speaker_id": "snli:test:5594", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5597", "candidate_text": "well, a man is jumping over rocks in order to escape being attacked by a wild boar", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5597:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5441:premise", "snli:test:5597:premise", "snli:test:6712:premise", "snli:test:4233:premise", "style:personachat:test:211:self:1:positive:0"], "evidence_texts": ["The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "you are never to young to be in love , my dear ."], "speaker_id": "snli:test:5597", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5597", "candidate_text": "A man is jumping over rocks in order to escape being attacked by a wild boar.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5597:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5441:premise", "snli:test:5597:premise", "snli:test:6712:premise", "snli:test:4233:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A man who wears glasses, a blue shirt and a Chicago White Sox cap holds a young boy who also wears a blue shirt.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5597", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5604", "candidate_text": "well, the girl sits at the kitchen table, decorating the christmas cookies while her mother looks on", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5604:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5604:premise", "snli:test:4914:premise", "snli:test:6271:premise", "snli:test:4867:premise", "style:personachat:test:211:self:2:positive:0", "style:personachat:test:211:self:2:positive:1"], "evidence_texts": ["A young girl running on grass", "Two boys in running clothes stretching.", "Two men work outside polishing shoes.", "People shopping at an outside market", "i was married 4 times , and lost three of them in accidents .", "you are never to young to be in love , my dear ."], "speaker_id": "snli:test:5604", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5604", "candidate_text": "I wonder whether the girl sits at the kitchen table, decorating the christmas cookies while her mother looks on?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5604:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5604:premise", "snli:test:4914:premise", "snli:test:6271:premise", "snli:test:4867:premise", "style:personachat:test:790:self:2:positive:0", "style:personachat:test:790:self:2:positive:1"], "evidence_texts": ["A young girl running on grass", "Two boys in running clothes stretching.", "Two men work outside polishing shoes.", "People shopping at an outside market", "lol have you ever done a reading for them or yourself ?", "lol how many do you do daily ? or it is not a limit ?"], "speaker_id": "snli:test:5604", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5610", "candidate_text": "well, a person crossing a busy street intersection", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5610:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5490:premise", "snli:test:6260:premise", "snli:test:5610:premise", "snli:test:5099:premise", "style:personachat:test:211:self:5:positive:0", "style:personachat:test:211:self:5:positive:1", "style:personachat:test:211:self:5:positive:2", "style:personachat:test:211:self:5:positive:3", "style:personachat:test:211:self:5:positive:4"], "evidence_texts": ["A couple walking their dog.", "Two men shining peoples shoes.", "pedestrian walking on the street", "A child hugs a birdhouse.", "are you single ? i can fake a british accent if it makes me more attractive .", "as long as you do not bring any mangoes . i am seriously allergic .", "not sure what you tried to say there . have you been married ?", "i was married 4 times , and lost three of them in accidents .", "you are never to young to be in love , my dear ."], "speaker_id": "snli:test:5610", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5610", "candidate_text": "Honestly, A person crossing a busy street intersection!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5610:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5490:premise", "snli:test:6260:premise", "snli:test:5610:premise", "snli:test:5099:premise", "style:personachat:test:587:self:2:positive:0", "style:personachat:test:587:self:2:positive:1"], "evidence_texts": ["A couple walking their dog.", "Two men shining peoples shoes.", "pedestrian walking on the street", "A child hugs a birdhouse.", "my sister lives with me . i also have a dog .", "it is ! what do you look like ?"], "speaker_id": "snli:test:5610", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5614", "candidate_text": "well, the people are waiting at their table for the waitress to bring their food", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5614:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5553:premise", "snli:test:5628:premise", "snli:test:5614:premise", "snli:test:5530:premise", "style:personachat:test:211:self:7:positive:0", "style:personachat:test:211:self:7:positive:1", "style:personachat:test:211:self:7:positive:2", "style:personachat:test:211:self:7:positive:3", "style:personachat:test:211:self:7:positive:4", "style:personachat:test:211:self:7:positive:5", "style:personachat:test:211:self:7:positive:6"], "evidence_texts": ["Two men scrimmage in soccer, as a referee looks on.", "A group of people traveling in a small wooden boat.", "People ordering and picking up food at a restaurant window.", "Two girls wearing hats are running through snow drifts outside.", "you can bring ralph over . i am a dog stylist . eccentric is what i am .", "no , i do like to collect civil war antiques . i put alot of time into that .", "are you single ? i can fake a british accent if it makes me more attractive .", "as long as you do not bring any mangoes . i am seriously allergic .", "not sure what you tried to say there . have you been married ?", "i was married 4 times , and lost three of them in accidents .", "you are never to young to be in love , my dear ."], "speaker_id": "snli:test:5614", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5614", "candidate_text": "The people are waiting at their table for the waitress to bring their food.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5614:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5553:premise", "snli:test:5628:premise", "snli:test:5614:premise", "snli:test:5530:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men scrimmage in soccer, as a referee looks on.", "A group of people traveling in a small wooden boat.", "People ordering and picking up food at a restaurant window.", "Two girls wearing hats are running through snow drifts outside.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5614", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5628", "candidate_text": "Honestly, A group of people on a sunken boat!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5628:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5628:premise", "snli:test:5614:premise", "snli:test:5773:premise", "snli:test:5553:premise", "style:personachat:test:212:self:1:positive:0"], "evidence_texts": ["A group of people traveling in a small wooden boat.", "People ordering and picking up food at a restaurant window.", "A young male sitting in front of a computer monitor", "Two men scrimmage in soccer, as a referee looks on.", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:5628", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5628", "candidate_text": "I wonder whether a group of people on a sunken boat?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5628:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5628:premise", "snli:test:5614:premise", "snli:test:5773:premise", "snli:test:5553:premise", "style:personachat:test:793:self:1:positive:0"], "evidence_texts": ["A group of people traveling in a small wooden boat.", "People ordering and picking up food at a restaurant window.", "A young male sitting in front of a computer monitor", "Two men scrimmage in soccer, as a referee looks on.", "no , since i just graduated i have not had time to adopt any . you ?"], "speaker_id": "snli:test:5628", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5648", "candidate_text": "Honestly, The band is playing punk rock music!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5648:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5648:premise", "snli:test:5527:premise", "snli:test:5681:premise", "snli:test:5524:premise", "style:personachat:test:212:self:2:positive:0", "style:personachat:test:212:self:2:positive:1"], "evidence_texts": ["A four piece band plays a song on a medium-size stage.", "A person stands between two giant statues that look like faces.", "a teenager laying down with paper crafts strewn about the floor", "A child enjoying themselves on a yellow slide at a park.", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:5648", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5648", "candidate_text": "well, the band is playing punk rock music", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5648:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5648:premise", "snli:test:5527:premise", "snli:test:5681:premise", "snli:test:5524:premise", "style:personachat:test:356:self:2:positive:0", "style:personachat:test:356:self:2:positive:1"], "evidence_texts": ["A four piece band plays a song on a medium-size stage.", "A person stands between two giant statues that look like faces.", "a teenager laying down with paper crafts strewn about the floor", "A child enjoying themselves on a yellow slide at a park.", "i have visited several times , i prefer hawaii", "sorry , no . i only date those in my profession ."], "speaker_id": "snli:test:5648", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5652", "candidate_text": "Honestly, People in costumes!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5652:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5652:premise", "snli:test:5277:premise", "snli:test:5770:premise", "snli:test:5074:premise", "style:personachat:test:212:self:5:positive:0", "style:personachat:test:212:self:5:positive:1", "style:personachat:test:212:self:5:positive:2", "style:personachat:test:212:self:5:positive:3", "style:personachat:test:212:self:5:positive:4"], "evidence_texts": ["Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "what is your favorite color ? mine is purple .", "i do see things differently ! what do you do for work ?", "i help people everyday in a hospital .", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:5652", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5652", "candidate_text": "People in costumes.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5652:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5652:premise", "snli:test:5277:premise", "snli:test:5770:premise", "snli:test:5074:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "A young ladies squatting at hillside sipping on a drink the other one with the white purse is talking pictures.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5652", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5665", "candidate_text": "Honestly, There is one female!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5665:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5516:premise", "snli:test:5665:premise", "snli:test:5412:premise", "snli:test:5672:premise", "style:personachat:test:212:self:6:positive:0", "style:personachat:test:212:self:6:positive:1", "style:personachat:test:212:self:6:positive:2", "style:personachat:test:212:self:6:positive:3", "style:personachat:test:212:self:6:positive:4", "style:personachat:test:212:self:6:positive:5"], "evidence_texts": ["A person standing next to an old dark building.", "One female and two male musicians holding musical equipment.", "A water scene with a sunset in the background.", "A big dog catches a ball on his nose", "omg ! i love one tree hill", "what is your favorite color ? mine is purple .", "i do see things differently ! what do you do for work ?", "i help people everyday in a hospital .", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:5665", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5665", "candidate_text": "I wonder whether there is one female?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5665:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5516:premise", "snli:test:5665:premise", "snli:test:5412:premise", "snli:test:5672:premise", "style:personachat:test:793:self:7:positive:0", "style:personachat:test:793:self:7:positive:1", "style:personachat:test:793:self:7:positive:2", "style:personachat:test:793:self:7:positive:3", "style:personachat:test:793:self:7:positive:4", "style:personachat:test:793:self:7:positive:5", "style:personachat:test:793:self:7:positive:6"], "evidence_texts": ["A person standing next to an old dark building.", "One female and two male musicians holding musical equipment.", "A water scene with a sunset in the background.", "A big dog catches a ball on his nose", "love it . that is the one thing i miss since graduating college . you ?", "nursing . it was super hard because i spent a lot of time playing soccer too .", "yes , i was great . do you play sports ?", "how cool . i wish i had time . i am stuck in an office all day", "hopefully . whats your name ?", "lucy ! like the tv show . nice to meet you", "no , since i just graduated i have not had time to adopt any . you ?"], "speaker_id": "snli:test:5665", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5667", "candidate_text": "well, they are musicans", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5667:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5672:premise", "snli:test:5516:premise", "snli:test:5688:premise", "snli:test:5667:premise", "style:personachat:test:213:self:1:positive:0"], "evidence_texts": ["A big dog catches a ball on his nose", "A person standing next to an old dark building.", "Man with an orange bandanna is playing the flute.", "One female and two male musicians holding musical equipment.", "we like to do different things do you have pets"], "speaker_id": "snli:test:5667", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5667", "candidate_text": "Honestly, They are musicans!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5667:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5672:premise", "snli:test:5516:premise", "snli:test:5688:premise", "snli:test:5667:premise", "style:personachat:test:588:self:7:positive:0", "style:personachat:test:588:self:7:positive:1", "style:personachat:test:588:self:7:positive:2", "style:personachat:test:588:self:7:positive:3", "style:personachat:test:588:self:7:positive:4", "style:personachat:test:588:self:7:positive:5", "style:personachat:test:588:self:7:positive:6"], "evidence_texts": ["A big dog catches a ball on his nose", "A person standing next to an old dark building.", "Man with an orange bandanna is playing the flute.", "One female and two male musicians holding musical equipment.", "i actually was not able to get an elk tag this season .", "i might go and see metallica with the money instead .", "what kind of opportunity do you have ?", "i would be willing to help ! i also have management experience from my job .", "nope , i am not allergic to anything ! you ?", "i love doing anything outside , especially in the snow ! what about you ?", "those are pretty cool hobbies ! do you have any siblings ?"], "speaker_id": "snli:test:5667", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5669", "candidate_text": "well, a group of people posing in front of a body of water", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5669:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5448:premise", "snli:test:5669:premise", "snli:test:5705:premise", "snli:test:5419:premise", "style:personachat:test:213:self:2:positive:0", "style:personachat:test:213:self:2:positive:1"], "evidence_texts": ["Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A group of people taking pictures on a walkway in front of a large body of water.", "A woman standing with smile on her face and she is holding a wood in her hand", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "i like to hang out with friends", "we like to do different things do you have pets"], "speaker_id": "snli:test:5669", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5669", "candidate_text": "A group of people posing in front of a body of water.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5669:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5448:premise", "snli:test:5669:premise", "snli:test:5705:premise", "snli:test:5419:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "A group of people taking pictures on a walkway in front of a large body of water.", "A woman standing with smile on her face and she is holding a wood in her hand", "A blond woman wearing a pearl bracelet is about to kiss a man wearing a white shirt.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5669", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5672", "candidate_text": "well, a puppy is rolling on the ground", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5672:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5672:premise", "snli:test:5667:premise", "snli:test:5688:premise", "snli:test:5665:premise", "style:personachat:test:213:self:5:positive:0", "style:personachat:test:213:self:5:positive:1", "style:personachat:test:213:self:5:positive:2", "style:personachat:test:213:self:5:positive:3", "style:personachat:test:213:self:5:positive:4"], "evidence_texts": ["A big dog catches a ball on his nose", "One female and two male musicians holding musical equipment.", "Man with an orange bandanna is playing the flute.", "One female and two male musicians holding musical equipment.", "that is nice i teach kindergarten", "just two years its rewarding but the pay is low", "yeah that is definitely a bonus", "i like to hang out with friends", "we like to do different things do you have pets"], "speaker_id": "snli:test:5672", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5672", "candidate_text": "I wonder whether a puppy is rolling on the ground?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5672:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5672:premise", "snli:test:5667:premise", "snli:test:5688:premise", "snli:test:5665:premise", "style:personachat:test:794:self:5:positive:0", "style:personachat:test:794:self:5:positive:1", "style:personachat:test:794:self:5:positive:2", "style:personachat:test:794:self:5:positive:3", "style:personachat:test:794:self:5:positive:4"], "evidence_texts": ["A big dog catches a ball on his nose", "One female and two male musicians holding musical equipment.", "Man with an orange bandanna is playing the flute.", "One female and two male musicians holding musical equipment.", "how fun ! i grew up on a farm so i love animals . where did you grow up ?", "do you work in the fitness industry ?", "interesting ! did you go to college for that ?", "oh . i graduated from usc .", "no . do you have pets other than your cats ?"], "speaker_id": "snli:test:5672", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5681", "candidate_text": "well, the teen is laying down in a mess", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5681:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5648:premise", "snli:test:5681:premise", "snli:test:5709:premise", "snli:test:5527:premise", "style:personachat:test:213:self:6:positive:0", "style:personachat:test:213:self:6:positive:1", "style:personachat:test:213:self:6:positive:2", "style:personachat:test:213:self:6:positive:3", "style:personachat:test:213:self:6:positive:4", "style:personachat:test:213:self:6:positive:5"], "evidence_texts": ["A four piece band plays a song on a medium-size stage.", "a teenager laying down with paper crafts strewn about the floor", "A young boy hanging on a pole smiling at the camera.", "A person stands between two giant statues that look like faces.", "i am good just getting back from work", "that is nice i teach kindergarten", "just two years its rewarding but the pay is low", "yeah that is definitely a bonus", "i like to hang out with friends", "we like to do different things do you have pets"], "speaker_id": "snli:test:5681", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5681", "candidate_text": "Honestly, The teen is laying down in a mess!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5681:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5648:premise", "snli:test:5681:premise", "snli:test:5709:premise", "snli:test:5527:premise", "style:personachat:test:593:self:5:positive:0", "style:personachat:test:593:self:5:positive:1", "style:personachat:test:593:self:5:positive:2", "style:personachat:test:593:self:5:positive:3", "style:personachat:test:593:self:5:positive:4"], "evidence_texts": ["A four piece band plays a song on a medium-size stage.", "a teenager laying down with paper crafts strewn about the floor", "A young boy hanging on a pole smiling at the camera.", "A person stands between two giant statues that look like faces.", "oh ? for yourself or someone else ?", "wow congrats ! i worked at a carny through college", "i love my sister . but her dog terrifies me .", "she has a mean terrier ! and maybe .", "the dog is a wire haired terrier and he is terrifying ! most dogs scare me ."], "speaker_id": "snli:test:5681", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5688", "candidate_text": "well, sad man with an orange bandanna is playing the flute", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5688:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5688:premise", "snli:test:5672:premise", "snli:test:5785:premise", "snli:test:5667:premise", "style:personachat:test:214:self:1:positive:0"], "evidence_texts": ["Man with an orange bandanna is playing the flute.", "A big dog catches a ball on his nose", "A man with a cigarette sits on the stoop.", "One female and two male musicians holding musical equipment.", "thank you and its the most amazing feeling in the world"], "speaker_id": "snli:test:5688", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5688", "candidate_text": "Sad Man with an orange bandanna is playing the flute.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5688:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5688:premise", "snli:test:5672:premise", "snli:test:5785:premise", "snli:test:5667:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Man with an orange bandanna is playing the flute.", "A big dog catches a ball on his nose", "A man with a cigarette sits on the stoop.", "One female and two male musicians holding musical equipment.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5688", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5705", "candidate_text": "well, a woman stands holding her baby and crying", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5705:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5669:premise", "snli:test:5705:premise", "snli:test:5725:premise", "snli:test:5448:premise", "style:personachat:test:214:self:2:positive:0", "style:personachat:test:214:self:2:positive:1"], "evidence_texts": ["A group of people taking pictures on a walkway in front of a large body of water.", "A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "i just got married recently , my wife had our daughter a few months ago", "thank you and its the most amazing feeling in the world"], "speaker_id": "snli:test:5705", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5705", "candidate_text": "I wonder whether a woman stands holding her baby and crying?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5705:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5669:premise", "snli:test:5705:premise", "snli:test:5725:premise", "snli:test:5448:premise", "style:personachat:test:806:self:2:positive:0", "style:personachat:test:806:self:2:positive:1"], "evidence_texts": ["A group of people taking pictures on a walkway in front of a large body of water.", "A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "Two young girls with long brunette hair, standing in front of a stone wall playing a game.", "have two cats . do you travel a lot ?", "italian food is good . your favorite dish ?"], "speaker_id": "snli:test:5705", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5709", "candidate_text": "well, the young boy is smiling", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5709:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5681:premise", "snli:test:5709:premise", "snli:test:6035:premise", "snli:test:5648:premise", "style:personachat:test:214:self:5:positive:0", "style:personachat:test:214:self:5:positive:1", "style:personachat:test:214:self:5:positive:2", "style:personachat:test:214:self:5:positive:3", "style:personachat:test:214:self:5:positive:4"], "evidence_texts": ["a teenager laying down with paper crafts strewn about the floor", "A young boy hanging on a pole smiling at the camera.", "Three people on skis are standing behind a no skiing sign.", "A four piece band plays a song on a medium-size stage.", "i might do that do you have any animals ?", "oh that is understandable i do not know what i would do if i did not have my dog ginger", "i know how you feel , i build model cars", "i just got married recently , my wife had our daughter a few months ago", "thank you and its the most amazing feeling in the world"], "speaker_id": "snli:test:5709", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5709", "candidate_text": "Honestly, The young boy is smiling!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5709:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5681:premise", "snli:test:5709:premise", "snli:test:6035:premise", "snli:test:5648:premise", "style:personachat:test:604:self:2:positive:0", "style:personachat:test:604:self:2:positive:1"], "evidence_texts": ["a teenager laying down with paper crafts strewn about the floor", "A young boy hanging on a pole smiling at the camera.", "Three people on skis are standing behind a no skiing sign.", "A four piece band plays a song on a medium-size stage.", "very true . what else do you like to do ?", "i m sure it is ! we are our worst critic after all ."], "speaker_id": "snli:test:5709", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5716", "candidate_text": "well, a built man wearing a tshirt", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5716:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5716:premise", "snli:test:5594:premise", "snli:test:5907:premise", "snli:test:5167:premise", "style:personachat:test:214:self:7:positive:0", "style:personachat:test:214:self:7:positive:1", "style:personachat:test:214:self:7:positive:2", "style:personachat:test:214:self:7:positive:3", "style:personachat:test:214:self:7:positive:4", "style:personachat:test:214:self:7:positive:5", "style:personachat:test:214:self:7:positive:6"], "evidence_texts": ["A football layer wearing a red shirt.", "A man in glasses in a basement.", "A girl is blowing a huge bubble.", "An old man holds up a sign.", "hey i am good how about you", "i never heard of them actually", "i might do that do you have any animals ?", "oh that is understandable i do not know what i would do if i did not have my dog ginger", "i know how you feel , i build model cars", "i just got married recently , my wife had our daughter a few months ago", "thank you and its the most amazing feeling in the world"], "speaker_id": "snli:test:5716", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5716", "candidate_text": "A built man wearing a tshirt.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5716:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5716:premise", "snli:test:5594:premise", "snli:test:5907:premise", "snli:test:5167:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A football layer wearing a red shirt.", "A man in glasses in a basement.", "A girl is blowing a huge bubble.", "An old man holds up a sign.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5716", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5722", "candidate_text": "well, a young african child playing soccer", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5722:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5429:premise", "snli:test:5818:premise", "snli:test:5722:premise", "snli:test:5278:premise", "style:personachat:test:215:self:1:positive:0"], "evidence_texts": ["an asian man in a black tracksuit is watching students doing exercised on basketball court.", "Child plays with pigeon in busy public area while group of people watch him play.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "i will learn about that one day . i am more interested in ice currently ."], "speaker_id": "snli:test:5722", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5722", "candidate_text": "I wonder whether a young african child playing soccer?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5722:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5429:premise", "snli:test:5818:premise", "snli:test:5722:premise", "snli:test:5278:premise", "style:personachat:test:807:self:1:positive:0"], "evidence_texts": ["an asian man in a black tracksuit is watching students doing exercised on basketball court.", "Child plays with pigeon in busy public area while group of people watch him play.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A young man standing on one foot in colorful bowling shoes after throwing the ball.", "1 younger brother . you guys get along alright ?"], "speaker_id": "snli:test:5722", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5725", "candidate_text": "well, many visitors have fun at the latin celebration", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5725:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5705:premise", "snli:test:5725:premise", "snli:test:5669:premise", "snli:test:6218:premise", "style:personachat:test:215:self:2:positive:0", "style:personachat:test:215:self:2:positive:1"], "evidence_texts": ["A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of people taking pictures on a walkway in front of a large body of water.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "if you ever have a question about rocks , i am your person . i used to teach .", "i will learn about that one day . i am more interested in ice currently ."], "speaker_id": "snli:test:5725", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5725", "candidate_text": "Honestly, Many visitors have fun at the Latin celebration!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5725:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5705:premise", "snli:test:5725:premise", "snli:test:5669:premise", "snli:test:6218:premise", "style:personachat:test:609:self:1:positive:0"], "evidence_texts": ["A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of people taking pictures on a walkway in front of a large body of water.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "well , fishing as well ! except i am not allowed to eat shellfish"], "speaker_id": "snli:test:5725", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5726", "candidate_text": "well, the city is completely devoid of people", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5726:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6218:premise", "snli:test:5705:premise", "snli:test:5726:premise", "snli:test:6219:premise", "style:personachat:test:215:self:5:positive:0", "style:personachat:test:215:self:5:positive:1", "style:personachat:test:215:self:5:positive:2", "style:personachat:test:215:self:5:positive:3", "style:personachat:test:215:self:5:positive:4"], "evidence_texts": ["A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "fun ! we must live in totally different climates . i have been out fishing all day .", "i mostly use the animals for food . polar bears taste really good and are fun to hunt .", "a big white bear that eats meat . wow penguins ! we have those around here .", "if you ever have a question about rocks , i am your person . i used to teach .", "i will learn about that one day . i am more interested in ice currently ."], "speaker_id": "snli:test:5726", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5726", "candidate_text": "The city is completely devoid of people.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5726:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6218:premise", "snli:test:5705:premise", "snli:test:5726:premise", "snli:test:6219:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A woman standing with smile on her face and she is holding a wood in her hand", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5726", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5732", "candidate_text": "well, bride running away from her wedding", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5732:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5732:premise", "snli:test:5351:premise", "snli:test:7110:premise", "snli:test:1634:premise", "style:personachat:test:215:self:6:positive:0", "style:personachat:test:215:self:6:positive:1", "style:personachat:test:215:self:6:positive:2", "style:personachat:test:215:self:6:positive:3", "style:personachat:test:215:self:6:positive:4", "style:personachat:test:215:self:6:positive:5"], "evidence_texts": ["A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "good morning to you ! no , i ice skate to college .", "fun ! we must live in totally different climates . i have been out fishing all day .", "i mostly use the animals for food . polar bears taste really good and are fun to hunt .", "a big white bear that eats meat . wow penguins ! we have those around here .", "if you ever have a question about rocks , i am your person . i used to teach .", "i will learn about that one day . i am more interested in ice currently ."], "speaker_id": "snli:test:5732", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5732", "candidate_text": "I wonder whether bride running away from her wedding?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5732:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5732:premise", "snli:test:5351:premise", "snli:test:7110:premise", "snli:test:1634:premise", "style:personachat:test:807:self:7:positive:0", "style:personachat:test:807:self:7:positive:1", "style:personachat:test:807:self:7:positive:2", "style:personachat:test:807:self:7:positive:3", "style:personachat:test:807:self:7:positive:4", "style:personachat:test:807:self:7:positive:5", "style:personachat:test:807:self:7:positive:6"], "evidence_texts": ["A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A young boy sits outside in a grassy field with his back turned towards the viewer- he is holding a wooden toy that can be spun between your hands and launched into the air.", "getting ready for bed it is almost 900 here . what are you doing ?", "i used to play the drums back in my day . what do you play ?", "i loved a singer once back in afghanistan .", "i was a military man . stationed there 2 years .", "great way to serve . i would still be active if i had not lost my leg .", "thank you for that . you are correct . i get along without it .", "1 younger brother . you guys get along alright ?"], "speaker_id": "snli:test:5732", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5744", "candidate_text": "Honestly, Both people fell off their bike!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5744:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5744:premise", "snli:test:5205:premise", "snli:test:4953:premise", "snli:test:6164:premise", "style:personachat:test:216:self:1:positive:0"], "evidence_texts": ["A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:5744", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5744", "candidate_text": "well, both people fell off their bike", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5744:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5744:premise", "snli:test:5205:premise", "snli:test:4953:premise", "snli:test:6164:premise", "style:personachat:test:361:self:1:positive:0"], "evidence_texts": ["A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "Blond-haired girl wearing a bicycle helmet, a black t-shirt that reads \"Twizzlers\" and a printed dog tag necklace around her neck.", "A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "i love the fall also ."], "speaker_id": "snli:test:5744", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5745", "candidate_text": "Honestly, A person fell off their bike!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5745:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6164:premise", "snli:test:5745:premise", "snli:test:5205:premise", "snli:test:6221:premise", "style:personachat:test:216:self:2:positive:0", "style:personachat:test:216:self:2:positive:1"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:5745", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5745", "candidate_text": "A person fell off their bike.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5745:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6164:premise", "snli:test:5745:premise", "snli:test:5205:premise", "snli:test:6221:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "Two women look at a table with assorted items on it in some sort of market that others are currently browsing.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5745", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5756", "candidate_text": "Honestly, A male is surrounded with other human beings having a large snake around his chest!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5756:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5756:premise", "snli:test:5220:premise", "snli:test:5891:premise", "snli:test:4920:premise", "style:personachat:test:216:self:5:positive:0", "style:personachat:test:216:self:5:positive:1", "style:personachat:test:216:self:5:positive:2", "style:personachat:test:216:self:5:positive:3", "style:personachat:test:216:self:5:positive:4"], "evidence_texts": ["A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "sorry to hear that ! must be hard", "i am hoping retire soon maybe i can write then", "construction , my left leg hurts every day", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:5756", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5756", "candidate_text": "I wonder whether a male is surrounded with other human beings having a large snake around his chest?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5756:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5756:premise", "snli:test:5220:premise", "snli:test:5891:premise", "snli:test:4920:premise", "style:personachat:test:809:self:5:positive:0", "style:personachat:test:809:self:5:positive:1", "style:personachat:test:809:self:5:positive:2", "style:personachat:test:809:self:5:positive:3", "style:personachat:test:809:self:5:positive:4"], "evidence_texts": ["A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "a soccer player wearing a red uniform walks off the field after a game near a parking lot.", "whoa big money , i have been at the same job for years", "i enjoy onions , they are my favorite , what is your favorite food", "yeah italian is great , tell me something else interesting about yourself .", "i do not like my family or my dad , he is a drunk", "so do you like your family ?"], "speaker_id": "snli:test:5756", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5767", "candidate_text": "Honestly, A man is holding a leash on someone else dog!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5767:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5433:premise", "snli:test:5899:premise", "snli:test:5247:premise", "snli:test:5767:premise", "style:personachat:test:216:self:7:positive:0", "style:personachat:test:216:self:7:positive:1", "style:personachat:test:216:self:7:positive:2", "style:personachat:test:216:self:7:positive:3", "style:personachat:test:216:self:7:positive:4", "style:personachat:test:216:self:7:positive:5", "style:personachat:test:216:self:7:positive:6"], "evidence_texts": ["Four men wearing orange vests are putting in a steel beam on the ground.", "A woman is holding a microphone in one hand and her mouth is open.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "A dog with a purple leash is held by a woman wearing white shoes.", "i am good . yikes i hate spiders too", "nope and no kids either . i love my horses like kids", "sorry to hear that ! must be hard", "i am hoping retire soon maybe i can write then", "construction , my left leg hurts every day", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:5767", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5767", "candidate_text": "well, a man is holding a leash on someone else dog", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5767:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5433:premise", "snli:test:5899:premise", "snli:test:5247:premise", "snli:test:5767:premise", "style:personachat:test:361:self:7:positive:0", "style:personachat:test:361:self:7:positive:1", "style:personachat:test:361:self:7:positive:2", "style:personachat:test:361:self:7:positive:3", "style:personachat:test:361:self:7:positive:4", "style:personachat:test:361:self:7:positive:5", "style:personachat:test:361:self:7:positive:6"], "evidence_texts": ["Four men wearing orange vests are putting in a steel beam on the ground.", "A woman is holding a microphone in one hand and her mouth is open.", "A woman is walking with a full shopping cart away from a Trader Joe's.", "A dog with a purple leash is held by a woman wearing white shoes.", "not really . i like to study , usually biology at the university .", "so you understand ! i love to study ! i am not much of a drinker .", "so you want to be a nurse ? i would like to be a marine biologist .", "oh ok . are you from the usa ?", "i am also , i live in california .", "it is really nice . do you like michigan ? i guess it gets cold there ?", "i love the fall also ."], "speaker_id": "snli:test:5767", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5770", "candidate_text": "I wonder whether she talks to her boyfriend about plans that night?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5770:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5652:premise", "snli:test:5277:premise", "snli:test:5770:premise", "snli:test:5990:premise", "style:personachat:test:217:self:1:positive:0"], "evidence_texts": ["Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:5770", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5770", "candidate_text": "Honestly, She talks to her boyfriend about plans that night!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5770:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5652:premise", "snli:test:5277:premise", "snli:test:5770:premise", "snli:test:5990:premise", "style:personachat:test:614:self:6:positive:0", "style:personachat:test:614:self:6:positive:1", "style:personachat:test:614:self:6:positive:2", "style:personachat:test:614:self:6:positive:3", "style:personachat:test:614:self:6:positive:4", "style:personachat:test:614:self:6:positive:5"], "evidence_texts": ["Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "Four youth and an elder man and woman playing in water just up to the bottom of man's swim trunks.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "hey there ! a little tired , just finished my mile walk , you ?", "i am about to do the same , curl up with my siamese cat", "almost one am , what bout you ?", "southern ish area of texas .", "what is the weather like there ?", "switch places with me ! it is too hot and sunny here"], "speaker_id": "snli:test:5770", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5772", "candidate_text": "I wonder whether she has a conversation on her phone outdoors?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5772:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5772:premise", "snli:test:5990:premise", "snli:test:5652:premise", "snli:test:6002:premise", "style:personachat:test:217:self:2:positive:0", "style:personachat:test:217:self:2:positive:1"], "evidence_texts": ["A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "what kinds of things do you like to build ?", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:5772", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5772", "candidate_text": "She has a conversation on her phone outdoors.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5772:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5772:premise", "snli:test:5990:premise", "snli:test:5652:premise", "snli:test:6002:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "Two men in costumes with fake carrot noses, top hats, sunglasses and white fur coats that contain white electrical lights.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5772", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5773", "candidate_text": "I wonder whether someone is at the movies with their friends?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5773:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5628:premise", "snli:test:5773:premise", "snli:test:5821:premise", "snli:test:5614:premise", "style:personachat:test:217:self:5:positive:0", "style:personachat:test:217:self:5:positive:1", "style:personachat:test:217:self:5:positive:2", "style:personachat:test:217:self:5:positive:3", "style:personachat:test:217:self:5:positive:4"], "evidence_texts": ["A group of people traveling in a small wooden boat.", "A young male sitting in front of a computer monitor", "Two men are assisting a small girl in a harness.", "People ordering and picking up food at a restaurant window.", "oh , i love to sing in the shower country mostly . what music do you like ?", "oh , my husband alex likes jazz .", "no , not really . i only dance if i drink . mostly , i stick to iced tea , though .", "what kinds of things do you like to build ?", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:5773", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5773", "candidate_text": "well, someone is at the movies with their friends", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5773:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5628:premise", "snli:test:5773:premise", "snli:test:5821:premise", "snli:test:5614:premise", "style:personachat:test:362:self:5:positive:0", "style:personachat:test:362:self:5:positive:1", "style:personachat:test:362:self:5:positive:2", "style:personachat:test:362:self:5:positive:3", "style:personachat:test:362:self:5:positive:4"], "evidence_texts": ["A group of people traveling in a small wooden boat.", "A young male sitting in front of a computer monitor", "Two men are assisting a small girl in a harness.", "People ordering and picking up food at a restaurant window.", "yes as long as the campsite is not near lakes or rivers ! do you ?", "yes , ever since i was a kid . what about you ? any crazy phobias ?", "that is good ! what are you doing right now ? i am watching the game", "14 10 . my boys the ohio state buckeyes are up by 4 ! what music ?", "i love prince . especially purple rain . i love prince and purple so it works out"], "speaker_id": "snli:test:5773", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5779", "candidate_text": "I wonder whether the older men are sitting inside drinking tea?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5779:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6441:premise", "snli:test:5556:premise", "snli:test:734:premise", "snli:test:5779:premise", "style:personachat:test:217:self:7:positive:0", "style:personachat:test:217:self:7:positive:1", "style:personachat:test:217:self:7:positive:2", "style:personachat:test:217:self:7:positive:3", "style:personachat:test:217:self:7:positive:4", "style:personachat:test:217:self:7:positive:5", "style:personachat:test:217:self:7:positive:6"], "evidence_texts": ["Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "hi john . jessie here . i am from canada . where are you from ?", "just recently to my high school sweetheart . we are both 19 .", "oh , i love to sing in the shower country mostly . what music do you like ?", "oh , my husband alex likes jazz .", "no , not really . i only dance if i drink . mostly , i stick to iced tea , though .", "what kinds of things do you like to build ?", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:5779", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5779", "candidate_text": "Honestly, The older men are sitting inside drinking tea!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5779:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6441:premise", "snli:test:5556:premise", "snli:test:734:premise", "snli:test:5779:premise", "style:personachat:test:615:self:5:positive:0", "style:personachat:test:615:self:5:positive:1", "style:personachat:test:615:self:5:positive:2", "style:personachat:test:615:self:5:positive:3", "style:personachat:test:615:self:5:positive:4"], "evidence_texts": ["Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "lol . but for real , what does she do ?", "yes , my favorite is rap . you ?", "that is awesome ! what is your favorite food ?", "i was going to suggest bringing vegan sushi my fave to our rock out session . xd", "get some beef jerky or some wings for you too . but i am not touching it !"], "speaker_id": "snli:test:5779", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5780", "candidate_text": "I wonder whether the older gentlemen are outside?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5780:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5780:premise", "snli:test:734:premise", "snli:test:6441:premise", "snli:test:6938:premise", "style:personachat:test:218:self:1:positive:0"], "evidence_texts": ["Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "do you enjoy what you do ?"], "speaker_id": "snli:test:5780", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5780", "candidate_text": "The older gentlemen are outside.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5780:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5780:premise", "snli:test:734:premise", "snli:test:6441:premise", "snli:test:6938:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5780", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5785", "candidate_text": "I wonder whether a man rolls a cigarette and lights it?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5785:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5688:premise", "snli:test:5785:premise", "snli:test:5911:premise", "snli:test:5672:premise", "style:personachat:test:218:self:2:positive:0", "style:personachat:test:218:self:2:positive:1"], "evidence_texts": ["Man with an orange bandanna is playing the flute.", "A man with a cigarette sits on the stoop.", "A female camera operator is capturing some off-camera activity.", "A big dog catches a ball on his nose", "i think about that too . bums me out", "do you enjoy what you do ?"], "speaker_id": "snli:test:5785", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5785", "candidate_text": "well, a man rolls a cigarette and lights it", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5785:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5688:premise", "snli:test:5785:premise", "snli:test:5911:premise", "snli:test:5672:premise", "style:personachat:test:367:self:2:positive:0", "style:personachat:test:367:self:2:positive:1"], "evidence_texts": ["Man with an orange bandanna is playing the flute.", "A man with a cigarette sits on the stoop.", "A female camera operator is capturing some off-camera activity.", "A big dog catches a ball on his nose", "no i am 40 but i played games when i was a kid . i still have my atari", "they are good . they like staying with me . they said i am a lot cooler ."], "speaker_id": "snli:test:5785", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5789", "candidate_text": "I wonder whether the man is flying a plane?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5789:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5560:premise", "snli:test:5789:premise", "snli:test:5811:premise", "snli:test:5173:premise", "style:personachat:test:218:self:5:positive:0", "style:personachat:test:218:self:5:positive:1", "style:personachat:test:218:self:5:positive:2", "style:personachat:test:218:self:5:positive:3", "style:personachat:test:218:self:5:positive:4"], "evidence_texts": ["A group of three people are digging a large trench in a yard while another person walks close by.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "they sure are . what do you do for fun", "music is key to life . remember that .", "thanks . i worry about the environment sometimes too", "i think about that too . bums me out", "do you enjoy what you do ?"], "speaker_id": "snli:test:5789", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5789", "candidate_text": "Honestly, the man is flying a plane!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5789:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5560:premise", "snli:test:5789:premise", "snli:test:5811:premise", "snli:test:5173:premise", "style:personachat:test:619:self:2:positive:0", "style:personachat:test:619:self:2:positive:1"], "evidence_texts": ["A group of three people are digging a large trench in a yard while another person walks close by.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "Bearded black dressed male performer is displaying his skill by swinging a fire line in each hand in arcs.", "cool . you just said you got done playing ?", "wow medical marvel ! fastest surgery ever"], "speaker_id": "snli:test:5789", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5811", "candidate_text": "I wonder whether a woman is lost and looks at directions mounted on the wall?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5811:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5789:premise", "snli:test:6805:premise", "snli:test:5560:premise", "snli:test:5811:premise", "style:personachat:test:218:self:7:positive:0", "style:personachat:test:218:self:7:positive:1", "style:personachat:test:218:self:7:positive:2", "style:personachat:test:218:self:7:positive:3", "style:personachat:test:218:self:7:positive:4", "style:personachat:test:218:self:7:positive:5", "style:personachat:test:218:self:7:positive:6"], "evidence_texts": ["An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A group of three people are digging a large trench in a yard while another person walks close by.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "good evening here . lol . good how are you", "sorry that must suck . i recently got a new cat that cheers me up", "they sure are . what do you do for fun", "music is key to life . remember that .", "thanks . i worry about the environment sometimes too", "i think about that too . bums me out", "do you enjoy what you do ?"], "speaker_id": "snli:test:5811", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5811", "candidate_text": "A woman is lost and looks at directions mounted on the wall.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5811:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5789:premise", "snli:test:6805:premise", "snli:test:5560:premise", "snli:test:5811:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A group of three people are digging a large trench in a yard while another person walks close by.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5811", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5817", "candidate_text": "well, the photographer is a professional", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5817:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5535:premise", "snli:test:5817:premise", "snli:test:5969:premise", "snli:test:5200:premise", "style:personachat:test:219:self:1:positive:0"], "evidence_texts": ["Two young teenage girls hiking through the snow on a cold winter day.", "A photographer is using a tripod to take a picture of a sunset.", "Two men with graying beards share a laugh while enjoying the nice weather.", "A gray-haired man rides a bike in front of a bus advertising Ironman.", "i enjoy scientific books like stiff by mary roach"], "speaker_id": "snli:test:5817", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5817", "candidate_text": "Honestly, The photographer is a professional!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5817:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5535:premise", "snli:test:5817:premise", "snli:test:5969:premise", "snli:test:5200:premise", "style:personachat:test:619:self:7:positive:0", "style:personachat:test:619:self:7:positive:1", "style:personachat:test:619:self:7:positive:2", "style:personachat:test:619:self:7:positive:3", "style:personachat:test:619:self:7:positive:4", "style:personachat:test:619:self:7:positive:5", "style:personachat:test:619:self:7:positive:6"], "evidence_texts": ["Two young teenage girls hiking through the snow on a cold winter day.", "A photographer is using a tripod to take a picture of a sunset.", "Two men with graying beards share a laugh while enjoying the nice weather.", "A gray-haired man rides a bike in front of a bus advertising Ironman.", "i did , how are you doing ?", "oh cool , i grew up playing frisbee in north dakota", "i had some donuts at my advertising job i started last year", "oh no , you better take it easy", "yeah good thing you played recently", "cool . you just said you got done playing ?", "wow medical marvel ! fastest surgery ever"], "speaker_id": "snli:test:5817", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5818", "candidate_text": "well, a group of people take turns carrying an injured child", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5818:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5722:premise", "snli:test:5986:premise", "snli:test:5818:premise", "snli:test:5429:premise", "style:personachat:test:219:self:2:positive:0", "style:personachat:test:219:self:2:positive:1"], "evidence_texts": ["A young african child in a dirty yellow shirt carrying a bag on his head.", "A girl in a costume is performing a dance as other girls in costumes watch.", "Child plays with pigeon in busy public area while group of people watch him play.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "yes , i do in my free time", "i enjoy scientific books like stiff by mary roach"], "speaker_id": "snli:test:5818", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5818", "candidate_text": "A group of people take turns carrying an injured child.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5818:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5722:premise", "snli:test:5986:premise", "snli:test:5818:premise", "snli:test:5429:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young african child in a dirty yellow shirt carrying a bag on his head.", "A girl in a costume is performing a dance as other girls in costumes watch.", "Child plays with pigeon in busy public area while group of people watch him play.", "an asian man in a black tracksuit is watching students doing exercised on basketball court.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5818", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5821", "candidate_text": "well, there are two women helping a boy with a harness", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5821:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5773:premise", "snli:test:5821:premise", "snli:test:5628:premise", "snli:test:6066:premise", "style:personachat:test:219:self:5:positive:0", "style:personachat:test:219:self:5:positive:1", "style:personachat:test:219:self:5:positive:2", "style:personachat:test:219:self:5:positive:3", "style:personachat:test:219:self:5:positive:4"], "evidence_texts": ["A young male sitting in front of a computer monitor", "Two men are assisting a small girl in a harness.", "A group of people traveling in a small wooden boat.", "a blond woman kissing the cheek of a brown-haired man.", "i usually enjoy reading and just spending time with friends", "not really , we usually go shopping on the weekends", "that sounds great too me ! because of work i am always so tired", "yes , i do in my free time", "i enjoy scientific books like stiff by mary roach"], "speaker_id": "snli:test:5821", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5821", "candidate_text": "I wonder whether there are two women helping a boy with a harness?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5821:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5773:premise", "snli:test:5821:premise", "snli:test:5628:premise", "snli:test:6066:premise", "style:personachat:test:828:self:5:positive:0", "style:personachat:test:828:self:5:positive:1", "style:personachat:test:828:self:5:positive:2", "style:personachat:test:828:self:5:positive:3", "style:personachat:test:828:self:5:positive:4"], "evidence_texts": ["A young male sitting in front of a computer monitor", "Two men are assisting a small girl in a harness.", "A group of people traveling in a small wooden boat.", "a blond woman kissing the cheek of a brown-haired man.", "yes , i came from paraguay 10 years ago , perfected my english and now i am an engineer .", "south america . did you have lunch yet ?", "no i love my bacon too much", "that is a health choice too .", "what kind of music do you like ?"], "speaker_id": "snli:test:5821", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5823", "candidate_text": "well, the men tying the girl up", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5823:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6066:premise", "snli:test:5773:premise", "snli:test:6079:premise", "snli:test:5823:premise", "style:personachat:test:219:self:7:positive:0", "style:personachat:test:219:self:7:positive:1", "style:personachat:test:219:self:7:positive:2", "style:personachat:test:219:self:7:positive:3", "style:personachat:test:219:self:7:positive:4", "style:personachat:test:219:self:7:positive:5", "style:personachat:test:219:self:7:positive:6"], "evidence_texts": ["a blond woman kissing the cheek of a brown-haired man.", "A young male sitting in front of a computer monitor", "A brown dog plays in a deep pile of snow.", "Two men are assisting a small girl in a harness.", "that sounds interesting ! do you also like working out ?", "me too . i am kind of busy with work , so i only go once a week", "i usually enjoy reading and just spending time with friends", "not really , we usually go shopping on the weekends", "that sounds great too me ! because of work i am always so tired", "yes , i do in my free time", "i enjoy scientific books like stiff by mary roach"], "speaker_id": "snli:test:5823", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5823", "candidate_text": "Honestly, The men tying the girl up!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5823:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6066:premise", "snli:test:5773:premise", "snli:test:6079:premise", "snli:test:5823:premise", "style:personachat:test:620:self:5:positive:0", "style:personachat:test:620:self:5:positive:1", "style:personachat:test:620:self:5:positive:2", "style:personachat:test:620:self:5:positive:3", "style:personachat:test:620:self:5:positive:4"], "evidence_texts": ["a blond woman kissing the cheek of a brown-haired man.", "A young male sitting in front of a computer monitor", "A brown dog plays in a deep pile of snow.", "Two men are assisting a small girl in a harness.", "yes , i travelled to spain in high school but then my best friend took my boyfriend .", "haha , nah , he and i were starting to grow apart but i stopped talking to her .", "what do you mean , deal ?", "so you must be lonely then ?", "yeah , dogs rock ! my parents were gone a lot but my dog was always there ."], "speaker_id": "snli:test:5823", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5828", "candidate_text": "well, the person is driving the suv", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5828:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5828:premise", "snli:test:4494:premise", "snli:test:4177:premise", "snli:test:5917:premise", "style:personachat:test:220:self:1:positive:0"], "evidence_texts": ["A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "mit . i studied engineering but ended up pursuing writing instead ."], "speaker_id": "snli:test:5828", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5828", "candidate_text": "The person is driving the SUV.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5828:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5828:premise", "snli:test:4494:premise", "snli:test:4177:premise", "snli:test:5917:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A person of indeterminate sex wearing a dark hat, neutral pants, and a red shirt walking up a rocky slope at high altitude in the daytime.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5828", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5829", "candidate_text": "well, the person is about to go hiking", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5829:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5917:premise", "snli:test:5829:premise", "snli:test:4494:premise", "snli:test:5919:premise", "style:personachat:test:220:self:2:positive:0", "style:personachat:test:220:self:2:positive:1"], "evidence_texts": ["A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "where did you go to school ?", "mit . i studied engineering but ended up pursuing writing instead ."], "speaker_id": "snli:test:5829", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5829", "candidate_text": "I wonder whether the person is about to go hiking?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5829:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5917:premise", "snli:test:5829:premise", "snli:test:4494:premise", "snli:test:5919:premise", "style:personachat:test:834:self:2:positive:0", "style:personachat:test:834:self:2:positive:1"], "evidence_texts": ["A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A boy in a yellow t-shirt and pink sweater talks on a cellphone while riding a horse through a crowd of people who are looking on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "no i am currently single and do not want children and you ?", "what do you like to do in spare time ?"], "speaker_id": "snli:test:5829", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5830", "candidate_text": "well, a man with no shirt holds a stick", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5830:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5589:premise", "snli:test:6074:premise", "snli:test:5830:premise", "snli:test:5393:premise", "style:personachat:test:220:self:5:positive:0", "style:personachat:test:220:self:5:positive:1", "style:personachat:test:220:self:5:positive:2", "style:personachat:test:220:self:5:positive:3", "style:personachat:test:220:self:5:positive:4"], "evidence_texts": ["A surfer is riding a wave in a large body of water.", "A woman in her swimsuit is kicking a ball on the beach.", "A shirtless man with red paint on his torso holding a stick.", "A person, in a striped blue shirt and pants, is running along.", "yes , i write sci fi . do you enjoy reading ?", "that is nice . what do you do for work ?", "wonderful . my parents both had wonderful aides before they passed .", "where did you go to school ?", "mit . i studied engineering but ended up pursuing writing instead ."], "speaker_id": "snli:test:5830", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5830", "candidate_text": "Honestly, A man with no shirt holds a stick!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5830:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5589:premise", "snli:test:6074:premise", "snli:test:5830:premise", "snli:test:5393:premise", "style:personachat:test:624:self:2:positive:0", "style:personachat:test:624:self:2:positive:1"], "evidence_texts": ["A surfer is riding a wave in a large body of water.", "A woman in her swimsuit is kicking a ball on the beach.", "A shirtless man with red paint on his torso holding a stick.", "A person, in a striped blue shirt and pants, is running along.", "just frisbee . when i can i travel to bora bora . that is where they moved", "not old enough to meet new friends !"], "speaker_id": "snli:test:5830", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5891", "candidate_text": "well, 2 people are talking", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5891:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5756:premise", "snli:test:6011:premise", "snli:test:5220:premise", "snli:test:5891:premise", "style:personachat:test:220:self:6:positive:0", "style:personachat:test:220:self:6:positive:1", "style:personachat:test:220:self:6:positive:2", "style:personachat:test:220:self:6:positive:3", "style:personachat:test:220:self:6:positive:4", "style:personachat:test:220:self:6:positive:5"], "evidence_texts": ["A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "i am wonderful . i just finished my book . how are you ?", "yes , i write sci fi . do you enjoy reading ?", "that is nice . what do you do for work ?", "wonderful . my parents both had wonderful aides before they passed .", "where did you go to school ?", "mit . i studied engineering but ended up pursuing writing instead ."], "speaker_id": "snli:test:5891", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5891", "candidate_text": "2 people are talking.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5891:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5756:premise", "snli:test:6011:premise", "snli:test:5220:premise", "snli:test:5891:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "A woman wearing a straw hat is sitting among a variety of different foods while holding a drink.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5891", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5899", "candidate_text": "well, a woman laying down sleeping", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5899:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5767:premise", "snli:test:5977:premise", "snli:test:5433:premise", "snli:test:5899:premise", "style:personachat:test:221:self:1:positive:0"], "evidence_texts": ["A dog with a purple leash is held by a woman wearing white shoes.", "Two adults hold either side of a boat at the edge of the water.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A woman is holding a microphone in one hand and her mouth is open.", "yes , but they are all grown now ."], "speaker_id": "snli:test:5899", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5899", "candidate_text": "I wonder whether a woman laying down sleeping?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5899:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5767:premise", "snli:test:5977:premise", "snli:test:5433:premise", "snli:test:5899:premise", "style:personachat:test:837:self:1:positive:0"], "evidence_texts": ["A dog with a purple leash is held by a woman wearing white shoes.", "Two adults hold either side of a boat at the edge of the water.", "Four men wearing orange vests are putting in a steel beam on the ground.", "A woman is holding a microphone in one hand and her mouth is open.", "way to kill our dreams , huh ? lol"], "speaker_id": "snli:test:5899", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5907", "candidate_text": "well, a girl is expelling air", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5907:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5716:premise", "snli:test:6364:premise", "snli:test:5594:premise", "snli:test:5907:premise", "style:personachat:test:221:self:2:positive:0", "style:personachat:test:221:self:2:positive:1"], "evidence_texts": ["A football layer wearing a red shirt.", "an elderly woman walking down the street.", "A man in glasses in a basement.", "A girl is blowing a huge bubble.", "you dislike people that much . do you have children ?", "yes , but they are all grown now ."], "speaker_id": "snli:test:5907", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5907", "candidate_text": "Honestly, A girl is expelling air!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5907:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5716:premise", "snli:test:6364:premise", "snli:test:5594:premise", "snli:test:5907:premise", "style:personachat:test:627:self:1:positive:0"], "evidence_texts": ["A football layer wearing a red shirt.", "an elderly woman walking down the street.", "A man in glasses in a basement.", "A girl is blowing a huge bubble.", "i am craving cake and pizza ! !"], "speaker_id": "snli:test:5907", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5911", "candidate_text": "well, a lady is taking a picture", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5911:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5785:premise", "snli:test:5939:premise", "snli:test:5688:premise", "snli:test:5911:premise", "style:personachat:test:221:self:5:positive:0", "style:personachat:test:221:self:5:positive:1", "style:personachat:test:221:self:5:positive:2", "style:personachat:test:221:self:5:positive:3", "style:personachat:test:221:self:5:positive:4"], "evidence_texts": ["A man with a cigarette sits on the stoop.", "Two cars are being driven in a car race.", "Man with an orange bandanna is playing the flute.", "A female camera operator is capturing some off-camera activity.", "oh i love animals ! i have a cat he is siamese .", "yes , do you have any hobbies . i like to walk everyday at least a mile .", "i do like music . much easier to go to concerts now that i am retired .", "you dislike people that much . do you have children ?", "yes , but they are all grown now ."], "speaker_id": "snli:test:5911", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5911", "candidate_text": "A lady is taking a picture.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5911:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5785:premise", "snli:test:5939:premise", "snli:test:5688:premise", "snli:test:5911:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man with a cigarette sits on the stoop.", "Two cars are being driven in a car race.", "Man with an orange bandanna is playing the flute.", "A female camera operator is capturing some off-camera activity.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5911", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5917", "candidate_text": "well, a man is running", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5917:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5829:premise", "snli:test:5917:premise", "snli:test:5828:premise", "snli:test:5958:premise", "style:personachat:test:221:self:7:positive:0", "style:personachat:test:221:self:7:positive:1", "style:personachat:test:221:self:7:positive:2", "style:personachat:test:221:self:7:positive:3", "style:personachat:test:221:self:7:positive:4", "style:personachat:test:221:self:7:positive:5", "style:personachat:test:221:self:7:positive:6"], "evidence_texts": ["A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "hello , i am sewing some quilts for my nieces .", "that is amazing , what fo you do . i retired from banking .", "oh i love animals ! i have a cat he is siamese .", "yes , do you have any hobbies . i like to walk everyday at least a mile .", "i do like music . much easier to go to concerts now that i am retired .", "you dislike people that much . do you have children ?", "yes , but they are all grown now ."], "speaker_id": "snli:test:5917", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5917", "candidate_text": "I wonder whether a man is running?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5917:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5829:premise", "snli:test:5917:premise", "snli:test:5828:premise", "snli:test:5958:premise", "style:personachat:test:837:self:7:positive:0", "style:personachat:test:837:self:7:positive:1", "style:personachat:test:837:self:7:positive:2", "style:personachat:test:837:self:7:positive:3", "style:personachat:test:837:self:7:positive:4", "style:personachat:test:837:self:7:positive:5", "style:personachat:test:837:self:7:positive:6"], "evidence_texts": ["A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "i am good . how are you doing ?", "sounds time consuming . i just got home from work at the grocery store", "cool ! do you enjoy painting ? i do .", "i also enjoy horseback riding", "i own two beautiful horses , but cant see them as often as i would like .", "i wish ! my voice could shatter glass haha", "way to kill our dreams , huh ? lol"], "speaker_id": "snli:test:5917", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5919", "candidate_text": "Honestly, A referee is punching a goat!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5919:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5958:premise", "snli:test:5829:premise", "snli:test:6038:premise", "snli:test:5919:premise", "style:personachat:test:222:self:1:positive:0"], "evidence_texts": ["Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:5919", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5919", "candidate_text": "well, a referee is punching a goat", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5919:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5958:premise", "snli:test:5829:premise", "snli:test:6038:premise", "snli:test:5919:premise", "style:personachat:test:372:self:1:positive:0"], "evidence_texts": ["Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A person in an orange jacket getting out of the rear of a Toyota SUV, parked in front of a view of the ocean and mountains.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:5919", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5929", "candidate_text": "Honestly, There is a group of people, waiting at the bus stop!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5929:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5495:premise", "snli:test:5929:premise", "snli:test:6013:premise", "snli:test:5459:premise", "style:personachat:test:222:self:2:positive:0", "style:personachat:test:222:self:2:positive:1"], "evidence_texts": ["A small child with a backpack walks down a narrow road next to a painted wall.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "A black and white picture of a city street with a food vendor in the foreground.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "that sounds fun . i spend my free time on art .", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:5929", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5929", "candidate_text": "There is a group of people, waiting at the bus stop.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5929:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5495:premise", "snli:test:5929:premise", "snli:test:6013:premise", "snli:test:5459:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A small child with a backpack walks down a narrow road next to a painted wall.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "A black and white picture of a city street with a food vendor in the foreground.", "A young girl with a purple shirt and a pink hair band standing near the woods.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5929", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5939", "candidate_text": "Honestly, A Bugatti and a Lambourgini compete in a road race!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5939:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5911:premise", "snli:test:6253:premise", "snli:test:5785:premise", "snli:test:5939:premise", "style:personachat:test:222:self:5:positive:0", "style:personachat:test:222:self:5:positive:1", "style:personachat:test:222:self:5:positive:2", "style:personachat:test:222:self:5:positive:3", "style:personachat:test:222:self:5:positive:4"], "evidence_texts": ["A female camera operator is capturing some off-camera activity.", "A person sitting on a rock beside a waterfall.", "A man with a cigarette sits on the stoop.", "Two cars are being driven in a car race.", "i earn a paycheck at a research facility cleaning cages . what would you study ?", "i understand . luckily i decided to create my art from trash so it is free !", "cool ! what do you like to do in your free time ?", "that sounds fun . i spend my free time on art .", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:5939", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5939", "candidate_text": "I wonder whether a bugatti and a lambourgini compete in a road race?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5939:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5911:premise", "snli:test:6253:premise", "snli:test:5785:premise", "snli:test:5939:premise", "style:personachat:test:838:self:5:positive:0", "style:personachat:test:838:self:5:positive:1", "style:personachat:test:838:self:5:positive:2", "style:personachat:test:838:self:5:positive:3", "style:personachat:test:838:self:5:positive:4"], "evidence_texts": ["A female camera operator is capturing some off-camera activity.", "A person sitting on a rock beside a waterfall.", "A man with a cigarette sits on the stoop.", "Two cars are being driven in a car race.", "i love fantasy . i am a huge fan ! what kind of books do you like ?", "i wish to write a book someday . a book about dragons .", "i am 25 years old . i have an active imagination .", "yes , one of my jobs is in a comic shop . are you an athlete ?", "are you a personal trainer ? is that your second job ?"], "speaker_id": "snli:test:5939", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5958", "candidate_text": "Honestly, A person takes a photo!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5958:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5919:premise", "snli:test:6038:premise", "snli:test:5958:premise", "snli:test:5917:premise", "style:personachat:test:222:self:6:positive:0", "style:personachat:test:222:self:6:positive:1", "style:personachat:test:222:self:6:positive:2", "style:personachat:test:222:self:6:positive:3", "style:personachat:test:222:self:6:positive:4", "style:personachat:test:222:self:6:positive:5"], "evidence_texts": ["A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "i am good . i prefer artistic endeavors to scholarly ones . you ?", "i earn a paycheck at a research facility cleaning cages . what would you study ?", "i understand . luckily i decided to create my art from trash so it is free !", "cool ! what do you like to do in your free time ?", "that sounds fun . i spend my free time on art .", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:5958", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5958", "candidate_text": "well, a person takes a photo", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5958:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5919:premise", "snli:test:6038:premise", "snli:test:5958:premise", "snli:test:5917:premise", "style:personachat:test:372:self:7:positive:0", "style:personachat:test:372:self:7:positive:1", "style:personachat:test:372:self:7:positive:2", "style:personachat:test:372:self:7:positive:3", "style:personachat:test:372:self:7:positive:4", "style:personachat:test:372:self:7:positive:5", "style:personachat:test:372:self:7:positive:6"], "evidence_texts": ["A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "i am doing well . i just took a break from writing my new novel .", "it is actually a romance novel , but it has sci fi elements .", "it should be out by the end of the year . do you have any pets ?", "i have some cats as well . two of them . they keep me company since i am isolated .", "i live on an island near greece . so i am introverted , but by location not personality .", "i love it here . it is beautiful . i have two sisters , by the way .", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:5958", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5969", "candidate_text": "I wonder whether two women with leg hair share a laugh while enjoying the nice weather?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5969:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5817:premise", "snli:test:5969:premise", "snli:test:6167:premise", "snli:test:5535:premise", "style:personachat:test:223:self:1:positive:0"], "evidence_texts": ["A photographer is using a tripod to take a picture of a sunset.", "Two men with graying beards share a laugh while enjoying the nice weather.", "A big dog and a smaller dog fight over a piece of fabric.", "Two young teenage girls hiking through the snow on a cold winter day.", "what type of music do you like ?"], "speaker_id": "snli:test:5969", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5969", "candidate_text": "Honestly, Two women with leg hair share a laugh while enjoying the nice weather!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:5969:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5817:premise", "snli:test:5969:premise", "snli:test:6167:premise", "snli:test:5535:premise", "style:personachat:test:636:self:7:positive:0", "style:personachat:test:636:self:7:positive:1", "style:personachat:test:636:self:7:positive:2", "style:personachat:test:636:self:7:positive:3", "style:personachat:test:636:self:7:positive:4", "style:personachat:test:636:self:7:positive:5", "style:personachat:test:636:self:7:positive:6"], "evidence_texts": ["A photographer is using a tripod to take a picture of a sunset.", "Two men with graying beards share a laugh while enjoying the nice weather.", "A big dog and a smaller dog fight over a piece of fabric.", "Two young teenage girls hiking through the snow on a cold winter day.", "sure ! can always use more friends . tell me about you", "awesome ! i would love to be a fashion designer ! are you married ? kids ?", "congrats on graduating ! i am married but no kids yet . we want kids in the future !", "i had a friend in high school named lucy but she stole my boyfriend lol", "thanks ; hehe so what do you like to do ?", "absolutely ! my mom and dad were always gone growing up so i am used to eating junk .", "oh most definitely ! the pink ones are the best !"], "speaker_id": "snli:test:5969", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5977", "candidate_text": "I wonder whether two people are near water?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5977:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5899:premise", "snli:test:5977:premise", "snli:test:5767:premise", "snli:test:6027:premise", "style:personachat:test:223:self:2:positive:0", "style:personachat:test:223:self:2:positive:1"], "evidence_texts": ["A woman is holding a microphone in one hand and her mouth is open.", "Two adults hold either side of a boat at the edge of the water.", "A dog with a purple leash is held by a woman wearing white shoes.", "Navy men stand along a red carpet and salute as their superior walks by.", "a few dogs . tho i am not able to walk them like i should", "what type of music do you like ?"], "speaker_id": "snli:test:5977", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5977", "candidate_text": "Two people are near water.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5977:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5899:premise", "snli:test:5977:premise", "snli:test:5767:premise", "snli:test:6027:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman is holding a microphone in one hand and her mouth is open.", "Two adults hold either side of a boat at the edge of the water.", "A dog with a purple leash is held by a woman wearing white shoes.", "Navy men stand along a red carpet and salute as their superior walks by.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:5977", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5979", "candidate_text": "I wonder whether two people are in the ocean?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5979:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6027:premise", "snli:test:5899:premise", "snli:test:6070:premise", "snli:test:5979:premise", "style:personachat:test:223:self:5:positive:0", "style:personachat:test:223:self:5:positive:1", "style:personachat:test:223:self:5:positive:2", "style:personachat:test:223:self:5:positive:3", "style:personachat:test:223:self:5:positive:4"], "evidence_texts": ["Navy men stand along a red carpet and salute as their superior walks by.", "A woman is holding a microphone in one hand and her mouth is open.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "wow that is a busy beaver . i have 7 myself", "i am just a little rough around the edges for that . lol", "that sounds like fun . i was born for hard work", "a few dogs . tho i am not able to walk them like i should", "what type of music do you like ?"], "speaker_id": "snli:test:5979", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5979", "candidate_text": "well, two people are in the ocean", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5979:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6027:premise", "snli:test:5899:premise", "snli:test:6070:premise", "snli:test:5979:premise", "style:personachat:test:373:self:5:positive:0", "style:personachat:test:373:self:5:positive:1", "style:personachat:test:373:self:5:positive:2", "style:personachat:test:373:self:5:positive:3", "style:personachat:test:373:self:5:positive:4"], "evidence_texts": ["Navy men stand along a red carpet and salute as their superior walks by.", "A woman is holding a microphone in one hand and her mouth is open.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "i see blue and its really green .", "yes i am . i like to sing what about you ?", "i like to go to the karaoke bars", "are you dressing up for halloween ?", "i want to dress up like a jennie"], "speaker_id": "snli:test:5979", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5986", "candidate_text": "I wonder whether a girl is critiquing her friend's dance moves?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5986:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5818:premise", "snli:test:5722:premise", "snli:test:5986:premise", "snli:test:6060:premise", "style:personachat:test:223:self:7:positive:0", "style:personachat:test:223:self:7:positive:1", "style:personachat:test:223:self:7:positive:2", "style:personachat:test:223:self:7:positive:3", "style:personachat:test:223:self:7:positive:4", "style:personachat:test:223:self:7:positive:5", "style:personachat:test:223:self:7:positive:6"], "evidence_texts": ["Child plays with pigeon in busy public area while group of people watch him play.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A girl in a costume is performing a dance as other girls in costumes watch.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "sounds like a good idea . i am good . do you work", "that is a noble career . i haul load over the country .", "wow that is a busy beaver . i have 7 myself", "i am just a little rough around the edges for that . lol", "that sounds like fun . i was born for hard work", "a few dogs . tho i am not able to walk them like i should", "what type of music do you like ?"], "speaker_id": "snli:test:5986", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5986", "candidate_text": "Honestly, A girl is critiquing her friend's dance moves!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5986:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5818:premise", "snli:test:5722:premise", "snli:test:5986:premise", "snli:test:6060:premise", "style:personachat:test:637:self:5:positive:0", "style:personachat:test:637:self:5:positive:1", "style:personachat:test:637:self:5:positive:2", "style:personachat:test:637:self:5:positive:3", "style:personachat:test:637:self:5:positive:4"], "evidence_texts": ["Child plays with pigeon in busy public area while group of people watch him play.", "A young african child in a dirty yellow shirt carrying a bag on his head.", "A girl in a costume is performing a dance as other girls in costumes watch.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "i also attend school . what are you going to school for ?", "awesome ! music is what i am studying since i like it so much .", "i saw them in concert . could not see much since i am only five feet .", "did you come from a big family ? it is just me , parents and a sibling .", "how late ? my parents were also older . in their 50s !"], "speaker_id": "snli:test:5986", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:5988", "candidate_text": "well, a girl is dancing", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5988:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5988:premise", "snli:test:6060:premise", "snli:test:5818:premise", "snli:test:6085:premise", "style:personachat:test:224:self:1:positive:0"], "evidence_texts": ["A girl in a costume is performing a dance as other girls in costumes watch.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "Child plays with pigeon in busy public area while group of people watch him play.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "my youngest brother enjoys more of my genre the paranormal as well ."], "speaker_id": "snli:test:5988", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:5988", "candidate_text": "I wonder whether a girl is dancing?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5988:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5988:premise", "snli:test:6060:premise", "snli:test:5818:premise", "snli:test:6085:premise", "style:personachat:test:841:self:1:positive:0"], "evidence_texts": ["A girl in a costume is performing a dance as other girls in costumes watch.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "Child plays with pigeon in busy public area while group of people watch him play.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "what are you studying to become ?"], "speaker_id": "snli:test:5988", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:5990", "candidate_text": "well, a tall person in a hat", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5990:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5772:premise", "snli:test:6002:premise", "snli:test:5990:premise", "snli:test:5770:premise", "style:personachat:test:224:self:2:positive:0", "style:personachat:test:224:self:2:positive:1"], "evidence_texts": ["A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "my six older sisters love to watch horror and eat a pizza .", "my youngest brother enjoys more of my genre the paranormal as well ."], "speaker_id": "snli:test:5990", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:5990", "candidate_text": "Honestly, A tall person in a hat!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:5990:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5772:premise", "snli:test:6002:premise", "snli:test:5990:premise", "snli:test:5770:premise", "style:personachat:test:646:self:1:positive:0"], "evidence_texts": ["A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "fantasy videogames ! ! do you like them ?"], "speaker_id": "snli:test:5990", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6002", "candidate_text": "well, a man is putting a sign in the window to alert customers of an upcoming ice cream sale", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6002:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5990:premise", "snli:test:6678:premise", "snli:test:6002:premise", "snli:test:5772:premise", "style:personachat:test:224:self:5:positive:0", "style:personachat:test:224:self:5:positive:1", "style:personachat:test:224:self:5:positive:2", "style:personachat:test:224:self:5:positive:3", "style:personachat:test:224:self:5:positive:4"], "evidence_texts": ["Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "oh damn . tha a t is awesome ! i wish i could do that for a living .", "california ! in my spare time all i get to do is read novels of the supernatural .", "n o o way ! my siblings love to watch horror as well .", "my six older sisters love to watch horror and eat a pizza .", "my youngest brother enjoys more of my genre the paranormal as well ."], "speaker_id": "snli:test:6002", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6002", "candidate_text": "A man is putting a sign in the window to alert customers of an upcoming ice cream sale.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6002:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5990:premise", "snli:test:6678:premise", "snli:test:6002:premise", "snli:test:5772:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "A woman talks on a cellphone while sitting in front of blue railings that are in front of the ocean.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6002", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6011", "candidate_text": "well, a man takes a smoke break from work", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6011:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5891:premise", "snli:test:6393:premise", "snli:test:5756:premise", "snli:test:6011:premise", "style:personachat:test:224:self:7:positive:0", "style:personachat:test:224:self:7:positive:1", "style:personachat:test:224:self:7:positive:2", "style:personachat:test:224:self:7:positive:3", "style:personachat:test:224:self:7:positive:4", "style:personachat:test:224:self:7:positive:5", "style:personachat:test:224:self:7:positive:6"], "evidence_texts": ["2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "i am doing well . my name is rachel . yourself ?", "i am still attending at cypress . i binge eat a lot because of this .", "oh damn . tha a t is awesome ! i wish i could do that for a living .", "california ! in my spare time all i get to do is read novels of the supernatural .", "n o o way ! my siblings love to watch horror as well .", "my six older sisters love to watch horror and eat a pizza .", "my youngest brother enjoys more of my genre the paranormal as well ."], "speaker_id": "snli:test:6011", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6011", "candidate_text": "I wonder whether a man takes a smoke break from work?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6011:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5891:premise", "snli:test:6393:premise", "snli:test:5756:premise", "snli:test:6011:premise", "style:personachat:test:841:self:7:positive:0", "style:personachat:test:841:self:7:positive:1", "style:personachat:test:841:self:7:positive:2", "style:personachat:test:841:self:7:positive:3", "style:personachat:test:841:self:7:positive:4", "style:personachat:test:841:self:7:positive:5", "style:personachat:test:841:self:7:positive:6"], "evidence_texts": ["2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A blond boy stands among people in a shady setting with a large snake wrapped around his chest.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "i am scared of heights", "you like religion ? i do not", "i do lawyer work lol i am the lawyer", "i am here , like tall buildings ?", "where are you lol", "i hate them i am scared lol", "what are you studying to become ?"], "speaker_id": "snli:test:6011", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6013", "candidate_text": "well, the colors has disappeared form the world", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6013:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5929:premise", "snli:test:6013:premise", "snli:test:6291:premise", "snli:test:5495:premise", "style:personachat:test:225:self:1:positive:0"], "evidence_texts": ["4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "A black and white picture of a city street with a food vendor in the foreground.", "Two men in hard hats look at a building while on the side of a street.", "A small child with a backpack walks down a narrow road next to a painted wall.", "i bet that is not true . who would not want their child to be scientist"], "speaker_id": "snli:test:6013", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6013", "candidate_text": "Honestly, The colors has disappeared form the world!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6013:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5929:premise", "snli:test:6013:premise", "snli:test:6291:premise", "snli:test:5495:premise", "style:personachat:test:646:self:7:positive:0", "style:personachat:test:646:self:7:positive:1", "style:personachat:test:646:self:7:positive:2", "style:personachat:test:646:self:7:positive:3", "style:personachat:test:646:self:7:positive:4", "style:personachat:test:646:self:7:positive:5", "style:personachat:test:646:self:7:positive:6"], "evidence_texts": ["4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "A black and white picture of a city street with a food vendor in the foreground.", "Two men in hard hats look at a building while on the side of a street.", "A small child with a backpack walks down a narrow road next to a painted wall.", "hello ! i am well . being shy in class", "oh wow ! i am too shy to go to the gym . where do you go ?", "how many laps do you do ? i stay on my computer because i work all day", "wow ! i build model spaceships and we launch them from pools .", "its okay ! it was very hard to get my degree in computer science .", "now i just play a lot of games .", "fantasy videogames ! ! do you like them ?"], "speaker_id": "snli:test:6013", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6016", "candidate_text": "well, the man is old", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6016:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6016:premise", "snli:test:5577:premise", "snli:test:6721:premise", "snli:test:4450:premise", "style:personachat:test:225:self:2:positive:0", "style:personachat:test:225:self:2:positive:1"], "evidence_texts": ["A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "my parents would love you . i think i might try for a teaching degree", "i bet that is not true . who would not want their child to be scientist"], "speaker_id": "snli:test:6016", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6016", "candidate_text": "The man is old.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6016:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6016:premise", "snli:test:5577:premise", "snli:test:6721:premise", "snli:test:4450:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "The backs of a woman and her dog as they walk away from the camera with the dog on a red leash and her in a green tank top.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6016", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6027", "candidate_text": "well, some men are waving hello and speaking to their superior", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6027:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5979:premise", "snli:test:6070:premise", "snli:test:5977:premise", "snli:test:6027:premise", "style:personachat:test:225:self:5:positive:0", "style:personachat:test:225:self:5:positive:1", "style:personachat:test:225:self:5:positive:2", "style:personachat:test:225:self:5:positive:3", "style:personachat:test:225:self:5:positive:4"], "evidence_texts": ["Two adults hold either side of a boat at the edge of the water.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "Navy men stand along a red carpet and salute as their superior walks by.", "the snow makes it hard to walk the dogs , do you have pets ?", "that is always hard . i love pets . besides my 2 dogs there are 3 cats", "that sounds great . my parents both taught science", "my parents would love you . i think i might try for a teaching degree", "i bet that is not true . who would not want their child to be scientist"], "speaker_id": "snli:test:6027", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6027", "candidate_text": "I wonder whether some men are waving hello and speaking to their superior?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6027:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5979:premise", "snli:test:6070:premise", "snli:test:5977:premise", "snli:test:6027:premise", "style:personachat:test:844:self:5:positive:0", "style:personachat:test:844:self:5:positive:1", "style:personachat:test:844:self:5:positive:2", "style:personachat:test:844:self:5:positive:3", "style:personachat:test:844:self:5:positive:4"], "evidence_texts": ["Two adults hold either side of a boat at the edge of the water.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "Navy men stand along a red carpet and salute as their superior walks by.", "haha you seems very funny .", "i wish i have a sister . i have two brothers only .", "in my opinion you need high school .", "truck drives making good money .", "do you have a job now ?"], "speaker_id": "snli:test:6027", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6031", "candidate_text": "well, a man is trying to catch a flight home for christmas", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6031:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5558:premise", "snli:test:3104:premise", "snli:test:6962:premise", "snli:test:6031:premise", "style:personachat:test:225:self:7:positive:0", "style:personachat:test:225:self:7:positive:1", "style:personachat:test:225:self:7:positive:2", "style:personachat:test:225:self:7:positive:3", "style:personachat:test:225:self:7:positive:4", "style:personachat:test:225:self:7:positive:5", "style:personachat:test:225:self:7:positive:6"], "evidence_texts": ["A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "its a great night here in the mountains , you ?", "i do not mind chilly . i do not like snow though", "the snow makes it hard to walk the dogs , do you have pets ?", "that is always hard . i love pets . besides my 2 dogs there are 3 cats", "that sounds great . my parents both taught science", "my parents would love you . i think i might try for a teaching degree", "i bet that is not true . who would not want their child to be scientist"], "speaker_id": "snli:test:6031", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6031", "candidate_text": "Honestly, A man is trying to catch a flight home for Christmas!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6031:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5558:premise", "snli:test:3104:premise", "snli:test:6962:premise", "snli:test:6031:premise", "style:personachat:test:648:self:5:positive:0", "style:personachat:test:648:self:5:positive:1", "style:personachat:test:648:self:5:positive:2", "style:personachat:test:648:self:5:positive:3", "style:personachat:test:648:self:5:positive:4"], "evidence_texts": ["A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A woman in skirt, torn stockings and combat boots and a man in aviators and batman tank top walk down a city street.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "in new york . it is in a great town with film festivals .", "i have a two year old daughter . . she inherited my love of muffins .", "i tried getting a real estate licence but it seemed too tedious .", "i believe it . it seemed like something i would like .", "maybe i will revisit the idea in the future !"], "speaker_id": "snli:test:6031", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6032", "candidate_text": "Honestly, A man is sitting on a beach in Mexico!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6032:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6032:premise", "snli:test:6962:premise", "snli:test:5558:premise", "snli:test:6965:premise", "style:personachat:test:226:self:1:positive:0"], "evidence_texts": ["A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:6032", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6032", "candidate_text": "A man is sitting on a beach in Mexico.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6032:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6032:premise", "snli:test:6962:premise", "snli:test:5558:premise", "snli:test:6965:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a plaid shirt and brown pants is sitting on the sidewalk in front of some blankets while people walk by.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6032", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6035", "candidate_text": "Honestly, Three people sitting by a fire!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6035:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5709:premise", "snli:test:6119:premise", "snli:test:5681:premise", "snli:test:6035:premise", "style:personachat:test:226:self:2:positive:0", "style:personachat:test:226:self:2:positive:1"], "evidence_texts": ["A young boy hanging on a pole smiling at the camera.", "A young girl, is mid jump, while performing a dance routine.", "a teenager laying down with paper crafts strewn about the floor", "Three people on skis are standing behind a no skiing sign.", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:6035", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6035", "candidate_text": "I wonder whether three people sitting by a fire?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6035:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5709:premise", "snli:test:6119:premise", "snli:test:5681:premise", "snli:test:6035:premise", "style:personachat:test:845:self:2:positive:0", "style:personachat:test:845:self:2:positive:1"], "evidence_texts": ["A young boy hanging on a pole smiling at the camera.", "A young girl, is mid jump, while performing a dance routine.", "a teenager laying down with paper crafts strewn about the floor", "Three people on skis are standing behind a no skiing sign.", "so if you are only sort of human , what is the rest of you ?", "what ? oh hell , clearly i am studying too hard ."], "speaker_id": "snli:test:6035", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6038", "candidate_text": "Honestly, The child is sitting!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6038:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5958:premise", "snli:test:5919:premise", "snli:test:6038:premise", "snli:test:8099:premise", "style:personachat:test:226:self:5:positive:0", "style:personachat:test:226:self:5:positive:1", "style:personachat:test:226:self:5:positive:2", "style:personachat:test:226:self:5:positive:3", "style:personachat:test:226:self:5:positive:4"], "evidence_texts": ["Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "i work in advertising . how about you ?", "awww , how fulfilling ! so are you enjoying your pizza all alone or with friends ?", "just the significant other , or do you have kids of your own ?", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:6038", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6038", "candidate_text": "well, the child is sitting", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6038:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5958:premise", "snli:test:5919:premise", "snli:test:6038:premise", "snli:test:8099:premise", "style:personachat:test:381:self:5:positive:0", "style:personachat:test:381:self:5:positive:1", "style:personachat:test:381:self:5:positive:2", "style:personachat:test:381:self:5:positive:3", "style:personachat:test:381:self:5:positive:4"], "evidence_texts": ["Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A referee with a whistle in his mouth points to the right and watches as a player with the basketball bends down with a defensive player.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "sounds relaxing . what is your occupation ? i have been working advertising since last year .", "i see . where are you from . i am from north dakota .", "nice . do you go camping with your friends ?", "my best friend since kindergarten and i like to go fishing .", "with my dad . he loves fishing . my mom not so much ."], "speaker_id": "snli:test:6038", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6039", "candidate_text": "Honestly, The children are all holding up paper drawings!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6039:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8099:premise", "snli:test:5958:premise", "snli:test:8225:premise", "snli:test:6039:premise", "style:personachat:test:226:self:7:positive:0", "style:personachat:test:226:self:7:positive:1", "style:personachat:test:226:self:7:positive:2", "style:personachat:test:226:self:7:positive:3", "style:personachat:test:226:self:7:positive:4", "style:personachat:test:226:self:7:positive:5", "style:personachat:test:226:self:7:positive:6"], "evidence_texts": ["A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "i am great ! how are you doing on this fine evening ?", "cool . i only eat kosher meat so i have to make my own pizzas .", "i work in advertising . how about you ?", "awww , how fulfilling ! so are you enjoying your pizza all alone or with friends ?", "just the significant other , or do you have kids of your own ?", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:6039", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6039", "candidate_text": "The children are all holding up paper drawings.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6039:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8099:premise", "snli:test:5958:premise", "snli:test:8225:premise", "snli:test:6039:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "Along a paved walkway outdoors, a person in a black windbreaker takes a photo of a multi-piece fish statue while a small white dog looks on.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6039", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6048", "candidate_text": "I wonder whether two people are ballroom dancing?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6048:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6048:premise", "snli:test:5336:premise", "snli:test:6135:premise", "snli:test:5253:premise", "style:personachat:test:227:self:1:positive:0"], "evidence_texts": ["A young couple holding hands in the park.", "This woman is skiing down a snowy hill.", "A young couple thinking about their Christmas decorations.", "A fireman searching for something using a flashlight.", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:6048", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6048", "candidate_text": "Two people are ballroom dancing.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6048:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6048:premise", "snli:test:5336:premise", "snli:test:6135:premise", "snli:test:5253:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A young couple holding hands in the park.", "This woman is skiing down a snowy hill.", "A young couple thinking about their Christmas decorations.", "A fireman searching for something using a flashlight.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6048", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6060", "candidate_text": "I wonder whether the man is white?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6060:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6060:premise", "snli:test:5988:premise", "snli:test:6085:premise", "snli:test:5986:premise", "style:personachat:test:227:self:2:positive:0", "style:personachat:test:227:self:2:positive:1"], "evidence_texts": ["A middle-aged man in a collard shirt is raising his forearms next to his face.", "A girl in a costume is performing a dance as other girls in costumes watch.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A girl in a costume is performing a dance as other girls in costumes watch.", "thank you . do you have any children ?", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:6060", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6060", "candidate_text": "well, the man is white", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6060:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6060:premise", "snli:test:5988:premise", "snli:test:6085:premise", "snli:test:5986:premise", "style:personachat:test:382:self:2:positive:0", "style:personachat:test:382:self:2:positive:1"], "evidence_texts": ["A middle-aged man in a collard shirt is raising his forearms next to his face.", "A girl in a costume is performing a dance as other girls in costumes watch.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A girl in a costume is performing a dance as other girls in costumes watch.", "i understand what you mean !", "i like a bit . christ stapleton is good ."], "speaker_id": "snli:test:6060", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6066", "candidate_text": "I wonder whether the couple is displaying affection?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6066:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5823:premise", "snli:test:6079:premise", "snli:test:6066:premise", "snli:test:5821:premise", "style:personachat:test:227:self:5:positive:0", "style:personachat:test:227:self:5:positive:1", "style:personachat:test:227:self:5:positive:2", "style:personachat:test:227:self:5:positive:3", "style:personachat:test:227:self:5:positive:4"], "evidence_texts": ["Two men are assisting a small girl in a harness.", "A brown dog plays in a deep pile of snow.", "a blond woman kissing the cheek of a brown-haired man.", "Two men are assisting a small girl in a harness.", "i lifted weights in prison . i did some time for stealing .", "my parents do not care about me . they are hardly ever around .", "i would love to be a mom but i am trying to turn my life around .", "thank you . do you have any children ?", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:6066", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6066", "candidate_text": "Honestly, The couple is displaying affection!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6066:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5823:premise", "snli:test:6079:premise", "snli:test:6066:premise", "snli:test:5821:premise", "style:personachat:test:662:self:2:positive:0", "style:personachat:test:662:self:2:positive:1"], "evidence_texts": ["Two men are assisting a small girl in a harness.", "A brown dog plays in a deep pile of snow.", "a blond woman kissing the cheek of a brown-haired man.", "Two men are assisting a small girl in a harness.", "really ? i am surprised to hear that . where do you live ?", "you really do like the cold ! i prefer the heat ."], "speaker_id": "snli:test:6066", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6070", "candidate_text": "I wonder whether a man on a motorcycle?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6070:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6027:premise", "snli:test:6078:premise", "snli:test:6070:premise", "snli:test:5979:premise", "style:personachat:test:227:self:7:positive:0", "style:personachat:test:227:self:7:positive:1", "style:personachat:test:227:self:7:positive:2", "style:personachat:test:227:self:7:positive:3", "style:personachat:test:227:self:7:positive:4", "style:personachat:test:227:self:7:positive:5", "style:personachat:test:227:self:7:positive:6"], "evidence_texts": ["Navy men stand along a red carpet and salute as their superior walks by.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "i just finished bacon and pancakes .", "that is true . i am studying to become a fashion designer .", "i lifted weights in prison . i did some time for stealing .", "my parents do not care about me . they are hardly ever around .", "i would love to be a mom but i am trying to turn my life around .", "thank you . do you have any children ?", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:6070", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6070", "candidate_text": "A man on a motorcycle.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6070:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6027:premise", "snli:test:6078:premise", "snli:test:6070:premise", "snli:test:5979:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Navy men stand along a red carpet and salute as their superior walks by.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "Two adults hold either side of a boat at the edge of the water.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6070", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6074", "candidate_text": "well, a woman lounging on the beach", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6074:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5830:premise", "snli:test:6158:premise", "snli:test:6074:premise", "snli:test:5589:premise", "style:personachat:test:228:self:1:positive:0"], "evidence_texts": ["A shirtless man with red paint on his torso holding a stick.", "A woman carries a small, unhappy child in a busy street scene.", "A woman in her swimsuit is kicking a ball on the beach.", "A surfer is riding a wave in a large body of water.", "you should look it up because it is great ."], "speaker_id": "snli:test:6074", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6074", "candidate_text": "Honestly, A woman lounging on the beach!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6074:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:5830:premise", "snli:test:6158:premise", "snli:test:6074:premise", "snli:test:5589:premise", "style:personachat:test:662:self:7:positive:0", "style:personachat:test:662:self:7:positive:1", "style:personachat:test:662:self:7:positive:2", "style:personachat:test:662:self:7:positive:3", "style:personachat:test:662:self:7:positive:4", "style:personachat:test:662:self:7:positive:5", "style:personachat:test:662:self:7:positive:6"], "evidence_texts": ["A shirtless man with red paint on his torso holding a stick.", "A woman carries a small, unhappy child in a busy street scene.", "A woman in her swimsuit is kicking a ball on the beach.", "A surfer is riding a wave in a large body of water.", "i am great ! it is hot here in south carolina today .", "charleston . i moved here after graduation from usc .", "yes . i love it here . i belong to a horror film group .", "i went to school for teaching but i sing professionally now .", "that sounds illegal . are they not protected ?", "really ? i am surprised to hear that . where do you live ?", "you really do like the cold ! i prefer the heat ."], "speaker_id": "snli:test:6074", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6078", "candidate_text": "well, a man is climbing a ladder on a cliff in hopes of not falling to his death", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6078:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6070:premise", "snli:test:6251:premise", "snli:test:6078:premise", "snli:test:6027:premise", "style:personachat:test:228:self:2:positive:0", "style:personachat:test:228:self:2:positive:1"], "evidence_texts": ["A man in a hat is driving a horse-driven cart down a cobblestone street.", "A kid bored in a train with brown hair and his head lying down.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "Navy men stand along a red carpet and salute as their superior walks by.", "do you like the office ? it is my favorite show .", "you should look it up because it is great ."], "speaker_id": "snli:test:6078", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6078", "candidate_text": "A man is climbing a ladder on a cliff in hopes of not falling to his death.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6078:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6070:premise", "snli:test:6251:premise", "snli:test:6078:premise", "snli:test:6027:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man in a hat is driving a horse-driven cart down a cobblestone street.", "A kid bored in a train with brown hair and his head lying down.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "Navy men stand along a red carpet and salute as their superior walks by.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6078", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6079", "candidate_text": "well, a brown dog is running down the road chasing the mailman", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6079:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6066:premise", "snli:test:6121:premise", "snli:test:5823:premise", "snli:test:6079:premise", "style:personachat:test:228:self:5:positive:0", "style:personachat:test:228:self:5:positive:1", "style:personachat:test:228:self:5:positive:2", "style:personachat:test:228:self:5:positive:3", "style:personachat:test:228:self:5:positive:4"], "evidence_texts": ["a blond woman kissing the cheek of a brown-haired man.", "a young man pouring tea inside of a glass jar.", "Two men are assisting a small girl in a harness.", "A brown dog plays in a deep pile of snow.", "i love my jewish heritage .", "no yiddish is too difficult .", "the only language i speak is english", "do you like the office ? it is my favorite show .", "you should look it up because it is great ."], "speaker_id": "snli:test:6079", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6079", "candidate_text": "I wonder whether a brown dog is running down the road chasing the mailman?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6079:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6066:premise", "snli:test:6121:premise", "snli:test:5823:premise", "snli:test:6079:premise", "style:personachat:test:862:self:5:positive:0", "style:personachat:test:862:self:5:positive:1", "style:personachat:test:862:self:5:positive:2", "style:personachat:test:862:self:5:positive:3", "style:personachat:test:862:self:5:positive:4"], "evidence_texts": ["a blond woman kissing the cheek of a brown-haired man.", "a young man pouring tea inside of a glass jar.", "Two men are assisting a small girl in a harness.", "A brown dog plays in a deep pile of snow.", "oh man , i am overweight and am frustrated by it .", "i am from california ? do you have hobbies ?", "awesome ! . i have 3 kids as well ! do you enjoy being married ?", "i like my work but its frustrating because i work a different schedule from my husband .", "that is very true . where are you from ?"], "speaker_id": "snli:test:6079", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6085", "candidate_text": "well, the two women are walking down the shore drinking tea", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6085:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6060:premise", "snli:test:6085:premise", "snli:test:6523:premise", "snli:test:5988:premise", "style:personachat:test:228:self:7:positive:0", "style:personachat:test:228:self:7:positive:1", "style:personachat:test:228:self:7:positive:2", "style:personachat:test:228:self:7:positive:3", "style:personachat:test:228:self:7:positive:4", "style:personachat:test:228:self:7:positive:5", "style:personachat:test:228:self:7:positive:6"], "evidence_texts": ["A middle-aged man in a collard shirt is raising his forearms next to his face.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "A girl in a costume is performing a dance as other girls in costumes watch.", "doing well thanks . just got home from the synagogue .", "it is a jewish church . it is in manhattan .", "i love my jewish heritage .", "no yiddish is too difficult .", "the only language i speak is english", "do you like the office ? it is my favorite show .", "you should look it up because it is great ."], "speaker_id": "snli:test:6085", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6085", "candidate_text": "Honestly, The two women are walking down the shore drinking tea!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6085:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6060:premise", "snli:test:6085:premise", "snli:test:6523:premise", "snli:test:5988:premise", "style:personachat:test:677:self:5:positive:0", "style:personachat:test:677:self:5:positive:1", "style:personachat:test:677:self:5:positive:2", "style:personachat:test:677:self:5:positive:3", "style:personachat:test:677:self:5:positive:4"], "evidence_texts": ["A middle-aged man in a collard shirt is raising his forearms next to his face.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "A girl in a costume is performing a dance as other girls in costumes watch.", "sounds like an important job . what do you do for fun ?", "i really love to watch football every sunday .", "oh i like to eat cheese pizza while i watch as it is a favorite of mine .", "italian is quite good . anything in particular ?", "never ! have you started getting ready for the holidays yet ?"], "speaker_id": "snli:test:6085", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6119", "candidate_text": "well, a young girl is in the middle of jumping during her dance routine", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6119:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6035:premise", "snli:test:6238:premise", "snli:test:5709:premise", "snli:test:6119:premise", "style:personachat:test:229:self:1:positive:0"], "evidence_texts": ["Three people on skis are standing behind a no skiing sign.", "A man balances a unicycle using his face and both hands.", "A young boy hanging on a pole smiling at the camera.", "A young girl, is mid jump, while performing a dance routine.", "i agree . i love nature ."], "speaker_id": "snli:test:6119", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6119", "candidate_text": "A young girl is in the middle of jumping during her dance routine.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6119:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6035:premise", "snli:test:6238:premise", "snli:test:5709:premise", "snli:test:6119:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Three people on skis are standing behind a no skiing sign.", "A man balances a unicycle using his face and both hands.", "A young boy hanging on a pole smiling at the camera.", "A young girl, is mid jump, while performing a dance routine.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6119", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6121", "candidate_text": "well, a man is getting ready to drink ice tea", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6121:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6079:premise", "snli:test:6121:premise", "snli:test:6324:premise", "snli:test:6066:premise", "style:personachat:test:229:self:2:positive:0", "style:personachat:test:229:self:2:positive:1"], "evidence_texts": ["A brown dog plays in a deep pile of snow.", "a young man pouring tea inside of a glass jar.", "It looks like the clown has fallen off the horse.", "a blond woman kissing the cheek of a brown-haired man.", "it is beautiful . great for long walks with hubby and kids .", "i agree . i love nature ."], "speaker_id": "snli:test:6121", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6121", "candidate_text": "I wonder whether a man is getting ready to drink ice tea?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6121:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6079:premise", "snli:test:6121:premise", "snli:test:6324:premise", "snli:test:6066:premise", "style:personachat:test:871:self:2:positive:0", "style:personachat:test:871:self:2:positive:1"], "evidence_texts": ["A brown dog plays in a deep pile of snow.", "a young man pouring tea inside of a glass jar.", "It looks like the clown has fallen off the horse.", "a blond woman kissing the cheek of a brown-haired man.", "i work as a science fiction writer . what do you do ?", "i need a lawyer , lol . are you a good one ?"], "speaker_id": "snli:test:6121", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6135", "candidate_text": "well, the couple has their christmas tree out", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6135:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6048:premise", "snli:test:6293:premise", "snli:test:5336:premise", "snli:test:6135:premise", "style:personachat:test:229:self:5:positive:0", "style:personachat:test:229:self:5:positive:1", "style:personachat:test:229:self:5:positive:2", "style:personachat:test:229:self:5:positive:3", "style:personachat:test:229:self:5:positive:4"], "evidence_texts": ["A young couple holding hands in the park.", "A man looks back while laughing and walking.", "This woman is skiing down a snowy hill.", "A young couple thinking about their Christmas decorations.", "that is great that you play soccer . i am into fitness .", "i would love that work . do you swim ?", "yes . i swim every day . in the sea .", "it is beautiful . great for long walks with hubby and kids .", "i agree . i love nature ."], "speaker_id": "snli:test:6135", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6135", "candidate_text": "Honestly, The couple has their Christmas tree out!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6135:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6048:premise", "snli:test:6293:premise", "snli:test:5336:premise", "snli:test:6135:premise", "style:personachat:test:680:self:2:positive:0", "style:personachat:test:680:self:2:positive:1"], "evidence_texts": ["A young couple holding hands in the park.", "A man looks back while laughing and walking.", "This woman is skiing down a snowy hill.", "A young couple thinking about their Christmas decorations.", "it is very expensive . to expensive , i think", "yeah . also , hotels are very expensive as well !"], "speaker_id": "snli:test:6135", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6158", "candidate_text": "well, man walking a dog", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6158:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6074:premise", "snli:test:5830:premise", "snli:test:6158:premise", "snli:test:6176:premise", "style:personachat:test:229:self:7:positive:0", "style:personachat:test:229:self:7:positive:1", "style:personachat:test:229:self:7:positive:2", "style:personachat:test:229:self:7:positive:3", "style:personachat:test:229:self:7:positive:4", "style:personachat:test:229:self:7:positive:5", "style:personachat:test:229:self:7:positive:6"], "evidence_texts": ["A woman in her swimsuit is kicking a ball on the beach.", "A shirtless man with red paint on his torso holding a stick.", "A woman carries a small, unhappy child in a busy street scene.", "Several people talk and drink beer while sitting around a picnic table.", "i love my job as a kindergarten teacher . what do you do ?", "do you have children ? i love my kids .", "that is great that you play soccer . i am into fitness .", "i would love that work . do you swim ?", "yes . i swim every day . in the sea .", "it is beautiful . great for long walks with hubby and kids .", "i agree . i love nature ."], "speaker_id": "snli:test:6158", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6158", "candidate_text": "man walking a dog.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6158:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6074:premise", "snli:test:5830:premise", "snli:test:6158:premise", "snli:test:6176:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in her swimsuit is kicking a ball on the beach.", "A shirtless man with red paint on his torso holding a stick.", "A woman carries a small, unhappy child in a busy street scene.", "Several people talk and drink beer while sitting around a picnic table.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6158", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6159", "candidate_text": "I wonder whether woman carrying child to the store?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6159:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6176:premise", "snli:test:6074:premise", "snli:test:6196:premise", "snli:test:6159:premise", "style:personachat:test:230:self:1:positive:0"], "evidence_texts": ["Several people talk and drink beer while sitting around a picnic table.", "A woman in her swimsuit is kicking a ball on the beach.", "A brown and white dog is chained up near a red chair.", "A woman carries a small, unhappy child in a busy street scene.", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:6159", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6159", "candidate_text": "woman carrying child to the store.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6159:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6176:premise", "snli:test:6074:premise", "snli:test:6196:premise", "snli:test:6159:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Several people talk and drink beer while sitting around a picnic table.", "A woman in her swimsuit is kicking a ball on the beach.", "A brown and white dog is chained up near a red chair.", "A woman carries a small, unhappy child in a busy street scene.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6159", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6160", "candidate_text": "I wonder whether three of the men are talking on their phones?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6160:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6160:premise", "snli:test:3100:premise", "snli:test:3659:premise", "snli:test:2518:premise", "style:personachat:test:230:self:2:positive:0", "style:personachat:test:230:self:2:positive:1"], "evidence_texts": ["Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "yeah and why people are so mean . i live with my sis , but she ignores it .", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:6160", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6160", "candidate_text": "well, three of the men are talking on their phones", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6160:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6160:premise", "snli:test:3100:premise", "snli:test:3659:premise", "snli:test:2518:premise", "style:personachat:test:387:self:2:positive:0", "style:personachat:test:387:self:2:positive:1"], "evidence_texts": ["Multiple men fill the scene, one in a white button up shirt using his cellphone, the others wearing pink, orange, and green shirts engaged in various activities in the background.", "A man in a tie-dyed shirt and jeans is sitting on a bench with a dog and a guitar on his lap, as well as a harmonica near his mouth.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "An old man with a cowboy hat, cowboy boots, and blue jeans stands next to a cabin and looks off into the distance as the sun sets in the background.", "no , they are still around and helping me to pay for law school . lucky me !", "i am not . studying is keeping me very busy at the moment ."], "speaker_id": "snli:test:6160", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6164", "candidate_text": "I wonder whether the woman is playing soccer in the cobble stone streets of brazil?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6164:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6164:premise", "snli:test:5745:premise", "snli:test:6221:premise", "snli:test:5744:premise", "style:personachat:test:230:self:5:positive:0", "style:personachat:test:230:self:5:positive:1", "style:personachat:test:230:self:5:positive:2", "style:personachat:test:230:self:5:positive:3", "style:personachat:test:230:self:5:positive:4"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "woah ! that is cool . i am cleaning up after my dog ralph .", "i believe that . i am nervous about school with all the race stuff going on .", "my mom and dad are not here anymore , and being mixed i have questions .", "yeah and why people are so mean . i live with my sis , but she ignores it .", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:6164", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6164", "candidate_text": "Honestly, The woman is playing soccer in the cobble stone streets of Brazil!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6164:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6164:premise", "snli:test:5745:premise", "snli:test:6221:premise", "snli:test:5744:premise", "style:personachat:test:685:self:1:positive:0"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "definitely ! have you ridden any hybrid metal and wood ones ?"], "speaker_id": "snli:test:6164", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6167", "candidate_text": "I wonder whether two dogs fighting over a piece of fabric?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6167:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:5969:premise", "snli:test:5817:premise", "snli:test:6475:premise", "snli:test:6167:premise", "style:personachat:test:230:self:6:positive:0", "style:personachat:test:230:self:6:positive:1", "style:personachat:test:230:self:6:positive:2", "style:personachat:test:230:self:6:positive:3", "style:personachat:test:230:self:6:positive:4", "style:personachat:test:230:self:6:positive:5"], "evidence_texts": ["Two men with graying beards share a laugh while enjoying the nice weather.", "A photographer is using a tripod to take a picture of a sunset.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "A big dog and a smaller dog fight over a piece of fabric.", "that is cool ! what do you do there ? i just got home from dance", "woah ! that is cool . i am cleaning up after my dog ralph .", "i believe that . i am nervous about school with all the race stuff going on .", "my mom and dad are not here anymore , and being mixed i have questions .", "yeah and why people are so mean . i live with my sis , but she ignores it .", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:6167", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6167", "candidate_text": "Two dogs fighting over a piece of fabric.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6167:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5969:premise", "snli:test:5817:premise", "snli:test:6475:premise", "snli:test:6167:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men with graying beards share a laugh while enjoying the nice weather.", "A photographer is using a tripod to take a picture of a sunset.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "A big dog and a smaller dog fight over a piece of fabric.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6167", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6168", "candidate_text": "Honestly, A group of dogs fighting over a dead body!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6168:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6168:premise", "snli:test:6475:premise", "snli:test:5969:premise", "snli:test:6556:premise", "style:personachat:test:231:self:1:positive:0"], "evidence_texts": ["A big dog and a smaller dog fight over a piece of fabric.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "Two men with graying beards share a laugh while enjoying the nice weather.", "Three young people are sitting in a room and talking to one another.", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:6168", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6168", "candidate_text": "well, a group of dogs fighting over a dead body", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6168:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6168:premise", "snli:test:6475:premise", "snli:test:5969:premise", "snli:test:6556:premise", "style:personachat:test:388:self:1:positive:0"], "evidence_texts": ["A big dog and a smaller dog fight over a piece of fabric.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "Two men with graying beards share a laugh while enjoying the nice weather.", "Three young people are sitting in a room and talking to one another.", "i try to eat healthy . i work out a lot ."], "speaker_id": "snli:test:6168", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6176", "candidate_text": "Honestly, People talking and drinking while sitting at a table outside!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6176:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6176:premise", "snli:test:6159:premise", "snli:test:6196:premise", "snli:test:6158:premise", "style:personachat:test:231:self:2:positive:0", "style:personachat:test:231:self:2:positive:1"], "evidence_texts": ["Several people talk and drink beer while sitting around a picnic table.", "A woman carries a small, unhappy child in a busy street scene.", "A brown and white dog is chained up near a red chair.", "A woman carries a small, unhappy child in a busy street scene.", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:6176", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6176", "candidate_text": "People talking and drinking while sitting at a table outside.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6176:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6176:premise", "snli:test:6159:premise", "snli:test:6196:premise", "snli:test:6158:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Several people talk and drink beer while sitting around a picnic table.", "A woman carries a small, unhappy child in a busy street scene.", "A brown and white dog is chained up near a red chair.", "A woman carries a small, unhappy child in a busy street scene.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6176", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6184", "candidate_text": "Honestly, A man is riding in a spaceship to the moon!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6184:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6184:premise", "snli:test:5360:premise", "snli:test:6517:premise", "snli:test:5339:premise", "style:personachat:test:231:self:5:positive:0", "style:personachat:test:231:self:5:positive:1", "style:personachat:test:231:self:5:positive:2", "style:personachat:test:231:self:5:positive:3", "style:personachat:test:231:self:5:positive:4"], "evidence_texts": ["A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "i grew up in the last was . not sure what that means but it is true", "brr ! what do your parents do ? my dad was a firefighter and mom was a teacher", "wow ! i am not much into teaching . too torn up over my divorce .", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:6184", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6184", "candidate_text": "I wonder whether a man is riding in a spaceship to the moon?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6184:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6184:premise", "snli:test:5360:premise", "snli:test:6517:premise", "snli:test:5339:premise", "style:personachat:test:876:self:5:positive:0", "style:personachat:test:876:self:5:positive:1", "style:personachat:test:876:self:5:positive:2", "style:personachat:test:876:self:5:positive:3", "style:personachat:test:876:self:5:positive:4"], "evidence_texts": ["A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A lady sitting on a bench with a book on her lap with a boy and girl on either side of her.", "where are you from ? i grew up in south carolina", "did you got to college there ?", "i graduated recently . from usc . my parents pushed me into it , wanted better", "music is great ! my mom teaches that . my dad just tends the farm", "do you play with other musicians ?"], "speaker_id": "snli:test:6184", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6196", "candidate_text": "Honestly, The dog is chained to a chair!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6196:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6196:premise", "snli:test:6176:premise", "snli:test:6246:premise", "snli:test:6159:premise", "style:personachat:test:231:self:6:positive:0", "style:personachat:test:231:self:6:positive:1", "style:personachat:test:231:self:6:positive:2", "style:personachat:test:231:self:6:positive:3", "style:personachat:test:231:self:6:positive:4", "style:personachat:test:231:self:6:positive:5"], "evidence_texts": ["A brown and white dog is chained up near a red chair.", "Several people talk and drink beer while sitting around a picnic table.", "A man holding a pink cooler is talking to an older woman.", "A woman carries a small, unhappy child in a busy street scene.", "i enjoy spending time with my 3 girls not my ex though", "i grew up in the last was . not sure what that means but it is true", "brr ! what do your parents do ? my dad was a firefighter and mom was a teacher", "wow ! i am not much into teaching . too torn up over my divorce .", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:6196", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6196", "candidate_text": "well, the dog is chained to a chair", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6196:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6196:premise", "snli:test:6176:premise", "snli:test:6246:premise", "snli:test:6159:premise", "style:personachat:test:388:self:7:positive:0", "style:personachat:test:388:self:7:positive:1", "style:personachat:test:388:self:7:positive:2", "style:personachat:test:388:self:7:positive:3", "style:personachat:test:388:self:7:positive:4", "style:personachat:test:388:self:7:positive:5", "style:personachat:test:388:self:7:positive:6"], "evidence_texts": ["A brown and white dog is chained up near a red chair.", "Several people talk and drink beer while sitting around a picnic table.", "A man holding a pink cooler is talking to an older woman.", "A woman carries a small, unhappy child in a busy street scene.", "i am doing good . listening to some john denver .", "oh . what do you like to do ?", "i love horses . i grew up on a ranch .", "it was . i also like to garden and cook .", "what is your favorite thing to cook ?", "i use a lot of the veggies i grow to cook with .", "i try to eat healthy . i work out a lot ."], "speaker_id": "snli:test:6196", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6218", "candidate_text": "Honestly, A group of musicians!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6218:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:5726:premise", "snli:test:5725:premise", "snli:test:6218:premise", "snli:test:6266:premise", "style:personachat:test:232:self:1:positive:0"], "evidence_texts": ["The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:6218", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6218", "candidate_text": "A group of musicians.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6218:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5726:premise", "snli:test:5725:premise", "snli:test:6218:premise", "snli:test:6266:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6218", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6219", "candidate_text": "Honestly, A group of athletes!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6219:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6266:premise", "snli:test:5726:premise", "snli:test:6396:premise", "snli:test:6219:premise", "style:personachat:test:232:self:2:positive:0", "style:personachat:test:232:self:2:positive:1"], "evidence_texts": ["A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:6219", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6219", "candidate_text": "I wonder whether a group of athletes?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6219:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6266:premise", "snli:test:5726:premise", "snli:test:6396:premise", "snli:test:6219:premise", "style:personachat:test:880:self:2:positive:0", "style:personachat:test:880:self:2:positive:1"], "evidence_texts": ["A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "The festivities of the Latin celebration has brought many visitors and enthusiastic, colorful, performers to the city.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "nice do you like to go out to eat ? i enjoy finding good barbeque restaurants .", "what do you do when you go out ? i go to the gym and play racquetball ."], "speaker_id": "snli:test:6219", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6221", "candidate_text": "Honestly, The woman is wearing jeans!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6221:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6164:premise", "snli:test:6221:premise", "snli:test:6827:premise", "snli:test:5745:premise", "style:personachat:test:232:self:5:positive:0", "style:personachat:test:232:self:5:positive:1", "style:personachat:test:232:self:5:positive:2", "style:personachat:test:232:self:5:positive:3", "style:personachat:test:232:self:5:positive:4"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "jerry . i used to live in arizona . now i am living in new york .", "yes , i love to sing on my modeling jobs .", "what is your dream job ? mine is becoming an actress .", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:6221", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6221", "candidate_text": "well, the woman is wearing jeans", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6221:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6164:premise", "snli:test:6221:premise", "snli:test:6827:premise", "snli:test:5745:premise", "style:personachat:test:390:self:5:positive:0", "style:personachat:test:390:self:5:positive:1", "style:personachat:test:390:self:5:positive:2", "style:personachat:test:390:self:5:positive:3", "style:personachat:test:390:self:5:positive:4"], "evidence_texts": ["A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "A person is riding their motorbike and a person in a bright yellow jacket is watching someone fall off their bike.", "cool , i binge watch it", "what does lol i have 2 jobs like this , you ?", "i want to write books myself", "you must not be interested no i want to write about fantasy", "well you be safe goodnight okay"], "speaker_id": "snli:test:6221", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6238", "candidate_text": "Honestly, A man is working for a circus!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6238:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6119:premise", "snli:test:6413:premise", "snli:test:6238:premise", "snli:test:6035:premise", "style:personachat:test:232:self:6:positive:0", "style:personachat:test:232:self:6:positive:1", "style:personachat:test:232:self:6:positive:2", "style:personachat:test:232:self:6:positive:3", "style:personachat:test:232:self:6:positive:4", "style:personachat:test:232:self:6:positive:5"], "evidence_texts": ["A young girl, is mid jump, while performing a dance routine.", "A topless man straps surfboards on top of his blue car.", "A man balances a unicycle using his face and both hands.", "Three people on skis are standing behind a no skiing sign.", "i have a dog . it is perfect for my single life", "jerry . i used to live in arizona . now i am living in new york .", "yes , i love to sing on my modeling jobs .", "what is your dream job ? mine is becoming an actress .", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:6238", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6238", "candidate_text": "A man is working for a circus.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6238:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6119:premise", "snli:test:6413:premise", "snli:test:6238:premise", "snli:test:6035:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young girl, is mid jump, while performing a dance routine.", "A topless man straps surfboards on top of his blue car.", "A man balances a unicycle using his face and both hands.", "Three people on skis are standing behind a no skiing sign.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6238", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6246", "candidate_text": "well, a man talks to a woman", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6246:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6196:premise", "snli:test:6299:premise", "snli:test:6246:premise", "snli:test:6176:premise", "style:personachat:test:233:self:1:positive:0"], "evidence_texts": ["A brown and white dog is chained up near a red chair.", "A woman in a dress is singing and having a good time.", "A man holding a pink cooler is talking to an older woman.", "Several people talk and drink beer while sitting around a picnic table.", "that is funny . i used to live in california"], "speaker_id": "snli:test:6246", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6246", "candidate_text": "I wonder whether a man talks to a woman?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6246:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6196:premise", "snli:test:6299:premise", "snli:test:6246:premise", "snli:test:6176:premise", "style:personachat:test:882:self:1:positive:0"], "evidence_texts": ["A brown and white dog is chained up near a red chair.", "A woman in a dress is singing and having a good time.", "A man holding a pink cooler is talking to an older woman.", "Several people talk and drink beer while sitting around a picnic table.", "what types of things do you like to craft ?"], "speaker_id": "snli:test:6246", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6251", "candidate_text": "well, a child runs and screams violently in around the train", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6251:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6078:premise", "snli:test:6480:premise", "snli:test:6070:premise", "snli:test:6251:premise", "style:personachat:test:233:self:2:positive:0", "style:personachat:test:233:self:2:positive:1"], "evidence_texts": ["A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "Two young girls in bathing suits laugh while standing under a fountain of water.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "A kid bored in a train with brown hair and his head lying down.", "neat , i love a good detective show", "that is funny . i used to live in california"], "speaker_id": "snli:test:6251", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6251", "candidate_text": "Honestly, A child runs and screams violently in around the train!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6251:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6078:premise", "snli:test:6480:premise", "snli:test:6070:premise", "snli:test:6251:premise", "style:personachat:test:696:self:6:positive:0", "style:personachat:test:696:self:6:positive:1", "style:personachat:test:696:self:6:positive:2", "style:personachat:test:696:self:6:positive:3", "style:personachat:test:696:self:6:positive:4", "style:personachat:test:696:self:6:positive:5"], "evidence_texts": ["A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "Two young girls in bathing suits laugh while standing under a fountain of water.", "A man in a hat is driving a horse-driven cart down a cobblestone street.", "A kid bored in a train with brown hair and his head lying down.", "hi , i am wonderful and you ?", "really . . . i am sorry he or she is broken .", "lol , funny . i am listening to my favorite music , jazz . and you ?", "cool , blue roots . i have clients that color their pets hair blue .", "yes , part time . however , i work with children or a pediatrician .", "awesome ! i loved school . i enjoyed running and reading ."], "speaker_id": "snli:test:6251", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6253", "candidate_text": "well, a person is taking a break from walking", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6253:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5939:premise", "snli:test:6253:premise", "snli:test:6401:premise", "snli:test:5911:premise", "style:personachat:test:233:self:5:positive:0", "style:personachat:test:233:self:5:positive:1", "style:personachat:test:233:self:5:positive:2", "style:personachat:test:233:self:5:positive:3", "style:personachat:test:233:self:5:positive:4"], "evidence_texts": ["Two cars are being driven in a car race.", "A person sitting on a rock beside a waterfall.", "A group of men playing rugby on the sand.", "A female camera operator is capturing some off-camera activity.", "i have a cat . he likes to watch me play video games .", "yes . i am getting ready to read a book , love a good drama", "interesting . i am a vet , but i enjoy my musician job on the weekends", "neat , i love a good detective show", "that is funny . i used to live in california"], "speaker_id": "snli:test:6253", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6253", "candidate_text": "A person is taking a break from walking.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6253:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:5939:premise", "snli:test:6253:premise", "snli:test:6401:premise", "snli:test:5911:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two cars are being driven in a car race.", "A person sitting on a rock beside a waterfall.", "A group of men playing rugby on the sand.", "A female camera operator is capturing some off-camera activity.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6253", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6260", "candidate_text": "well, two men shining shoes in the plaza", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6260:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6260:premise", "snli:test:5610:premise", "snli:test:6575:premise", "snli:test:5490:premise", "style:personachat:test:233:self:6:positive:0", "style:personachat:test:233:self:6:positive:1", "style:personachat:test:233:self:6:positive:2", "style:personachat:test:233:self:6:positive:3", "style:personachat:test:233:self:6:positive:4", "style:personachat:test:233:self:6:positive:5"], "evidence_texts": ["Two men shining peoples shoes.", "pedestrian walking on the street", "A dog catching a Frisbee.", "A couple walking their dog.", "beautiful . my name is jordan and i am enjoying the florida sunshine", "i have a cat . he likes to watch me play video games .", "yes . i am getting ready to read a book , love a good drama", "interesting . i am a vet , but i enjoy my musician job on the weekends", "neat , i love a good detective show", "that is funny . i used to live in california"], "speaker_id": "snli:test:6260", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6260", "candidate_text": "I wonder whether two men shining shoes in the plaza?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6260:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6260:premise", "snli:test:5610:premise", "snli:test:6575:premise", "snli:test:5490:premise", "style:personachat:test:882:self:7:positive:0", "style:personachat:test:882:self:7:positive:1", "style:personachat:test:882:self:7:positive:2", "style:personachat:test:882:self:7:positive:3", "style:personachat:test:882:self:7:positive:4", "style:personachat:test:882:self:7:positive:5", "style:personachat:test:882:self:7:positive:6"], "evidence_texts": ["Two men shining peoples shoes.", "pedestrian walking on the street", "A dog catching a Frisbee.", "A couple walking their dog.", "hello , i am doing just fine . how about you ?", "that sounds like a good time .", "i love reading too ! what types of books do you like ?", "i prefer fiction and sci fi", "right now i am just working as a secretary , but i hope to move up", "beyond reading i really like to play soccer , i was on the team back in college", "what types of things do you like to craft ?"], "speaker_id": "snli:test:6260", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6266", "candidate_text": "well, the young man is getting ice cream", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6266:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6219:premise", "snli:test:6396:premise", "snli:test:6266:premise", "snli:test:6218:premise", "style:personachat:test:234:self:1:positive:0"], "evidence_texts": ["A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "haha . you are funny . i just have my two cats ."], "speaker_id": "snli:test:6266", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6266", "candidate_text": "Honestly, The young man is getting ice cream!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6266:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6219:premise", "snli:test:6396:premise", "snli:test:6266:premise", "snli:test:6218:premise", "style:personachat:test:698:self:5:positive:0", "style:personachat:test:698:self:5:positive:1", "style:personachat:test:698:self:5:positive:2", "style:personachat:test:698:self:5:positive:3", "style:personachat:test:698:self:5:positive:4"], "evidence_texts": ["A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "i take walks in my garden a few times a week .", "oh i love siamese cats , i teach their physiology .", "yes , but if i am not teaching i am traveling different ecosystems .", "it is okay , start small .", "oh , i think you should travel , i have been to 6 countries !"], "speaker_id": "snli:test:6266", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6271", "candidate_text": "well, some men are polishing the shoes of business men", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6271:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5604:premise", "snli:test:6286:premise", "snli:test:4914:premise", "snli:test:6271:premise", "style:personachat:test:234:self:2:positive:0", "style:personachat:test:234:self:2:positive:1"], "evidence_texts": ["A young girl running on grass", "A skateboarder sails over white steps.", "Two boys in running clothes stretching.", "Two men work outside polishing shoes.", "romance novels . do you have any children ?", "haha . you are funny . i just have my two cats ."], "speaker_id": "snli:test:6271", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6271", "candidate_text": "Some men are polishing the shoes of business men.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6271:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5604:premise", "snli:test:6286:premise", "snli:test:4914:premise", "snli:test:6271:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young girl running on grass", "A skateboarder sails over white steps.", "Two boys in running clothes stretching.", "Two men work outside polishing shoes.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6271", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6286", "candidate_text": "well, the skater was elated", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6286:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6271:premise", "snli:test:6302:premise", "snli:test:6286:premise", "snli:test:5604:premise", "style:personachat:test:234:self:5:positive:0", "style:personachat:test:234:self:5:positive:1", "style:personachat:test:234:self:5:positive:2", "style:personachat:test:234:self:5:positive:3", "style:personachat:test:234:self:5:positive:4"], "evidence_texts": ["Two men work outside polishing shoes.", "A dog is walking down steps.", "A skateboarder sails over white steps.", "A young girl running on grass", "that sucks . i am writing a romance novel . what are your hobbies ?", "i reside on an island far away and it has made me an introvert .", "no , i just want to focus on being a writer right now . are you ?", "romance novels . do you have any children ?", "haha . you are funny . i just have my two cats ."], "speaker_id": "snli:test:6286", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6286", "candidate_text": "I wonder whether the skater was elated?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6286:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6271:premise", "snli:test:6302:premise", "snli:test:6286:premise", "snli:test:5604:premise", "style:personachat:test:883:self:5:positive:0", "style:personachat:test:883:self:5:positive:1", "style:personachat:test:883:self:5:positive:2", "style:personachat:test:883:self:5:positive:3", "style:personachat:test:883:self:5:positive:4"], "evidence_texts": ["Two men work outside polishing shoes.", "A dog is walking down steps.", "A skateboarder sails over white steps.", "A young girl running on grass", "sail in , although my day trade is carpentry . what do you do for a living ?", "that sounds like fun . i would like to take my wife and kids to europe .", "yep . wish i had more friends , but they kept losing to me in poker .", "need to go to europe just for the new chances to win .", "i hear you . do have some other hobbies ?"], "speaker_id": "snli:test:6286", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6291", "candidate_text": "well, 2 male humans are outside with a hat on", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6291:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6013:premise", "snli:test:6453:premise", "snli:test:5929:premise", "snli:test:6291:premise", "style:personachat:test:234:self:6:positive:0", "style:personachat:test:234:self:6:positive:1", "style:personachat:test:234:self:6:positive:2", "style:personachat:test:234:self:6:positive:3", "style:personachat:test:234:self:6:positive:4", "style:personachat:test:234:self:6:positive:5"], "evidence_texts": ["A black and white picture of a city street with a food vendor in the foreground.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "Two men in hard hats look at a building while on the side of a street.", "i am great . just cuddling with my two cats . . and you ?", "that sucks . i am writing a romance novel . what are your hobbies ?", "i reside on an island far away and it has made me an introvert .", "no , i just want to focus on being a writer right now . are you ?", "romance novels . do you have any children ?", "haha . you are funny . i just have my two cats ."], "speaker_id": "snli:test:6291", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6291", "candidate_text": "Honestly, 2 male humans are outside with a hat on!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6291:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6013:premise", "snli:test:6453:premise", "snli:test:5929:premise", "snli:test:6291:premise", "style:personachat:test:701:self:2:positive:0", "style:personachat:test:701:self:2:positive:1"], "evidence_texts": ["A black and white picture of a city street with a food vendor in the foreground.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "4 children and 1 adult look at an armadillo on a grassy hill with 2 trees.", "Two men in hard hats look at a building while on the side of a street.", "sounds boring . i hate to read it sucks", "ah okay ! and not sure it hurts my ears . i prefer to cook and clean"], "speaker_id": "snli:test:6291", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6293", "candidate_text": "well, a man is falling down on the ground", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6293:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6135:premise", "snli:test:6293:premise", "snli:test:6408:premise", "snli:test:6048:premise", "style:personachat:test:235:self:1:positive:0"], "evidence_texts": ["A young couple thinking about their Christmas decorations.", "A man looks back while laughing and walking.", "A bundled up toddler is walking over snow.", "A young couple holding hands in the park.", "ooh i see . mine is dogs i am a wss"], "speaker_id": "snli:test:6293", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6293", "candidate_text": "A man is falling down on the ground.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6293:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6135:premise", "snli:test:6293:premise", "snli:test:6408:premise", "snli:test:6048:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A young couple thinking about their Christmas decorations.", "A man looks back while laughing and walking.", "A bundled up toddler is walking over snow.", "A young couple holding hands in the park.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6293", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6299", "candidate_text": "well, a woman is having a bad time singing", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6299:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6246:premise", "snli:test:6533:premise", "snli:test:6196:premise", "snli:test:6299:premise", "style:personachat:test:235:self:2:positive:0", "style:personachat:test:235:self:2:positive:1"], "evidence_texts": ["A man holding a pink cooler is talking to an older woman.", "a snowboarder wearing a green hat is riding along a metal railing.", "A brown and white dog is chained up near a red chair.", "A woman in a dress is singing and having a good time.", "is he dying i am sorry", "ooh i see . mine is dogs i am a wss"], "speaker_id": "snli:test:6299", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6299", "candidate_text": "I wonder whether a woman is having a bad time singing?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6299:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6246:premise", "snli:test:6533:premise", "snli:test:6196:premise", "snli:test:6299:premise", "style:personachat:test:887:self:2:positive:0", "style:personachat:test:887:self:2:positive:1"], "evidence_texts": ["A man holding a pink cooler is talking to an older woman.", "a snowboarder wearing a green hat is riding along a metal railing.", "A brown and white dog is chained up near a red chair.", "A woman in a dress is singing and having a good time.", "i love hiking , camping and cooking . love the outdoors .", "i listen to everything but mainly enjoy rock how about yourself ?"], "speaker_id": "snli:test:6299", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6302", "candidate_text": "well, a large dog is walking down the front steps", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6302:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6286:premise", "snli:test:6387:premise", "snli:test:6302:premise", "snli:test:6271:premise", "style:personachat:test:235:self:5:positive:0", "style:personachat:test:235:self:5:positive:1", "style:personachat:test:235:self:5:positive:2", "style:personachat:test:235:self:5:positive:3", "style:personachat:test:235:self:5:positive:4"], "evidence_texts": ["A skateboarder sails over white steps.", "Instructor on Basket weaving at fair", "A dog is walking down steps.", "Two men work outside polishing shoes.", "i like that too you drink ?", "i have scotch to chill out at night", "got any fears there buddy ?", "is he dying i am sorry", "ooh i see . mine is dogs i am a wss"], "speaker_id": "snli:test:6302", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6302", "candidate_text": "Honestly, A large dog is walking down the front steps!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6302:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6286:premise", "snli:test:6387:premise", "snli:test:6302:premise", "snli:test:6271:premise", "style:personachat:test:712:self:1:positive:0"], "evidence_texts": ["A skateboarder sails over white steps.", "Instructor on Basket weaving at fair", "A dog is walking down steps.", "Two men work outside polishing shoes.", "hah ! no . why did you quit being a republican ?"], "speaker_id": "snli:test:6302", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6309", "candidate_text": "well, people stand in front of the new display", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6309:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5058:premise", "snli:test:6309:premise", "snli:test:6389:premise", "snli:test:5007:premise", "style:personachat:test:235:self:7:positive:0", "style:personachat:test:235:self:7:positive:1", "style:personachat:test:235:self:7:positive:2", "style:personachat:test:235:self:7:positive:3", "style:personachat:test:235:self:7:positive:4", "style:personachat:test:235:self:7:positive:5", "style:personachat:test:235:self:7:positive:6"], "evidence_texts": ["A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "hey how is it going ?", "you do not like to move around", "i like that too you drink ?", "i have scotch to chill out at night", "got any fears there buddy ?", "is he dying i am sorry", "ooh i see . mine is dogs i am a wss"], "speaker_id": "snli:test:6309", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6309", "candidate_text": "People stand in front of the new display.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6309:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5058:premise", "snli:test:6309:premise", "snli:test:6389:premise", "snli:test:5007:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "A girl wearing a knit cap, gloves, winter jacket, snow pants and boots stands by a frozen waterfall, with a metal pole by her side.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6309", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6324", "candidate_text": "well, the sad clown fell off the horse at the state fair rodeo", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6324:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6324:premise", "snli:test:6121:premise", "snli:test:6444:premise", "snli:test:6079:premise", "style:personachat:test:236:self:1:positive:0"], "evidence_texts": ["It looks like the clown has fallen off the horse.", "a young man pouring tea inside of a glass jar.", "A little girl in a colorful shirt holding a flower.", "A brown dog plays in a deep pile of snow.", "not to big on it these days . pretty much just watch what i am eating really"], "speaker_id": "snli:test:6324", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6324", "candidate_text": "I wonder whether the sad clown fell off the horse at the state fair rodeo?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6324:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6324:premise", "snli:test:6121:premise", "snli:test:6444:premise", "snli:test:6079:premise", "style:personachat:test:888:self:1:positive:0"], "evidence_texts": ["It looks like the clown has fallen off the horse.", "a young man pouring tea inside of a glass jar.", "A little girl in a colorful shirt holding a flower.", "A brown dog plays in a deep pile of snow.", "oh , is she a romantic interest ? or just a friend ?"], "speaker_id": "snli:test:6324", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6364", "candidate_text": "well, an elderly woman is shown outside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6364:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6364:premise", "snli:test:5907:premise", "snli:test:6454:premise", "snli:test:5716:premise", "style:personachat:test:236:self:2:positive:0", "style:personachat:test:236:self:2:positive:1"], "evidence_texts": ["an elderly woman walking down the street.", "A girl is blowing a huge bubble.", "Naked children play on the muddy riverbank.", "A football layer wearing a red shirt.", "what are you principles you stick too ?", "not to big on it these days . pretty much just watch what i am eating really"], "speaker_id": "snli:test:6364", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6364", "candidate_text": "Honestly, An elderly woman is shown outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6364:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6364:premise", "snli:test:5907:premise", "snli:test:6454:premise", "snli:test:5716:premise", "style:personachat:test:712:self:7:positive:0", "style:personachat:test:712:self:7:positive:1", "style:personachat:test:712:self:7:positive:2", "style:personachat:test:712:self:7:positive:3", "style:personachat:test:712:self:7:positive:4", "style:personachat:test:712:self:7:positive:5", "style:personachat:test:712:self:7:positive:6"], "evidence_texts": ["an elderly woman walking down the street.", "A girl is blowing a huge bubble.", "Naked children play on the muddy riverbank.", "A football layer wearing a red shirt.", "i am the proud parent of three girls . currently looking for work .", "i would like to work more in public service like my parents .", "my mother would be proud of you , she was a teacher . i might try libraries .", "i think i would do well . after my divorce i started researching the peace corps .", "no politics for me . i would prefer a good heart concert instead .", "my father saved a few people from fires . . . i feel some sort of calling to help people .", "hah ! no . why did you quit being a republican ?"], "speaker_id": "snli:test:6364", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6387", "candidate_text": "well, a basket weaver teaching a class at the fair", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6387:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6302:premise", "snli:test:6616:premise", "snli:test:6286:premise", "snli:test:6387:premise", "style:personachat:test:236:self:5:positive:0", "style:personachat:test:236:self:5:positive:1", "style:personachat:test:236:self:5:positive:2", "style:personachat:test:236:self:5:positive:3", "style:personachat:test:236:self:5:positive:4"], "evidence_texts": ["A dog is walking down steps.", "A man wearing black carving stone.", "A skateboarder sails over white steps.", "Instructor on Basket weaving at fair", "not really more into civil war things . too many wives in the past", "that is very nice . what kind of diets ?", "do you eat anything special for it ?", "what are you principles you stick too ?", "not to big on it these days . pretty much just watch what i am eating really"], "speaker_id": "snli:test:6387", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6387", "candidate_text": "A basket weaver teaching a class at the fair.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6387:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6302:premise", "snli:test:6616:premise", "snli:test:6286:premise", "snli:test:6387:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A dog is walking down steps.", "A man wearing black carving stone.", "A skateboarder sails over white steps.", "Instructor on Basket weaving at fair", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6387", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6389", "candidate_text": "well, the guys are having a party at the park", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6389:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6309:premise", "snli:test:6389:premise", "snli:test:7234:premise", "snli:test:5058:premise", "style:personachat:test:236:self:7:positive:0", "style:personachat:test:236:self:7:positive:1", "style:personachat:test:236:self:7:positive:2", "style:personachat:test:236:self:7:positive:3", "style:personachat:test:236:self:7:positive:4", "style:personachat:test:236:self:7:positive:5", "style:personachat:test:236:self:7:positive:6"], "evidence_texts": ["A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "i am pretty good just got home", "no i work at a groomers tho they are like my pets .", "not really more into civil war things . too many wives in the past", "that is very nice . what kind of diets ?", "do you eat anything special for it ?", "what are you principles you stick too ?", "not to big on it these days . pretty much just watch what i am eating really"], "speaker_id": "snli:test:6389", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6389", "candidate_text": "I wonder whether the guys are having a party at the park?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6389:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6309:premise", "snli:test:6389:premise", "snli:test:7234:premise", "snli:test:5058:premise", "style:personachat:test:888:self:6:positive:0", "style:personachat:test:888:self:6:positive:1", "style:personachat:test:888:self:6:positive:2", "style:personachat:test:888:self:6:positive:3", "style:personachat:test:888:self:6:positive:4", "style:personachat:test:888:self:6:positive:5"], "evidence_texts": ["A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "great ! how are you doing ?", "oh no ! i am sorry . what is your favorite band ?", "i like rock , but i like a wide variety too", "oh neat ! i have a dog named ginger . do you have pets too ?", "they are so rewarding and are great company !", "oh , is she a romantic interest ? or just a friend ?"], "speaker_id": "snli:test:6389", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6393", "candidate_text": "well, a bald man sits in a red chair", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6393:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6011:premise", "snli:test:6393:premise", "snli:test:6650:premise", "snli:test:5891:premise", "style:personachat:test:237:self:1:positive:0"], "evidence_texts": ["A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "i love music . pink floyd is pretty great , especially while watching the wizard of oz ."], "speaker_id": "snli:test:6393", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6393", "candidate_text": "Honestly, A bald man sits in a red chair!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6393:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6011:premise", "snli:test:6393:premise", "snli:test:6650:premise", "snli:test:5891:premise", "style:personachat:test:716:self:5:positive:0", "style:personachat:test:716:self:5:positive:1", "style:personachat:test:716:self:5:positive:2", "style:personachat:test:716:self:5:positive:3", "style:personachat:test:716:self:5:positive:4"], "evidence_texts": ["A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "2 people with backpacks are at a table with a woman with a beige blazer sitting behind it.", "what church do you go to ? do they have candy ? i love candy !", "oh in which ways are you ?", "that is neat . i have a dog in school because he is so bad .", "do you ever take pictures of yourself hunting and post them ?", "yes all the time ! i like to post pictures of myself on face book ."], "speaker_id": "snli:test:6393", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6396", "candidate_text": "well, young woman are walking", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6396:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6396:premise", "snli:test:6266:premise", "snli:test:6528:premise", "snli:test:6219:premise", "style:personachat:test:237:self:2:positive:0", "style:personachat:test:237:self:2:positive:1"], "evidence_texts": ["A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "People checking out a car with all its doors open in a parking lot in the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "true . you must have lots of insight into both male and female .", "i love music . pink floyd is pretty great , especially while watching the wizard of oz ."], "speaker_id": "snli:test:6396", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6396", "candidate_text": "Young woman are walking.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6396:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6396:premise", "snli:test:6266:premise", "snli:test:6528:premise", "snli:test:6219:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "People checking out a car with all its doors open in a parking lot in the city.", "A group of young band members wearing black perform on instruments including the saxophone and the trombone.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6396", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6401", "candidate_text": "well, a rugby team plays a match at the beach", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6401:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6253:premise", "snli:test:6401:premise", "snli:test:6405:premise", "snli:test:5939:premise", "style:personachat:test:237:self:5:positive:0", "style:personachat:test:237:self:5:positive:1", "style:personachat:test:237:self:5:positive:2", "style:personachat:test:237:self:5:positive:3", "style:personachat:test:237:self:5:positive:4"], "evidence_texts": ["A person sitting on a rock beside a waterfall.", "A group of men playing rugby on the sand.", "Beautiful girl in white dress steps into the sunlight.", "Two cars are being driven in a car race.", "i am 20 years old and studying in college . and you ?", "i am studying marine biology . i love marine life and the ocean .", "agreed ! i am allergic to cats . what do you want to study in college ?", "true . you must have lots of insight into both male and female .", "i love music . pink floyd is pretty great , especially while watching the wizard of oz ."], "speaker_id": "snli:test:6401", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6401", "candidate_text": "I wonder whether a rugby team plays a match at the beach?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6401:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6253:premise", "snli:test:6401:premise", "snli:test:6405:premise", "snli:test:5939:premise", "style:personachat:test:890:self:5:positive:0", "style:personachat:test:890:self:5:positive:1", "style:personachat:test:890:self:5:positive:2", "style:personachat:test:890:self:5:positive:3", "style:personachat:test:890:self:5:positive:4"], "evidence_texts": ["A person sitting on a rock beside a waterfall.", "A group of men playing rugby on the sand.", "Beautiful girl in white dress steps into the sunlight.", "Two cars are being driven in a car race.", "sorry to hear . do you collect ? i do . my favorite is civil war antiques .", "that is funny . i sometimes fake a british accent to be funny and attractive .", "hilarious ! do you have any food allergies ? mine is mangoes .", "you are very lucky ! i wish i could eat mangoes . i would eat them everyday .", "oh i like those too . what is your favorite vegetable ? i love them all ."], "speaker_id": "snli:test:6401", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6405", "candidate_text": "well, it is late at night", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6405:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6401:premise", "snli:test:6405:premise", "snli:test:6410:premise", "snli:test:6253:premise", "style:personachat:test:237:self:6:positive:0", "style:personachat:test:237:self:6:positive:1", "style:personachat:test:237:self:6:positive:2", "style:personachat:test:237:self:6:positive:3", "style:personachat:test:237:self:6:positive:4", "style:personachat:test:237:self:6:positive:5"], "evidence_texts": ["A group of men playing rugby on the sand.", "Beautiful girl in white dress steps into the sunlight.", "A robed dancer holding fire stick entertains a crowd.", "A person sitting on a rock beside a waterfall.", "i am great ! i just got done with a big biology test . how are you ?", "i am 20 years old and studying in college . and you ?", "i am studying marine biology . i love marine life and the ocean .", "agreed ! i am allergic to cats . what do you want to study in college ?", "true . you must have lots of insight into both male and female .", "i love music . pink floyd is pretty great , especially while watching the wizard of oz ."], "speaker_id": "snli:test:6405", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6405", "candidate_text": "Honestly, It is late at night!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6405:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6401:premise", "snli:test:6405:premise", "snli:test:6410:premise", "snli:test:6253:premise", "style:personachat:test:735:self:2:positive:0", "style:personachat:test:735:self:2:positive:1"], "evidence_texts": ["A group of men playing rugby on the sand.", "Beautiful girl in white dress steps into the sunlight.", "A robed dancer holding fire stick entertains a crowd.", "A person sitting on a rock beside a waterfall.", "cool , where are you from ? i live in south carolina , born and raised .", "that would be so fun !"], "speaker_id": "snli:test:6405", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6408", "candidate_text": "well, a toddler walking in sand", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6408:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6408:premise", "snli:test:6293:premise", "snli:test:6425:premise", "snli:test:6135:premise", "style:personachat:test:238:self:1:positive:0"], "evidence_texts": ["A bundled up toddler is walking over snow.", "A man looks back while laughing and walking.", "A group of people celebrating in the street.", "A young couple thinking about their Christmas decorations.", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:6408", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6408", "candidate_text": "A toddler walking in sand.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6408:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6408:premise", "snli:test:6293:premise", "snli:test:6425:premise", "snli:test:6135:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A bundled up toddler is walking over snow.", "A man looks back while laughing and walking.", "A group of people celebrating in the street.", "A young couple thinking about their Christmas decorations.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6408", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6410", "candidate_text": "well, a man in pants and a t-shirt doing fire tricks", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6410:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6405:premise", "snli:test:6419:premise", "snli:test:6410:premise", "snli:test:6401:premise", "style:personachat:test:238:self:2:positive:0", "style:personachat:test:238:self:2:positive:1"], "evidence_texts": ["Beautiful girl in white dress steps into the sunlight.", "Two men heavily immersed in a conversation worth having.", "A robed dancer holding fire stick entertains a crowd.", "A group of men playing rugby on the sand.", "may i ask what you are studying ?", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:6410", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6410", "candidate_text": "I wonder whether a man in pants and a t-shirt doing fire tricks?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6410:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6405:premise", "snli:test:6419:premise", "snli:test:6410:premise", "snli:test:6401:premise", "style:personachat:test:891:self:2:positive:0", "style:personachat:test:891:self:2:positive:1"], "evidence_texts": ["Beautiful girl in white dress steps into the sunlight.", "Two men heavily immersed in a conversation worth having.", "A robed dancer holding fire stick entertains a crowd.", "A group of men playing rugby on the sand.", "it took me 5 years as one year i was part time .", "are you honor or soar student ?"], "speaker_id": "snli:test:6410", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6413", "candidate_text": "well, a man is heading out to the beach for a day of surfing", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6413:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6238:premise", "snli:test:6413:premise", "snli:test:6436:premise", "snli:test:6119:premise", "style:personachat:test:238:self:5:positive:0", "style:personachat:test:238:self:5:positive:1", "style:personachat:test:238:self:5:positive:2", "style:personachat:test:238:self:5:positive:3", "style:personachat:test:238:self:5:positive:4"], "evidence_texts": ["A man balances a unicycle using his face and both hands.", "A topless man straps surfboards on top of his blue car.", "A man and woman look through CD's at a street vendor.", "A young girl, is mid jump, while performing a dance routine.", "watching horror movies are one of my favorites and i like to lift weights on weekends", "every weekend i lift weight and watch a horror movie or two . need to go shopping .", "clothes . i am a tall guy and kind of big so shopping is not my favorite", "may i ask what you are studying ?", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:6413", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6413", "candidate_text": "Honestly, A man is heading out to the beach for a day of surfing!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6413:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6238:premise", "snli:test:6413:premise", "snli:test:6436:premise", "snli:test:6119:premise", "style:personachat:test:736:self:1:positive:0"], "evidence_texts": ["A man balances a unicycle using his face and both hands.", "A topless man straps surfboards on top of his blue car.", "A man and woman look through CD's at a street vendor.", "A young girl, is mid jump, while performing a dance routine.", "heavens no ! you ever read any books on rock climbing ?"], "speaker_id": "snli:test:6413", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6419", "candidate_text": "well, two men are talking", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6419:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6410:premise", "snli:test:6430:premise", "snli:test:6405:premise", "snli:test:6419:premise", "style:personachat:test:238:self:6:positive:0", "style:personachat:test:238:self:6:positive:1", "style:personachat:test:238:self:6:positive:2", "style:personachat:test:238:self:6:positive:3", "style:personachat:test:238:self:6:positive:4", "style:personachat:test:238:self:6:positive:5"], "evidence_texts": ["A robed dancer holding fire stick entertains a crowd.", "Two women wearing red are smiling at the camera.", "Beautiful girl in white dress steps into the sunlight.", "Two men heavily immersed in a conversation worth having.", "i am well . just finishing up making some barbecue . i love it .", "watching horror movies are one of my favorites and i like to lift weights on weekends", "every weekend i lift weight and watch a horror movie or two . need to go shopping .", "clothes . i am a tall guy and kind of big so shopping is not my favorite", "may i ask what you are studying ?", "that is awesome . my son thought about studying law ."], "speaker_id": "snli:test:6419", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6419", "candidate_text": "Two men are talking.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6419:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6410:premise", "snli:test:6430:premise", "snli:test:6405:premise", "snli:test:6419:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A robed dancer holding fire stick entertains a crowd.", "Two women wearing red are smiling at the camera.", "Beautiful girl in white dress steps into the sunlight.", "Two men heavily immersed in a conversation worth having.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6419", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6425", "candidate_text": "I wonder whether people are celebrating their friend's birthday?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6425:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6408:premise", "snli:test:6425:premise", "snli:test:6704:premise", "snli:test:6293:premise", "style:personachat:test:239:self:1:positive:0"], "evidence_texts": ["A bundled up toddler is walking over snow.", "A group of people celebrating in the street.", "The squirrel sits straight up in the grass.", "A man looks back while laughing and walking.", "what do you do for work ?"], "speaker_id": "snli:test:6425", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6425", "candidate_text": "People are celebrating their friend's birthday.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6425:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6408:premise", "snli:test:6425:premise", "snli:test:6704:premise", "snli:test:6293:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A bundled up toddler is walking over snow.", "A group of people celebrating in the street.", "The squirrel sits straight up in the grass.", "A man looks back while laughing and walking.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6425", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6430", "candidate_text": "I wonder whether smiling at the camera the two women wore red?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6430:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6419:premise", "snli:test:6499:premise", "snli:test:6430:premise", "snli:test:6410:premise", "style:personachat:test:239:self:2:positive:0", "style:personachat:test:239:self:2:positive:1"], "evidence_texts": ["Two men heavily immersed in a conversation worth having.", "A man siting on a bench with a briefcase.", "Two women wearing red are smiling at the camera.", "A robed dancer holding fire stick entertains a crowd.", "i do . i really just run so i can eat what ever i like .", "what do you do for work ?"], "speaker_id": "snli:test:6430", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6430", "candidate_text": "well, smiling at the camera the two women wore red", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6430:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6419:premise", "snli:test:6499:premise", "snli:test:6430:premise", "snli:test:6410:premise", "style:personachat:test:400:self:2:positive:0", "style:personachat:test:400:self:2:positive:1"], "evidence_texts": ["Two men heavily immersed in a conversation worth having.", "A man siting on a bench with a briefcase.", "Two women wearing red are smiling at the camera.", "A robed dancer holding fire stick entertains a crowd.", "no we must strive to do everything we can to save the planet", "oh every kind possible . also global warming alarms me"], "speaker_id": "snli:test:6430", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6436", "candidate_text": "I wonder whether a man and women laying down in bed?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6436:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6436:premise", "snli:test:6413:premise", "snli:test:6563:premise", "snli:test:6238:premise", "style:personachat:test:239:self:5:positive:0", "style:personachat:test:239:self:5:positive:1", "style:personachat:test:239:self:5:positive:2", "style:personachat:test:239:self:5:positive:3", "style:personachat:test:239:self:5:positive:4"], "evidence_texts": ["A man and woman look through CD's at a street vendor.", "A topless man straps surfboards on top of his blue car.", "A gymnastics performer is in mid-pose as she completes her jump.", "A man balances a unicycle using his face and both hands.", "i do not watch much tv , but i do enjoy archer .", "i enjoy blues more because i am from chicago . what do you like about jazz ?", "i love new york . i will be coming there in a few weeks for the marathon .", "i do . i really just run so i can eat what ever i like .", "what do you do for work ?"], "speaker_id": "snli:test:6436", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6436", "candidate_text": "Honestly, A man and women laying down in bed!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6436:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6436:premise", "snli:test:6413:premise", "snli:test:6563:premise", "snli:test:6238:premise", "style:personachat:test:751:self:1:positive:0"], "evidence_texts": ["A man and woman look through CD's at a street vendor.", "A topless man straps surfboards on top of his blue car.", "A gymnastics performer is in mid-pose as she completes her jump.", "A man balances a unicycle using his face and both hands.", "haha , that is actually my ideal life right there !"], "speaker_id": "snli:test:6436", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6441", "candidate_text": "I wonder whether many people are being led through the desert on the backs on camels?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6441:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:5556:premise", "snli:test:6441:premise", "snli:test:5779:premise", "snli:test:1694:premise", "style:personachat:test:239:self:7:positive:0", "style:personachat:test:239:self:7:positive:1", "style:personachat:test:239:self:7:positive:2", "style:personachat:test:239:self:7:positive:3", "style:personachat:test:239:self:7:positive:4", "style:personachat:test:239:self:7:positive:5", "style:personachat:test:239:self:7:positive:6"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "i am doing just fine . how are you today ?", "i am not particularly religious . why do you ask ?", "i do not watch much tv , but i do enjoy archer .", "i enjoy blues more because i am from chicago . what do you like about jazz ?", "i love new york . i will be coming there in a few weeks for the marathon .", "i do . i really just run so i can eat what ever i like .", "what do you do for work ?"], "speaker_id": "snli:test:6441", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6441", "candidate_text": "Many people are being led through the desert on the backs on camels.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6441:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:5556:premise", "snli:test:6441:premise", "snli:test:5779:premise", "snli:test:1694:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["At an outdoor event on a large table, several workers form an assembly line in which they prepare food, with many squeeze bottles and dispensers visible and a woman wearing plastic gloves in the foreground.", "Six people are riding camels and being led in a straight line across a gently rising sandy slope with mountains in the background, by a man walking and holding the tether of the lead camel.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "A lone person in a long, black trench coat and scarf and carrying a red bag goes past an apparently-closed cafe at night, with the text \"CAFE DE FLO\" visible on the cafe canopy banner.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6441", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6444", "candidate_text": "Honestly, The girl is holding a flower!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6444:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6444:premise", "snli:test:6324:premise", "snli:test:6450:premise", "snli:test:6121:premise", "style:personachat:test:240:self:1:positive:0"], "evidence_texts": ["A little girl in a colorful shirt holding a flower.", "It looks like the clown has fallen off the horse.", "Three people are posing on the side of a mountain.", "a young man pouring tea inside of a glass jar.", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:6444", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6444", "candidate_text": "well, the girl is holding a flower", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6444:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6444:premise", "snli:test:6324:premise", "snli:test:6450:premise", "snli:test:6121:premise", "style:personachat:test:401:self:1:positive:0"], "evidence_texts": ["A little girl in a colorful shirt holding a flower.", "It looks like the clown has fallen off the horse.", "Three people are posing on the side of a mountain.", "a young man pouring tea inside of a glass jar.", "yes i prefer alternative living"], "speaker_id": "snli:test:6444", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6450", "candidate_text": "Honestly, The three people are posing at the mountain!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6450:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6444:premise", "snli:test:6639:premise", "snli:test:6450:premise", "snli:test:6324:premise", "style:personachat:test:240:self:2:positive:0", "style:personachat:test:240:self:2:positive:1"], "evidence_texts": ["A little girl in a colorful shirt holding a flower.", "Two women in black enjoying a lunch in a restaurant.", "Three people are posing on the side of a mountain.", "It looks like the clown has fallen off the horse.", "he loves to eat . he will need that discount .", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:6450", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6450", "candidate_text": "The three people are posing at the mountain.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6450:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6444:premise", "snli:test:6639:premise", "snli:test:6450:premise", "snli:test:6324:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A little girl in a colorful shirt holding a flower.", "Two women in black enjoying a lunch in a restaurant.", "Three people are posing on the side of a mountain.", "It looks like the clown has fallen off the horse.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6450", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6453", "candidate_text": "Honestly, A child at a birthday party, plays with other children, laughing and screaming!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6453:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6291:premise", "snli:test:6453:premise", "snli:test:6863:premise", "snli:test:6013:premise", "style:personachat:test:240:self:5:positive:0", "style:personachat:test:240:self:5:positive:1", "style:personachat:test:240:self:5:positive:2", "style:personachat:test:240:self:5:positive:3", "style:personachat:test:240:self:5:positive:4"], "evidence_texts": ["Two men in hard hats look at a building while on the side of a street.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A black and white picture of a city street with a food vendor in the foreground.", "how about paint a picture of me eating onion rings ? i love onions . . . lol", "my dad is single . he is a doctor and i am his nurse in his practice .", "he might . which store do you work at ?", "he loves to eat . he will need that discount .", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:6453", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6453", "candidate_text": "I wonder whether a child at a birthday party, plays with other children, laughing and screaming?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6453:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6291:premise", "snli:test:6453:premise", "snli:test:6863:premise", "snli:test:6013:premise", "style:personachat:test:897:self:5:positive:0", "style:personachat:test:897:self:5:positive:1", "style:personachat:test:897:self:5:positive:2", "style:personachat:test:897:self:5:positive:3", "style:personachat:test:897:self:5:positive:4"], "evidence_texts": ["Two men in hard hats look at a building while on the side of a street.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A black and white picture of a city street with a food vendor in the foreground.", "i spent most my life in california sharks are grey . what parts do you sing ?", "neat . so do you perform or work in the industry ?", "if you can sing , go get on the voice ! or take up surfing .", "is there surfing around new york ?", "do not know why i thought that . where do you surf ?"], "speaker_id": "snli:test:6453", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6454", "candidate_text": "Honestly, THE RIVERBANK WAS PLAYED ON BY NUDITY OF KIDS!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6454:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6364:premise", "snli:test:6464:premise", "snli:test:6454:premise", "snli:test:5907:premise", "style:personachat:test:240:self:7:positive:0", "style:personachat:test:240:self:7:positive:1", "style:personachat:test:240:self:7:positive:2", "style:personachat:test:240:self:7:positive:3", "style:personachat:test:240:self:7:positive:4", "style:personachat:test:240:self:7:positive:5", "style:personachat:test:240:self:7:positive:6"], "evidence_texts": ["an elderly woman walking down the street.", "A man hammering nails on a shed.", "Naked children play on the muddy riverbank.", "A girl is blowing a huge bubble.", "sure ! but do not paint me with a cat , i hate cats .", "i do not like taylor swift , my dad is always listening to her music and i do not get along with my dad .", "how about paint a picture of me eating onion rings ? i love onions . . . lol", "my dad is single . he is a doctor and i am his nurse in his practice .", "he might . which store do you work at ?", "he loves to eat . he will need that discount .", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:6454", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6454", "candidate_text": "well, the riverbank was played on by nudity of kids", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6454:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6364:premise", "snli:test:6464:premise", "snli:test:6454:premise", "snli:test:5907:premise", "style:personachat:test:401:self:6:positive:0", "style:personachat:test:401:self:6:positive:1", "style:personachat:test:401:self:6:positive:2", "style:personachat:test:401:self:6:positive:3", "style:personachat:test:401:self:6:positive:4", "style:personachat:test:401:self:6:positive:5"], "evidence_texts": ["an elderly woman walking down the street.", "A man hammering nails on a shed.", "Naked children play on the muddy riverbank.", "A girl is blowing a huge bubble.", "hi my name is anne", "well i like to sing and sew clothes . i also love cats", "well , i own my own tiny home on wheels", "sometimes . most of my time goes to making my clothes", "that is cool . i cant stand high rise buildings", "yes i prefer alternative living"], "speaker_id": "snli:test:6454", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6464", "candidate_text": "I wonder whether the man is a handyman?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6464:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6464:premise", "snli:test:6454:premise", "snli:test:6520:premise", "snli:test:6364:premise", "style:personachat:test:241:self:1:positive:0"], "evidence_texts": ["A man hammering nails on a shed.", "Naked children play on the muddy riverbank.", "A man and a young girl swimming.", "an elderly woman walking down the street.", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:6464", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6464", "candidate_text": "Honestly, The man is a handyman!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6464:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6464:premise", "snli:test:6454:premise", "snli:test:6520:premise", "snli:test:6364:premise", "style:personachat:test:766:self:5:positive:0", "style:personachat:test:766:self:5:positive:1", "style:personachat:test:766:self:5:positive:2", "style:personachat:test:766:self:5:positive:3", "style:personachat:test:766:self:5:positive:4"], "evidence_texts": ["A man hammering nails on a shed.", "Naked children play on the muddy riverbank.", "A man and a young girl swimming.", "an elderly woman walking down the street.", "do you walk in your neighborhood ? i like to walk to my parents house .", "he is great . i am sure your vegetables are delicious .", "my parents live in the house i grew up in , just down the street .", "it is ! we go to church together every week and have dinner after .", "amen ! our country needs more god !"], "speaker_id": "snli:test:6464", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6475", "candidate_text": "I wonder whether a man driving a car?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6475:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6168:premise", "snli:test:6556:premise", "snli:test:6167:premise", "snli:test:6475:premise", "style:personachat:test:241:self:2:positive:0", "style:personachat:test:241:self:2:positive:1"], "evidence_texts": ["A big dog and a smaller dog fight over a piece of fabric.", "Three young people are sitting in a room and talking to one another.", "A big dog and a smaller dog fight over a piece of fabric.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "they are an excellent snack with my show .", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:6475", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6475", "candidate_text": "A man driving a car.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6475:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6168:premise", "snli:test:6556:premise", "snli:test:6167:premise", "snli:test:6475:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A big dog and a smaller dog fight over a piece of fabric.", "Three young people are sitting in a room and talking to one another.", "A big dog and a smaller dog fight over a piece of fabric.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6475", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6480", "candidate_text": "I wonder whether two young girls are riding roller coasters?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6480:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6480:premise", "snli:test:6251:premise", "snli:test:6573:premise", "snli:test:6078:premise", "style:personachat:test:241:self:5:positive:0", "style:personachat:test:241:self:5:positive:1", "style:personachat:test:241:self:5:positive:2", "style:personachat:test:241:self:5:positive:3", "style:personachat:test:241:self:5:positive:4"], "evidence_texts": ["Two young girls in bathing suits laugh while standing under a fountain of water.", "A kid bored in a train with brown hair and his head lying down.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "it is . i used to live in ohio . are you jewish ?", "nothing wrong with that , i was only going to ask if you knew of a deli .", "i do at times , i mainly wanted pickles . the office is almost on .", "they are an excellent snack with my show .", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:6480", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6480", "candidate_text": "well, two young girls are riding roller coasters", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6480:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6480:premise", "snli:test:6251:premise", "snli:test:6573:premise", "snli:test:6078:premise", "style:personachat:test:402:self:5:positive:0", "style:personachat:test:402:self:5:positive:1", "style:personachat:test:402:self:5:positive:2", "style:personachat:test:402:self:5:positive:3", "style:personachat:test:402:self:5:positive:4"], "evidence_texts": ["Two young girls in bathing suits laugh while standing under a fountain of water.", "A kid bored in a train with brown hair and his head lying down.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A person is climbing a wooden ladder on a very steep, jagged, rock cliff.", "drama and romance goes together", "i understand , where are you from", "i live near the beach", "yeah california is warm most of the tiem", "oh yeah , well florida is warm right now tho"], "speaker_id": "snli:test:6480", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6499", "candidate_text": "I wonder whether the man is waiting for the bus to take him to the airport?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6499:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6430:premise", "snli:test:6419:premise", "snli:test:6511:premise", "snli:test:6499:premise", "style:personachat:test:241:self:7:positive:0", "style:personachat:test:241:self:7:positive:1", "style:personachat:test:241:self:7:positive:2", "style:personachat:test:241:self:7:positive:3", "style:personachat:test:241:self:7:positive:4", "style:personachat:test:241:self:7:positive:5", "style:personachat:test:241:self:7:positive:6"], "evidence_texts": ["Two women wearing red are smiling at the camera.", "Two men heavily immersed in a conversation worth having.", "Group of men standing in a meeting room, talking.", "A man siting on a bench with a briefcase.", "hello , i am doing great , how are you this evening ?", "photography , i do not drive in nyc , so i take photos whenever i walk .", "it is . i used to live in ohio . are you jewish ?", "nothing wrong with that , i was only going to ask if you knew of a deli .", "i do at times , i mainly wanted pickles . the office is almost on .", "they are an excellent snack with my show .", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:6499", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6499", "candidate_text": "Honestly, The man is waiting for the bus to take him to the airport!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6499:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6430:premise", "snli:test:6419:premise", "snli:test:6511:premise", "snli:test:6499:premise", "style:personachat:test:767:self:2:positive:0", "style:personachat:test:767:self:2:positive:1"], "evidence_texts": ["Two women wearing red are smiling at the camera.", "Two men heavily immersed in a conversation worth having.", "Group of men standing in a meeting room, talking.", "A man siting on a bench with a briefcase.", "we will have to get to get together and have my robot meet yours !", "nice name ! my girlfriend is calling me , have to go !"], "speaker_id": "snli:test:6499", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6500", "candidate_text": "well, the man is walking up the stairs", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6500:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6500:premise", "snli:test:6511:premise", "snli:test:6430:premise", "snli:test:6513:premise", "style:personachat:test:242:self:1:positive:0"], "evidence_texts": ["A man siting on a bench with a briefcase.", "Group of men standing in a meeting room, talking.", "Two women wearing red are smiling at the camera.", "Group of men standing in a meeting room, talking.", "drake , jay z and my favorite beyonce"], "speaker_id": "snli:test:6500", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6500", "candidate_text": "I wonder whether the man is walking up the stairs?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6500:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6500:premise", "snli:test:6511:premise", "snli:test:6430:premise", "snli:test:6513:premise", "style:personachat:test:900:self:1:positive:0"], "evidence_texts": ["A man siting on a bench with a briefcase.", "Group of men standing in a meeting room, talking.", "Two women wearing red are smiling at the camera.", "Group of men standing in a meeting room, talking.", "is your hair long or short ?"], "speaker_id": "snli:test:6500", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6511", "candidate_text": "well, a group of men are discussing business", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6511:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6500:premise", "snli:test:6499:premise", "snli:test:6729:premise", "snli:test:6511:premise", "style:personachat:test:242:self:2:positive:0", "style:personachat:test:242:self:2:positive:1"], "evidence_texts": ["A man siting on a bench with a briefcase.", "A man siting on a bench with a briefcase.", "A dog jumps to catch a red ball outside.", "Group of men standing in a meeting room, talking.", "no , mostly hip hop , it is my passion", "drake , jay z and my favorite beyonce"], "speaker_id": "snli:test:6511", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6511", "candidate_text": "Honestly, A group of men are discussing business!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6511:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6500:premise", "snli:test:6499:premise", "snli:test:6729:premise", "snli:test:6511:premise", "style:personachat:test:767:self:7:positive:0", "style:personachat:test:767:self:7:positive:1", "style:personachat:test:767:self:7:positive:2", "style:personachat:test:767:self:7:positive:3", "style:personachat:test:767:self:7:positive:4", "style:personachat:test:767:self:7:positive:5", "style:personachat:test:767:self:7:positive:6"], "evidence_texts": ["A man siting on a bench with a briefcase.", "A man siting on a bench with a briefcase.", "A dog jumps to catch a red ball outside.", "Group of men standing in a meeting room, talking.", "i love blue hair , my live in girlfriend has blue hair too !", "maryland , studying law at johns hopkins", "that is cool ; do you have pets ? my dog max says hi .", "robots are great , i take mine on all my travel adventures .", "yes , better than some humans !", "we will have to get to get together and have my robot meet yours !", "nice name ! my girlfriend is calling me , have to go !"], "speaker_id": "snli:test:6511", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6513", "candidate_text": "well, a group of men are talking to each other", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6513:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6729:premise", "snli:test:6513:premise", "snli:test:6500:premise", "snli:test:6772:premise", "style:personachat:test:242:self:5:positive:0", "style:personachat:test:242:self:5:positive:1", "style:personachat:test:242:self:5:positive:2", "style:personachat:test:242:self:5:positive:3", "style:personachat:test:242:self:5:positive:4"], "evidence_texts": ["A dog jumps to catch a red ball outside.", "Group of men standing in a meeting room, talking.", "A man siting on a bench with a briefcase.", "Scientists frantically research ways to finally eliminate male baldness.", "how exciting , i just got home from walking my dog", "i like them , but i only have a dog , because my sister is allergic to cats .", "i have been practicing my dance", "no , mostly hip hop , it is my passion", "drake , jay z and my favorite beyonce"], "speaker_id": "snli:test:6513", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6513", "candidate_text": "A group of men are talking to each other.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6513:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6729:premise", "snli:test:6513:premise", "snli:test:6500:premise", "snli:test:6772:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A dog jumps to catch a red ball outside.", "Group of men standing in a meeting room, talking.", "A man siting on a bench with a briefcase.", "Scientists frantically research ways to finally eliminate male baldness.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6513", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6517", "candidate_text": "well, the woman is yelling at the package", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6517:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6184:premise", "snli:test:6517:premise", "snli:test:5360:premise", "snli:test:6529:premise", "style:personachat:test:242:self:6:positive:0", "style:personachat:test:242:self:6:positive:1", "style:personachat:test:242:self:6:positive:2", "style:personachat:test:242:self:6:positive:3", "style:personachat:test:242:self:6:positive:4", "style:personachat:test:242:self:6:positive:5"], "evidence_texts": ["A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "hello , what have you been doing ?", "how exciting , i just got home from walking my dog", "i like them , but i only have a dog , because my sister is allergic to cats .", "i have been practicing my dance", "no , mostly hip hop , it is my passion", "drake , jay z and my favorite beyonce"], "speaker_id": "snli:test:6517", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6517", "candidate_text": "I wonder whether the woman is yelling at the package?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6517:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6184:premise", "snli:test:6517:premise", "snli:test:5360:premise", "snli:test:6529:premise", "style:personachat:test:900:self:7:positive:0", "style:personachat:test:900:self:7:positive:1", "style:personachat:test:900:self:7:positive:2", "style:personachat:test:900:self:7:positive:3", "style:personachat:test:900:self:7:positive:4", "style:personachat:test:900:self:7:positive:5", "style:personachat:test:900:self:7:positive:6"], "evidence_texts": ["A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A woman with a vest and red shirt is holding her hand up above a black and white dog that is jumping.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "good . is english your native tongue ? it is mine .", "i work as an engineer , but i cannot stand any bugs that i find while working .", "that sounds very creative . do you like music ? bach is my favorite .", "how cool you make sculptures . i love bacon and tried to make a sculpture with some .", "i also watch cooking shows ! great way to get recipe ideas .", "i have a small mole on my chin . what color is your hair ?", "is your hair long or short ?"], "speaker_id": "snli:test:6517", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6518", "candidate_text": "I wonder whether the woman is about to open the package?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6518:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6529:premise", "snli:test:6184:premise", "snli:test:6518:premise", "snli:test:6552:premise", "style:personachat:test:243:self:1:positive:0"], "evidence_texts": ["A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:6518", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6518", "candidate_text": "well, the woman is about to open the package", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6518:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6529:premise", "snli:test:6184:premise", "snli:test:6518:premise", "snli:test:6552:premise", "style:personachat:test:406:self:1:positive:0"], "evidence_texts": ["A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "A man with white hair wearing a gray suit speaks into a microphone with a metal jug sitting in front of him.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "well i like john denver"], "speaker_id": "snli:test:6518", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6520", "candidate_text": "I wonder whether a man and girl swim?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6520:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6520:premise", "snli:test:6464:premise", "snli:test:6673:premise", "snli:test:6454:premise", "style:personachat:test:243:self:2:positive:0", "style:personachat:test:243:self:2:positive:1"], "evidence_texts": ["A man and a young girl swimming.", "A man hammering nails on a shed.", "Two butchers are walking through their shop.", "Naked children play on the muddy riverbank.", "are you a pet groomer ?", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:6520", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6520", "candidate_text": "Honestly, a man and girl swim!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6520:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6520:premise", "snli:test:6464:premise", "snli:test:6673:premise", "snli:test:6454:premise", "style:personachat:test:769:self:7:positive:0", "style:personachat:test:769:self:7:positive:1", "style:personachat:test:769:self:7:positive:2", "style:personachat:test:769:self:7:positive:3", "style:personachat:test:769:self:7:positive:4", "style:personachat:test:769:self:7:positive:5", "style:personachat:test:769:self:7:positive:6"], "evidence_texts": ["A man and a young girl swimming.", "A man hammering nails on a shed.", "Two butchers are walking through their shop.", "Naked children play on the muddy riverbank.", "not great . just watched the news like i do every night . so much hatred .", "no i have not but i care a lot about politics .", "that is not good . do you like to read ? i read whenever possible .", "my profession is in it .", "yeah , i can say the same here .", "yes i do . my favorite is football .", "i love watching movies ! i like reading more though ."], "speaker_id": "snli:test:6520", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6523", "candidate_text": "I wonder whether a woman hurdler trips over a hurdle?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6523:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6085:premise", "snli:test:6793:premise", "snli:test:6060:premise", "snli:test:6523:premise", "style:personachat:test:243:self:5:positive:0", "style:personachat:test:243:self:5:positive:1", "style:personachat:test:243:self:5:positive:2", "style:personachat:test:243:self:5:positive:3", "style:personachat:test:243:self:5:positive:4"], "evidence_texts": ["Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "that is cool . are you married ?", "wow , sounds like you are a black widower . . .", "yes , i have a mini aussie shepherd . you ?", "are you a pet groomer ?", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:6523", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6523", "candidate_text": "A woman hurdler trips over a hurdle.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6523:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6085:premise", "snli:test:6793:premise", "snli:test:6060:premise", "snli:test:6523:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "A middle-aged man in a collard shirt is raising his forearms next to his face.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6523", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6528", "candidate_text": "I wonder whether people are admiring the car?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6528:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6528:premise", "snli:test:6396:premise", "snli:test:6645:premise", "snli:test:6266:premise", "style:personachat:test:243:self:6:positive:0", "style:personachat:test:243:self:6:positive:1", "style:personachat:test:243:self:6:positive:2", "style:personachat:test:243:self:6:positive:3", "style:personachat:test:243:self:6:positive:4", "style:personachat:test:243:self:6:positive:5"], "evidence_texts": ["People checking out a car with all its doors open in a parking lot in the city.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "of course ! do you like watching movies ?", "that is cool . are you married ?", "wow , sounds like you are a black widower . . .", "yes , i have a mini aussie shepherd . you ?", "are you a pet groomer ?", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:6528", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6528", "candidate_text": "well, people are admiring the car", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6528:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6528:premise", "snli:test:6396:premise", "snli:test:6645:premise", "snli:test:6266:premise", "style:personachat:test:406:self:7:positive:0", "style:personachat:test:406:self:7:positive:1", "style:personachat:test:406:self:7:positive:2", "style:personachat:test:406:self:7:positive:3", "style:personachat:test:406:self:7:positive:4", "style:personachat:test:406:self:7:positive:5", "style:personachat:test:406:self:7:positive:6"], "evidence_texts": ["People checking out a car with all its doors open in a parking lot in the city.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A young man is standing near his bike outside a shop called, \"Sara's Old Fashioned Ice Cream.\"", "it is a pretty chilly night", "i would say country since i have a big garden", "i love the gym , i go daily .", "i do not lol i grew up on a horse ranch", "nothing like a fresh home cooked meal especially garden fresh", "yea it is gmo there a lot", "well i like john denver"], "speaker_id": "snli:test:6528", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6529", "candidate_text": "I wonder whether the man is forty?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6529:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6518:premise", "snli:test:6529:premise", "snli:test:6552:premise", "snli:test:6517:premise", "style:personachat:test:244:self:1:positive:0"], "evidence_texts": ["A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "i have two boys . any other family ?"], "speaker_id": "snli:test:6529", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6529", "candidate_text": "Honestly, the man is forty!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6529:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6518:premise", "snli:test:6529:premise", "snli:test:6552:premise", "snli:test:6517:premise", "style:personachat:test:775:self:5:positive:0", "style:personachat:test:775:self:5:positive:1", "style:personachat:test:775:self:5:positive:2", "style:personachat:test:775:self:5:positive:3", "style:personachat:test:775:self:5:positive:4"], "evidence_texts": ["A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "that is great ! is that what you do for a living ?", "i train dogs and run an obedience school", "the ones i train are so cute ! i take a lot of pics with them", "nice ! what do you do in your free time ?", "that is nice . i love sweets so i bake a lot . then i dance it off !"], "speaker_id": "snli:test:6529", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6533", "candidate_text": "I wonder whether a snowboarder in a green hat rides along a metal railing in a competition?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6533:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6299:premise", "snli:test:6533:premise", "snli:test:6844:premise", "snli:test:6246:premise", "style:personachat:test:244:self:2:positive:0", "style:personachat:test:244:self:2:positive:1"], "evidence_texts": ["A woman in a dress is singing and having a good time.", "a snowboarder wearing a green hat is riding along a metal railing.", "The three young children played with some toilet paper in the grass.", "A man holding a pink cooler is talking to an older woman.", "i eat granola every day ! do you have kids ?", "i have two boys . any other family ?"], "speaker_id": "snli:test:6533", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6533", "candidate_text": "A snowboarder in a green hat rides along a metal railing in a competition.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6533:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6299:premise", "snli:test:6533:premise", "snli:test:6844:premise", "snli:test:6246:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman in a dress is singing and having a good time.", "a snowboarder wearing a green hat is riding along a metal railing.", "The three young children played with some toilet paper in the grass.", "A man holding a pink cooler is talking to an older woman.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6533", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6552", "candidate_text": "I wonder whether a girl drinks a milkshake?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6552:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6552:premise", "snli:test:6529:premise", "snli:test:6587:premise", "snli:test:6518:premise", "style:personachat:test:244:self:5:positive:0", "style:personachat:test:244:self:5:positive:1", "style:personachat:test:244:self:5:positive:2", "style:personachat:test:244:self:5:positive:3", "style:personachat:test:244:self:5:positive:4"], "evidence_texts": ["A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "cutting their hair used to be a kind of hobby . do you have any hobbies ?", "well i have exotic fish ! sort of similiar . u have other hobbies ?", "collect 50s decorations . what did you have for breakfast today ?", "i eat granola every day ! do you have kids ?", "i have two boys . any other family ?"], "speaker_id": "snli:test:6552", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6552", "candidate_text": "well, a girl drinks a milkshake", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6552:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6552:premise", "snli:test:6529:premise", "snli:test:6587:premise", "snli:test:6518:premise", "style:personachat:test:407:self:5:positive:0", "style:personachat:test:407:self:5:positive:1", "style:personachat:test:407:self:5:positive:2", "style:personachat:test:407:self:5:positive:3", "style:personachat:test:407:self:5:positive:4"], "evidence_texts": ["A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "A woman smiles while she opens a red package in front of a table with yarn and glass of liquid on it.", "i took my dogs to the dog park .", "i have two . a husky and a pitbull .", "lol awesome . what did you study ?", "sweet . what do you do with your degree ?", "i work at a local animal shelter . i want to go to school to be a vet ."], "speaker_id": "snli:test:6552", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6556", "candidate_text": "I wonder whether there are people indoors?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6556:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6556:premise", "snli:test:6475:premise", "snli:test:6778:premise", "snli:test:6168:premise", "style:personachat:test:244:self:6:positive:0", "style:personachat:test:244:self:6:positive:1", "style:personachat:test:244:self:6:positive:2", "style:personachat:test:244:self:6:positive:3", "style:personachat:test:244:self:6:positive:4", "style:personachat:test:244:self:6:positive:5"], "evidence_texts": ["Three young people are sitting in a room and talking to one another.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A big dog and a smaller dog fight over a piece of fabric.", "i am ok , a little tired from school . how are you ?", "cutting their hair used to be a kind of hobby . do you have any hobbies ?", "well i have exotic fish ! sort of similiar . u have other hobbies ?", "collect 50s decorations . what did you have for breakfast today ?", "i eat granola every day ! do you have kids ?", "i have two boys . any other family ?"], "speaker_id": "snli:test:6556", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6556", "candidate_text": "Honestly, There are people indoors!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6556:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6556:premise", "snli:test:6475:premise", "snli:test:6778:premise", "snli:test:6168:premise", "style:personachat:test:780:self:2:positive:0", "style:personachat:test:780:self:2:positive:1"], "evidence_texts": ["Three young people are sitting in a room and talking to one another.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A big dog and a smaller dog fight over a piece of fabric.", "i have extras i can give them ! not the green ones that is my favorite color !", "absolutely ! the collection is pretty large . what is your favorite color ?"], "speaker_id": "snli:test:6556", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6563", "candidate_text": "well, a gymnastics performer is stretching", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6563:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6436:premise", "snli:test:6413:premise", "snli:test:6578:premise", "snli:test:6563:premise", "style:personachat:test:245:self:1:positive:0"], "evidence_texts": ["A man and woman look through CD's at a street vendor.", "A topless man straps surfboards on top of his blue car.", "A beautiful bride walking on a sidewalk with her new husband.", "A gymnastics performer is in mid-pose as she completes her jump.", "yeah , but it is all good . dynamite comes in small packages ."], "speaker_id": "snli:test:6563", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6563", "candidate_text": "I wonder whether a gymnastics performer is stretching?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6563:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6436:premise", "snli:test:6413:premise", "snli:test:6578:premise", "snli:test:6563:premise", "style:personachat:test:912:self:1:positive:0"], "evidence_texts": ["A man and woman look through CD's at a street vendor.", "A topless man straps surfboards on top of his blue car.", "A beautiful bride walking on a sidewalk with her new husband.", "A gymnastics performer is in mid-pose as she completes her jump.", "good , currently cuddling with my five animals . you ?"], "speaker_id": "snli:test:6563", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6564", "candidate_text": "well, a gymnast is completing her jump in the finals", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6564:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6564:premise", "snli:test:6578:premise", "snli:test:6436:premise", "snli:test:6604:premise", "style:personachat:test:245:self:2:positive:0", "style:personachat:test:245:self:2:positive:1"], "evidence_texts": ["A gymnastics performer is in mid-pose as she completes her jump.", "A beautiful bride walking on a sidewalk with her new husband.", "A man and woman look through CD's at a street vendor.", "A small girl in a green shirt plays on a swing.", "haha . i get asked to get stuff off the top self all the time !", "yeah , but it is all good . dynamite comes in small packages ."], "speaker_id": "snli:test:6564", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6564", "candidate_text": "Honestly, A gymnast is completing her jump in the finals!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6564:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6564:premise", "snli:test:6578:premise", "snli:test:6436:premise", "snli:test:6604:premise", "style:personachat:test:780:self:6:positive:0", "style:personachat:test:780:self:6:positive:1", "style:personachat:test:780:self:6:positive:2", "style:personachat:test:780:self:6:positive:3", "style:personachat:test:780:self:6:positive:4", "style:personachat:test:780:self:6:positive:5"], "evidence_texts": ["A gymnastics performer is in mid-pose as she completes her jump.", "A beautiful bride walking on a sidewalk with her new husband.", "A man and woman look through CD's at a street vendor.", "A small girl in a green shirt plays on a swing.", "that sounds lovely ! have you ever taken them to the zoo ? i love zoos !", "what kind of cat ? i do not have any pets .", "no need to worry , i am a nurse not a cop !", "how nice . sometimes i donate teddy bears to old folks homes i have a large collection", "i have extras i can give them ! not the green ones that is my favorite color !", "absolutely ! the collection is pretty large . what is your favorite color ?"], "speaker_id": "snli:test:6564", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6573", "candidate_text": "well, a young boy climbs an oak tree", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6573:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6480:premise", "snli:test:6573:premise", "snli:test:6612:premise", "snli:test:6251:premise", "style:personachat:test:245:self:5:positive:0", "style:personachat:test:245:self:5:positive:1", "style:personachat:test:245:self:5:positive:2", "style:personachat:test:245:self:5:positive:3", "style:personachat:test:245:self:5:positive:4"], "evidence_texts": ["Two young girls in bathing suits laugh while standing under a fountain of water.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A kid bored in a train with brown hair and his head lying down.", "right on . what else do you like ? i love grilling barbecue .", "that is sounds really good right now . i eat a lot of protein cause i do weekend lifts .", "i run awkward , i am a super tall and pretty big fella .", "haha . i get asked to get stuff off the top self all the time !", "yeah , but it is all good . dynamite comes in small packages ."], "speaker_id": "snli:test:6573", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6573", "candidate_text": "A young boy climbs an oak tree.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6573:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6480:premise", "snli:test:6573:premise", "snli:test:6612:premise", "snli:test:6251:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two young girls in bathing suits laugh while standing under a fountain of water.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A kid bored in a train with brown hair and his head lying down.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6573", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6575", "candidate_text": "well, a puppy has leapt into the air to grab a frisbee", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6575:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6260:premise", "snli:test:6590:premise", "snli:test:5610:premise", "snli:test:6575:premise", "style:personachat:test:245:self:7:positive:0", "style:personachat:test:245:self:7:positive:1", "style:personachat:test:245:self:7:positive:2", "style:personachat:test:245:self:7:positive:3", "style:personachat:test:245:self:7:positive:4", "style:personachat:test:245:self:7:positive:5", "style:personachat:test:245:self:7:positive:6"], "evidence_texts": ["Two men shining peoples shoes.", "People trying to get warm.", "pedestrian walking on the street", "A dog catching a Frisbee.", "i am ok . long day of warehouse and forklift work . you ?", "i bet . kicking back and watching horror movies now . haha", "right on . what else do you like ? i love grilling barbecue .", "that is sounds really good right now . i eat a lot of protein cause i do weekend lifts .", "i run awkward , i am a super tall and pretty big fella .", "haha . i get asked to get stuff off the top self all the time !", "yeah , but it is all good . dynamite comes in small packages ."], "speaker_id": "snli:test:6575", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6575", "candidate_text": "I wonder whether a puppy has leapt into the air to grab a frisbee?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6575:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6260:premise", "snli:test:6590:premise", "snli:test:5610:premise", "snli:test:6575:premise", "style:personachat:test:912:self:7:positive:0", "style:personachat:test:912:self:7:positive:1", "style:personachat:test:912:self:7:positive:2", "style:personachat:test:912:self:7:positive:3", "style:personachat:test:912:self:7:positive:4", "style:personachat:test:912:self:7:positive:5", "style:personachat:test:912:self:7:positive:6"], "evidence_texts": ["Two men shining peoples shoes.", "People trying to get warm.", "pedestrian walking on the street", "A dog catching a Frisbee.", "good . just got done volunteering for the animal shelter . you ?", "my parents taught and they always use to make me watch . so boring", "i would rather hike the mountains near my house and watch the wildlife .", "what do you like to do ? i love to listen to barry manila", "what are you currently reading ? i like to read about animals", "interesting . i would vote if barry manila was running !", "good , currently cuddling with my five animals . you ?"], "speaker_id": "snli:test:6575", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6578", "candidate_text": "well, newlyweds sitting on a park bench", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6578:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6564:premise", "snli:test:6604:premise", "snli:test:6578:premise", "snli:test:6563:premise", "style:personachat:test:246:self:1:positive:0"], "evidence_texts": ["A gymnastics performer is in mid-pose as she completes her jump.", "A small girl in a green shirt plays on a swing.", "A beautiful bride walking on a sidewalk with her new husband.", "A gymnastics performer is in mid-pose as she completes her jump.", "i am sorry to hear that . hope they were both there for you ."], "speaker_id": "snli:test:6578", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6578", "candidate_text": "Honestly, Newlyweds sitting on a park bench!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6578:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6564:premise", "snli:test:6604:premise", "snli:test:6578:premise", "snli:test:6563:premise", "style:personachat:test:797:self:5:positive:0", "style:personachat:test:797:self:5:positive:1", "style:personachat:test:797:self:5:positive:2", "style:personachat:test:797:self:5:positive:3", "style:personachat:test:797:self:5:positive:4"], "evidence_texts": ["A gymnastics performer is in mid-pose as she completes her jump.", "A small girl in a green shirt plays on a swing.", "A beautiful bride walking on a sidewalk with her new husband.", "A gymnastics performer is in mid-pose as she completes her jump.", "yeah i am kind of anti consumer , make my own clothes , adopt stray cats . you ?", "do you have any hobbies ?", "building guitars . i also like tinkering with electric guitar electronics", "you must have some talents . do you win at poker often ?", "winning at poker is an enviable talent !"], "speaker_id": "snli:test:6578", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6587", "candidate_text": "well, a man wears a white tommy hilfiger shirt", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6587:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6552:premise", "snli:test:7118:premise", "snli:test:6529:premise", "snli:test:6587:premise", "style:personachat:test:246:self:2:positive:0", "style:personachat:test:246:self:2:positive:1"], "evidence_texts": ["A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "awesome . any brothers and sisters ? i have a few and a dog charlie too .", "i am sorry to hear that . hope they were both there for you ."], "speaker_id": "snli:test:6587", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6587", "candidate_text": "A man wears a white Tommy Hilfiger shirt.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6587:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6552:premise", "snli:test:7118:premise", "snli:test:6529:premise", "snli:test:6587:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "A man in a dark long rainproof jacket but dry hair walks to the left in a city that has recently rained.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6587", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6590", "candidate_text": "well, people are sleeping", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6590:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6575:premise", "snli:test:7169:premise", "snli:test:6590:premise", "snli:test:6260:premise", "style:personachat:test:246:self:5:positive:0", "style:personachat:test:246:self:5:positive:1", "style:personachat:test:246:self:5:positive:2", "style:personachat:test:246:self:5:positive:3", "style:personachat:test:246:self:5:positive:4"], "evidence_texts": ["A dog catching a Frisbee.", "cheerleaders pose for the camera.", "People trying to get warm.", "Two men shining peoples shoes.", "my dad in australia grills good steak .", "not really . the ocean is more his spiritual guide .", "is it a pinto ? i had one of those before becoming a stock broker .", "awesome . any brothers and sisters ? i have a few and a dog charlie too .", "i am sorry to hear that . hope they were both there for you ."], "speaker_id": "snli:test:6590", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6590", "candidate_text": "I wonder whether people are sleeping?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6590:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6575:premise", "snli:test:7169:premise", "snli:test:6590:premise", "snli:test:6260:premise", "style:personachat:test:913:self:5:positive:0", "style:personachat:test:913:self:5:positive:1", "style:personachat:test:913:self:5:positive:2", "style:personachat:test:913:self:5:positive:3", "style:personachat:test:913:self:5:positive:4"], "evidence_texts": ["A dog catching a Frisbee.", "cheerleaders pose for the camera.", "People trying to get warm.", "Two men shining peoples shoes.", "one of my brothers just got engaged , the other is single like me .", "growing up in wisconsin , i played hockey all the time .", "my job as a managerial consultant keeps me from playing any these days .", "it can be . it is very demanding .", "i went to dartmouth , yourself ?"], "speaker_id": "snli:test:6590", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6604", "candidate_text": "well, old lady sitting on a chair", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6604:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6604:premise", "snli:test:6578:premise", "snli:test:6817:premise", "snli:test:6564:premise", "style:personachat:test:246:self:6:positive:0", "style:personachat:test:246:self:6:positive:1", "style:personachat:test:246:self:6:positive:2", "style:personachat:test:246:self:6:positive:3", "style:personachat:test:246:self:6:positive:4", "style:personachat:test:246:self:6:positive:5"], "evidence_texts": ["A small girl in a green shirt plays on a swing.", "A beautiful bride walking on a sidewalk with her new husband.", "A man wearing protective gear grinds down a large metal object.", "A gymnastics performer is in mid-pose as she completes her jump.", "eating some yummy eggplant parmigiana . it is so good .", "my dad in australia grills good steak .", "not really . the ocean is more his spiritual guide .", "is it a pinto ? i had one of those before becoming a stock broker .", "awesome . any brothers and sisters ? i have a few and a dog charlie too .", "i am sorry to hear that . hope they were both there for you ."], "speaker_id": "snli:test:6604", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6604", "candidate_text": "Honestly, old lady sitting on a chair!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6604:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6604:premise", "snli:test:6578:premise", "snli:test:6817:premise", "snli:test:6564:premise", "style:personachat:test:802:self:2:positive:0", "style:personachat:test:802:self:2:positive:1"], "evidence_texts": ["A small girl in a green shirt plays on a swing.", "A beautiful bride walking on a sidewalk with her new husband.", "A man wearing protective gear grinds down a large metal object.", "A gymnastics performer is in mid-pose as she completes her jump.", "i have always been pretty athletic so it works ! what other hobbies do you enjoy ?", "nothing wrong with that ! who could pass up relaxing activities ?"], "speaker_id": "snli:test:6604", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6612", "candidate_text": "well, a man wearing headphones working on a sculpture", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6612:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6612:premise", "snli:test:6573:premise", "snli:test:6919:premise", "snli:test:6480:premise", "style:personachat:test:247:self:1:positive:0"], "evidence_texts": ["A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "Two young girls in bathing suits laugh while standing under a fountain of water.", "i know how to make a delicious bowl of captain crunch"], "speaker_id": "snli:test:6612", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6612", "candidate_text": "a man wearing headphones working on a sculpture.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6612:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6612:premise", "snli:test:6573:premise", "snli:test:6919:premise", "snli:test:6480:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "Two young girls in bathing suits laugh while standing under a fountain of water.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6612", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6616", "candidate_text": "well, the man is wearing white", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6616:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6616:premise", "snli:test:6387:premise", "snli:test:6971:premise", "snli:test:6302:premise", "style:personachat:test:247:self:2:positive:0", "style:personachat:test:247:self:2:positive:1"], "evidence_texts": ["A man wearing black carving stone.", "Instructor on Basket weaving at fair", "Two men walking in summer attire.", "A dog is walking down steps.", "you cannot beat home made italian food and a big game of sports", "i know how to make a delicious bowl of captain crunch"], "speaker_id": "snli:test:6616", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6616", "candidate_text": "I wonder whether the man is wearing white?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6616:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6616:premise", "snli:test:6387:premise", "snli:test:6971:premise", "snli:test:6302:premise", "style:personachat:test:915:self:2:positive:0", "style:personachat:test:915:self:2:positive:1"], "evidence_texts": ["A man wearing black carving stone.", "Instructor on Basket weaving at fair", "Two men walking in summer attire.", "A dog is walking down steps.", "i understand . organized religion is not for everyone .", "oh really ? i did not know that ."], "speaker_id": "snli:test:6616", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6639", "candidate_text": "well, the restaurant is crowded", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6639:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6450:premise", "snli:test:6647:premise", "snli:test:6444:premise", "snli:test:6639:premise", "style:personachat:test:247:self:5:positive:0", "style:personachat:test:247:self:5:positive:1", "style:personachat:test:247:self:5:positive:2", "style:personachat:test:247:self:5:positive:3", "style:personachat:test:247:self:5:positive:4"], "evidence_texts": ["Three people are posing on the side of a mountain.", "Six people appearing to be planting something in the ground.", "A little girl in a colorful shirt holding a flower.", "Two women in black enjoying a lunch in a restaurant.", "i would not fight you , instead i would have you over for italian food", "you can bring the family . do they like to sing ?", "lets get a my family against your family softball game going", "you cannot beat home made italian food and a big game of sports", "i know how to make a delicious bowl of captain crunch"], "speaker_id": "snli:test:6639", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6639", "candidate_text": "Honestly, The restaurant is crowded!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6639:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6450:premise", "snli:test:6647:premise", "snli:test:6444:premise", "snli:test:6639:premise", "style:personachat:test:813:self:1:positive:0"], "evidence_texts": ["Three people are posing on the side of a mountain.", "Six people appearing to be planting something in the ground.", "A little girl in a colorful shirt holding a flower.", "Two women in black enjoying a lunch in a restaurant.", "sounds very nice . i want to travel to italy . i love italian food !"], "speaker_id": "snli:test:6639", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6645", "candidate_text": "well, the woman is holding an instrument", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6645:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6528:premise", "snli:test:6645:premise", "snli:test:6652:premise", "snli:test:6396:premise", "style:personachat:test:247:self:7:positive:0", "style:personachat:test:247:self:7:positive:1", "style:personachat:test:247:self:7:positive:2", "style:personachat:test:247:self:7:positive:3", "style:personachat:test:247:self:7:positive:4", "style:personachat:test:247:self:7:positive:5", "style:personachat:test:247:self:7:positive:6"], "evidence_texts": ["People checking out a car with all its doors open in a parking lot in the city.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "i have been doing taekwondo since i was a kid , but enjoy softball now", "i am not a violent person really . i am in the choir at church !", "i would not fight you , instead i would have you over for italian food", "you can bring the family . do they like to sing ?", "lets get a my family against your family softball game going", "you cannot beat home made italian food and a big game of sports", "i know how to make a delicious bowl of captain crunch"], "speaker_id": "snli:test:6645", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6645", "candidate_text": "The woman is holding an instrument.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6645:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6528:premise", "snli:test:6645:premise", "snli:test:6652:premise", "snli:test:6396:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["People checking out a car with all its doors open in a parking lot in the city.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A wall hung with multicolored crosses is visible behind the group of young men as they converse.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6645", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6647", "candidate_text": "well, six people are at the movies", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6647:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6639:premise", "snli:test:6669:premise", "snli:test:6450:premise", "snli:test:6647:premise", "style:personachat:test:248:self:1:positive:0"], "evidence_texts": ["Two women in black enjoying a lunch in a restaurant.", "A man playing a large stringed instrument on the street.", "Three people are posing on the side of a mountain.", "Six people appearing to be planting something in the ground.", "how do you usually support him"], "speaker_id": "snli:test:6647", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6647", "candidate_text": "I wonder whether six people are at the movies?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6647:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6639:premise", "snli:test:6669:premise", "snli:test:6450:premise", "snli:test:6647:premise", "style:personachat:test:917:self:1:positive:0"], "evidence_texts": ["Two women in black enjoying a lunch in a restaurant.", "A man playing a large stringed instrument on the street.", "Three people are posing on the side of a mountain.", "Six people appearing to be planting something in the ground.", "that is a good movie what is your favorite show on netflix ?"], "speaker_id": "snli:test:6647", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6650", "candidate_text": "well, a man with no tattoos is getting a massage", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6650:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6393:premise", "snli:test:6671:premise", "snli:test:6650:premise", "snli:test:6011:premise", "style:personachat:test:248:self:2:positive:0", "style:personachat:test:248:self:2:positive:1"], "evidence_texts": ["A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "oh that is so cool", "how do you usually support him"], "speaker_id": "snli:test:6650", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6650", "candidate_text": "Honestly, A man with no tattoos is getting a massage!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6650:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6393:premise", "snli:test:6671:premise", "snli:test:6650:premise", "snli:test:6011:premise", "style:personachat:test:813:self:7:positive:0", "style:personachat:test:813:self:7:positive:1", "style:personachat:test:813:self:7:positive:2", "style:personachat:test:813:self:7:positive:3", "style:personachat:test:813:self:7:positive:4", "style:personachat:test:813:self:7:positive:5", "style:personachat:test:813:self:7:positive:6"], "evidence_texts": ["A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "A man is holding a cigarette outdoors in front of a blue wall with a sink on it.", "i am doing pretty good . just got back from a nice long walk . you ?", "ugh i am sorry to hear that . what do you want to do for work ?", "what do you like to do for fun ?", "that is very nice . i love spending time with my big pet dog .", "maybe take a nice long walk to think . that always helps me .", "no , but i love to read books about traveling so i often read about it .", "sounds very nice . i want to travel to italy . i love italian food !"], "speaker_id": "snli:test:6650", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6652", "candidate_text": "well, a man in his van because he caught his victim", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6652:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6652:premise", "snli:test:6645:premise", "snli:test:6655:premise", "snli:test:6528:premise", "style:personachat:test:248:self:5:positive:0", "style:personachat:test:248:self:5:positive:1", "style:personachat:test:248:self:5:positive:2", "style:personachat:test:248:self:5:positive:3", "style:personachat:test:248:self:5:positive:4"], "evidence_texts": ["A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "People checking out a car with all its doors open in a parking lot in the city.", "the family the happier just like my garden is this you", "yes that s why i love i love the passion of teaching biology in college", "kind of business does your husband have i like walking", "oh that is so cool", "how do you usually support him"], "speaker_id": "snli:test:6652", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6652", "candidate_text": "a man in his van because he caught his victim.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6652:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6652:premise", "snli:test:6645:premise", "snli:test:6655:premise", "snli:test:6528:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "People checking out a car with all its doors open in a parking lot in the city.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6652", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6655", "candidate_text": "well, two people are near a sonoma store", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6655:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6652:premise", "snli:test:6645:premise", "snli:test:7282:premise", "snli:test:6655:premise", "style:personachat:test:248:self:7:positive:0", "style:personachat:test:248:self:7:positive:1", "style:personachat:test:248:self:7:positive:2", "style:personachat:test:248:self:7:positive:3", "style:personachat:test:248:self:7:positive:4", "style:personachat:test:248:self:7:positive:5", "style:personachat:test:248:self:7:positive:6"], "evidence_texts": ["A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "you mean good and i m planning for my next travel", "so six country is the total of them", "the family the happier just like my garden is this you", "yes that s why i love i love the passion of teaching biology in college", "kind of business does your husband have i like walking", "oh that is so cool", "how do you usually support him"], "speaker_id": "snli:test:6655", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6655", "candidate_text": "I wonder whether two people are near a sonoma store?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6655:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6652:premise", "snli:test:6645:premise", "snli:test:7282:premise", "snli:test:6655:premise", "style:personachat:test:917:self:7:positive:0", "style:personachat:test:917:self:7:positive:1", "style:personachat:test:917:self:7:positive:2", "style:personachat:test:917:self:7:positive:3", "style:personachat:test:917:self:7:positive:4", "style:personachat:test:917:self:7:positive:5", "style:personachat:test:917:self:7:positive:6"], "evidence_texts": ["A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman with red-hair and a leather jacket holds a harp in front of a snowy forest.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "i just stay home all the time and do not do physical things", "no i do not i stay inside and watch movies all the time", "i have 1 dog and 2 cats with my husband he is my world", "they are bankers how about yourself ?", "does your place sell ice cream it is my favorite food , what is yours ?", "pizza is the perfect food to watch movies with , what is your favorite movie ?", "that is a good movie what is your favorite show on netflix ?"], "speaker_id": "snli:test:6655", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6657", "candidate_text": "Honestly, A woman has her daughter in a stroller!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6657:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7282:premise", "snli:test:6657:premise", "snli:test:6652:premise", "snli:test:7488:premise", "style:personachat:test:249:self:1:positive:0"], "evidence_texts": ["A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "i met my wife at a wedding !"], "speaker_id": "snli:test:6657", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6657", "candidate_text": "well, a woman has her daughter in a stroller", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6657:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7282:premise", "snli:test:6657:premise", "snli:test:6652:premise", "snli:test:7488:premise", "style:personachat:test:416:self:1:positive:0"], "evidence_texts": ["A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "A man with a latex glove and black t-shit is standing in front of a Chevy van.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "i stand right at five feet tall ."], "speaker_id": "snli:test:6657", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6669", "candidate_text": "Honestly, There is a man entertaining outdoors!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6669:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6647:premise", "snli:test:6669:premise", "snli:test:6685:premise", "snli:test:6639:premise", "style:personachat:test:249:self:2:positive:0", "style:personachat:test:249:self:2:positive:1"], "evidence_texts": ["Six people appearing to be planting something in the ground.", "A man playing a large stringed instrument on the street.", "Three people are riding a carriage pulled by four horses.", "Two women in black enjoying a lunch in a restaurant.", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:6669", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6669", "candidate_text": "There is a man entertaining outdoors.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6669:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6647:premise", "snli:test:6669:premise", "snli:test:6685:premise", "snli:test:6639:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Six people appearing to be planting something in the ground.", "A man playing a large stringed instrument on the street.", "Three people are riding a carriage pulled by four horses.", "Two women in black enjoying a lunch in a restaurant.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6669", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6671", "candidate_text": "Honestly, A woman not paying attention to the camera!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6671:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6650:premise", "snli:test:7473:premise", "snli:test:6393:premise", "snli:test:6671:premise", "style:personachat:test:249:self:5:positive:0", "style:personachat:test:249:self:5:positive:1", "style:personachat:test:249:self:5:positive:2", "style:personachat:test:249:self:5:positive:3", "style:personachat:test:249:self:5:positive:4"], "evidence_texts": ["A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "yes ! what kind of work do you do indoors ?", "i would put wierd stickers all over the walls ! do you have any hobbies ?", "i collect stickers , i try all kinds of strange food . can you recommend any strange food ?", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:6671", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6671", "candidate_text": "I wonder whether a woman not paying attention to the camera?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6671:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6650:premise", "snli:test:7473:premise", "snli:test:6393:premise", "snli:test:6671:premise", "style:personachat:test:919:self:5:positive:0", "style:personachat:test:919:self:5:positive:1", "style:personachat:test:919:self:5:positive:2", "style:personachat:test:919:self:5:positive:3", "style:personachat:test:919:self:5:positive:4"], "evidence_texts": ["A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A person with braided hair and a gray sweatshirt covers his mouth and sits in a purple chair.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "purple . i have a turtle , do you like any animals ?", "that is pretty cool , any favorite sports ?", "no , not at all . spiritual maybe , but not much else . what about you ?", "what is your favorite thing to do ?", "do you enjoy it ? my mother was a cna , emotionally painful for her ."], "speaker_id": "snli:test:6671", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6673", "candidate_text": "Honestly, The owners of a very famous meat market are getting ready to open up shop!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6673:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6520:premise", "snli:test:6673:premise", "snli:test:7276:premise", "snli:test:6464:premise", "style:personachat:test:249:self:7:positive:0", "style:personachat:test:249:self:7:positive:1", "style:personachat:test:249:self:7:positive:2", "style:personachat:test:249:self:7:positive:3", "style:personachat:test:249:self:7:positive:4", "style:personachat:test:249:self:7:positive:5", "style:personachat:test:249:self:7:positive:6"], "evidence_texts": ["A man and a young girl swimming.", "Two butchers are walking through their shop.", "Two girls are kneeling on the ground.", "A man hammering nails on a shed.", "i just got home from a nice nature hike . do you like the outdoors ?", "hiking gives me inspiration for my drawing class . do you like art ?", "yes ! what kind of work do you do indoors ?", "i would put wierd stickers all over the walls ! do you have any hobbies ?", "i collect stickers , i try all kinds of strange food . can you recommend any strange food ?", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:6673", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6673", "candidate_text": "well, the owners of a very famous meat market are getting ready to open up shop", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6673:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6520:premise", "snli:test:6673:premise", "snli:test:7276:premise", "snli:test:6464:premise", "style:personachat:test:416:self:7:positive:0", "style:personachat:test:416:self:7:positive:1", "style:personachat:test:416:self:7:positive:2", "style:personachat:test:416:self:7:positive:3", "style:personachat:test:416:self:7:positive:4", "style:personachat:test:416:self:7:positive:5", "style:personachat:test:416:self:7:positive:6"], "evidence_texts": ["A man and a young girl swimming.", "Two butchers are walking through their shop.", "Two girls are kneeling on the ground.", "A man hammering nails on a shed.", "i am alright . except my feet , they are a little frog like .", "no . i dream of kids to make art with .", "i see . do you have any trash i could have for my art ?", "what kind of fish things do you have ?", "aquariums , those remind me of the cages where i work .", "is anything very tall , i am pretty short", "i stand right at five feet tall ."], "speaker_id": "snli:test:6673", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6678", "candidate_text": "well, a man in a white shirt is making a speech in front of a crowd", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6678:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6002:premise", "snli:test:7181:premise", "snli:test:6678:premise", "snli:test:5990:premise", "style:personachat:test:250:self:1:positive:0"], "evidence_texts": ["A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "ahh i see . motorcycles are fun but other drivers are dangerous and do not see you ."], "speaker_id": "snli:test:6678", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6678", "candidate_text": "A man in a white shirt is making a speech in front of a crowd.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6678:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6002:premise", "snli:test:7181:premise", "snli:test:6678:premise", "snli:test:5990:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "Women in a red hat and beige shoes standing with hands behind her back in front of a cement wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6678", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6681", "candidate_text": "well, girls can stand", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6681:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:4812:premise", "snli:test:6681:premise", "snli:test:9840:premise", "snli:test:3469:premise", "style:personachat:test:250:self:2:positive:0", "style:personachat:test:250:self:2:positive:1"], "evidence_texts": ["Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "like travel or be a super hero ?", "ahh i see . motorcycles are fun but other drivers are dangerous and do not see you ."], "speaker_id": "snli:test:6681", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6681", "candidate_text": "I wonder whether girls can stand?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6681:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4812:premise", "snli:test:6681:premise", "snli:test:9840:premise", "snli:test:3469:premise", "style:personachat:test:924:self:2:positive:0", "style:personachat:test:924:self:2:positive:1"], "evidence_texts": ["Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "A man wearing dark jeans, an orange shirt, and an orange apron is next to a bunch of plants and flowers and a woman with a blue top.", "do you like computers or anything like that ?", "what do you like about them ?"], "speaker_id": "snli:test:6681", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6685", "candidate_text": "well, the horses are scrawny", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6685:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6669:premise", "snli:test:6685:premise", "snli:test:6816:premise", "snli:test:6647:premise", "style:personachat:test:250:self:5:positive:0", "style:personachat:test:250:self:5:positive:1", "style:personachat:test:250:self:5:positive:2", "style:personachat:test:250:self:5:positive:3", "style:personachat:test:250:self:5:positive:4"], "evidence_texts": ["A man playing a large stringed instrument on the street.", "Three people are riding a carriage pulled by four horses.", "Two workers in green uniforms are standing in an alleyway.", "Six people appearing to be planting something in the ground.", "well , after dinner , my friends and i will play out of doors . how was gym ?", "you enough to daydream about living in the ocean with long , wavy hair and a flexible tail .", "my error young not you is the first word . do you have day dreams ?", "like travel or be a super hero ?", "ahh i see . motorcycles are fun but other drivers are dangerous and do not see you ."], "speaker_id": "snli:test:6685", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6685", "candidate_text": "Honestly, The horses are scrawny!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6685:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6669:premise", "snli:test:6685:premise", "snli:test:6816:premise", "snli:test:6647:premise", "style:personachat:test:821:self:1:positive:0"], "evidence_texts": ["A man playing a large stringed instrument on the street.", "Three people are riding a carriage pulled by four horses.", "Two workers in green uniforms are standing in an alleyway.", "Six people appearing to be planting something in the ground.", "it makes me a better gamer !"], "speaker_id": "snli:test:6685", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6704", "candidate_text": "well, a squirrel sits in the grass", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6704:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6704:premise", "snli:test:6425:premise", "snli:test:6756:premise", "snli:test:6408:premise", "style:personachat:test:250:self:7:positive:0", "style:personachat:test:250:self:7:positive:1", "style:personachat:test:250:self:7:positive:2", "style:personachat:test:250:self:7:positive:3", "style:personachat:test:250:self:7:positive:4", "style:personachat:test:250:self:7:positive:5", "style:personachat:test:250:self:7:positive:6"], "evidence_texts": ["The squirrel sits straight up in the grass.", "A group of people celebrating in the street.", "Construction workers on lifts working on a building.", "A bundled up toddler is walking over snow.", "hello to you , how was your day ?", "cereal with milk , then school most of the day . and you ?", "well , after dinner , my friends and i will play out of doors . how was gym ?", "you enough to daydream about living in the ocean with long , wavy hair and a flexible tail .", "my error young not you is the first word . do you have day dreams ?", "like travel or be a super hero ?", "ahh i see . motorcycles are fun but other drivers are dangerous and do not see you ."], "speaker_id": "snli:test:6704", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6704", "candidate_text": "A squirrel sits in the grass.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6704:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6704:premise", "snli:test:6425:premise", "snli:test:6756:premise", "snli:test:6408:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["The squirrel sits straight up in the grass.", "A group of people celebrating in the street.", "Construction workers on lifts working on a building.", "A bundled up toddler is walking over snow.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6704", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6712", "candidate_text": "well, a tall person standing", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6712:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6712:premise", "snli:test:5597:premise", "snli:test:5441:premise", "snli:test:8300:premise", "style:personachat:test:251:self:1:positive:0"], "evidence_texts": ["An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "how old are you if you do not mind to share"], "speaker_id": "snli:test:6712", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6712", "candidate_text": "I wonder whether a tall person standing?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6712:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6712:premise", "snli:test:5597:premise", "snli:test:5441:premise", "snli:test:8300:premise", "style:personachat:test:934:self:1:positive:0"], "evidence_texts": ["An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "The blond young man who had been splattered with paint colors of blue, orange, and green held up his camera to take a picture.", "A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "so , what do you do for a living ?"], "speaker_id": "snli:test:6712", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6713", "candidate_text": "well, nobody is standing", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6713:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8300:premise", "snli:test:6713:premise", "snli:test:5597:premise", "snli:test:9745:premise", "style:personachat:test:251:self:2:positive:0", "style:personachat:test:251:self:2:positive:1"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "day time at 3 pm", "how old are you if you do not mind to share"], "speaker_id": "snli:test:6713", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6713", "candidate_text": "Honestly, Nobody is standing!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6713:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8300:premise", "snli:test:6713:premise", "snli:test:5597:premise", "snli:test:9745:premise", "style:personachat:test:821:self:7:positive:0", "style:personachat:test:821:self:7:positive:1", "style:personachat:test:821:self:7:positive:2", "style:personachat:test:821:self:7:positive:3", "style:personachat:test:821:self:7:positive:4", "style:personachat:test:821:self:7:positive:5", "style:personachat:test:821:self:7:positive:6"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A young man wearing khaki pants and a checked shirt is jumping over large rocks with green mountains and a lagoon in the background.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "hi ! i am kylee . doing well . nothing too exciting tonight", "waiting for my next model spaceship to be delivered .", "not too hard , just fun while i am not playing videogames", "anything really , but mostly warcraft . what are your hobbies ?", "i wish i could draw . my parents were more scientific . mom is a doctor", "i work in computer science , so not both for me lol", "it makes me a better gamer !"], "speaker_id": "snli:test:6713", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6721", "candidate_text": "well, the photographer is taking a picture of an old man", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6721:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6016:premise", "snli:test:6721:premise", "snli:test:5577:premise", "snli:test:781:premise", "style:personachat:test:251:self:5:positive:0", "style:personachat:test:251:self:5:positive:1", "style:personachat:test:251:self:5:positive:2", "style:personachat:test:251:self:5:positive:3", "style:personachat:test:251:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "if in marriage has no trust better divorce .", "are you male or female ?", "having kids is hard i have a little nephew i imagine myself with kids lol", "day time at 3 pm", "how old are you if you do not mind to share"], "speaker_id": "snli:test:6721", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6721", "candidate_text": "The photographer is taking a picture of an old man.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6721:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6016:premise", "snli:test:6721:premise", "snli:test:5577:premise", "snli:test:781:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "A young woman with brown hair and an elderly man with gray hair and a sweater jump in the air on a snowy hill with snowshoes on their feet.", "A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6721", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6722", "candidate_text": "well, the photographer is taking a photo of a dog", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6722:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:781:premise", "snli:test:6016:premise", "snli:test:6722:premise", "snli:test:1307:premise", "style:personachat:test:251:self:7:positive:0", "style:personachat:test:251:self:7:positive:1", "style:personachat:test:251:self:7:positive:2", "style:personachat:test:251:self:7:positive:3", "style:personachat:test:251:self:7:positive:4", "style:personachat:test:251:self:7:positive:5", "style:personachat:test:251:self:7:positive:6"], "evidence_texts": ["A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "i am fine what about you ?", "no i am not married . i loved a guy for 4 years and he let me", "if in marriage has no trust better divorce .", "are you male or female ?", "having kids is hard i have a little nephew i imagine myself with kids lol", "day time at 3 pm", "how old are you if you do not mind to share"], "speaker_id": "snli:test:6722", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6722", "candidate_text": "I wonder whether the photographer is taking a photo of a dog?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6722:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:781:premise", "snli:test:6016:premise", "snli:test:6722:premise", "snli:test:1307:premise", "style:personachat:test:934:self:7:positive:0", "style:personachat:test:934:self:7:positive:1", "style:personachat:test:934:self:7:positive:2", "style:personachat:test:934:self:7:positive:3", "style:personachat:test:934:self:7:positive:4", "style:personachat:test:934:self:7:positive:5", "style:personachat:test:934:self:7:positive:6"], "evidence_texts": ["A person dressed in white and black winter clothing leaps a narrow, water-filled ditch from one frost-covered field to another, where a female dressed in black coat and pants awaits.", "A man in a blue shirt and tan shorts has his feet in the steps of an exercise bicycle and is working on the control panel of the machine.", "A photographer and assistant take a photo of a man in a suit and a woman in a dress in front of large white columns and a yellow wall.", "Someone with a long pole that appears to have a spike on the end is pointing it at a red balloon on the other side of the wall from him.", "we ate way earlier . always by 630 . how about you ?", "sounds yummy ! do you work tomorrow ?", "i love traveling . i often quilt while my husband drives .", "i am too old for much of that anymore . we prefer visiting great american landmarks .", "we keep it to great american landmarks . we love seeing american history .", "we have a tiny lapdog . she likes to go on road trips with us .", "so , what do you do for a living ?"], "speaker_id": "snli:test:6722", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6729", "candidate_text": "well, the dog likes to play catch", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6729:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6513:premise", "snli:test:6729:premise", "snli:test:6772:premise", "snli:test:6511:premise", "style:personachat:test:252:self:1:positive:0"], "evidence_texts": ["Group of men standing in a meeting room, talking.", "A dog jumps to catch a red ball outside.", "Scientists frantically research ways to finally eliminate male baldness.", "Group of men standing in a meeting room, talking.", "right by the back porch"], "speaker_id": "snli:test:6729", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6729", "candidate_text": "Honestly, the dog likes to play catch!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6729:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6513:premise", "snli:test:6729:premise", "snli:test:6772:premise", "snli:test:6511:premise", "style:personachat:test:827:self:5:positive:0", "style:personachat:test:827:self:5:positive:1", "style:personachat:test:827:self:5:positive:2", "style:personachat:test:827:self:5:positive:3", "style:personachat:test:827:self:5:positive:4"], "evidence_texts": ["Group of men standing in a meeting room, talking.", "A dog jumps to catch a red ball outside.", "Scientists frantically research ways to finally eliminate male baldness.", "Group of men standing in a meeting room, talking.", "you do not know the half of it . i am also colorblind .", "that is alright . it is not all doom and gloom though . i love singing karaoke .", "oh cool . where in europe was that ? i lived in london earlier .", "paris is nice . i love barcelona better though .", "never been to israel . so much history there !"], "speaker_id": "snli:test:6729", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6756", "candidate_text": "well, one man is painting a billboard", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6756:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6756:premise", "snli:test:6704:premise", "snli:test:6910:premise", "snli:test:6425:premise", "style:personachat:test:252:self:2:positive:0", "style:personachat:test:252:self:2:positive:1"], "evidence_texts": ["Construction workers on lifts working on a building.", "The squirrel sits straight up in the grass.", "This is a clown at an elementary school.", "A group of people celebrating in the street.", "i camp in my backyard", "right by the back porch"], "speaker_id": "snli:test:6756", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6756", "candidate_text": "One man is painting a billboard.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6756:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6756:premise", "snli:test:6704:premise", "snli:test:6910:premise", "snli:test:6425:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Construction workers on lifts working on a building.", "The squirrel sits straight up in the grass.", "This is a clown at an elementary school.", "A group of people celebrating in the street.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6756", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6772", "candidate_text": "well, scientists are working on a cure", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6772:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6772:premise", "snli:test:6729:premise", "snli:test:6856:premise", "snli:test:6513:premise", "style:personachat:test:252:self:5:positive:0", "style:personachat:test:252:self:5:positive:1", "style:personachat:test:252:self:5:positive:2", "style:personachat:test:252:self:5:positive:3", "style:personachat:test:252:self:5:positive:4"], "evidence_texts": ["Scientists frantically research ways to finally eliminate male baldness.", "A dog jumps to catch a red ball outside.", "Men wearing painted clothes are standing in the rain.", "Group of men standing in a meeting room, talking.", "my dad just retired 2 years ago .", "my mom is a doctor at a senior citizen home .", "i go camping 6 times a year and hiking every month .", "i camp in my backyard", "right by the back porch"], "speaker_id": "snli:test:6772", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6772", "candidate_text": "I wonder whether scientists are working on a cure?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6772:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6772:premise", "snli:test:6729:premise", "snli:test:6856:premise", "snli:test:6513:premise", "style:personachat:test:937:self:5:positive:0", "style:personachat:test:937:self:5:positive:1", "style:personachat:test:937:self:5:positive:2", "style:personachat:test:937:self:5:positive:3", "style:personachat:test:937:self:5:positive:4"], "evidence_texts": ["Scientists frantically research ways to finally eliminate male baldness.", "A dog jumps to catch a red ball outside.", "Men wearing painted clothes are standing in the rain.", "Group of men standing in a meeting room, talking.", "oh ok . what is your favorite animal ?", "i love horses ! i love to get my nails done in my spare time .", "i like pink floyd too ! i am working right now at my fathers car dealership .", "i like him . he is great when he hosts the oscars .", "i never saw his kids . do you want to go to college ?"], "speaker_id": "snli:test:6772", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6778", "candidate_text": "well, a mountaineer is standing on a mountain", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6778:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6556:premise", "snli:test:7032:premise", "snli:test:6778:premise", "snli:test:6475:premise", "style:personachat:test:252:self:7:positive:0", "style:personachat:test:252:self:7:positive:1", "style:personachat:test:252:self:7:positive:2", "style:personachat:test:252:self:7:positive:3", "style:personachat:test:252:self:7:positive:4", "style:personachat:test:252:self:7:positive:5", "style:personachat:test:252:self:7:positive:6"], "evidence_texts": ["Three young people are sitting in a room and talking to one another.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "good evening how are you ?", "i love to eat red meat .", "my dad just retired 2 years ago .", "my mom is a doctor at a senior citizen home .", "i go camping 6 times a year and hiking every month .", "i camp in my backyard", "right by the back porch"], "speaker_id": "snli:test:6778", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6778", "candidate_text": "Honestly, A mountaineer is standing on a mountain!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6778:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6556:premise", "snli:test:7032:premise", "snli:test:6778:premise", "snli:test:6475:premise", "style:personachat:test:836:self:2:positive:0", "style:personachat:test:836:self:2:positive:1"], "evidence_texts": ["Three young people are sitting in a room and talking to one another.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A police officer standing next his police motorbike holding a yellow safety jacket.", "i love seafood . it is so good .", "goodness ! that sounds amazing ! i like caviar and oysters"], "speaker_id": "snli:test:6778", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6790", "candidate_text": "well, woman adjusting her eyeglasses after reading", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6790:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:3820:premise", "snli:test:9479:premise", "snli:test:6790:premise", "snli:test:3819:premise", "style:personachat:test:253:self:1:positive:0"], "evidence_texts": ["Woman is looking something.", "A boy plays basketball.", "Woman adjusting her eyeglasses.", "Woman is looking something.", "how old are you i am 33"], "speaker_id": "snli:test:6790", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6790", "candidate_text": "Woman adjusting her eyeglasses after reading.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6790:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3820:premise", "snli:test:9479:premise", "snli:test:6790:premise", "snli:test:3819:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Woman is looking something.", "A boy plays basketball.", "Woman adjusting her eyeglasses.", "Woman is looking something.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6790", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6793", "candidate_text": "well, a happy man is picking up a pear", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6793:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6523:premise", "snli:test:6793:premise", "snli:test:6894:premise", "snli:test:6085:premise", "style:personachat:test:253:self:2:positive:0", "style:personachat:test:253:self:2:positive:1"], "evidence_texts": ["A man in a competitive field sport jumps into sand and his glasses fall off.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "i have to brothers and two sisters", "how old are you i am 33"], "speaker_id": "snli:test:6793", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6793", "candidate_text": "I wonder whether a happy man is picking up a pear?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6793:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6523:premise", "snli:test:6793:premise", "snli:test:6894:premise", "snli:test:6085:premise", "style:personachat:test:941:self:2:positive:0", "style:personachat:test:941:self:2:positive:1"], "evidence_texts": ["A man in a competitive field sport jumps into sand and his glasses fall off.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "Two women drink coffee and chat while sitting on a blanket near a rocky shore.", "i am part of a noise band , we are hoping to play in vegas", "what kind of show do you have ?"], "speaker_id": "snli:test:6793", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6805", "candidate_text": "well, a young smiling girl is holding a stuffed animal in one hand", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6805:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:5811:premise", "snli:test:6805:premise", "snli:test:6836:premise", "snli:test:5789:premise", "style:personachat:test:253:self:5:positive:0", "style:personachat:test:253:self:5:positive:1", "style:personachat:test:253:self:5:positive:2", "style:personachat:test:253:self:5:positive:3", "style:personachat:test:253:self:5:positive:4"], "evidence_texts": ["A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "where are you from i am from new york", "i am doing ok are you male or female", "i am male and this is a wired hit to do", "i have to brothers and two sisters", "how old are you i am 33"], "speaker_id": "snli:test:6805", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6805", "candidate_text": "Honestly, A young smiling girl is holding a stuffed animal in one hand!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6805:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:5811:premise", "snli:test:6805:premise", "snli:test:6836:premise", "snli:test:5789:premise", "style:personachat:test:842:self:1:positive:0"], "evidence_texts": ["A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "An elderly man with glasses, a red ball cap and backpack is standing in front of an open doorway.", "i have heard about those in our human studies class !"], "speaker_id": "snli:test:6805", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6816", "candidate_text": "well, the workers are wearing black", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6816:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6816:premise", "snli:test:6685:premise", "snli:test:6822:premise", "snli:test:6669:premise", "style:personachat:test:253:self:7:positive:0", "style:personachat:test:253:self:7:positive:1", "style:personachat:test:253:self:7:positive:2", "style:personachat:test:253:self:7:positive:3", "style:personachat:test:253:self:7:positive:4", "style:personachat:test:253:self:7:positive:5", "style:personachat:test:253:self:7:positive:6"], "evidence_texts": ["Two workers in green uniforms are standing in an alleyway.", "Three people are riding a carriage pulled by four horses.", "Two rescue guys looking at a guy on a stretcher", "A man playing a large stringed instrument on the street.", "hello how are you doing", "is your name tara by any chance", "where are you from i am from new york", "i am doing ok are you male or female", "i am male and this is a wired hit to do", "i have to brothers and two sisters", "how old are you i am 33"], "speaker_id": "snli:test:6816", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6816", "candidate_text": "The workers are wearing black.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6816:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6816:premise", "snli:test:6685:premise", "snli:test:6822:premise", "snli:test:6669:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two workers in green uniforms are standing in an alleyway.", "Three people are riding a carriage pulled by four horses.", "Two rescue guys looking at a guy on a stretcher", "A man playing a large stringed instrument on the street.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6816", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6817", "candidate_text": "I wonder whether the man is using a grinder wearing red protective gear?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6817:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6604:premise", "snli:test:6817:premise", "snli:test:6847:premise", "snli:test:6578:premise", "style:personachat:test:254:self:1:positive:0"], "evidence_texts": ["A small girl in a green shirt plays on a swing.", "A man wearing protective gear grinds down a large metal object.", "Two males and one female playing in a fountain of water.", "A beautiful bride walking on a sidewalk with her new husband.", "do you build model cars ?"], "speaker_id": "snli:test:6817", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6817", "candidate_text": "The man is using a grinder wearing red protective gear.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6817:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6604:premise", "snli:test:6817:premise", "snli:test:6847:premise", "snli:test:6578:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A small girl in a green shirt plays on a swing.", "A man wearing protective gear grinds down a large metal object.", "Two males and one female playing in a fountain of water.", "A beautiful bride walking on a sidewalk with her new husband.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6817", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6822", "candidate_text": "I wonder whether a man on a stretcher after falling from the third floor?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6822:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6816:premise", "snli:test:7001:premise", "snli:test:6822:premise", "snli:test:6685:premise", "style:personachat:test:254:self:2:positive:0", "style:personachat:test:254:self:2:positive:1"], "evidence_texts": ["Two workers in green uniforms are standing in an alleyway.", "A man and a child are laughing at each other.", "Two rescue guys looking at a guy on a stretcher", "Three people are riding a carriage pulled by four horses.", "have you seen the movie titanic ? it is my favorite .", "do you build model cars ?"], "speaker_id": "snli:test:6822", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6822", "candidate_text": "well, a man on a stretcher after falling from the third floor", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6822:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6816:premise", "snli:test:7001:premise", "snli:test:6822:premise", "snli:test:6685:premise", "style:personachat:test:424:self:2:positive:0", "style:personachat:test:424:self:2:positive:1"], "evidence_texts": ["Two workers in green uniforms are standing in an alleyway.", "A man and a child are laughing at each other.", "Two rescue guys looking at a guy on a stretcher", "Three people are riding a carriage pulled by four horses.", "that is awesome . i guess your parents rubbed off on you .", "it is a healthy life to live ."], "speaker_id": "snli:test:6822", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6827", "candidate_text": "I wonder whether three kids in chairs taking tests?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6827:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6221:premise", "snli:test:7219:premise", "snli:test:6164:premise", "snli:test:6827:premise", "style:personachat:test:254:self:5:positive:0", "style:personachat:test:254:self:5:positive:1", "style:personachat:test:254:self:5:positive:2", "style:personachat:test:254:self:5:positive:3", "style:personachat:test:254:self:5:positive:4"], "evidence_texts": ["A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "i prefer mustangs and jeeps . my baby does not go in either .", "no , i do not . i am a photographer .", "that is none of your business , and very rude to ask .", "have you seen the movie titanic ? it is my favorite .", "do you build model cars ?"], "speaker_id": "snli:test:6827", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6827", "candidate_text": "Honestly, three kids in chairs taking tests!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6827:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6221:premise", "snli:test:7219:premise", "snli:test:6164:premise", "snli:test:6827:premise", "style:personachat:test:856:self:1:positive:0"], "evidence_texts": ["A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "A woman in a bright pink and green skirt carries a large wicker basket on her back up a stone-cobbled street.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "no fox news again ! my dad and brother would really dislike you"], "speaker_id": "snli:test:6827", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6836", "candidate_text": "I wonder whether two people running a race?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6836:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6836:premise", "snli:test:6805:premise", "snli:test:6841:premise", "snli:test:5811:premise", "style:personachat:test:254:self:7:positive:0", "style:personachat:test:254:self:7:positive:1", "style:personachat:test:254:self:7:positive:2", "style:personachat:test:254:self:7:positive:3", "style:personachat:test:254:self:7:positive:4", "style:personachat:test:254:self:7:positive:5", "style:personachat:test:254:self:7:positive:6"], "evidence_texts": ["A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "hello , ginger and i want to live in florida .", "it is raining , we are getting the tail end of nate .", "i prefer mustangs and jeeps . my baby does not go in either .", "no , i do not . i am a photographer .", "that is none of your business , and very rude to ask .", "have you seen the movie titanic ? it is my favorite .", "do you build model cars ?"], "speaker_id": "snli:test:6836", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6836", "candidate_text": "two people running a race.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6836:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6836:premise", "snli:test:6805:premise", "snli:test:6841:premise", "snli:test:5811:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A woman examines a poster mounted on the wall of a tunnel or hallway lit by overhead fluorescent lights.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6836", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6841", "candidate_text": "well, a crowd of people gathers in a street n winter time", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6841:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6836:premise", "snli:test:6841:premise", "snli:test:6805:premise", "snli:test:6906:premise", "style:personachat:test:255:self:1:positive:0"], "evidence_texts": ["A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "wish i had the time for that . sounds relaxing ."], "speaker_id": "snli:test:6841", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6841", "candidate_text": "Honestly, A crowd of people gathers in a street n winter time!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6841:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6836:premise", "snli:test:6841:premise", "snli:test:6805:premise", "snli:test:6906:premise", "style:personachat:test:856:self:5:positive:0", "style:personachat:test:856:self:5:positive:1", "style:personachat:test:856:self:5:positive:2", "style:personachat:test:856:self:5:positive:3", "style:personachat:test:856:self:5:positive:4"], "evidence_texts": ["A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A young smiling girl is holding a stuffed animal in one hand and a candy apple in the other.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "oh no i hate fox news . so do my dad and brother", "no i do not . too busy going out for sushi to have kids . you ?", "nothing with fish since i am a vegan . how many children do you have ?", "you must be busy ! can you still work out with 3 kids ?", "no fox news again ! my dad and brother would really dislike you"], "speaker_id": "snli:test:6841", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6842", "candidate_text": "well, sasquatch is riding a bike on the street", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6842:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6906:premise", "snli:test:6836:premise", "snli:test:6842:premise", "snli:test:7150:premise", "style:personachat:test:255:self:2:positive:0", "style:personachat:test:255:self:2:positive:1"], "evidence_texts": ["Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "so true ! what else do you do in your spare time other than knit ?", "wish i had the time for that . sounds relaxing ."], "speaker_id": "snli:test:6842", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6842", "candidate_text": "Sasquatch is riding a bike on the street.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6842:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6906:premise", "snli:test:6836:premise", "snli:test:6842:premise", "snli:test:7150:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A man with arms folded in front of his chest sitting next to at least one other person, laughing.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6842", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6844", "candidate_text": "well, the three young children are playing inside", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6844:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6844:premise", "snli:test:6533:premise", "snli:test:6299:premise", "snli:test:7126:premise", "style:personachat:test:255:self:5:positive:0", "style:personachat:test:255:self:5:positive:1", "style:personachat:test:255:self:5:positive:2", "style:personachat:test:255:self:5:positive:3", "style:personachat:test:255:self:5:positive:4"], "evidence_texts": ["The three young children played with some toilet paper in the grass.", "a snowboarder wearing a green hat is riding along a metal railing.", "A woman in a dress is singing and having a good time.", "Family sitting down in a field of grass while others walk by.", "those actually are not too hard to accommodate for and can be cost effective sometimes .", "it is what i do every day . except sundays . that day is for the lord .", "i agree . things would be a lot better in this country if people put god first .", "so true ! what else do you do in your spare time other than knit ?", "wish i had the time for that . sounds relaxing ."], "speaker_id": "snli:test:6844", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6844", "candidate_text": "I wonder whether the three young children are playing inside?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6844:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6844:premise", "snli:test:6533:premise", "snli:test:6299:premise", "snli:test:7126:premise", "style:personachat:test:950:self:5:positive:0", "style:personachat:test:950:self:5:positive:1", "style:personachat:test:950:self:5:positive:2", "style:personachat:test:950:self:5:positive:3", "style:personachat:test:950:self:5:positive:4"], "evidence_texts": ["The three young children played with some toilet paper in the grass.", "a snowboarder wearing a green hat is riding along a metal railing.", "A woman in a dress is singing and having a good time.", "Family sitting down in a field of grass while others walk by.", "yes , very nice . my 2 sons love to hike buttermilk falls .", "i try to not eat many sweets to set a good example for my boys .", "i am single mom so other than hiking , a lot of running my boys around to sports", "it is great . what about yourself ?", "i love dancing . what style ?"], "speaker_id": "snli:test:6844", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6845", "candidate_text": "well, the three young children are throwing the toilet paper around", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6845:positive", "evidence_budget": 14, "evidence_ids": ["snli:test:7126:premise", "snli:test:6845:premise", "snli:test:6533:premise", "snli:test:7206:premise", "style:personachat:test:255:self:10:positive:0", "style:personachat:test:255:self:10:positive:1", "style:personachat:test:255:self:10:positive:2", "style:personachat:test:255:self:10:positive:3", "style:personachat:test:255:self:10:positive:4", "style:personachat:test:255:self:10:positive:5", "style:personachat:test:255:self:10:positive:6", "style:personachat:test:255:self:10:positive:7", "style:personachat:test:255:self:10:positive:8", "style:personachat:test:255:self:10:positive:9"], "evidence_texts": ["Family sitting down in a field of grass while others walk by.", "The three young children played with some toilet paper in the grass.", "a snowboarder wearing a green hat is riding along a metal railing.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "doing pretty good . spent most the night at church . you ?", "well that is just too sweet . how did you get involved in that ?", "that is such a touching story . most people would not be as generous as you .", "me too ! i help build houses . it is a family business .", "really ? i have built a couple of those myself . not difficult at all !", "those actually are not too hard to accommodate for and can be cost effective sometimes .", "it is what i do every day . except sundays . that day is for the lord .", "i agree . things would be a lot better in this country if people put god first .", "so true ! what else do you do in your spare time other than knit ?", "wish i had the time for that . sounds relaxing ."], "speaker_id": "snli:test:6845", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6845", "candidate_text": "Honestly, The three young children are throwing the toilet paper around!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6845:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7126:premise", "snli:test:6845:premise", "snli:test:6533:premise", "snli:test:7206:premise", "style:personachat:test:863:self:2:positive:0", "style:personachat:test:863:self:2:positive:1"], "evidence_texts": ["Family sitting down in a field of grass while others walk by.", "The three young children played with some toilet paper in the grass.", "a snowboarder wearing a green hat is riding along a metal railing.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "that is awesome of you . i might start doing that now that i am retired .", "i bet ! i will end up with a zoo at my house !"], "speaker_id": "snli:test:6845", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6847", "candidate_text": "well, there are three ladies playing in the fountain", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6847:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6817:premise", "snli:test:6967:premise", "snli:test:6604:premise", "snli:test:6847:premise", "style:personachat:test:256:self:1:positive:0"], "evidence_texts": ["A man wearing protective gear grinds down a large metal object.", "Little boy in a green sweatshirt playing with his toy train.", "A small girl in a green shirt plays on a swing.", "Two males and one female playing in a fountain of water.", "lol . . i have an 88 cutlass that needs to be restored"], "speaker_id": "snli:test:6847", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6847", "candidate_text": "There are three ladies playing in the fountain.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6847:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6817:premise", "snli:test:6967:premise", "snli:test:6604:premise", "snli:test:6847:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing protective gear grinds down a large metal object.", "Little boy in a green sweatshirt playing with his toy train.", "A small girl in a green shirt plays on a swing.", "Two males and one female playing in a fountain of water.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6847", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6856", "candidate_text": "well, men are dancing in the rain", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6856:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6856:premise", "snli:test:6772:premise", "snli:test:7012:premise", "snli:test:6729:premise", "style:personachat:test:256:self:2:positive:0", "style:personachat:test:256:self:2:positive:1"], "evidence_texts": ["Men wearing painted clothes are standing in the rain.", "Scientists frantically research ways to finally eliminate male baldness.", "A boy with no shirt is standing in water.", "A dog jumps to catch a red ball outside.", "i bet they did , that is crazy", "lol . . i have an 88 cutlass that needs to be restored"], "speaker_id": "snli:test:6856", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6856", "candidate_text": "I wonder whether men are dancing in the rain?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6856:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6856:premise", "snli:test:6772:premise", "snli:test:7012:premise", "snli:test:6729:premise", "style:personachat:test:955:self:2:positive:0", "style:personachat:test:955:self:2:positive:1"], "evidence_texts": ["Men wearing painted clothes are standing in the rain.", "Scientists frantically research ways to finally eliminate male baldness.", "A boy with no shirt is standing in water.", "A dog jumps to catch a red ball outside.", "yes i did an in ohio", "yes it is where do you work ?"], "speaker_id": "snli:test:6856", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6863", "candidate_text": "well, a woman is covering her eyes from the bright concert lights", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6863:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6453:premise", "snli:test:6291:premise", "snli:test:6915:premise", "snli:test:6863:premise", "style:personachat:test:256:self:5:positive:0", "style:personachat:test:256:self:5:positive:1", "style:personachat:test:256:self:5:positive:2", "style:personachat:test:256:self:5:positive:3", "style:personachat:test:256:self:5:positive:4"], "evidence_texts": ["Young girl in an bathing suit falling off the end of a slide into a pool.", "Two men in hard hats look at a building while on the side of a street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "cool . . . i just had dinner with my parents and my sister", "oh i hate sea food", "how awful , that is why i dine in mostly", "i bet they did , that is crazy", "lol . . i have an 88 cutlass that needs to be restored"], "speaker_id": "snli:test:6863", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6863", "candidate_text": "Honestly, A woman is covering her eyes from the bright concert lights!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6863:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6453:premise", "snli:test:6291:premise", "snli:test:6915:premise", "snli:test:6863:premise", "style:personachat:test:870:self:1:positive:0"], "evidence_texts": ["Young girl in an bathing suit falling off the end of a slide into a pool.", "Two men in hard hats look at a building while on the side of a street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "bbq ! you would be a great voiceover actor . i write cookbooks for dieters ."], "speaker_id": "snli:test:6863", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6864", "candidate_text": "well, a woman is attempting to recognize a friend she thinks is approaching", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6864:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6864:premise", "snli:test:6915:premise", "snli:test:6453:premise", "snli:test:7042:premise", "style:personachat:test:256:self:6:positive:0", "style:personachat:test:256:self:6:positive:1", "style:personachat:test:256:self:6:positive:2", "style:personachat:test:256:self:6:positive:3", "style:personachat:test:256:self:6:positive:4", "style:personachat:test:256:self:6:positive:5"], "evidence_texts": ["A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "hi , i am good . how is life ?", "cool . . . i just had dinner with my parents and my sister", "oh i hate sea food", "how awful , that is why i dine in mostly", "i bet they did , that is crazy", "lol . . i have an 88 cutlass that needs to be restored"], "speaker_id": "snli:test:6864", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6864", "candidate_text": "A woman is attempting to recognize a friend she thinks is approaching.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6864:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6864:premise", "snli:test:6915:premise", "snli:test:6453:premise", "snli:test:7042:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "Young girl in an bathing suit falling off the end of a slide into a pool.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6864", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6894", "candidate_text": "well, people are in a bike race", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6894:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6793:premise", "snli:test:7571:premise", "snli:test:6894:premise", "snli:test:6523:premise", "style:personachat:test:257:self:1:positive:0"], "evidence_texts": ["A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Two men watching one man make a golf putt on the set of Sports Center.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:6894", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6894", "candidate_text": "I wonder whether people are in a bike race?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6894:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6793:premise", "snli:test:7571:premise", "snli:test:6894:premise", "snli:test:6523:premise", "style:personachat:test:959:self:1:positive:0"], "evidence_texts": ["A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Two men watching one man make a golf putt on the set of Sports Center.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "A man in a competitive field sport jumps into sand and his glasses fall off.", "oh do you like it ?"], "speaker_id": "snli:test:6894", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6906", "candidate_text": "well, an intense game of soccer at the world cup", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6906:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6842:premise", "snli:test:7150:premise", "snli:test:6906:premise", "snli:test:6841:premise", "style:personachat:test:257:self:2:positive:0", "style:personachat:test:257:self:2:positive:1"], "evidence_texts": ["A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "what kids are you trying to help ?", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:6906", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6906", "candidate_text": "Honestly, An intense game of soccer at the World Cup!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6906:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6842:premise", "snli:test:7150:premise", "snli:test:6906:premise", "snli:test:6841:premise", "style:personachat:test:870:self:7:positive:0", "style:personachat:test:870:self:7:positive:1", "style:personachat:test:870:self:7:positive:2", "style:personachat:test:870:self:7:positive:3", "style:personachat:test:870:self:7:positive:4", "style:personachat:test:870:self:7:positive:5", "style:personachat:test:870:self:7:positive:6"], "evidence_texts": ["A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "great ! hello , i cook but i am not a chef . you have pets ?", "awww . i am a screen write and read plays . you workout ?", "oh okay . i run and take long walks for exercises . are you dieting ?", "no but i wish i did . i own a dieting business that is why i asked . you sing ?", "get out ! me too ! grace and kelly are my cats . whats your fave car ?", "thank you . john john is my dog . she help me exercise . you act ?", "bbq ! you would be a great voiceover actor . i write cookbooks for dieters ."], "speaker_id": "snli:test:6906", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6910", "candidate_text": "well, the children at this school defintely all love clowns", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6910:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6756:premise", "snli:test:6925:premise", "snli:test:6910:premise", "snli:test:6704:premise", "style:personachat:test:257:self:5:positive:0", "style:personachat:test:257:self:5:positive:1", "style:personachat:test:257:self:5:positive:2", "style:personachat:test:257:self:5:positive:3", "style:personachat:test:257:self:5:positive:4"], "evidence_texts": ["Construction workers on lifts working on a building.", "A little dark-haired girl runs on the beach.", "This is a clown at an elementary school.", "The squirrel sits straight up in the grass.", "i just got off work .", "i am looking for another job myself .", "my parents did not want me either .", "what kids are you trying to help ?", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:6910", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6910", "candidate_text": "The children at this school defintely all love clowns.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6910:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6756:premise", "snli:test:6925:premise", "snli:test:6910:premise", "snli:test:6704:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Construction workers on lifts working on a building.", "A little dark-haired girl runs on the beach.", "This is a clown at an elementary school.", "The squirrel sits straight up in the grass.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6910", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6915", "candidate_text": "well, a woman got married", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6915:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6864:premise", "snli:test:7042:premise", "snli:test:6863:premise", "snli:test:6915:premise", "style:personachat:test:257:self:6:positive:0", "style:personachat:test:257:self:6:positive:1", "style:personachat:test:257:self:6:positive:2", "style:personachat:test:257:self:6:positive:3", "style:personachat:test:257:self:6:positive:4", "style:personachat:test:257:self:6:positive:5"], "evidence_texts": ["A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "pretty good . how are you ?", "i just got off work .", "i am looking for another job myself .", "my parents did not want me either .", "what kids are you trying to help ?", "yeah , it is sad that there are kids who do not have their basic needs met ."], "speaker_id": "snli:test:6915", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6915", "candidate_text": "I wonder whether a woman got married?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6915:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6864:premise", "snli:test:7042:premise", "snli:test:6863:premise", "snli:test:6915:premise", "style:personachat:test:959:self:7:positive:0", "style:personachat:test:959:self:7:positive:1", "style:personachat:test:959:self:7:positive:2", "style:personachat:test:959:self:7:positive:3", "style:personachat:test:959:self:7:positive:4", "style:personachat:test:959:self:7:positive:5", "style:personachat:test:959:self:7:positive:6"], "evidence_texts": ["A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "hey i am doing great what about you", "o is playing poker hard ? i have been wondering for a while", "hmm i might try it soon i am gonna be 21 soon", "oh i know , i know way too many casino addicts to get addicted", "i do not think i will be able to i have a baby but 21st birthday so why not", "true very true so what are you doing", "oh do you like it ?"], "speaker_id": "snli:test:6915", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6919", "candidate_text": "well, the young woman is wearing a helmet", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6919:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6612:premise", "snli:test:7332:premise", "snli:test:6573:premise", "snli:test:6919:premise", "style:personachat:test:258:self:1:positive:0"], "evidence_texts": ["A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A little boy and his father talking to a man dressed as Santa Claws.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "cool , i like to sing , dance , talk and workout ."], "speaker_id": "snli:test:6919", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6919", "candidate_text": "Honestly, The young woman is wearing a helmet!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6919:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6612:premise", "snli:test:7332:premise", "snli:test:6573:premise", "snli:test:6919:premise", "style:personachat:test:885:self:5:positive:0", "style:personachat:test:885:self:5:positive:1", "style:personachat:test:885:self:5:positive:2", "style:personachat:test:885:self:5:positive:3", "style:personachat:test:885:self:5:positive:4"], "evidence_texts": ["A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "A little boy and his father talking to a man dressed as Santa Claws.", "A young boy wearing red Croc sandals climbs a tree with healthy, green leaves.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "awesome . do you play on second life ? i do and i love it !", "i wish i could do that since living in texas i see them a lot .", "that would be nice ! i think those cars are cool . old or newer versions ?", "wow i would love to see that collection ! do you drive them ?", "oh my that is a very early start to a life of crime !"], "speaker_id": "snli:test:6919", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6925", "candidate_text": "well, the girl with the dark hair is running in the sand", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6925:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6910:premise", "snli:test:6925:premise", "snli:test:6756:premise", "snli:test:6943:premise", "style:personachat:test:258:self:2:positive:0", "style:personachat:test:258:self:2:positive:1"], "evidence_texts": ["This is a clown at an elementary school.", "A little dark-haired girl runs on the beach.", "Construction workers on lifts working on a building.", "Person parasailing with a board in the ocean.", "do you play any instruments and do others do it with you ?", "cool , i like to sing , dance , talk and workout ."], "speaker_id": "snli:test:6925", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6925", "candidate_text": "The girl with the dark hair is running in the sand.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6925:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6910:premise", "snli:test:6925:premise", "snli:test:6756:premise", "snli:test:6943:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["This is a clown at an elementary school.", "A little dark-haired girl runs on the beach.", "Construction workers on lifts working on a building.", "Person parasailing with a board in the ocean.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6925", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6926", "candidate_text": "well, the dark haired girl is running away from a wave", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6926:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6943:premise", "snli:test:6910:premise", "snli:test:6926:premise", "snli:test:6998:premise", "style:personachat:test:258:self:5:positive:0", "style:personachat:test:258:self:5:positive:1", "style:personachat:test:258:self:5:positive:2", "style:personachat:test:258:self:5:positive:3", "style:personachat:test:258:self:5:positive:4"], "evidence_texts": ["Person parasailing with a board in the ocean.", "This is a clown at an elementary school.", "A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "oh ok that is very mean that people are picking on you .", "what type of music do you like ?", "that is very creative , do you dance to it ?", "do you play any instruments and do others do it with you ?", "cool , i like to sing , dance , talk and workout ."], "speaker_id": "snli:test:6926", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6926", "candidate_text": "I wonder whether the dark haired girl is running away from a wave?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6926:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6943:premise", "snli:test:6910:premise", "snli:test:6926:premise", "snli:test:6998:premise", "style:personachat:test:963:self:5:positive:0", "style:personachat:test:963:self:5:positive:1", "style:personachat:test:963:self:5:positive:2", "style:personachat:test:963:self:5:positive:3", "style:personachat:test:963:self:5:positive:4"], "evidence_texts": ["Person parasailing with a board in the ocean.", "This is a clown at an elementary school.", "A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "i love animals . i grew up around horses in north dakota .", "i love fresh food . i can stuff all the time .", "not very much . work keeps me pretty busy .", "i recently changed jobs , started in advertising last year .", "that would a fun job . what do you do for fun ?"], "speaker_id": "snli:test:6926", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6938", "candidate_text": "well, several young models in leather coats and boots are giving the cameramen a very good parade", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6938:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:734:premise", "snli:test:2891:premise", "snli:test:6938:premise", "snli:test:5780:premise", "style:personachat:test:258:self:7:positive:0", "style:personachat:test:258:self:7:positive:1", "style:personachat:test:258:self:7:positive:2", "style:personachat:test:258:self:7:positive:3", "style:personachat:test:258:self:7:positive:4", "style:personachat:test:258:self:7:positive:5", "style:personachat:test:258:self:7:positive:6"], "evidence_texts": ["A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "hi i am doing good , how are you doing and how was school ?", "oh no that is not very nice , what grade are you in ?", "oh ok that is very mean that people are picking on you .", "what type of music do you like ?", "that is very creative , do you dance to it ?", "do you play any instruments and do others do it with you ?", "cool , i like to sing , dance , talk and workout ."], "speaker_id": "snli:test:6938", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6938", "candidate_text": "Honestly, Several young models in leather coats and boots are giving the cameramen a very good parade!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6938:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:734:premise", "snli:test:2891:premise", "snli:test:6938:premise", "snli:test:5780:premise", "style:personachat:test:892:self:2:positive:0", "style:personachat:test:892:self:2:positive:1"], "evidence_texts": ["A man wearing a short-sleeved blue shirt and carrying a blue backpack while using snow walking sticks treks through the snow with a woman wearing a long-sleeved blue shirt and black pants also using snow walking sticks.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "A woman in a black leather shirt trench-coat and black leather calf-high boots and carrying a colorful bag with a shoulder strap faces away from the camera with a yellow post visible just in front of her.", "Two distinguished older gentlemen stand outside a house, trees in background, both with white hair, the taller man having his arm about the other man's shoulder as they face the camera from behind their corrective lenses.", "a dog and 2 cute kitties how about you ?", "like the show on nickelodeon !"], "speaker_id": "snli:test:6938", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6943", "candidate_text": "well, this person is in the ocean", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6943:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6926:premise", "snli:test:6998:premise", "snli:test:6925:premise", "snli:test:6943:premise", "style:personachat:test:259:self:1:positive:0"], "evidence_texts": ["A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "A little dark-haired girl runs on the beach.", "Person parasailing with a board in the ocean.", "in the northwest , where i go to college"], "speaker_id": "snli:test:6943", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6943", "candidate_text": "This person is in the ocean.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6943:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6926:premise", "snli:test:6998:premise", "snli:test:6925:premise", "snli:test:6943:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "A little dark-haired girl runs on the beach.", "Person parasailing with a board in the ocean.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6943", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6962", "candidate_text": "well, a man is resting in a hammock", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6962:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6032:premise", "snli:test:6965:premise", "snli:test:6962:premise", "snli:test:6031:premise", "style:personachat:test:259:self:2:positive:0", "style:personachat:test:259:self:2:positive:1"], "evidence_texts": ["A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "i just started jujitsu , i may have to start ultimate too", "in the northwest , where i go to college"], "speaker_id": "snli:test:6962", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6962", "candidate_text": "I wonder whether a man is resting in a hammock?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6962:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6032:premise", "snli:test:6965:premise", "snli:test:6962:premise", "snli:test:6031:premise", "style:personachat:test:965:self:2:positive:0", "style:personachat:test:965:self:2:positive:1"], "evidence_texts": ["A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "i do not like anything sweet like shakes . do you have any pets ?", "my wife made me get rid of my snake . are you married ?"], "speaker_id": "snli:test:6962", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6965", "candidate_text": "well, a tall person in a uniform", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6965:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6962:premise", "snli:test:6965:premise", "snli:test:7261:premise", "snli:test:6032:premise", "style:personachat:test:259:self:5:positive:0", "style:personachat:test:259:self:5:positive:1", "style:personachat:test:259:self:5:positive:2", "style:personachat:test:259:self:5:positive:3", "style:personachat:test:259:self:5:positive:4"], "evidence_texts": ["A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "sounds fun , is it a video game ? i like playing poker .", "oh ! that makes more sense haha . i have never played", "i love running , so i may have to check that out", "i just started jujitsu , i may have to start ultimate too", "in the northwest , where i go to college"], "speaker_id": "snli:test:6965", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6965", "candidate_text": "Honestly, A tall person in a uniform!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6965:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6962:premise", "snli:test:6965:premise", "snli:test:7261:premise", "snli:test:6032:premise", "style:personachat:test:921:self:1:positive:0"], "evidence_texts": ["A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "A man in a suit stands on a walkway with is suitcase with a large display in front of him announcing airline flights.", "really ? you are missing out for sure !"], "speaker_id": "snli:test:6965", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6967", "candidate_text": "well, a boy is playing with toys", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6967:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6847:premise", "snli:test:6974:premise", "snli:test:6817:premise", "snli:test:6967:premise", "style:personachat:test:259:self:6:positive:0", "style:personachat:test:259:self:6:positive:1", "style:personachat:test:259:self:6:positive:2", "style:personachat:test:259:self:6:positive:3", "style:personachat:test:259:self:6:positive:4", "style:personachat:test:259:self:6:positive:5"], "evidence_texts": ["Two males and one female playing in a fountain of water.", "An older man sleeping in a folding chair on the street.", "A man wearing protective gear grinds down a large metal object.", "Little boy in a green sweatshirt playing with his toy train.", "great , getting ready for midterms . you ?", "sounds fun , is it a video game ? i like playing poker .", "oh ! that makes more sense haha . i have never played", "i love running , so i may have to check that out", "i just started jujitsu , i may have to start ultimate too", "in the northwest , where i go to college"], "speaker_id": "snli:test:6967", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6967", "candidate_text": "A boy is playing with toys.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6967:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6847:premise", "snli:test:6974:premise", "snli:test:6817:premise", "snli:test:6967:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two males and one female playing in a fountain of water.", "An older man sleeping in a folding chair on the street.", "A man wearing protective gear grinds down a large metal object.", "Little boy in a green sweatshirt playing with his toy train.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6967", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6971", "candidate_text": "I wonder whether two men are walking?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6971:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6616:premise", "snli:test:7188:premise", "snli:test:6387:premise", "snli:test:6971:premise", "style:personachat:test:260:self:1:positive:0"], "evidence_texts": ["A man wearing black carving stone.", "construction worker working on a structure", "Instructor on Basket weaving at fair", "Two men walking in summer attire.", "cool do you like to watch movies ?"], "speaker_id": "snli:test:6971", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6971", "candidate_text": "Two men are walking.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6971:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6616:premise", "snli:test:7188:premise", "snli:test:6387:premise", "snli:test:6971:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing black carving stone.", "construction worker working on a structure", "Instructor on Basket weaving at fair", "Two men walking in summer attire.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6971", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6974", "candidate_text": "I wonder whether a man is sleeping outside?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6974:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6967:premise", "snli:test:6847:premise", "snli:test:6974:premise", "snli:test:7065:premise", "style:personachat:test:260:self:2:positive:0", "style:personachat:test:260:self:2:positive:1"], "evidence_texts": ["Little boy in a green sweatshirt playing with his toy train.", "Two males and one female playing in a fountain of water.", "An older man sleeping in a folding chair on the street.", "A man laying down in middle of street during heavy traffic.", "that does i love the olympics especially the summer ones", "cool do you like to watch movies ?"], "speaker_id": "snli:test:6974", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6974", "candidate_text": "well, a man is sleeping outside", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6974:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6967:premise", "snli:test:6847:premise", "snli:test:6974:premise", "snli:test:7065:premise", "style:personachat:test:433:self:2:positive:0", "style:personachat:test:433:self:2:positive:1"], "evidence_texts": ["Little boy in a green sweatshirt playing with his toy train.", "Two males and one female playing in a fountain of water.", "An older man sleeping in a folding chair on the street.", "A man laying down in middle of street during heavy traffic.", "does the train go visit my three cats and dogs that are too", "not really that how do you practice it"], "speaker_id": "snli:test:6974", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:6975", "candidate_text": "I wonder whether a man is looking at the ground?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6975:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7065:premise", "snli:test:6967:premise", "snli:test:7066:premise", "snli:test:6975:premise", "style:personachat:test:260:self:5:positive:0", "style:personachat:test:260:self:5:positive:1", "style:personachat:test:260:self:5:positive:2", "style:personachat:test:260:self:5:positive:3", "style:personachat:test:260:self:5:positive:4"], "evidence_texts": ["A man laying down in middle of street during heavy traffic.", "Little boy in a green sweatshirt playing with his toy train.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "i love music and especially bluegrass which i can play good", "i had a tail when i was born but had it cut off", "i had super long hair . do you like sports ?", "that does i love the olympics especially the summer ones", "cool do you like to watch movies ?"], "speaker_id": "snli:test:6975", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6975", "candidate_text": "Honestly, A man is looking at the ground!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6975:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7065:premise", "snli:test:6967:premise", "snli:test:7066:premise", "snli:test:6975:premise", "style:personachat:test:931:self:1:positive:0"], "evidence_texts": ["A man laying down in middle of street during heavy traffic.", "Little boy in a green sweatshirt playing with his toy train.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "absolutely ! but it is nice to have some alone time too ."], "speaker_id": "snli:test:6975", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6998", "candidate_text": "I wonder whether three gals are doing their makeup?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6998:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6943:premise", "snli:test:6926:premise", "snli:test:6998:premise", "snli:test:7055:premise", "style:personachat:test:260:self:7:positive:0", "style:personachat:test:260:self:7:positive:1", "style:personachat:test:260:self:7:positive:2", "style:personachat:test:260:self:7:positive:3", "style:personachat:test:260:self:7:positive:4", "style:personachat:test:260:self:7:positive:5", "style:personachat:test:260:self:7:positive:6"], "evidence_texts": ["Person parasailing with a board in the ocean.", "A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "A moving truck with a ramp still attached.", "i am joe and a pro skater", "i live in fl also . i used to work for hollister back in school", "i love music and especially bluegrass which i can play good", "i had a tail when i was born but had it cut off", "i had super long hair . do you like sports ?", "that does i love the olympics especially the summer ones", "cool do you like to watch movies ?"], "speaker_id": "snli:test:6998", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:6998", "candidate_text": "Three gals are doing their makeup.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:6998:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6943:premise", "snli:test:6926:premise", "snli:test:6998:premise", "snli:test:7055:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Person parasailing with a board in the ocean.", "A little dark-haired girl runs on the beach.", "Two fans are cheering at a sporting event.", "A moving truck with a ramp still attached.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:6998", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:6999", "candidate_text": "Honestly, Two people are making noises!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6999:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7055:premise", "snli:test:6943:premise", "snli:test:7074:premise", "snli:test:6999:premise", "style:personachat:test:261:self:1:positive:0"], "evidence_texts": ["A moving truck with a ramp still attached.", "Person parasailing with a board in the ocean.", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:6999", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:6999", "candidate_text": "well, two people are making noises", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:6999:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7055:premise", "snli:test:6943:premise", "snli:test:7074:premise", "snli:test:6999:premise", "style:personachat:test:434:self:1:positive:0"], "evidence_texts": ["A moving truck with a ramp still attached.", "Person parasailing with a board in the ocean.", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "no not that but how to care for people , animals , etc ,"], "speaker_id": "snli:test:6999", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7001", "candidate_text": "Honestly, A man and his daughter are making funny faces!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7001:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6822:premise", "snli:test:7001:premise", "snli:test:7059:premise", "snli:test:6816:premise", "style:personachat:test:261:self:2:positive:0", "style:personachat:test:261:self:2:positive:1"], "evidence_texts": ["Two rescue guys looking at a guy on a stretcher", "A man and a child are laughing at each other.", "A boy taking a shot while his team watches anxiously.", "Two workers in green uniforms are standing in an alleyway.", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:7001", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7001", "candidate_text": "A man and his daughter are making funny faces.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7001:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6822:premise", "snli:test:7001:premise", "snli:test:7059:premise", "snli:test:6816:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two rescue guys looking at a guy on a stretcher", "A man and a child are laughing at each other.", "A boy taking a shot while his team watches anxiously.", "Two workers in green uniforms are standing in an alleyway.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7001", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7012", "candidate_text": "Honestly, He is outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7012:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7012:premise", "snli:test:6856:premise", "snli:test:6772:premise", "snli:test:7075:premise", "style:personachat:test:261:self:5:positive:0", "style:personachat:test:261:self:5:positive:1", "style:personachat:test:261:self:5:positive:2", "style:personachat:test:261:self:5:positive:3", "style:personachat:test:261:self:5:positive:4"], "evidence_texts": ["A boy with no shirt is standing in water.", "Men wearing painted clothes are standing in the rain.", "Scientists frantically research ways to finally eliminate male baldness.", "A man in summer clothing skiing on thin snow.", "i listen to pop music", "my dad owns a car dealership across town", "no but i get a discount", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:7012", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7012", "candidate_text": "I wonder whether he is outside?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7012:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7012:premise", "snli:test:6856:premise", "snli:test:6772:premise", "snli:test:7075:premise", "style:personachat:test:2:self:5:positive:0", "style:personachat:test:2:self:5:positive:1", "style:personachat:test:2:self:5:positive:2", "style:personachat:test:2:self:5:positive:3", "style:personachat:test:2:self:5:positive:4"], "evidence_texts": ["A boy with no shirt is standing in water.", "Men wearing painted clothes are standing in the rain.", "Scientists frantically research ways to finally eliminate male baldness.", "A man in summer clothing skiing on thin snow.", "very cool . do you have pets ?", "yes . two dogs and a cat . they are my babies .", "i live in texas . i love riding my bike here .", "yes i am . i have always been a christian .", "oh really ? so you do not go to church often ?"], "speaker_id": "snli:test:7012", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7013", "candidate_text": "Honestly, The boy is sitting on the couch!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7013:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7075:premise", "snli:test:7013:premise", "snli:test:6856:premise", "snli:test:7247:premise", "style:personachat:test:261:self:7:positive:0", "style:personachat:test:261:self:7:positive:1", "style:personachat:test:261:self:7:positive:2", "style:personachat:test:261:self:7:positive:3", "style:personachat:test:261:self:7:positive:4", "style:personachat:test:261:self:7:positive:5", "style:personachat:test:261:self:7:positive:6"], "evidence_texts": ["A man in summer clothing skiing on thin snow.", "A boy with no shirt is standing in water.", "Men wearing painted clothes are standing in the rain.", "Two people climbing up some sort of wore structure.", "i love music ! i also love horses", "i do not . i would get sweaty", "i listen to pop music", "my dad owns a car dealership across town", "no but i get a discount", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:7013", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7013", "candidate_text": "well, the boy is sitting on the couch", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7013:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7075:premise", "snli:test:7013:premise", "snli:test:6856:premise", "snli:test:7247:premise", "style:personachat:test:434:self:7:positive:0", "style:personachat:test:434:self:7:positive:1", "style:personachat:test:434:self:7:positive:2", "style:personachat:test:434:self:7:positive:3", "style:personachat:test:434:self:7:positive:4", "style:personachat:test:434:self:7:positive:5", "style:personachat:test:434:self:7:positive:6"], "evidence_texts": ["A man in summer clothing skiing on thin snow.", "A boy with no shirt is standing in water.", "Men wearing painted clothes are standing in the rain.", "Two people climbing up some sort of wore structure.", "i am good , how about you ? had dinner yet ?", "wow , me , too . mine was portuguese style .", "my mother was a nurse in the army and she taught me to cook .", "clams steamed with potatoes , sausage , and spices .", "ha ha , teenager food . mom taught me how to make my own clothing too ,", "yes , and about how to take care of our environment , too .", "no not that but how to care for people , animals , etc ,"], "speaker_id": "snli:test:7013", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7032", "candidate_text": "well, a person laying down", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7032:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7032:premise", "snli:test:6778:premise", "snli:test:7092:premise", "snli:test:6556:premise", "style:personachat:test:262:self:1:positive:0"], "evidence_texts": ["A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A woman in a yellow shirt is carrying two full red grocery bags.", "Three young people are sitting in a room and talking to one another.", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:7032", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7032", "candidate_text": "A person laying down.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7032:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7032:premise", "snli:test:6778:premise", "snli:test:7092:premise", "snli:test:6556:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A mountaineer about to descend down a mountain with a blue helmet on.", "A woman in a yellow shirt is carrying two full red grocery bags.", "Three young people are sitting in a room and talking to one another.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7032", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7042", "candidate_text": "well, a woman is pointing her camera upwards, while standing at a wall", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7042:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6915:premise", "snli:test:7134:premise", "snli:test:7042:premise", "snli:test:6864:premise", "style:personachat:test:262:self:2:positive:0", "style:personachat:test:262:self:2:positive:1"], "evidence_texts": ["A woman in a white wedding dress is being dressed and fitted by two other women.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "i am number two next to brothers also", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:7042", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7042", "candidate_text": "I wonder whether a woman is pointing her camera upwards, while standing at a wall?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7042:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6915:premise", "snli:test:7134:premise", "snli:test:7042:premise", "snli:test:6864:premise", "style:personachat:test:3:self:2:positive:0", "style:personachat:test:3:self:2:positive:1"], "evidence_texts": ["A woman in a white wedding dress is being dressed and fitted by two other women.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A woman on the sidewalk blocks the sun from her eyes to peer down the street.", "yes get on it now", "anytime are you okay now ?"], "speaker_id": "snli:test:7042", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7055", "candidate_text": "well, a moving truck being disassembled", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7055:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6999:premise", "snli:test:7074:premise", "snli:test:6998:premise", "snli:test:7055:premise", "style:personachat:test:262:self:5:positive:0", "style:personachat:test:262:self:5:positive:1", "style:personachat:test:262:self:5:positive:2", "style:personachat:test:262:self:5:positive:3", "style:personachat:test:262:self:5:positive:4"], "evidence_texts": ["Two fans are cheering at a sporting event.", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "A moving truck with a ramp still attached.", "you never heard of south carolina", "o yes that in columbia i live a bout 80 miles", "that ok , i really under that", "i am number two next to brothers also", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:7055", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7055", "candidate_text": "Honestly, A moving truck being disassembled!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7055:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6999:premise", "snli:test:7074:premise", "snli:test:6998:premise", "snli:test:7055:premise", "style:personachat:test:933:self:1:positive:0"], "evidence_texts": ["Two fans are cheering at a sporting event.", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "A moving truck with a ramp still attached.", "your mom must be very brave !"], "speaker_id": "snli:test:7055", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7059", "candidate_text": "well, a boy is on his bike", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7059:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7001:premise", "snli:test:7176:premise", "snli:test:6822:premise", "snli:test:7059:premise", "style:personachat:test:262:self:7:positive:0", "style:personachat:test:262:self:7:positive:1", "style:personachat:test:262:self:7:positive:2", "style:personachat:test:262:self:7:positive:3", "style:personachat:test:262:self:7:positive:4", "style:personachat:test:262:self:7:positive:5", "style:personachat:test:262:self:7:positive:6"], "evidence_texts": ["A man and a child are laughing at each other.", "A woman and little girl look into a display case.", "Two rescue guys looking at a guy on a stretcher", "A boy taking a shot while his team watches anxiously.", "hi , i am from the south , what about you ?", "i have a mother and five sisters , how about you ?", "you never heard of south carolina", "o yes that in columbia i live a bout 80 miles", "that ok , i really under that", "i am number two next to brothers also", "yes , it a lots of fun , what music do you like ."], "speaker_id": "snli:test:7059", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7059", "candidate_text": "A boy is on his bike.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7059:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7001:premise", "snli:test:7176:premise", "snli:test:6822:premise", "snli:test:7059:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man and a child are laughing at each other.", "A woman and little girl look into a display case.", "Two rescue guys looking at a guy on a stretcher", "A boy taking a shot while his team watches anxiously.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7059", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7065", "candidate_text": "well, the traffic was not to bad today the guy made it home early and took a nap", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7065:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6975:premise", "snli:test:7065:premise", "snli:test:7066:premise", "snli:test:6974:premise", "style:personachat:test:263:self:1:positive:0"], "evidence_texts": ["An older man sleeping in a folding chair on the street.", "A man laying down in middle of street during heavy traffic.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "florida , i like to run and swim"], "speaker_id": "snli:test:7065", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7065", "candidate_text": "I wonder whether the traffic was not to bad today the guy made it home early and took a nap?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7065:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6975:premise", "snli:test:7065:premise", "snli:test:7066:premise", "snli:test:6974:premise", "style:personachat:test:4:self:1:positive:0"], "evidence_texts": ["An older man sleeping in a folding chair on the street.", "A man laying down in middle of street during heavy traffic.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:7065", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7066", "candidate_text": "well, some girls are watching cartoons", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7066:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7065:premise", "snli:test:7202:premise", "snli:test:7066:premise", "snli:test:6975:premise", "style:personachat:test:263:self:2:positive:0", "style:personachat:test:263:self:2:positive:1"], "evidence_texts": ["A man laying down in middle of street during heavy traffic.", "A woman playing with play equipment while a child looks on.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "where are you from ? no on the kids", "florida , i like to run and swim"], "speaker_id": "snli:test:7066", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7066", "candidate_text": "Honestly, Some girls are watching cartoons!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7066:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7065:premise", "snli:test:7202:premise", "snli:test:7066:premise", "snli:test:6975:premise", "style:personachat:test:933:self:7:positive:0", "style:personachat:test:933:self:7:positive:1", "style:personachat:test:933:self:7:positive:2", "style:personachat:test:933:self:7:positive:3", "style:personachat:test:933:self:7:positive:4", "style:personachat:test:933:self:7:positive:5", "style:personachat:test:933:self:7:positive:6"], "evidence_texts": ["A man laying down in middle of street during heavy traffic.", "A woman playing with play equipment while a child looks on.", "3 Olympic winners posing for a photo on an advertised background", "An older man sleeping in a folding chair on the street.", "i am doing well , i am taking a break from building a tree house .", "i have been building tree houses for over 40 years now , the tree is safe . .", "why are you worried about the world ?", "yeah , humans have been rough on our plant . do you have any other hobbies ?", "do you knit any of your clothes ? i am a knitter .", "i often get funny looks when people see a 40 year old man knitting !", "your mom must be very brave !"], "speaker_id": "snli:test:7066", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7074", "candidate_text": "well, cowboy riding his horse", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7074:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7055:premise", "snli:test:7124:premise", "snli:test:7074:premise", "snli:test:6999:premise", "style:personachat:test:263:self:5:positive:0", "style:personachat:test:263:self:5:positive:1", "style:personachat:test:263:self:5:positive:2", "style:personachat:test:263:self:5:positive:3", "style:personachat:test:263:self:5:positive:4"], "evidence_texts": ["A moving truck with a ramp still attached.", "A number of cute asian children playing tambourines", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "nothing lately , do you have any hobbies ?", "nothing it just sounds good , my fears of snakes and spiders", "i have no kids , and i still am unmarried", "where are you from ? no on the kids", "florida , i like to run and swim"], "speaker_id": "snli:test:7074", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7074", "candidate_text": "cowboy riding his horse.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7074:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7055:premise", "snli:test:7124:premise", "snli:test:7074:premise", "snli:test:6999:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A moving truck with a ramp still attached.", "A number of cute asian children playing tambourines", "A cowboy riding a horse at a rodeo.", "Two fans are cheering at a sporting event.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7074", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7075", "candidate_text": "well, a man is skiing on a warm day", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7075:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7013:premise", "snli:test:7247:premise", "snli:test:7012:premise", "snli:test:7075:premise", "style:personachat:test:263:self:7:positive:0", "style:personachat:test:263:self:7:positive:1", "style:personachat:test:263:self:7:positive:2", "style:personachat:test:263:self:7:positive:3", "style:personachat:test:263:self:7:positive:4", "style:personachat:test:263:self:7:positive:5", "style:personachat:test:263:self:7:positive:6"], "evidence_texts": ["A boy with no shirt is standing in water.", "Two people climbing up some sort of wore structure.", "A boy with no shirt is standing in water.", "A man in summer clothing skiing on thin snow.", "i am well and you ? i write science fiction as a hobby", "i have a 8 year degree in engineering from mit", "nothing lately , do you have any hobbies ?", "nothing it just sounds good , my fears of snakes and spiders", "i have no kids , and i still am unmarried", "where are you from ? no on the kids", "florida , i like to run and swim"], "speaker_id": "snli:test:7075", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7075", "candidate_text": "I wonder whether a man is skiing on a warm day?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7075:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7013:premise", "snli:test:7247:premise", "snli:test:7012:premise", "snli:test:7075:premise", "style:personachat:test:4:self:7:positive:0", "style:personachat:test:4:self:7:positive:1", "style:personachat:test:4:self:7:positive:2", "style:personachat:test:4:self:7:positive:3", "style:personachat:test:4:self:7:positive:4", "style:personachat:test:4:self:7:positive:5", "style:personachat:test:4:self:7:positive:6"], "evidence_texts": ["A boy with no shirt is standing in water.", "Two people climbing up some sort of wore structure.", "A boy with no shirt is standing in water.", "A man in summer clothing skiing on thin snow.", "good , do you like animals ?", "i have 2 dogs , they are great , where do you work ?", "are you afraid of heights ? i certainly am", "sure i like tv , what do you watch ?", "i do not have time to watch tv , i am an attorney so i work alot", "i certainly am , i am part native american , i live here in oklahoma", "well i do not like heights very much and i love animals . what about you ?"], "speaker_id": "snli:test:7075", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7092", "candidate_text": "well, a lady has her arms full", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7092:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7092:premise", "snli:test:7032:premise", "snli:test:7140:premise", "snli:test:6778:premise", "style:personachat:test:264:self:1:positive:0"], "evidence_texts": ["A woman in a yellow shirt is carrying two full red grocery bags.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A mountaineer about to descend down a mountain with a blue helmet on.", "not really happy but we have to hang in there and do our best ."], "speaker_id": "snli:test:7092", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7092", "candidate_text": "Honestly, A lady has her arms full!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7092:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7092:premise", "snli:test:7032:premise", "snli:test:7140:premise", "snli:test:6778:premise", "style:personachat:test:936:self:5:positive:0", "style:personachat:test:936:self:5:positive:1", "style:personachat:test:936:self:5:positive:2", "style:personachat:test:936:self:5:positive:3", "style:personachat:test:936:self:5:positive:4"], "evidence_texts": ["A woman in a yellow shirt is carrying two full red grocery bags.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A mountaineer about to descend down a mountain with a blue helmet on.", "used to , but the homework for my studies burnt me out gonna be a lawyer", "mostly about my adventures . i love traveling . what do you do for fun ?", "ooh , sounds fun ! have you been doing that for a long time ?", "same . i much prefer the company of my dog over people !", "cute ! what are their names ?"], "speaker_id": "snli:test:7092", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7110", "candidate_text": "well, a man in a suit with gold buttons and a name tag gestures with his hands", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7110:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:5732:premise", "snli:test:908:premise", "snli:test:7110:premise", "snli:test:5351:premise", "style:personachat:test:264:self:2:positive:0", "style:personachat:test:264:self:2:positive:1"], "evidence_texts": ["A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "great i bet you do beautiful hair styles yes i sing lovely .", "not really happy but we have to hang in there and do our best ."], "speaker_id": "snli:test:7110", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7110", "candidate_text": "A man in a suit with gold buttons and a name tag gestures with his hands.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7110:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:5732:premise", "snli:test:908:premise", "snli:test:7110:premise", "snli:test:5351:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A smiling bride sits in a swing with her smiling groom standing behind her posing for the male photographer while a boy holding a bottled drink and another boy wearing a green shirt observe.", "A black man wearing a red belt and a white man wearing a blue belt are pictured in the act of practicing martial arts inside a building with a woman looking on in the background.", "A man in a suit with gold buttons and a name tag gestures with his hands as he speaks to another man in a dress shirt who is holding a beverage in a glass.", "A stoplight displaying two bicycle symbols, a left-turning green arrow and a red forward signal stands in the foreground while a man in a white hat and blue shirt bikes along a red path.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7110", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7118", "candidate_text": "well, the girl is outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7118:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6587:premise", "snli:test:6552:premise", "snli:test:7118:premise", "snli:test:7211:premise", "style:personachat:test:264:self:5:positive:0", "style:personachat:test:264:self:5:positive:1", "style:personachat:test:264:self:5:positive:2", "style:personachat:test:264:self:5:positive:3", "style:personachat:test:264:self:5:positive:4"], "evidence_texts": ["Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "i like having three dogs love animals i used to aide .", "i love being a hair stylist poor unicorn that is so funny .", "i like looking at football games on sundays yes hair is cool .", "great i bet you do beautiful hair styles yes i sing lovely .", "not really happy but we have to hang in there and do our best ."], "speaker_id": "snli:test:7118", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7118", "candidate_text": "I wonder whether the girl is outside?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7118:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6587:premise", "snli:test:6552:premise", "snli:test:7118:premise", "snli:test:7211:premise", "style:personachat:test:5:self:5:positive:0", "style:personachat:test:5:self:5:positive:1", "style:personachat:test:5:self:5:positive:2", "style:personachat:test:5:self:5:positive:3", "style:personachat:test:5:self:5:positive:4"], "evidence_texts": ["Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "A girl with curly hair and wearing a patriotic shirt is looking at a monarch butterfly sitting on her left index finger.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "i get that ! i am a doctor . pediatrics to be exact . do anything for fun ?", "me too ! well , i do not have any dogs , but i run everyday !", "yes , it helps me clear my mind . i also love to read to escape reality sometimes too", "california . where do you live ?", "lol . i am scared of heights too . do you like to listen to music ?"], "speaker_id": "snli:test:7118", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7119", "candidate_text": "well, the girl is on a beach", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7119:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7211:premise", "snli:test:6587:premise", "snli:test:7610:premise", "snli:test:7119:premise", "style:personachat:test:264:self:7:positive:0", "style:personachat:test:264:self:7:positive:1", "style:personachat:test:264:self:7:positive:2", "style:personachat:test:264:self:7:positive:3", "style:personachat:test:264:self:7:positive:4", "style:personachat:test:264:self:7:positive:5", "style:personachat:test:264:self:7:positive:6"], "evidence_texts": ["An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "hi how are you doing ? yes i like to eat beef chicken and pork .", "my best seasonal holiday is celebrating christmas . oh he will be okay .", "i like having three dogs love animals i used to aide .", "i love being a hair stylist poor unicorn that is so funny .", "i like looking at football games on sundays yes hair is cool .", "great i bet you do beautiful hair styles yes i sing lovely .", "not really happy but we have to hang in there and do our best ."], "speaker_id": "snli:test:7119", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7119", "candidate_text": "Honestly, The girl is on a beach!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7119:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7211:premise", "snli:test:6587:premise", "snli:test:7610:premise", "snli:test:7119:premise", "style:personachat:test:949:self:2:positive:0", "style:personachat:test:949:self:2:positive:1"], "evidence_texts": ["An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "Red capped male dressed in white shirt and dark shorts is standing on small green boat located water in tall grassy bank.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "any that involve traveling . we are trying to get to every state", "you should travel . it is so great !"], "speaker_id": "snli:test:7119", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7124", "candidate_text": "well, a group of cute asian children are beating on tambourines", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7124:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7124:premise", "snli:test:7074:premise", "snli:test:7199:premise", "snli:test:7055:premise", "style:personachat:test:265:self:1:positive:0"], "evidence_texts": ["A number of cute asian children playing tambourines", "A cowboy riding a horse at a rodeo.", "City street crowded with sports fans wearing orange.", "A moving truck with a ramp still attached.", "wow ; that is sad and unique . i hope things get better for you ."], "speaker_id": "snli:test:7124", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7124", "candidate_text": "A group of cute Asian children are beating on tambourines.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7124:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7124:premise", "snli:test:7074:premise", "snli:test:7199:premise", "snli:test:7055:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A number of cute asian children playing tambourines", "A cowboy riding a horse at a rodeo.", "City street crowded with sports fans wearing orange.", "A moving truck with a ramp still attached.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7124", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7126", "candidate_text": "well, two parents and two children are sitting in a field of grass", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7126:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6845:premise", "snli:test:7206:premise", "snli:test:7126:premise", "snli:test:6844:premise", "style:personachat:test:265:self:2:positive:0", "style:personachat:test:265:self:2:positive:1"], "evidence_texts": ["The three young children played with some toilet paper in the grass.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "Family sitting down in a field of grass while others walk by.", "The three young children played with some toilet paper in the grass.", "why , did something hurt you before ; i am sorry if that happened .", "wow ; that is sad and unique . i hope things get better for you ."], "speaker_id": "snli:test:7126", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7126", "candidate_text": "I wonder whether two parents and two children are sitting in a field of grass?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7126:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6845:premise", "snli:test:7206:premise", "snli:test:7126:premise", "snli:test:6844:premise", "style:personachat:test:8:self:2:positive:0", "style:personachat:test:8:self:2:positive:1"], "evidence_texts": ["The three young children played with some toilet paper in the grass.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "Family sitting down in a field of grass while others walk by.", "The three young children played with some toilet paper in the grass.", "i have a cat and a dog you would not be scared of my dog", "i am a secretary like my mom hard living and you ?"], "speaker_id": "snli:test:7126", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7134", "candidate_text": "well, a squirrel is sleeping in the tree", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7134:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7042:premise", "snli:test:7524:premise", "snli:test:7134:premise", "snli:test:6915:premise", "style:personachat:test:265:self:5:positive:0", "style:personachat:test:265:self:5:positive:1", "style:personachat:test:265:self:5:positive:2", "style:personachat:test:265:self:5:positive:3", "style:personachat:test:265:self:5:positive:4"], "evidence_texts": ["A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A man wearing a hat sitting on the side of a hill looking towards the water.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "any job is a good job . you should be proud .", "the good thing is online courses can help you advance if you like .", "ok , well i like my golden retriever named ginger and watching titanic again .", "why , did something hurt you before ; i am sorry if that happened .", "wow ; that is sad and unique . i hope things get better for you ."], "speaker_id": "snli:test:7134", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7134", "candidate_text": "Honestly, A squirrel is sleeping in the tree!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7134:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7042:premise", "snli:test:7524:premise", "snli:test:7134:premise", "snli:test:6915:premise", "style:personachat:test:964:self:1:positive:0"], "evidence_texts": ["A woman with long hair pointing her camera skyward as she stands on a brick wall.", "A man wearing a hat sitting on the side of a hill looking towards the water.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A woman in a white wedding dress is being dressed and fitted by two other women.", "omg i am ! guess how tall i am"], "speaker_id": "snli:test:7134", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7140", "candidate_text": "well, a little girl is wearing a dress", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7140:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7140:premise", "snli:test:7092:premise", "snli:test:7288:premise", "snli:test:7032:premise", "style:personachat:test:265:self:7:positive:0", "style:personachat:test:265:self:7:positive:1", "style:personachat:test:265:self:7:positive:2", "style:personachat:test:265:self:7:positive:3", "style:personachat:test:265:self:7:positive:4", "style:personachat:test:265:self:7:positive:5", "style:personachat:test:265:self:7:positive:6"], "evidence_texts": ["A little girl wearing a yellow dress is sitting on the white rocks.", "A woman in a yellow shirt is carrying two full red grocery bags.", "A boy and a girl stand in a median amongst Mike Huckabee signs.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "i am a work at home father and it is good ; especially since i became a dad .", "what kind of work do you do ?", "any job is a good job . you should be proud .", "the good thing is online courses can help you advance if you like .", "ok , well i like my golden retriever named ginger and watching titanic again .", "why , did something hurt you before ; i am sorry if that happened .", "wow ; that is sad and unique . i hope things get better for you ."], "speaker_id": "snli:test:7140", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7140", "candidate_text": "A little girl is wearing a dress.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7140:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7140:premise", "snli:test:7092:premise", "snli:test:7288:premise", "snli:test:7032:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A little girl wearing a yellow dress is sitting on the white rocks.", "A woman in a yellow shirt is carrying two full red grocery bags.", "A boy and a girl stand in a median amongst Mike Huckabee signs.", "A person in sunglasses, a scarf, a shirt, and jeans is walking outside.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7140", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7150", "candidate_text": "well, men talking, while catching fish", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7150:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6906:premise", "snli:test:7229:premise", "snli:test:7150:premise", "snli:test:6842:premise", "style:personachat:test:266:self:1:positive:0"], "evidence_texts": ["Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "that sounds like it might be difficult"], "speaker_id": "snli:test:7150", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7150", "candidate_text": "I wonder whether men talking, while catching fish?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7150:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6906:premise", "snli:test:7229:premise", "snli:test:7150:premise", "snli:test:6842:premise", "style:personachat:test:12:self:1:positive:0"], "evidence_texts": ["Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "A crowd of people in winter garb gather in the street against a backdrop of bare trees and snow.", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:7150", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7169", "candidate_text": "well, they are at a photo shoot", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7169:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6590:premise", "snli:test:7169:premise", "snli:test:7417:premise", "snli:test:6575:premise", "style:personachat:test:266:self:2:positive:0", "style:personachat:test:266:self:2:positive:1"], "evidence_texts": ["People trying to get warm.", "cheerleaders pose for the camera.", "Two friends having a laugh.", "A dog catching a Frisbee.", "i am only five foot", "that sounds like it might be difficult"], "speaker_id": "snli:test:7169", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7169", "candidate_text": "Honestly, They are at a photo shoot!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7169:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:6590:premise", "snli:test:7169:premise", "snli:test:7417:premise", "snli:test:6575:premise", "style:personachat:test:964:self:7:positive:0", "style:personachat:test:964:self:7:positive:1", "style:personachat:test:964:self:7:positive:2", "style:personachat:test:964:self:7:positive:3", "style:personachat:test:964:self:7:positive:4", "style:personachat:test:964:self:7:positive:5", "style:personachat:test:964:self:7:positive:6"], "evidence_texts": ["People trying to get warm.", "cheerleaders pose for the camera.", "Two friends having a laugh.", "A dog catching a Frisbee.", "i am pretty bored actually . listening to music .", "a lot of love , grace , kindness , scatting and dancing .", "i was hoping you knew what i was talking about ! school , no work !", "do i ? maybe a little to ficken", "no like the emphasis on do like duh , who does not love bbq", "homer simpson ? obviously , who else is homer , but explain !", "omg i am ! guess how tall i am"], "speaker_id": "snli:test:7169", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7176", "candidate_text": "well, a women and little girl are looking at something", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7176:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7176:premise", "snli:test:7059:premise", "snli:test:7259:premise", "snli:test:7001:premise", "style:personachat:test:266:self:5:positive:0", "style:personachat:test:266:self:5:positive:1", "style:personachat:test:266:self:5:positive:2", "style:personachat:test:266:self:5:positive:3", "style:personachat:test:266:self:5:positive:4"], "evidence_texts": ["A woman and little girl look into a display case.", "A boy taking a shot while his team watches anxiously.", "Woman standing in a store with an item in hand.", "A man and a child are laughing at each other.", "just my parents and my older sibling", "i like to make music and you ?", "ghosts scare me but i think that is because i am so small", "i am only five foot", "that sounds like it might be difficult"], "speaker_id": "snli:test:7176", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7176", "candidate_text": "A women and little girl are looking at something.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7176:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7176:premise", "snli:test:7059:premise", "snli:test:7259:premise", "snli:test:7001:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman and little girl look into a display case.", "A boy taking a shot while his team watches anxiously.", "Woman standing in a store with an item in hand.", "A man and a child are laughing at each other.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7176", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7181", "candidate_text": "well, a photographer is taking many photos of a hockey game", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7181:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:6678:premise", "snli:test:7181:premise", "snli:test:7411:premise", "snli:test:6002:premise", "style:personachat:test:266:self:7:positive:0", "style:personachat:test:266:self:7:positive:1", "style:personachat:test:266:self:7:positive:2", "style:personachat:test:266:self:7:positive:3", "style:personachat:test:266:self:7:positive:4", "style:personachat:test:266:self:7:positive:5", "style:personachat:test:266:self:7:positive:6"], "evidence_texts": ["A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "i am a music teacher , what are you ?", "do you have a big family ?", "just my parents and my older sibling", "i like to make music and you ?", "ghosts scare me but i think that is because i am so small", "i am only five foot", "that sounds like it might be difficult"], "speaker_id": "snli:test:7181", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7181", "candidate_text": "I wonder whether a photographer is taking many photos of a hockey game?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7181:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:6678:premise", "snli:test:7181:premise", "snli:test:7411:premise", "snli:test:6002:premise", "style:personachat:test:12:self:6:positive:0", "style:personachat:test:12:self:6:positive:1", "style:personachat:test:12:self:6:positive:2", "style:personachat:test:12:self:6:positive:3", "style:personachat:test:12:self:6:positive:4", "style:personachat:test:12:self:6:positive:5"], "evidence_texts": ["A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "A man wearing shorts and a blue t-shirt is applying signage to the display window of an ice cream shop.", "i am great . i am watching one tree hill . that is my favorite show", "is it the video where they are wearing purple ? best color ever", "the one at the hospital ? i am a nurse at a pediatric one", "i do not believe in sheltering animals . i am a catholic", "you could make them into a pizza . i love to eat pizza so i would buy", "why not ? it sounds like a good plan"], "speaker_id": "snli:test:7181", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7188", "candidate_text": "I wonder whether a man is working on a computer?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7188:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7188:premise", "snli:test:6971:premise", "snli:test:8585:premise", "snli:test:6616:premise", "style:personachat:test:267:self:1:positive:0"], "evidence_texts": ["construction worker working on a structure", "Two men walking in summer attire.", "White dog playing in the snow.", "A man wearing black carving stone.", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:7188", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7188", "candidate_text": "well, a man is working on a computer", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7188:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7188:premise", "snli:test:6971:premise", "snli:test:8585:premise", "snli:test:6616:premise", "style:personachat:test:447:self:1:positive:0"], "evidence_texts": ["construction worker working on a structure", "Two men walking in summer attire.", "White dog playing in the snow.", "A man wearing black carving stone.", "yes they are very healthy ."], "speaker_id": "snli:test:7188", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7199", "candidate_text": "I wonder whether the sports fans are wearing orange shirts?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7199:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7124:premise", "snli:test:7291:premise", "snli:test:7074:premise", "snli:test:7199:premise", "style:personachat:test:267:self:2:positive:0", "style:personachat:test:267:self:2:positive:1"], "evidence_texts": ["A number of cute asian children playing tambourines", "A man is hanging sausages in a market.", "A cowboy riding a horse at a rodeo.", "City street crowded with sports fans wearing orange.", "dance is great exercise for a young one like yourself !", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:7199", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7199", "candidate_text": "Honestly, The sports fans are wearing orange shirts!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7199:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7124:premise", "snli:test:7291:premise", "snli:test:7074:premise", "snli:test:7199:premise", "style:personachat:test:21:self:7:positive:0", "style:personachat:test:21:self:7:positive:1", "style:personachat:test:21:self:7:positive:2", "style:personachat:test:21:self:7:positive:3", "style:personachat:test:21:self:7:positive:4", "style:personachat:test:21:self:7:positive:5", "style:personachat:test:21:self:7:positive:6"], "evidence_texts": ["A number of cute asian children playing tambourines", "A man is hanging sausages in a market.", "A cowboy riding a horse at a rodeo.", "City street crowded with sports fans wearing orange.", "just as fantastic as i possibly can be and yourself ?", "what are you studying , may i ask ?", "cool i am in jail right now so can i ask you some questions ?", "so i am here for selling sea monkeys like you see in comic books", "what do you mean ? i got convicted for mail fraud cuz sea monkeys do not exist", "yes i thought they were real too , i got duped by my supplier", "yeah i had this tiny little nun public defender she slapped witnesses with a ruler !"], "speaker_id": "snli:test:7199", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7202", "candidate_text": "I wonder whether child watches woman playing with equipment?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7202:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7066:premise", "snli:test:7269:premise", "snli:test:7202:premise", "snli:test:7065:premise", "style:personachat:test:267:self:5:positive:0", "style:personachat:test:267:self:5:positive:1", "style:personachat:test:267:self:5:positive:2", "style:personachat:test:267:self:5:positive:3", "style:personachat:test:267:self:5:positive:4"], "evidence_texts": ["3 Olympic winners posing for a photo on an advertised background", "The tennis player spikes the tennis ball on the boundary line.", "A woman playing with play equipment while a child looks on.", "A man laying down in middle of street during heavy traffic.", "sorry to hear that . my mother is recently passed ; her and pa were proud immigrants .", "cool ! where is edris island ?", "i am an old baby boomer . not too great at dancing anymore .", "dance is great exercise for a young one like yourself !", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:7202", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7202", "candidate_text": "Child watches woman playing with equipment.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7202:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7066:premise", "snli:test:7269:premise", "snli:test:7202:premise", "snli:test:7065:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["3 Olympic winners posing for a photo on an advertised background", "The tennis player spikes the tennis ball on the boundary line.", "A woman playing with play equipment while a child looks on.", "A man laying down in middle of street during heavy traffic.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7202", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7206", "candidate_text": "I wonder whether two small boys draw picturees of trees outside in the grass?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7206:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7126:premise", "snli:test:7540:premise", "snli:test:7206:premise", "snli:test:6845:premise", "style:personachat:test:267:self:7:positive:0", "style:personachat:test:267:self:7:positive:1", "style:personachat:test:267:self:7:positive:2", "style:personachat:test:267:self:7:positive:3", "style:personachat:test:267:self:7:positive:4", "style:personachat:test:267:self:7:positive:5", "style:personachat:test:267:self:7:positive:6"], "evidence_texts": ["Family sitting down in a field of grass while others walk by.", "A brown and white dog holds a tennis ball in his mouth.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "The three young children played with some toilet paper in the grass.", "i am good , just relaxing with my cross stitch project . how about you ?", "oh , cool . my father lives with me as well . he was in the military and in war .", "sorry to hear that . my mother is recently passed ; her and pa were proud immigrants .", "cool ! where is edris island ?", "i am an old baby boomer . not too great at dancing anymore .", "dance is great exercise for a young one like yourself !", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:7206", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7206", "candidate_text": "well, two small boys draw picturees of trees outside in the grass", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7206:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7126:premise", "snli:test:7540:premise", "snli:test:7206:premise", "snli:test:6845:premise", "style:personachat:test:447:self:7:positive:0", "style:personachat:test:447:self:7:positive:1", "style:personachat:test:447:self:7:positive:2", "style:personachat:test:447:self:7:positive:3", "style:personachat:test:447:self:7:positive:4", "style:personachat:test:447:self:7:positive:5", "style:personachat:test:447:self:7:positive:6"], "evidence_texts": ["Family sitting down in a field of grass while others walk by.", "A brown and white dog holds a tennis ball in his mouth.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "The three young children played with some toilet paper in the grass.", "hello ! i am fine and you ?", "i live in connecticut . you ?", "i am looking for a job as i finished with college . you ?", "my major was for business administration . do you study ?", "wow that is really a tough thing to do .", "yes i love meat , but mostly i prefer chicken .", "yes they are very healthy ."], "speaker_id": "snli:test:7206", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7211", "candidate_text": "I wonder whether the teenager is sliding down the railing?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7211:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7119:premise", "snli:test:7610:premise", "snli:test:7118:premise", "snli:test:7211:premise", "style:personachat:test:268:self:1:positive:0"], "evidence_texts": ["A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:7211", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7211", "candidate_text": "Honestly, The teenager is sliding down the railing!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7211:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7119:premise", "snli:test:7610:premise", "snli:test:7118:premise", "snli:test:7211:premise", "style:personachat:test:29:self:5:positive:0", "style:personachat:test:29:self:5:positive:1", "style:personachat:test:29:self:5:positive:2", "style:personachat:test:29:self:5:positive:3", "style:personachat:test:29:self:5:positive:4"], "evidence_texts": ["A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "that is a great industry . what kind of music do you like ?", "yes . i love to cook and i am looking for someone to build me some cooling racks .", "me too what is your favorite ? chevelle ? cadillac ? oldsmobile ?", "i prefer to cook vegan food . what do you like to eat ?", "well they are not good for you but they taste great ! do you like french fries ?"], "speaker_id": "snli:test:7211", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7219", "candidate_text": "I wonder whether a man is being carried by birds?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7219:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6827:premise", "snli:test:7482:premise", "snli:test:6221:premise", "snli:test:7219:premise", "style:personachat:test:268:self:2:positive:0", "style:personachat:test:268:self:2:positive:1"], "evidence_texts": ["Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "haha yes you do have bad ideas", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:7219", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7219", "candidate_text": "A man is being carried by birds.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7219:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6827:premise", "snli:test:7482:premise", "snli:test:6221:premise", "snli:test:7219:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "A woman in a gray sweatshirt is sitting on the floor with her small dog that is wrapped in a blanket.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7219", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7229", "candidate_text": "I wonder whether a woman adjusts her bra?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7229:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7150:premise", "snli:test:7229:premise", "snli:test:7285:premise", "snli:test:6906:premise", "style:personachat:test:268:self:5:positive:0", "style:personachat:test:268:self:5:positive:1", "style:personachat:test:268:self:5:positive:2", "style:personachat:test:268:self:5:positive:3", "style:personachat:test:268:self:5:positive:4"], "evidence_texts": ["A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "i do not have a job", "what do you mean by this ?", "lol no i will not do this . it is illegal", "haha yes you do have bad ideas", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:7229", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7229", "candidate_text": "well, a woman adjusts her bra", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7229:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7150:premise", "snli:test:7229:premise", "snli:test:7285:premise", "snli:test:6906:premise", "style:personachat:test:448:self:5:positive:0", "style:personachat:test:448:self:5:positive:1", "style:personachat:test:448:self:5:positive:2", "style:personachat:test:448:self:5:positive:3", "style:personachat:test:448:self:5:positive:4"], "evidence_texts": ["A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "Three men are playing soccer, two of them in blue uniforms and the other in a bright green uniform.", "five ? . wow . what do you do for a living ? .", "okay . i cut hair for living . what is your favorite food ? .", "nice . mine is cheese pizza . have you ever tried pho ? .", "cool . i love eating hot noodles on cold days especially during holidays like christmas .", "christmas is the best holiday for me . i am catholic ."], "speaker_id": "snli:test:7229", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7234", "candidate_text": "I wonder whether three boys hold baseball gloves?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7234:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:6389:premise", "snli:test:6309:premise", "snli:test:7234:premise", "snli:test:5058:premise", "style:personachat:test:268:self:6:positive:0", "style:personachat:test:268:self:6:positive:1", "style:personachat:test:268:self:6:positive:2", "style:personachat:test:268:self:6:positive:3", "style:personachat:test:268:self:6:positive:4", "style:personachat:test:268:self:6:positive:5"], "evidence_texts": ["Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "i am fine i am worry as next month i have to pay loan .", "i do not have a job", "what do you mean by this ?", "lol no i will not do this . it is illegal", "haha yes you do have bad ideas", "i hate this thing and i am against this . do you want to be ?"], "speaker_id": "snli:test:7234", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7234", "candidate_text": "Honestly, Three boys hold baseball gloves!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7234:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6389:premise", "snli:test:6309:premise", "snli:test:7234:premise", "snli:test:5058:premise", "style:personachat:test:32:self:2:positive:0", "style:personachat:test:32:self:2:positive:1"], "evidence_texts": ["Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A man dressed in a light blue shirt dumping items from a bin into another bin, while standing in a room full of food donations.", "i do but not now", "i do not especially becoming a certified nurse !"], "speaker_id": "snli:test:7234", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7237", "candidate_text": "Honestly, Three boys are preparing for their big game that is about to start!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7237:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6389:premise", "snli:test:7237:premise", "snli:test:6309:premise", "snli:test:7643:premise", "style:personachat:test:269:self:1:positive:0"], "evidence_texts": ["Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:7237", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7237", "candidate_text": "I wonder whether three boys are preparing for their big game that is about to start?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7237:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:6389:premise", "snli:test:7237:premise", "snli:test:6309:premise", "snli:test:7643:premise", "style:personachat:test:22:self:1:positive:0"], "evidence_texts": ["Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A large display of artifacts are in a large room and around a very large display are six people with two people near wooden benches.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:7237", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7238", "candidate_text": "Honestly, The three boys swing baseball bats!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7238:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7643:premise", "snli:test:6389:premise", "snli:test:7238:premise", "snli:test:8358:premise", "style:personachat:test:269:self:2:positive:0", "style:personachat:test:269:self:2:positive:1"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:7238", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7238", "candidate_text": "well, the three boys swing baseball bats", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7238:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7643:premise", "snli:test:6389:premise", "snli:test:7238:premise", "snli:test:8358:premise", "style:personachat:test:449:self:2:positive:0", "style:personachat:test:449:self:2:positive:1"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "Two men, one in a blue shirt and one in a green shirt analyze a piece of paper in the middle of a workshop area.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "just thinking of all of those green house gases makes me feel like i cant breathe .", "i am committed to reusing fabric in the clothes i sew myself ."], "speaker_id": "snli:test:7238", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7239", "candidate_text": "Honestly, Three boys play volleyball as others watch them!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7239:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7643:premise", "snli:test:8358:premise", "snli:test:8406:premise", "snli:test:7239:premise", "style:personachat:test:269:self:5:positive:0", "style:personachat:test:269:self:5:positive:1", "style:personachat:test:269:self:5:positive:2", "style:personachat:test:269:self:5:positive:3", "style:personachat:test:269:self:5:positive:4"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "i have two small dogs , max and marsha and a cat named ginger .", "we do , though mostly they ignore us . what do you do for work john ?", "nice ! i work with the elderly helping them care for their daily needs .", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:7239", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7239", "candidate_text": "Three boys play volleyball as others watch them.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7239:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7643:premise", "snli:test:8358:premise", "snli:test:8406:premise", "snli:test:7239:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7239", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7247", "candidate_text": "Honestly, Two people race up a structure!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7247:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7075:premise", "snli:test:7250:premise", "snli:test:7013:premise", "snli:test:7247:premise", "style:personachat:test:269:self:7:positive:0", "style:personachat:test:269:self:7:positive:1", "style:personachat:test:269:self:7:positive:2", "style:personachat:test:269:self:7:positive:3", "style:personachat:test:269:self:7:positive:4", "style:personachat:test:269:self:7:positive:5", "style:personachat:test:269:self:7:positive:6"], "evidence_texts": ["A man in summer clothing skiing on thin snow.", "Two men with tan hats working in a garden.", "A boy with no shirt is standing in water.", "Two people climbing up some sort of wore structure.", "hi john my name is marie , nice to meet you .", "oh yes i love animals . how about you ?", "i have two small dogs , max and marsha and a cat named ginger .", "we do , though mostly they ignore us . what do you do for work john ?", "nice ! i work with the elderly helping them care for their daily needs .", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:7247", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7247", "candidate_text": "I wonder whether two people race up a structure?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7247:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7075:premise", "snli:test:7250:premise", "snli:test:7013:premise", "snli:test:7247:premise", "style:personachat:test:22:self:7:positive:0", "style:personachat:test:22:self:7:positive:1", "style:personachat:test:22:self:7:positive:2", "style:personachat:test:22:self:7:positive:3", "style:personachat:test:22:self:7:positive:4", "style:personachat:test:22:self:7:positive:5", "style:personachat:test:22:self:7:positive:6"], "evidence_texts": ["A man in summer clothing skiing on thin snow.", "Two men with tan hats working in a garden.", "A boy with no shirt is standing in water.", "Two people climbing up some sort of wore structure.", "i am well an you ? like dogs .", "names ? i have 3 dogs", "awe how old are they ?", "i went here in ohio like pizza ?", "what is your favorite holiday ?", "my fav is christmas i love that day", "both i just love it , need your hair done ?"], "speaker_id": "snli:test:7247", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7250", "candidate_text": "well, two men are tending to the garden", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7250:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7247:premise", "snli:test:7395:premise", "snli:test:7250:premise", "snli:test:7075:premise", "style:personachat:test:270:self:1:positive:0"], "evidence_texts": ["Two people climbing up some sort of wore structure.", "The gentleman is speaking while the others are listening.", "Two men with tan hats working in a garden.", "A man in summer clothing skiing on thin snow.", "i myself enjoy pokey , long runs , and then wine with a bath ; d"], "speaker_id": "snli:test:7250", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7250", "candidate_text": "Honestly, Two men are tending to the garden!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7250:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7247:premise", "snli:test:7395:premise", "snli:test:7250:premise", "snli:test:7075:premise", "style:personachat:test:36:self:5:positive:0", "style:personachat:test:36:self:5:positive:1", "style:personachat:test:36:self:5:positive:2", "style:personachat:test:36:self:5:positive:3", "style:personachat:test:36:self:5:positive:4"], "evidence_texts": ["Two people climbing up some sort of wore structure.", "The gentleman is speaking while the others are listening.", "Two men with tan hats working in a garden.", "A man in summer clothing skiing on thin snow.", "i hope you feel better soon . what do you like to do for fun ?", "i love to dance . and bake i love sweets !", "how fun ! what a great place to take pictures !", "woah ! salty ! i bake every weekend , so luckily i work out regularly !", "that sounds like hard work !"], "speaker_id": "snli:test:7250", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7259", "candidate_text": "well, a woman is in the store", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7259:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7176:premise", "snli:test:7265:premise", "snli:test:7059:premise", "snli:test:7259:premise", "style:personachat:test:270:self:2:positive:0", "style:personachat:test:270:self:2:positive:1"], "evidence_texts": ["A woman and little girl look into a display case.", "Two green painted women perform for a crowd of onlookers.", "A boy taking a shot while his team watches anxiously.", "Woman standing in a store with an item in hand.", "he is a popular musician , haha ! what hobbies do you have ?", "i myself enjoy pokey , long runs , and then wine with a bath ; d"], "speaker_id": "snli:test:7259", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7259", "candidate_text": "A woman is in the store.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7259:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7176:premise", "snli:test:7265:premise", "snli:test:7059:premise", "snli:test:7259:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman and little girl look into a display case.", "Two green painted women perform for a crowd of onlookers.", "A boy taking a shot while his team watches anxiously.", "Woman standing in a store with an item in hand.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7259", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7261", "candidate_text": "well, the woman is very good at hula hooping", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7261:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6965:premise", "snli:test:7261:premise", "snli:test:7349:premise", "snli:test:6962:premise", "style:personachat:test:270:self:5:positive:0", "style:personachat:test:270:self:5:positive:1", "style:personachat:test:270:self:5:positive:2", "style:personachat:test:270:self:5:positive:3", "style:personachat:test:270:self:5:positive:4"], "evidence_texts": ["A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "freelance work for now . practicing jujitsu to maybe teach one day !", "i am just starting out so it can be . strengthening them is important , though", "i am 23 years old , and listen to ed sheeran religiously", "he is a popular musician , haha ! what hobbies do you have ?", "i myself enjoy pokey , long runs , and then wine with a bath ; d"], "speaker_id": "snli:test:7261", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7261", "candidate_text": "I wonder whether the woman is very good at hula hooping?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7261:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6965:premise", "snli:test:7261:premise", "snli:test:7349:premise", "snli:test:6962:premise", "style:personachat:test:26:self:5:positive:0", "style:personachat:test:26:self:5:positive:1", "style:personachat:test:26:self:5:positive:2", "style:personachat:test:26:self:5:positive:3", "style:personachat:test:26:self:5:positive:4"], "evidence_texts": ["A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "A man is resting in a small stream with a hat over his head while the little waterfall is pouring in the background.", "folk music , its kind of like storytelling . people dig it , you listen to music ?", "i am average height , i am the shortest of my 3 sisters . you ?", "do you have any siblings ?", "do you work ? or your parents work ?", "college ? studying anything in particular ?"], "speaker_id": "snli:test:7261", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7265", "candidate_text": "well, people are nearby", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7265:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7259:premise", "snli:test:7265:premise", "snli:test:7176:premise", "snli:test:7281:premise", "style:personachat:test:270:self:7:positive:0", "style:personachat:test:270:self:7:positive:1", "style:personachat:test:270:self:7:positive:2", "style:personachat:test:270:self:7:positive:3", "style:personachat:test:270:self:7:positive:4", "style:personachat:test:270:self:7:positive:5", "style:personachat:test:270:self:7:positive:6"], "evidence_texts": ["Woman standing in a store with an item in hand.", "Two green painted women perform for a crowd of onlookers.", "A woman and little girl look into a display case.", "A drunk looking woman is hugging a drunk looking man.", "i am actually quite comfortable with my living situation right now !", "i absolutely will , and are you asking about hobbies or my career ?", "freelance work for now . practicing jujitsu to maybe teach one day !", "i am just starting out so it can be . strengthening them is important , though", "i am 23 years old , and listen to ed sheeran religiously", "he is a popular musician , haha ! what hobbies do you have ?", "i myself enjoy pokey , long runs , and then wine with a bath ; d"], "speaker_id": "snli:test:7265", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7265", "candidate_text": "Honestly, People are nearby!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7265:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7259:premise", "snli:test:7265:premise", "snli:test:7176:premise", "snli:test:7281:premise", "style:personachat:test:45:self:2:positive:0", "style:personachat:test:45:self:2:positive:1"], "evidence_texts": ["Woman standing in a store with an item in hand.", "Two green painted women perform for a crowd of onlookers.", "A woman and little girl look into a display case.", "A drunk looking woman is hugging a drunk looking man.", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:7265", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7266", "candidate_text": "I wonder whether irises bloom paisley?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7266:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7281:premise", "snli:test:7259:premise", "snli:test:7266:premise", "snli:test:7340:premise", "style:personachat:test:271:self:1:positive:0"], "evidence_texts": ["A drunk looking woman is hugging a drunk looking man.", "Woman standing in a store with an item in hand.", "Two green painted women perform for a crowd of onlookers.", "A young blond boy is eating cake with a spoon.", "yeah its helped me become a specialist in the army . what law do you study ?"], "speaker_id": "snli:test:7266", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7266", "candidate_text": "Honestly, Irises bloom paisley!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7266:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7281:premise", "snli:test:7259:premise", "snli:test:7266:premise", "snli:test:7340:premise", "style:personachat:test:45:self:5:positive:0", "style:personachat:test:45:self:5:positive:1", "style:personachat:test:45:self:5:positive:2", "style:personachat:test:45:self:5:positive:3", "style:personachat:test:45:self:5:positive:4"], "evidence_texts": ["A drunk looking woman is hugging a drunk looking man.", "Woman standing in a store with an item in hand.", "Two green painted women perform for a crowd of onlookers.", "A young blond boy is eating cake with a spoon.", "kindergarten would be fun ! do you play any sports", "i played soccer in college ! we won a cup one season", "that sounds like fun ! i just almost choked on a piece of candy", "i have never met anyone who knew that . do you like candy", "i like all candy ! those are pretty good . sour gummy worms are my favorite"], "speaker_id": "snli:test:7266", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7269", "candidate_text": "I wonder whether the tennis player is winning the game?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7269:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7202:premise", "snli:test:7269:premise", "snli:test:7363:premise", "snli:test:7066:premise", "style:personachat:test:271:self:2:positive:0", "style:personachat:test:271:self:2:positive:1"], "evidence_texts": ["A woman playing with play equipment while a child looks on.", "The tennis player spikes the tennis ball on the boundary line.", "A women's volleyball team plays in a brown and green gymnasium.", "3 Olympic winners posing for a photo on an advertised background", "it is unique . i can see camouflaged movement better than other people can though !", "yeah its helped me become a specialist in the army . what law do you study ?"], "speaker_id": "snli:test:7269", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7269", "candidate_text": "The tennis player is winning the game.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7269:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7202:premise", "snli:test:7269:premise", "snli:test:7363:premise", "snli:test:7066:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman playing with play equipment while a child looks on.", "The tennis player spikes the tennis ball on the boundary line.", "A women's volleyball team plays in a brown and green gymnasium.", "3 Olympic winners posing for a photo on an advertised background", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7269", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7276", "candidate_text": "I wonder whether two girls kneel together on the ground?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7276:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7276:premise", "snli:test:6673:premise", "snli:test:6520:premise", "snli:test:7321:premise", "style:personachat:test:271:self:5:positive:0", "style:personachat:test:271:self:5:positive:1", "style:personachat:test:271:self:5:positive:2", "style:personachat:test:271:self:5:positive:3", "style:personachat:test:271:self:5:positive:4"], "evidence_texts": ["Two girls are kneeling on the ground.", "Two butchers are walking through their shop.", "A man and a young girl swimming.", "Four young boys flexing for the camera.", "that is a good approach . what do you do for fun ? i love karaoke !", "hiking is good for you ! its tough for me though cause i am colorblind .", "yeah everything . i just see black and white . it has benefits though .", "it is unique . i can see camouflaged movement better than other people can though !", "yeah its helped me become a specialist in the army . what law do you study ?"], "speaker_id": "snli:test:7276", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7276", "candidate_text": "well, two girls kneel together on the ground", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7276:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7276:premise", "snli:test:6673:premise", "snli:test:6520:premise", "snli:test:7321:premise", "style:personachat:test:454:self:5:positive:0", "style:personachat:test:454:self:5:positive:1", "style:personachat:test:454:self:5:positive:2", "style:personachat:test:454:self:5:positive:3", "style:personachat:test:454:self:5:positive:4"], "evidence_texts": ["Two girls are kneeling on the ground.", "Two butchers are walking through their shop.", "A man and a young girl swimming.", "Four young boys flexing for the camera.", "it happened when my father and brother went skiing . i would rather eat sushi", "i make vegan pizza and sushi among other vegan dishes .", "i like xbox workout programs and youtube . hardly watch tv anymore", "although i do watch local news streaming via facebook .", "yes so many privacy issues with internet and facebook . i might do that too"], "speaker_id": "snli:test:7276", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7277", "candidate_text": "I wonder whether two girls kneel on the ground looking for a ring one had dropped?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7277:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7321:premise", "snli:test:7277:premise", "snli:test:6673:premise", "snli:test:7368:premise", "style:personachat:test:271:self:7:positive:0", "style:personachat:test:271:self:7:positive:1", "style:personachat:test:271:self:7:positive:2", "style:personachat:test:271:self:7:positive:3", "style:personachat:test:271:self:7:positive:4", "style:personachat:test:271:self:7:positive:5", "style:personachat:test:271:self:7:positive:6"], "evidence_texts": ["Four young boys flexing for the camera.", "Two girls are kneeling on the ground.", "Two butchers are walking through their shop.", "A young lady toying with her watch.", "hi josh , i am frank . i am on leave from the army .", "no siblings , my mother was not in my life . what do you like to do ?", "that is a good approach . what do you do for fun ? i love karaoke !", "hiking is good for you ! its tough for me though cause i am colorblind .", "yeah everything . i just see black and white . it has benefits though .", "it is unique . i can see camouflaged movement better than other people can though !", "yeah its helped me become a specialist in the army . what law do you study ?"], "speaker_id": "snli:test:7277", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7277", "candidate_text": "Honestly, Two girls kneel on the ground looking for a ring one had dropped!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7277:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7321:premise", "snli:test:7277:premise", "snli:test:6673:premise", "snli:test:7368:premise", "style:personachat:test:50:self:2:positive:0", "style:personachat:test:50:self:2:positive:1"], "evidence_texts": ["Four young boys flexing for the camera.", "Two girls are kneeling on the ground.", "Two butchers are walking through their shop.", "A young lady toying with her watch.", "do you like horror flicks ? those are my favorite !", "chuckie is a creepy little guy . i love mikey myers and the brute force !"], "speaker_id": "snli:test:7277", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7281", "candidate_text": "I wonder whether a drunk looking woman in a yankees shirt is hugging a drunk looking man in a red sox shirt?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7281:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7266:premise", "snli:test:7281:premise", "snli:test:7340:premise", "snli:test:7265:premise", "style:personachat:test:272:self:1:positive:0"], "evidence_texts": ["Two green painted women perform for a crowd of onlookers.", "A drunk looking woman is hugging a drunk looking man.", "A young blond boy is eating cake with a spoon.", "Two green painted women perform for a crowd of onlookers.", "that is very nice , what is your favorite meal ?"], "speaker_id": "snli:test:7281", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7281", "candidate_text": "A drunk looking woman in a Yankees shirt is hugging a drunk looking man in a Red Sox shirt.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7281:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7266:premise", "snli:test:7281:premise", "snli:test:7340:premise", "snli:test:7265:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two green painted women perform for a crowd of onlookers.", "A drunk looking woman is hugging a drunk looking man.", "A young blond boy is eating cake with a spoon.", "Two green painted women perform for a crowd of onlookers.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7281", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7282", "candidate_text": "I wonder whether the people are from the same family?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7282:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6657:premise", "snli:test:7488:premise", "snli:test:7282:premise", "snli:test:6655:premise", "style:personachat:test:272:self:2:positive:0", "style:personachat:test:272:self:2:positive:1"], "evidence_texts": ["Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "i do a lot of hiking my my best friend from kindergarten , you ?", "that is very nice , what is your favorite meal ?"], "speaker_id": "snli:test:7282", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7282", "candidate_text": "well, the people are from the same family", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7282:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6657:premise", "snli:test:7488:premise", "snli:test:7282:premise", "snli:test:6655:premise", "style:personachat:test:455:self:2:positive:0", "style:personachat:test:455:self:2:positive:1"], "evidence_texts": ["Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "is that anywhere near caldwell ?", "i have a friend who lives there ."], "speaker_id": "snli:test:7282", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7285", "candidate_text": "I wonder whether a man tries to tackle a player in a nfl game?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7285:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7229:premise", "snli:test:7285:premise", "snli:test:7315:premise", "snli:test:7150:premise", "style:personachat:test:272:self:5:positive:0", "style:personachat:test:272:self:5:positive:1", "style:personachat:test:272:self:5:positive:2", "style:personachat:test:272:self:5:positive:3", "style:personachat:test:272:self:5:positive:4"], "evidence_texts": ["A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "have you ever taught in north dakota ? i am from there", "i hear they have a lot of jewish people like myself there", "i guess that is the case in most states", "i do a lot of hiking my my best friend from kindergarten , you ?", "that is very nice , what is your favorite meal ?"], "speaker_id": "snli:test:7285", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7285", "candidate_text": "Honestly, A man tries to tackle a player in a NFL game!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7285:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7229:premise", "snli:test:7285:premise", "snli:test:7315:premise", "snli:test:7150:premise", "style:personachat:test:69:self:1:positive:0"], "evidence_texts": ["A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A man wearing a blue jacket fishing in a river having a conversation with his friend with no shirt.", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:7285", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7288", "candidate_text": "I wonder whether the boy and girl stand on the lakeshore?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7288:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7288:premise", "snli:test:7140:premise", "snli:test:7303:premise", "snli:test:7092:premise", "style:personachat:test:272:self:7:positive:0", "style:personachat:test:272:self:7:positive:1", "style:personachat:test:272:self:7:positive:2", "style:personachat:test:272:self:7:positive:3", "style:personachat:test:272:self:7:positive:4", "style:personachat:test:272:self:7:positive:5", "style:personachat:test:272:self:7:positive:6"], "evidence_texts": ["A boy and a girl stand in a median amongst Mike Huckabee signs.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A person holds a bunch of brush while walking down a stone path.", "A woman in a yellow shirt is carrying two full red grocery bags.", "very good , just remembering when i traveled abroad for studies", "i work in advertising , i started this year , so still getting my feet wet , you ?", "have you ever taught in north dakota ? i am from there", "i hear they have a lot of jewish people like myself there", "i guess that is the case in most states", "i do a lot of hiking my my best friend from kindergarten , you ?", "that is very nice , what is your favorite meal ?"], "speaker_id": "snli:test:7288", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7288", "candidate_text": "The boy and girl stand on the lakeshore.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7288:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7288:premise", "snli:test:7140:premise", "snli:test:7303:premise", "snli:test:7092:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A boy and a girl stand in a median amongst Mike Huckabee signs.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A person holds a bunch of brush while walking down a stone path.", "A woman in a yellow shirt is carrying two full red grocery bags.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7288", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7291", "candidate_text": "well, a sad man is hanging sausages in a market", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7291:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7199:premise", "snli:test:7415:premise", "snli:test:7124:premise", "snli:test:7291:premise", "style:personachat:test:273:self:1:positive:0"], "evidence_texts": ["City street crowded with sports fans wearing orange.", "Rose petals being thrown at an asian couple.", "A number of cute asian children playing tambourines", "A man is hanging sausages in a market.", "i aspire to be a chef someday ."], "speaker_id": "snli:test:7291", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7291", "candidate_text": "Honestly, A sad man is hanging sausages in a market!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7291:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7199:premise", "snli:test:7415:premise", "snli:test:7124:premise", "snli:test:7291:premise", "style:personachat:test:69:self:5:positive:0", "style:personachat:test:69:self:5:positive:1", "style:personachat:test:69:self:5:positive:2", "style:personachat:test:69:self:5:positive:3", "style:personachat:test:69:self:5:positive:4"], "evidence_texts": ["City street crowded with sports fans wearing orange.", "Rose petals being thrown at an asian couple.", "A number of cute asian children playing tambourines", "A man is hanging sausages in a market.", "i want to visit italy one day .", "do you cook italian food ?", "i understand , i am from the middle east .", "no , i have never been there . i was born in pennsylvania , usa .", "i do , i want to travel the world ! i know it sounds cliche ."], "speaker_id": "snli:test:7291", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7303", "candidate_text": "well, the person is doing a job for someone", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7303:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7288:premise", "snli:test:7140:premise", "snli:test:7334:premise", "snli:test:7303:premise", "style:personachat:test:273:self:2:positive:0", "style:personachat:test:273:self:2:positive:1"], "evidence_texts": ["A boy and a girl stand in a median amongst Mike Huckabee signs.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "romance ones are my favorite too !", "i aspire to be a chef someday ."], "speaker_id": "snli:test:7303", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7303", "candidate_text": "The person is doing a job for someone.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7303:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7288:premise", "snli:test:7140:premise", "snli:test:7334:premise", "snli:test:7303:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A boy and a girl stand in a median amongst Mike Huckabee signs.", "A little girl wearing a yellow dress is sitting on the white rocks.", "A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7303", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7305", "candidate_text": "well, the person is walking outside", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7305:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7334:premise", "snli:test:7305:premise", "snli:test:7288:premise", "snli:test:7354:premise", "style:personachat:test:273:self:5:positive:0", "style:personachat:test:273:self:5:positive:1", "style:personachat:test:273:self:5:positive:2", "style:personachat:test:273:self:5:positive:3", "style:personachat:test:273:self:5:positive:4"], "evidence_texts": ["A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "A boy and a girl stand in a median amongst Mike Huckabee signs.", "An adorable child leaping to block a soccer ball from entering the goal.", "yeah dogs scare me . i prefer horses too", "hope youre okay . i just turned 32 and am still single . that hurts too haha", "to each their own . i love to read . do you ?", "romance ones are my favorite too !", "i aspire to be a chef someday ."], "speaker_id": "snli:test:7305", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7305", "candidate_text": "I wonder whether the person is walking outside?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7305:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7334:premise", "snli:test:7305:premise", "snli:test:7288:premise", "snli:test:7354:premise", "style:personachat:test:37:self:5:positive:0", "style:personachat:test:37:self:5:positive:1", "style:personachat:test:37:self:5:positive:2", "style:personachat:test:37:self:5:positive:3", "style:personachat:test:37:self:5:positive:4"], "evidence_texts": ["A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "A boy and a girl stand in a median amongst Mike Huckabee signs.", "An adorable child leaping to block a soccer ball from entering the goal.", "that is interesting . i am also a vegan and a vet . i love animals .", "a sister in california , where we grew up . i am in florida now . you ?", "yes ! i rap on the weekends myself too . my friends like country , boo .", "i did not even know gucci was getting married . who to ?", "do they have any kids together ?"], "speaker_id": "snli:test:7305", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7315", "candidate_text": "well, a member of the army walks his dog", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7315:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7285:premise", "snli:test:7229:premise", "snli:test:7493:premise", "snli:test:7315:premise", "style:personachat:test:273:self:7:positive:0", "style:personachat:test:273:self:7:positive:1", "style:personachat:test:273:self:7:positive:2", "style:personachat:test:273:self:7:positive:3", "style:personachat:test:273:self:7:positive:4", "style:personachat:test:273:self:7:positive:5", "style:personachat:test:273:self:7:positive:6"], "evidence_texts": ["A man from one football team trying to dodge a tackle from a player from the opposing football team.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "hi . not too bad . how about you", "glad to hear it . do you like dogs ?", "yeah dogs scare me . i prefer horses too", "hope youre okay . i just turned 32 and am still single . that hurts too haha", "to each their own . i love to read . do you ?", "romance ones are my favorite too !", "i aspire to be a chef someday ."], "speaker_id": "snli:test:7315", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7315", "candidate_text": "Honestly, A member of the army walks his dog!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7315:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7285:premise", "snli:test:7229:premise", "snli:test:7493:premise", "snli:test:7315:premise", "style:personachat:test:70:self:2:positive:0", "style:personachat:test:70:self:2:positive:1"], "evidence_texts": ["A man from one football team trying to dodge a tackle from a player from the opposing football team.", "A man with a pinstriped shirt and tan slacks adjust his tie while looking at a apple laptop computer.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "i think he is great , but most of the other teachers where i work disagree .", "you might try reading . that is one of my favorite things !"], "speaker_id": "snli:test:7315", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7316", "candidate_text": "well, a member of the army walks a german shepard", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7316:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7316:premise", "snli:test:7493:premise", "snli:test:7285:premise", "snli:test:7543:premise", "style:personachat:test:274:self:1:positive:0"], "evidence_texts": ["A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "i am going to start auburn university in august ."], "speaker_id": "snli:test:7316", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7316", "candidate_text": "A member of the army walks a German Shepard.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7316:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7316:premise", "snli:test:7493:premise", "snli:test:7285:premise", "snli:test:7543:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "A man from one football team trying to dodge a tackle from a player from the opposing football team.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7316", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7321", "candidate_text": "well, for boys are flexing", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7321:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7277:premise", "snli:test:7321:premise", "snli:test:7368:premise", "snli:test:7276:premise", "style:personachat:test:274:self:2:positive:0", "style:personachat:test:274:self:2:positive:1"], "evidence_texts": ["Two girls are kneeling on the ground.", "Four young boys flexing for the camera.", "A young lady toying with her watch.", "Two girls are kneeling on the ground.", "that is cool . i love listening to music in my volkswagen beetle .", "i am going to start auburn university in august ."], "speaker_id": "snli:test:7321", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7321", "candidate_text": "I wonder whether for boys are flexing?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7321:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7277:premise", "snli:test:7321:premise", "snli:test:7368:premise", "snli:test:7276:premise", "style:personachat:test:39:self:2:positive:0", "style:personachat:test:39:self:2:positive:1"], "evidence_texts": ["Two girls are kneeling on the ground.", "Four young boys flexing for the camera.", "A young lady toying with her watch.", "Two girls are kneeling on the ground.", "i like pizza too . the sauce is the best part .", "is that like a alfredo sauce ?"], "speaker_id": "snli:test:7321", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7332", "candidate_text": "well, a boy and his father discuss christmas with santa claus", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7332:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7332:premise", "snli:test:6919:premise", "snli:test:7619:premise", "snli:test:6612:premise", "style:personachat:test:274:self:5:positive:0", "style:personachat:test:274:self:5:positive:1", "style:personachat:test:274:self:5:positive:2", "style:personachat:test:274:self:5:positive:3", "style:personachat:test:274:self:5:positive:4"], "evidence_texts": ["A little boy and his father talking to a man dressed as Santa Claws.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "A man is working in an art studio and is surrounded by art supplies.", "A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "i do not really do that kind of stuff . i am only 17 .", "no i do not . i am going to graduate high school in june .", "i do not know who he is ?", "that is cool . i love listening to music in my volkswagen beetle .", "i am going to start auburn university in august ."], "speaker_id": "snli:test:7332", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7332", "candidate_text": "Honestly, A boy and his father discuss Christmas with Santa Claus!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7332:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7332:premise", "snli:test:6919:premise", "snli:test:7619:premise", "snli:test:6612:premise", "style:personachat:test:72:self:1:positive:0"], "evidence_texts": ["A little boy and his father talking to a man dressed as Santa Claws.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "A man is working in an art studio and is surrounded by art supplies.", "A man wearing noise-canceling headphones and a breathing mask works on a white sculpture.", "i love my kids , but i love reading as well !"], "speaker_id": "snli:test:7332", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7334", "candidate_text": "well, a man is getting gas for his car while he talks on his cellphone", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7334:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7305:premise", "snli:test:7354:premise", "snli:test:7334:premise", "snli:test:7303:premise", "style:personachat:test:274:self:7:positive:0", "style:personachat:test:274:self:7:positive:1", "style:personachat:test:274:self:7:positive:2", "style:personachat:test:274:self:7:positive:3", "style:personachat:test:274:self:7:positive:4", "style:personachat:test:274:self:7:positive:5", "style:personachat:test:274:self:7:positive:6"], "evidence_texts": ["A person holds a bunch of brush while walking down a stone path.", "An adorable child leaping to block a soccer ball from entering the goal.", "A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "that is good . what is mills cooking ?", "i am not really into cooking . i like fashion ! i love to clothes and shopping !", "i do not really do that kind of stuff . i am only 17 .", "no i do not . i am going to graduate high school in june .", "i do not know who he is ?", "that is cool . i love listening to music in my volkswagen beetle .", "i am going to start auburn university in august ."], "speaker_id": "snli:test:7334", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7334", "candidate_text": "A man is getting gas for his car while he talks on his cellphone.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7334:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7305:premise", "snli:test:7354:premise", "snli:test:7334:premise", "snli:test:7303:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A person holds a bunch of brush while walking down a stone path.", "An adorable child leaping to block a soccer ball from entering the goal.", "A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7334", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7340", "candidate_text": "Honestly, a boy is eating cake!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7340:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7340:premise", "snli:test:7281:premise", "snli:test:7266:premise", "snli:test:7499:premise", "style:personachat:test:275:self:1:positive:0"], "evidence_texts": ["A young blond boy is eating cake with a spoon.", "A drunk looking woman is hugging a drunk looking man.", "Two green painted women perform for a crowd of onlookers.", "A guy in a red jacket is snowboarding in midair.", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:7340", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7340", "candidate_text": "I wonder whether a boy is eating cake?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7340:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7340:premise", "snli:test:7281:premise", "snli:test:7266:premise", "snli:test:7499:premise", "style:personachat:test:40:self:1:positive:0"], "evidence_texts": ["A young blond boy is eating cake with a spoon.", "A drunk looking woman is hugging a drunk looking man.", "Two green painted women perform for a crowd of onlookers.", "A guy in a red jacket is snowboarding in midair.", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:7340", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7341", "candidate_text": "Honestly, a girl is eating cake!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7341:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7499:premise", "snli:test:7341:premise", "snli:test:7281:premise", "snli:test:7579:premise", "style:personachat:test:275:self:2:positive:0", "style:personachat:test:275:self:2:positive:1"], "evidence_texts": ["A guy in a red jacket is snowboarding in midair.", "A young blond boy is eating cake with a spoon.", "A drunk looking woman is hugging a drunk looking man.", "A black and white dog jumps for a tennis ball.", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:7341", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7341", "candidate_text": "well, a girl is eating cake", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7341:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7499:premise", "snli:test:7341:premise", "snli:test:7281:premise", "snli:test:7579:premise", "style:personachat:test:459:self:2:positive:0", "style:personachat:test:459:self:2:positive:1"], "evidence_texts": ["A guy in a red jacket is snowboarding in midair.", "A young blond boy is eating cake with a spoon.", "A drunk looking woman is hugging a drunk looking man.", "A black and white dog jumps for a tennis ball.", "wow , that is a lot of sisters !", "i do like pink floyd . i like all sorts of music except rap ."], "speaker_id": "snli:test:7341", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7349", "candidate_text": "Honestly, Books are strewn on a table as women look through them!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7349:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7261:premise", "snli:test:7349:premise", "snli:test:7684:premise", "snli:test:6965:premise", "style:personachat:test:275:self:5:positive:0", "style:personachat:test:275:self:5:positive:1", "style:personachat:test:275:self:5:positive:2", "style:personachat:test:275:self:5:positive:3", "style:personachat:test:275:self:5:positive:4"], "evidence_texts": ["A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "my wife and 3 boys would skin me alive !", "i am so busy doing carpentry that i do not meet many people as friends .", "wow ! get back here , charlie ! are you vegetarian ?", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:7349", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7349", "candidate_text": "Books are strewn on a table as women look through them.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7349:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7261:premise", "snli:test:7349:premise", "snli:test:7684:premise", "snli:test:6965:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "A female softball player in a black and red uniform with a red batting helmet is holding her bat waiting for the pitch.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7349", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7354", "candidate_text": "Honestly, A child blocks a goal!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7354:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7334:premise", "snli:test:7305:premise", "snli:test:7354:premise", "snli:test:7406:premise", "style:personachat:test:275:self:7:positive:0", "style:personachat:test:275:self:7:positive:1", "style:personachat:test:275:self:7:positive:2", "style:personachat:test:275:self:7:positive:3", "style:personachat:test:275:self:7:positive:4", "style:personachat:test:275:self:7:positive:5", "style:personachat:test:275:self:7:positive:6"], "evidence_texts": ["A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "An adorable child leaping to block a soccer ball from entering the goal.", "There is a green trash truck in road with a person sweeping sidewalk.", "hi there ! want to play some poker ? i love it !", "we cant drink alcohol while playing , though . i quit 5 years back .", "my wife and 3 boys would skin me alive !", "i am so busy doing carpentry that i do not meet many people as friends .", "wow ! get back here , charlie ! are you vegetarian ?", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:7354", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7354", "candidate_text": "I wonder whether a child blocks a goal?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7354:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7334:premise", "snli:test:7305:premise", "snli:test:7354:premise", "snli:test:7406:premise", "style:personachat:test:40:self:7:positive:0", "style:personachat:test:40:self:7:positive:1", "style:personachat:test:40:self:7:positive:2", "style:personachat:test:40:self:7:positive:3", "style:personachat:test:40:self:7:positive:4", "style:personachat:test:40:self:7:positive:5", "style:personachat:test:40:self:7:positive:6"], "evidence_texts": ["A man is talking on a cellphone while filling his car with gas.", "A person holds a bunch of brush while walking down a stone path.", "An adorable child leaping to block a soccer ball from entering the goal.", "There is a green trash truck in road with a person sweeping sidewalk.", "hey , not bad . hunting , that is cool . do you hunt for sport or meat ?", "awesome . i love meat . i eat a high protein meat based diet .", "not me , but maybe for my brother . are you a realtor ?", "sweet . i do not work for a company , i do bodybuilding for a living .", "indeed ! so do you have any family ?", "i see ! so besides my brother i have my mom . she is a secretary .", "yes , not married and no kids , so my brother and mom are all i have . you ?"], "speaker_id": "snli:test:7354", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7355", "candidate_text": "well, a child blocks a goal with his hockey stick", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7355:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7406:premise", "snli:test:7334:premise", "snli:test:7552:premise", "snli:test:7355:premise", "style:personachat:test:276:self:1:positive:0"], "evidence_texts": ["There is a green trash truck in road with a person sweeping sidewalk.", "A man is talking on a cellphone while filling his car with gas.", "A black and brown dog is playing with a brown and white dog.", "An adorable child leaping to block a soccer ball from entering the goal.", "yes i love watching football on sundays"], "speaker_id": "snli:test:7355", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7355", "candidate_text": "Honestly, A child blocks a goal with his hockey stick!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7355:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7406:premise", "snli:test:7334:premise", "snli:test:7552:premise", "snli:test:7355:premise", "style:personachat:test:74:self:5:positive:0", "style:personachat:test:74:self:5:positive:1", "style:personachat:test:74:self:5:positive:2", "style:personachat:test:74:self:5:positive:3", "style:personachat:test:74:self:5:positive:4"], "evidence_texts": ["There is a green trash truck in road with a person sweeping sidewalk.", "A man is talking on a cellphone while filling his car with gas.", "A black and brown dog is playing with a brown and white dog.", "An adorable child leaping to block a soccer ball from entering the goal.", "great ! what is your favorite animal ? i love riding horses on my dads farm", "awe , i love turtles too . any hobbies ?", "cool . my favorite hobby is watching horror films", "no , can not say that i have . you ?", "wow ! and here i am . my mom is just a teacher in south carolina . xd"], "speaker_id": "snli:test:7355", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7363", "candidate_text": "well, a volleyball team is performing in a national competition", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7363:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7269:premise", "snli:test:7478:premise", "snli:test:7202:premise", "snli:test:7363:premise", "style:personachat:test:276:self:2:positive:0", "style:personachat:test:276:self:2:positive:1"], "evidence_texts": ["The tennis player spikes the tennis ball on the boundary line.", "A man in renaissance clothing is sitting on a brick wall.", "A woman playing with play equipment while a child looks on.", "A women's volleyball team plays in a brown and green gymnasium.", "i do not i am bored", "yes i love watching football on sundays"], "speaker_id": "snli:test:7363", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7363", "candidate_text": "A volleyball team is performing in a national competition.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7363:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7269:premise", "snli:test:7478:premise", "snli:test:7202:premise", "snli:test:7363:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The tennis player spikes the tennis ball on the boundary line.", "A man in renaissance clothing is sitting on a brick wall.", "A woman playing with play equipment while a child looks on.", "A women's volleyball team plays in a brown and green gymnasium.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7363", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7368", "candidate_text": "well, a woman is fidgeting with her wallet", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7368:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7368:premise", "snli:test:7321:premise", "snli:test:7374:premise", "snli:test:7277:premise", "style:personachat:test:276:self:5:positive:0", "style:personachat:test:276:self:5:positive:1", "style:personachat:test:276:self:5:positive:2", "style:personachat:test:276:self:5:positive:3", "style:personachat:test:276:self:5:positive:4"], "evidence_texts": ["A young lady toying with her watch.", "Four young boys flexing for the camera.", "Two girls looking into camera and smiling", "Two girls are kneeling on the ground.", "yeah tell me about it . i own 3 dogs", "which one were you planning on attending ?", "ok well let me know", "i do not i am bored", "yes i love watching football on sundays"], "speaker_id": "snli:test:7368", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7368", "candidate_text": "I wonder whether a woman is fidgeting with her wallet?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7368:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7368:premise", "snli:test:7321:premise", "snli:test:7374:premise", "snli:test:7277:premise", "style:personachat:test:42:self:5:positive:0", "style:personachat:test:42:self:5:positive:1", "style:personachat:test:42:self:5:positive:2", "style:personachat:test:42:self:5:positive:3", "style:personachat:test:42:self:5:positive:4"], "evidence_texts": ["A young lady toying with her watch.", "Four young boys flexing for the camera.", "Two girls looking into camera and smiling", "Two girls are kneeling on the ground.", "i have an aquarium . i keep exotic fish ! and i collect also .", "i do not get a chance . i kill time with my vintage collection . from 1950s", "decorations mainly . that sounds very exciting . alone or with family ?", "the nemo looking one . i stare at him when i eat my granola every morning", "where do you go to college ?"], "speaker_id": "snli:test:7368", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7374", "candidate_text": "well, two girls grin into the camera", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7374:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7368:premise", "snli:test:7420:premise", "snli:test:7374:premise", "snli:test:7321:premise", "style:personachat:test:276:self:7:positive:0", "style:personachat:test:276:self:7:positive:1", "style:personachat:test:276:self:7:positive:2", "style:personachat:test:276:self:7:positive:3", "style:personachat:test:276:self:7:positive:4", "style:personachat:test:276:self:7:positive:5", "style:personachat:test:276:self:7:positive:6"], "evidence_texts": ["A young lady toying with her watch.", "A man and a woman are laughing.", "Two girls looking into camera and smiling", "Four young boys flexing for the camera.", "hi . are you having a good day ?", "do you have any pets ?", "yeah tell me about it . i own 3 dogs", "which one were you planning on attending ?", "ok well let me know", "i do not i am bored", "yes i love watching football on sundays"], "speaker_id": "snli:test:7374", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7374", "candidate_text": "Honestly, Two girls grin into the camera!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7374:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7368:premise", "snli:test:7420:premise", "snli:test:7374:premise", "snli:test:7321:premise", "style:personachat:test:84:self:2:positive:0", "style:personachat:test:84:self:2:positive:1"], "evidence_texts": ["A young lady toying with her watch.", "A man and a woman are laughing.", "Two girls looking into camera and smiling", "Four young boys flexing for the camera.", "do you sell cars too", "why ? are you broke ? i like having dinner at home every day !"], "speaker_id": "snli:test:7374", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7395", "candidate_text": "well, the man is not talking", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7395:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7250:premise", "snli:test:7675:premise", "snli:test:7247:premise", "snli:test:7395:premise", "style:personachat:test:277:self:1:positive:0"], "evidence_texts": ["Two men with tan hats working in a garden.", "A man and women sleeping outside on a bench.", "Two people climbing up some sort of wore structure.", "The gentleman is speaking while the others are listening.", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:7395", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7395", "candidate_text": "The man is not talking.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7395:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7250:premise", "snli:test:7675:premise", "snli:test:7247:premise", "snli:test:7395:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two men with tan hats working in a garden.", "A man and women sleeping outside on a bench.", "Two people climbing up some sort of wore structure.", "The gentleman is speaking while the others are listening.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7395", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7406", "candidate_text": "well, a woman mops", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7406:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7355:premise", "snli:test:7552:premise", "snli:test:7406:premise", "snli:test:7354:premise", "style:personachat:test:277:self:2:positive:0", "style:personachat:test:277:self:2:positive:1"], "evidence_texts": ["An adorable child leaping to block a soccer ball from entering the goal.", "A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "An adorable child leaping to block a soccer ball from entering the goal.", "i love dogs as well , and otters . they are like water dogs .", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:7406", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7406", "candidate_text": "I wonder whether a woman mops?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7406:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7355:premise", "snli:test:7552:premise", "snli:test:7406:premise", "snli:test:7354:premise", "style:personachat:test:43:self:2:positive:0", "style:personachat:test:43:self:2:positive:1"], "evidence_texts": ["An adorable child leaping to block a soccer ball from entering the goal.", "A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "An adorable child leaping to block a soccer ball from entering the goal.", "great , how about colors ? my favorite is blue", "great , anything else you can tell me about yourself ?"], "speaker_id": "snli:test:7406", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7409", "candidate_text": "well, gandalf and the red toy crab do battle in the streets", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7409:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:3481:premise", "snli:test:7409:premise", "snli:test:9099:premise", "snli:test:4435:premise", "style:personachat:test:277:self:5:positive:0", "style:personachat:test:277:self:5:positive:1", "style:personachat:test:277:self:5:positive:2", "style:personachat:test:277:self:5:positive:3", "style:personachat:test:277:self:5:positive:4"], "evidence_texts": ["A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "i am disabled , and not able to work . it makes me feel good to give back .", "i like to listen to music .", "i am vegan , and i love to bake cakes .", "i love dogs as well , and otters . they are like water dogs .", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:7409", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7409", "candidate_text": "Honestly, Gandalf and the red toy crab do battle in the streets!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7409:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:3481:premise", "snli:test:7409:premise", "snli:test:9099:premise", "snli:test:4435:premise", "style:personachat:test:88:self:1:positive:0"], "evidence_texts": ["A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "i think i would choose the drums . i do not have metal music type hair , ha !"], "speaker_id": "snli:test:7409", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7410", "candidate_text": "well, the older bearded man is indeed gandalf the grey", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7410:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:3481:premise", "snli:test:9099:premise", "snli:test:7410:premise", "snli:test:4435:premise", "style:personachat:test:277:self:7:positive:0", "style:personachat:test:277:self:7:positive:1", "style:personachat:test:277:self:7:positive:2", "style:personachat:test:277:self:7:positive:3", "style:personachat:test:277:self:7:positive:4", "style:personachat:test:277:self:7:positive:5", "style:personachat:test:277:self:7:positive:6"], "evidence_texts": ["A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "hello , how are you tonight ? i do too and love to cook .", "i would too , i am knitting tiny hats for the babies in the icu", "i am disabled , and not able to work . it makes me feel good to give back .", "i like to listen to music .", "i am vegan , and i love to bake cakes .", "i love dogs as well , and otters . they are like water dogs .", "tiramisu . it is my absolute favorite ."], "speaker_id": "snli:test:7410", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7410", "candidate_text": "The older bearded man is indeed Gandalf the Grey.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7410:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:3481:premise", "snli:test:9099:premise", "snli:test:7410:premise", "snli:test:4435:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "Young men drink coffee and read books just outside some buildings on a sunny day; an older bearded man in a dark blue shirt glances at a younger man in a striped shirt, who has a red toy crab on the table next to him.", "A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7410", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7411", "candidate_text": "I wonder whether men are throwing books and talking outside in a courtyard?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7411:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7181:premise", "snli:test:7437:premise", "snli:test:6678:premise", "snli:test:7411:premise", "style:personachat:test:278:self:1:positive:0"], "evidence_texts": ["A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:7411", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7411", "candidate_text": "Men are throwing books and talking outside in a courtyard.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7411:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7181:premise", "snli:test:7437:premise", "snli:test:6678:premise", "snli:test:7411:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A man in a white shirt speaks into a microphone as other men gather behind him at a public event.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7411", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7415", "candidate_text": "I wonder whether a black couple are eating lunch?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7415:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7291:premise", "snli:test:7771:premise", "snli:test:7199:premise", "snli:test:7415:premise", "style:personachat:test:278:self:2:positive:0", "style:personachat:test:278:self:2:positive:1"], "evidence_texts": ["A man is hanging sausages in a market.", "A happy woman who is preparing for dinner.", "City street crowded with sports fans wearing orange.", "Rose petals being thrown at an asian couple.", "yes i am . . . located in south carolina", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:7415", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7415", "candidate_text": "well, a black couple are eating lunch", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7415:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7291:premise", "snli:test:7771:premise", "snli:test:7199:premise", "snli:test:7415:premise", "style:personachat:test:465:self:2:positive:0", "style:personachat:test:465:self:2:positive:1"], "evidence_texts": ["A man is hanging sausages in a market.", "A happy woman who is preparing for dinner.", "City street crowded with sports fans wearing orange.", "Rose petals being thrown at an asian couple.", "i am more of a graphic design artist actually . what do you write ?", "that is great . you should write one about my favorite band rush"], "speaker_id": "snli:test:7415", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7417", "candidate_text": "I wonder whether two friends are laughing at people on the street?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7417:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7169:premise", "snli:test:7417:premise", "snli:test:7714:premise", "snli:test:6590:premise", "style:personachat:test:278:self:5:positive:0", "style:personachat:test:278:self:5:positive:1", "style:personachat:test:278:self:5:positive:2", "style:personachat:test:278:self:5:positive:3", "style:personachat:test:278:self:5:positive:4"], "evidence_texts": ["cheerleaders pose for the camera.", "Two friends having a laugh.", "Child swimming in a pool.", "People trying to get warm.", "after a hard day i listen to some oldies", "ok , oldies and barbeque restaurants are my favorite", "well i am retiring soon like in five years so i will be visiting many", "yes i am . . . located in south carolina", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:7417", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7417", "candidate_text": "Honestly, Two friends are laughing at people on the street!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7417:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7169:premise", "snli:test:7417:premise", "snli:test:7714:premise", "snli:test:6590:premise", "style:personachat:test:105:self:1:positive:0"], "evidence_texts": ["cheerleaders pose for the camera.", "Two friends having a laugh.", "Child swimming in a pool.", "People trying to get warm.", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:7417", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7420", "candidate_text": "I wonder whether a man and woman are laughing at a joke?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7420:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7420:premise", "snli:test:7374:premise", "snli:test:7656:premise", "snli:test:7368:premise", "style:personachat:test:278:self:6:positive:0", "style:personachat:test:278:self:6:positive:1", "style:personachat:test:278:self:6:positive:2", "style:personachat:test:278:self:6:positive:3", "style:personachat:test:278:self:6:positive:4", "style:personachat:test:278:self:6:positive:5"], "evidence_texts": ["A man and a woman are laughing.", "Two girls looking into camera and smiling", "Two men renovate a brick wall outside.", "A young lady toying with her watch.", "i am great and you", "after a hard day i listen to some oldies", "ok , oldies and barbeque restaurants are my favorite", "well i am retiring soon like in five years so i will be visiting many", "yes i am . . . located in south carolina", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:7420", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7420", "candidate_text": "A man and woman are laughing at a joke.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7420:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7420:premise", "snli:test:7374:premise", "snli:test:7656:premise", "snli:test:7368:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man and a woman are laughing.", "Two girls looking into camera and smiling", "Two men renovate a brick wall outside.", "A young lady toying with her watch.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7420", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7437", "candidate_text": "well, a mom takes a break in a park", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7437:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7411:premise", "snli:test:7437:premise", "snli:test:8640:premise", "snli:test:7181:premise", "style:personachat:test:279:self:1:positive:0"], "evidence_texts": ["A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "i have heard that i joined the army right out of high school"], "speaker_id": "snli:test:7437", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7437", "candidate_text": "Honestly, a mom takes a break in a park!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7437:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7411:premise", "snli:test:7437:premise", "snli:test:8640:premise", "snli:test:7181:premise", "style:personachat:test:105:self:5:positive:0", "style:personachat:test:105:self:5:positive:1", "style:personachat:test:105:self:5:positive:2", "style:personachat:test:105:self:5:positive:3", "style:personachat:test:105:self:5:positive:4"], "evidence_texts": ["A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A photographer is taking photos of a hockey game, but he is protecting himself from harm by wearing a helmet.", "they played in the school band . do you play any instruments ?", "i should have ! i slacked off ended up being a administrative assistant . do you have kids ?", "i am going to have a polka band play when i retire in 5 years !", "tell me more about yourself so i can recognize you there !", "lol . . . you are a interesting person , it has been nice to meet you !"], "speaker_id": "snli:test:7437", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7473", "candidate_text": "well, a child plays with a sled in the snow while dressed warmly", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7473:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6671:premise", "snli:test:7473:premise", "snli:test:7887:premise", "snli:test:6650:premise", "style:personachat:test:279:self:2:positive:0", "style:personachat:test:279:self:2:positive:1"], "evidence_texts": ["A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "wow that seems like a hard job to have", "i have heard that i joined the army right out of high school"], "speaker_id": "snli:test:7473", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7473", "candidate_text": "A child plays with a sled in the snow while dressed warmly.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7473:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6671:premise", "snli:test:7473:premise", "snli:test:7887:premise", "snli:test:6650:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "A man with a tattoo on his arm staring to the side with vehicles and buildings behind him.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7473", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7478", "candidate_text": "well, the man is at a faire", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7478:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7363:premise", "snli:test:7269:premise", "snli:test:7478:premise", "snli:test:7507:premise", "style:personachat:test:279:self:5:positive:0", "style:personachat:test:279:self:5:positive:1", "style:personachat:test:279:self:5:positive:2", "style:personachat:test:279:self:5:positive:3", "style:personachat:test:279:self:5:positive:4"], "evidence_texts": ["A women's volleyball team plays in a brown and green gymnasium.", "The tennis player spikes the tennis ball on the boundary line.", "A man in renaissance clothing is sitting on a brick wall.", "Four women standing around a yellow table are looking at laptops.", "i target practice but the others read books and write home . i cant write home", "i am from puerto rico so its hard for me to get letters to them", "yeah it is a great thing . what do you do for a living", "wow that seems like a hard job to have", "i have heard that i joined the army right out of high school"], "speaker_id": "snli:test:7478", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7478", "candidate_text": "I wonder whether the man is at a faire?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7478:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7363:premise", "snli:test:7269:premise", "snli:test:7478:premise", "snli:test:7507:premise", "style:personachat:test:48:self:5:positive:0", "style:personachat:test:48:self:5:positive:1", "style:personachat:test:48:self:5:positive:2", "style:personachat:test:48:self:5:positive:3", "style:personachat:test:48:self:5:positive:4"], "evidence_texts": ["A women's volleyball team plays in a brown and green gymnasium.", "The tennis player spikes the tennis ball on the boundary line.", "A man in renaissance clothing is sitting on a brick wall.", "Four women standing around a yellow table are looking at laptops.", "yeah my dog lassie and my cat oliver", "i am my dads world lol , he is a retired cop", "is your kid in your class ? that would be fun", "i love to go running , its my favorite passion . i wish i ran for a living", "something about sweating in public gets me pumped ya know ?"], "speaker_id": "snli:test:7478", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7479", "candidate_text": "well, the man is in jeans", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7479:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7507:premise", "snli:test:7363:premise", "snli:test:7602:premise", "snli:test:7479:premise", "style:personachat:test:279:self:7:positive:0", "style:personachat:test:279:self:7:positive:1", "style:personachat:test:279:self:7:positive:2", "style:personachat:test:279:self:7:positive:3", "style:personachat:test:279:self:7:positive:4", "style:personachat:test:279:self:7:positive:5", "style:personachat:test:279:self:7:positive:6"], "evidence_texts": ["Four women standing around a yellow table are looking at laptops.", "A women's volleyball team plays in a brown and green gymnasium.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "hello i am doing okay how are you", "i just turned 20 and am stationed in south korea .", "i target practice but the others read books and write home . i cant write home", "i am from puerto rico so its hard for me to get letters to them", "yeah it is a great thing . what do you do for a living", "wow that seems like a hard job to have", "i have heard that i joined the army right out of high school"], "speaker_id": "snli:test:7479", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7479", "candidate_text": "Honestly, the man is in jeans!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7479:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7507:premise", "snli:test:7363:premise", "snli:test:7602:premise", "snli:test:7479:premise", "style:personachat:test:119:self:2:positive:0", "style:personachat:test:119:self:2:positive:1"], "evidence_texts": ["Four women standing around a yellow table are looking at laptops.", "A women's volleyball team plays in a brown and green gymnasium.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "that was my next guess ! my ex wife has a similar one . you been married before ?", "ahhh , great choice ! my divorce was tough but i am always on the go anyway ."], "speaker_id": "snli:test:7479", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7482", "candidate_text": "well, a girl is climbing up to see scenery", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7482:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7219:premise", "snli:test:7565:premise", "snli:test:7482:premise", "snli:test:6827:premise", "style:personachat:test:280:self:1:positive:0"], "evidence_texts": ["A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "being homeschooled is safer these days ."], "speaker_id": "snli:test:7482", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7482", "candidate_text": "A girl is climbing up to see scenery.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7482:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7219:premise", "snli:test:7565:premise", "snli:test:7482:premise", "snli:test:6827:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "Three little kids on a swing set; one is upside down, one hanging from the top bar, and on a swing.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7482", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7488", "candidate_text": "well, the horse has four legs", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7488:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7488:premise", "snli:test:7282:premise", "snli:test:7798:premise", "snli:test:6657:premise", "style:personachat:test:280:self:2:positive:0", "style:personachat:test:280:self:2:positive:1"], "evidence_texts": ["A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "must be cool having blue eyes .", "being homeschooled is safer these days ."], "speaker_id": "snli:test:7488", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7488", "candidate_text": "I wonder whether the horse has four legs?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7488:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7488:premise", "snli:test:7282:premise", "snli:test:7798:premise", "snli:test:6657:premise", "style:personachat:test:53:self:2:positive:0", "style:personachat:test:53:self:2:positive:1"], "evidence_texts": ["A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "Two people standing behind a Sonoma display and a woman with a stroller on the other side.", "helps me stay healthy and avoid thinking about my ex wife", "i sure hope so , anything interesting going on in your life ?"], "speaker_id": "snli:test:7488", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7493", "candidate_text": "well, a woman is putting on mascara", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7493:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7316:premise", "snli:test:7493:premise", "snli:test:7543:premise", "snli:test:7315:premise", "style:personachat:test:280:self:5:positive:0", "style:personachat:test:280:self:5:positive:1", "style:personachat:test:280:self:5:positive:2", "style:personachat:test:280:self:5:positive:3", "style:personachat:test:280:self:5:positive:4"], "evidence_texts": ["A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "i am very short nearly five feet .", "in my family it is just my parents , my sister and myself .", "i wish i had more than one sibling .", "must be cool having blue eyes .", "being homeschooled is safer these days ."], "speaker_id": "snli:test:7493", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7493", "candidate_text": "Honestly, A woman is putting on mascara!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7493:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7316:premise", "snli:test:7493:premise", "snli:test:7543:premise", "snli:test:7315:premise", "style:personachat:test:127:self:1:positive:0"], "evidence_texts": ["A member of the army is walking with a German Shepard while the dog has something in has mouth.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:7493", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7499", "candidate_text": "well, a guy is in a hot, sunny place", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7499:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7341:premise", "snli:test:7579:premise", "snli:test:7340:premise", "snli:test:7499:premise", "style:personachat:test:280:self:7:positive:0", "style:personachat:test:280:self:7:positive:1", "style:personachat:test:280:self:7:positive:2", "style:personachat:test:280:self:7:positive:3", "style:personachat:test:280:self:7:positive:4", "style:personachat:test:280:self:7:positive:5", "style:personachat:test:280:self:7:positive:6"], "evidence_texts": ["A young blond boy is eating cake with a spoon.", "A black and white dog jumps for a tennis ball.", "A young blond boy is eating cake with a spoon.", "A guy in a red jacket is snowboarding in midair.", "good evening how are you doing ?", "i love to listen to music ?", "i am very short nearly five feet .", "in my family it is just my parents , my sister and myself .", "i wish i had more than one sibling .", "must be cool having blue eyes .", "being homeschooled is safer these days ."], "speaker_id": "snli:test:7499", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7499", "candidate_text": "A guy is in a hot, sunny place.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7499:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7341:premise", "snli:test:7579:premise", "snli:test:7340:premise", "snli:test:7499:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young blond boy is eating cake with a spoon.", "A black and white dog jumps for a tennis ball.", "A young blond boy is eating cake with a spoon.", "A guy in a red jacket is snowboarding in midair.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7499", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7507", "candidate_text": "well, a group of women shopping for a new laptop", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7507:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7479:premise", "snli:test:7602:premise", "snli:test:7478:premise", "snli:test:7507:premise", "style:personachat:test:281:self:1:positive:0"], "evidence_texts": ["A man in renaissance clothing is sitting on a brick wall.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "Four women standing around a yellow table are looking at laptops.", "this is true . now days is hard to find a job even you have degree"], "speaker_id": "snli:test:7507", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7507", "candidate_text": "I wonder whether a group of women shopping for a new laptop?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7507:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7479:premise", "snli:test:7602:premise", "snli:test:7478:premise", "snli:test:7507:premise", "style:personachat:test:60:self:1:positive:0"], "evidence_texts": ["A man in renaissance clothing is sitting on a brick wall.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "Four women standing around a yellow table are looking at laptops.", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:7507", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7524", "candidate_text": "well, the man wearing a hat is seated, slightly elevated, facing the water", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7524:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7524:premise", "snli:test:7134:premise", "snli:test:7742:premise", "snli:test:7042:premise", "style:personachat:test:281:self:2:positive:0", "style:personachat:test:281:self:2:positive:1"], "evidence_texts": ["A man wearing a hat sitting on the side of a hill looking towards the water.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A little girl in a pink hat is in a lush green field walking an ox.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "you should start college you will have a better job", "this is true . now days is hard to find a job even you have degree"], "speaker_id": "snli:test:7524", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7524", "candidate_text": "Honestly, The man wearing a hat is seated, slightly elevated, facing the water!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7524:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7524:premise", "snli:test:7134:premise", "snli:test:7742:premise", "snli:test:7042:premise", "style:personachat:test:127:self:7:positive:0", "style:personachat:test:127:self:7:positive:1", "style:personachat:test:127:self:7:positive:2", "style:personachat:test:127:self:7:positive:3", "style:personachat:test:127:self:7:positive:4", "style:personachat:test:127:self:7:positive:5", "style:personachat:test:127:self:7:positive:6"], "evidence_texts": ["A man wearing a hat sitting on the side of a hill looking towards the water.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "A little girl in a pink hat is in a lush green field walking an ox.", "A woman with long hair pointing her camera skyward as she stands on a brick wall.", "i am not really into sports . i like robots though !", "that is cool , where do you live ?", "i live out here in california", "i love the water , it so much fun to go swimming in the ocean", "that is kind of boring , but to each their own !", "i was homeschooled , that is pretty boring too", "that is not boring at all ! i have blue hair"], "speaker_id": "snli:test:7524", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7540", "candidate_text": "well, a dog is playing fetch with his owner", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7540:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7540:premise", "snli:test:7206:premise", "snli:test:7126:premise", "snli:test:7667:premise", "style:personachat:test:281:self:5:positive:0", "style:personachat:test:281:self:5:positive:1", "style:personachat:test:281:self:5:positive:2", "style:personachat:test:281:self:5:positive:3", "style:personachat:test:281:self:5:positive:4"], "evidence_texts": ["A brown and white dog holds a tennis ball in his mouth.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "Family sitting down in a field of grass while others walk by.", "A man on stilts is performing a magical trick while onlookers enjoy.", "oh it is ok . i do not work yet do you ?", "how old are you ?", "i am 24 . did you finish high school ?", "you should start college you will have a better job", "this is true . now days is hard to find a job even you have degree"], "speaker_id": "snli:test:7540", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7540", "candidate_text": "A dog is playing fetch with his owner.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7540:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7540:premise", "snli:test:7206:premise", "snli:test:7126:premise", "snli:test:7667:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A brown and white dog holds a tennis ball in his mouth.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "Family sitting down in a field of grass while others walk by.", "A man on stilts is performing a magical trick while onlookers enjoy.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7540", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7542", "candidate_text": "well, a dog mouth holds a retrieved ball", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7542:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7667:premise", "snli:test:7206:premise", "snli:test:7542:premise", "snli:test:7679:premise", "style:personachat:test:281:self:7:positive:0", "style:personachat:test:281:self:7:positive:1", "style:personachat:test:281:self:7:positive:2", "style:personachat:test:281:self:7:positive:3", "style:personachat:test:281:self:7:positive:4", "style:personachat:test:281:self:7:positive:5", "style:personachat:test:281:self:7:positive:6"], "evidence_texts": ["A man on stilts is performing a magical trick while onlookers enjoy.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "hey . i am fine and what about you ?", "what does weel means ? i never heard of this", "oh it is ok . i do not work yet do you ?", "how old are you ?", "i am 24 . did you finish high school ?", "you should start college you will have a better job", "this is true . now days is hard to find a job even you have degree"], "speaker_id": "snli:test:7542", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7542", "candidate_text": "I wonder whether a dog mouth holds a retrieved ball?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7542:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7667:premise", "snli:test:7206:premise", "snli:test:7542:premise", "snli:test:7679:premise", "style:personachat:test:60:self:7:positive:0", "style:personachat:test:60:self:7:positive:1", "style:personachat:test:60:self:7:positive:2", "style:personachat:test:60:self:7:positive:3", "style:personachat:test:60:self:7:positive:4", "style:personachat:test:60:self:7:positive:5", "style:personachat:test:60:self:7:positive:6"], "evidence_texts": ["A man on stilts is performing a magical trick while onlookers enjoy.", "Two small boys dressed in tuxedos sitting on a red carpeted floor.", "A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "yes , i would like that .", "i am studying to be an investment broker .", "i like the atmosphere that it offers . what led you to nursing ?", "good a reason as any . does she work at a hospital ?", "do you also want to work with babies ?", "that is sweet of you . do you like working out ?", "that sounds like fun . where do you go ?"], "speaker_id": "snli:test:7542", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7543", "candidate_text": "well, two man walking on the road in rain", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7543:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7493:premise", "snli:test:7637:premise", "snli:test:7316:premise", "snli:test:7543:premise", "style:personachat:test:282:self:1:positive:0"], "evidence_texts": ["A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "yes we do practice a lot and its some bit of fun also ."], "speaker_id": "snli:test:7543", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7543", "candidate_text": "Honestly, Two man walking on the road in rain!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7543:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7493:premise", "snli:test:7637:premise", "snli:test:7316:premise", "snli:test:7543:premise", "style:personachat:test:130:self:5:positive:0", "style:personachat:test:130:self:5:positive:1", "style:personachat:test:130:self:5:positive:2", "style:personachat:test:130:self:5:positive:3", "style:personachat:test:130:self:5:positive:4"], "evidence_texts": ["A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A member of the army is walking with a German Shepard while the dog has something in has mouth.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "ah , so you probably like shopping for clothes like i do ?", "even at my age , i am not married . single is my status .", "that is cool ! my hair is brown .", "well that is good . what color is her hair ?", "that is too funny ! no , i do not ."], "speaker_id": "snli:test:7543", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7552", "candidate_text": "well, two dogs sleep on the floor", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7552:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7552:premise", "snli:test:7406:premise", "snli:test:7355:premise", "snli:test:7777:premise", "style:personachat:test:282:self:2:positive:0", "style:personachat:test:282:self:2:positive:1"], "evidence_texts": ["A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "An adorable child leaping to block a soccer ball from entering the goal.", "The man in the black hat has his back to a theater sign.", "i do sing in a church choir only .", "yes we do practice a lot and its some bit of fun also ."], "speaker_id": "snli:test:7552", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7552", "candidate_text": "Two dogs sleep on the floor.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7552:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7552:premise", "snli:test:7406:premise", "snli:test:7355:premise", "snli:test:7777:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "An adorable child leaping to block a soccer ball from entering the goal.", "The man in the black hat has his back to a theater sign.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7552", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7553", "candidate_text": "well, two dogs play", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7553:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7777:premise", "snli:test:7553:premise", "snli:test:7406:premise", "snli:test:7779:premise", "style:personachat:test:282:self:5:positive:0", "style:personachat:test:282:self:5:positive:1", "style:personachat:test:282:self:5:positive:2", "style:personachat:test:282:self:5:positive:3", "style:personachat:test:282:self:5:positive:4"], "evidence_texts": ["The man in the black hat has his back to a theater sign.", "A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "The man in the black hat has his back to a theater sign.", "that sounds good i would also want to move but it is tricky for me .", "okay am an artist specializing in graffiti", "i do surfing but not a lot and fishing in my free time .", "i do sing in a church choir only .", "yes we do practice a lot and its some bit of fun also ."], "speaker_id": "snli:test:7553", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7553", "candidate_text": "I wonder whether two dogs play?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7553:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7777:premise", "snli:test:7553:premise", "snli:test:7406:premise", "snli:test:7779:premise", "style:personachat:test:63:self:5:positive:0", "style:personachat:test:63:self:5:positive:1", "style:personachat:test:63:self:5:positive:2", "style:personachat:test:63:self:5:positive:3", "style:personachat:test:63:self:5:positive:4"], "evidence_texts": ["The man in the black hat has his back to a theater sign.", "A black and brown dog is playing with a brown and white dog.", "There is a green trash truck in road with a person sweeping sidewalk.", "The man in the black hat has his back to a theater sign.", "i am a warehouse worker . i cook a great bbq .", "keep your head up ! i work the forklift . what work do you do ?", "that is good ! people laugh because my name is joe shirley .", "hmmm . got for it ! is six four two twenty tall and big ?", "i see . i have a cookbook coming out . you write ?"], "speaker_id": "snli:test:7553", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7565", "candidate_text": "well, a man is wearing a suit", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7565:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7482:premise", "snli:test:7565:premise", "snli:test:7803:premise", "snli:test:7219:premise", "style:personachat:test:282:self:7:positive:0", "style:personachat:test:282:self:7:positive:1", "style:personachat:test:282:self:7:positive:2", "style:personachat:test:282:self:7:positive:3", "style:personachat:test:282:self:7:positive:4", "style:personachat:test:282:self:7:positive:5", "style:personachat:test:282:self:7:positive:6"], "evidence_texts": ["A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "fine and am just sipping my wine at the moment .", "am from georgia usa and you ?", "that sounds good i would also want to move but it is tricky for me .", "okay am an artist specializing in graffiti", "i do surfing but not a lot and fishing in my free time .", "i do sing in a church choir only .", "yes we do practice a lot and its some bit of fun also ."], "speaker_id": "snli:test:7565", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7565", "candidate_text": "Honestly, A man is wearing a suit!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7565:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7482:premise", "snli:test:7565:premise", "snli:test:7803:premise", "snli:test:7219:premise", "style:personachat:test:133:self:2:positive:0", "style:personachat:test:133:self:2:positive:1"], "evidence_texts": ["A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A picture of an artwork that depicts a man being flown by birds that is in font of a construction area.", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:7565", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7571", "candidate_text": "I wonder whether people watching sports?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7571:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:6894:premise", "snli:test:7625:premise", "snli:test:6793:premise", "snli:test:7571:premise", "style:personachat:test:283:self:1:positive:0"], "evidence_texts": ["Three people on bikes are coming over a ridge, while a man in orange watches.", "A person dressed in black and red protective gear drives an ATV on a track.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Two men watching one man make a golf putt on the set of Sports Center.", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:7571", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7571", "candidate_text": "Honestly, People watching sports!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7571:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6894:premise", "snli:test:7625:premise", "snli:test:6793:premise", "snli:test:7571:premise", "style:personachat:test:133:self:5:positive:0", "style:personachat:test:133:self:5:positive:1", "style:personachat:test:133:self:5:positive:2", "style:personachat:test:133:self:5:positive:3", "style:personachat:test:133:self:5:positive:4"], "evidence_texts": ["Three people on bikes are coming over a ridge, while a man in orange watches.", "A person dressed in black and red protective gear drives an ATV on a track.", "A man pushing a hand-truck of boxes is bending over to pick up a pear.", "Two men watching one man make a golf putt on the set of Sports Center.", "do you like to cook ? i make homemade gourmet cat food , and people food , too !", "i love cooking ! and , i am pretty good at it , too ! that explains the extra pounds !", "if i was taller , i might feel better about my body . what do you do ?", "it is good to take your time to decide . what do you study ?", "i love playing music with the kiddies and the kitties !"], "speaker_id": "snli:test:7571", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7579", "candidate_text": "I wonder whether a border collie is jumping to catching a toy?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7579:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7499:premise", "snli:test:7859:premise", "snli:test:7341:premise", "snli:test:7579:premise", "style:personachat:test:283:self:2:positive:0", "style:personachat:test:283:self:2:positive:1"], "evidence_texts": ["A guy in a red jacket is snowboarding in midair.", "a group of construction workers busy refinishing a subway station.", "A young blond boy is eating cake with a spoon.", "A black and white dog jumps for a tennis ball.", "yes , but i don t tell this to many people , i usually keep it to myself", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:7579", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7579", "candidate_text": "A border collie is jumping to catching a toy.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7579:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7499:premise", "snli:test:7859:premise", "snli:test:7341:premise", "snli:test:7579:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A guy in a red jacket is snowboarding in midair.", "a group of construction workers busy refinishing a subway station.", "A young blond boy is eating cake with a spoon.", "A black and white dog jumps for a tennis ball.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7579", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7602", "candidate_text": "I wonder whether a scientist is inspecting a new strain of virus?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7602:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7507:premise", "snli:test:7660:premise", "snli:test:7602:premise", "snli:test:7479:premise", "style:personachat:test:283:self:5:positive:0", "style:personachat:test:283:self:5:positive:1", "style:personachat:test:283:self:5:positive:2", "style:personachat:test:283:self:5:positive:3", "style:personachat:test:283:self:5:positive:4"], "evidence_texts": ["Four women standing around a yellow table are looking at laptops.", "A brown dog sniffing at the fence of a brown bull.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "i m old so i can t join them now , hope to retire soon", "accountant . sitting down hurts my left leg", "no . i ll ride some horses when it hurts , it helps .", "yes , but i don t tell this to many people , i usually keep it to myself", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:7602", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7602", "candidate_text": "well, a scientist is inspecting a new strain of virus", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7602:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7507:premise", "snli:test:7660:premise", "snli:test:7602:premise", "snli:test:7479:premise", "style:personachat:test:475:self:5:positive:0", "style:personachat:test:475:self:5:positive:1", "style:personachat:test:475:self:5:positive:2", "style:personachat:test:475:self:5:positive:3", "style:personachat:test:475:self:5:positive:4"], "evidence_texts": ["Four women standing around a yellow table are looking at laptops.", "A brown dog sniffing at the fence of a brown bull.", "A man in a lab coat is looking through a microscope.", "A man in renaissance clothing is sitting on a brick wall.", "i do , as long as there are no dogs there . they terrify me .", "i just never cared for them . i am more of a cat person .", "i am your stereotypical 30 something single gal with her cats and her books . haha", "do read a lot ? i love reading , i like classic romance novels the most .", "i have read a few true crime novels , i enjoyed them ."], "speaker_id": "snli:test:7602", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7610", "candidate_text": "I wonder whether a man and a woman are performing a song together?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7610:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7211:premise", "snli:test:7795:premise", "snli:test:7610:premise", "snli:test:7119:premise", "style:personachat:test:283:self:7:positive:0", "style:personachat:test:283:self:7:positive:1", "style:personachat:test:283:self:7:positive:2", "style:personachat:test:283:self:7:positive:3", "style:personachat:test:283:self:7:positive:4", "style:personachat:test:283:self:7:positive:5", "style:personachat:test:283:self:7:positive:6"], "evidence_texts": ["An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "hi , i m fine thanks , you ?", "good , i love the army .", "i m old so i can t join them now , hope to retire soon", "accountant . sitting down hurts my left leg", "no . i ll ride some horses when it hurts , it helps .", "yes , but i don t tell this to many people , i usually keep it to myself", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:7610", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7610", "candidate_text": "Honestly, A man and a woman are performing a song together!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7610:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7211:premise", "snli:test:7795:premise", "snli:test:7610:premise", "snli:test:7119:premise", "style:personachat:test:137:self:2:positive:0", "style:personachat:test:137:self:2:positive:1"], "evidence_texts": ["An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A small girl with short brown hair is pushing a pink scooter over grass with brown leaves with threes in the background", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:7610", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7619", "candidate_text": "well, a man is at an auto shop", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7619:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7332:premise", "snli:test:7851:premise", "snli:test:6919:premise", "snli:test:7619:premise", "style:personachat:test:284:self:1:positive:0"], "evidence_texts": ["A little boy and his father talking to a man dressed as Santa Claws.", "A woman in a white tennis outfit is jumping up to hit the ball.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "A man is working in an art studio and is surrounded by art supplies.", "that is why i am trying to pursue a career in fashion"], "speaker_id": "snli:test:7619", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7619", "candidate_text": "I wonder whether a man is at an auto shop?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7619:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7332:premise", "snli:test:7851:premise", "snli:test:6919:premise", "snli:test:7619:premise", "style:personachat:test:66:self:1:positive:0"], "evidence_texts": ["A little boy and his father talking to a man dressed as Santa Claws.", "A woman in a white tennis outfit is jumping up to hit the ball.", "A young woman skateboarding on an outdoor basketball court in front of a mural.", "A man is working in an art studio and is surrounded by art supplies.", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:7619", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7625", "candidate_text": "well, there is a person driving in a tournament", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7625:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7571:premise", "snli:test:7625:premise", "snli:test:7732:premise", "snli:test:6894:premise", "style:personachat:test:284:self:2:positive:0", "style:personachat:test:284:self:2:positive:1"], "evidence_texts": ["Two men watching one man make a golf putt on the set of Sports Center.", "A person dressed in black and red protective gear drives an ATV on a track.", "A young child climbing a stack of logs even though the sign warns against it.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "makes sense . i never had any direction", "that is why i am trying to pursue a career in fashion"], "speaker_id": "snli:test:7625", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7625", "candidate_text": "Honestly, There is a person driving in a tournament!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7625:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7571:premise", "snli:test:7625:premise", "snli:test:7732:premise", "snli:test:6894:premise", "style:personachat:test:137:self:6:positive:0", "style:personachat:test:137:self:6:positive:1", "style:personachat:test:137:self:6:positive:2", "style:personachat:test:137:self:6:positive:3", "style:personachat:test:137:self:6:positive:4", "style:personachat:test:137:self:6:positive:5"], "evidence_texts": ["Two men watching one man make a golf putt on the set of Sports Center.", "A person dressed in black and red protective gear drives an ATV on a track.", "A young child climbing a stack of logs even though the sign warns against it.", "Three people on bikes are coming over a ridge, while a man in orange watches.", "i love all cute things ! babies are on the top of my list now", "right ? ! i have actually been knitting hats for babies in hospitals for last few years", "i am a few years older , whats your favorite music ? mine is country !", "that would be awesome , i want to be a chef since i love cooking", "sounds like a plan ! it will be a vegan menu though !", "yes i would love to hear a secret !"], "speaker_id": "snli:test:7625", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7637", "candidate_text": "well, well dressed looter", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7637:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7543:premise", "snli:test:7637:premise", "snli:test:7861:premise", "snli:test:7493:premise", "style:personachat:test:284:self:5:positive:0", "style:personachat:test:284:self:5:positive:1", "style:personachat:test:284:self:5:positive:2", "style:personachat:test:284:self:5:positive:3", "style:personachat:test:284:self:5:positive:4"], "evidence_texts": ["The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "oh okay cool i want to be a fashion designer !", "sounds fun . . my parents were never around while i was young", "oh okay . . i was locked up for stealing once . . ever gone to jail ?", "makes sense . i never had any direction", "that is why i am trying to pursue a career in fashion"], "speaker_id": "snli:test:7637", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7637", "candidate_text": "Well dressed looter.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7637:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7543:premise", "snli:test:7637:premise", "snli:test:7861:premise", "snli:test:7493:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A bearded man, using a wooden pole, stirs bowl-like objects around in a large metal box filled with water.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7637", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7643", "candidate_text": "well, a woman near a table", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7643:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7239:premise", "snli:test:8358:premise", "snli:test:7238:premise", "snli:test:7643:premise", "style:personachat:test:284:self:6:positive:0", "style:personachat:test:284:self:6:positive:1", "style:personachat:test:284:self:6:positive:2", "style:personachat:test:284:self:6:positive:3", "style:personachat:test:284:self:6:positive:4", "style:personachat:test:284:self:6:positive:5"], "evidence_texts": ["a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "i am doing alright , buti really wish i had kids . .", "oh okay cool i want to be a fashion designer !", "sounds fun . . my parents were never around while i was young", "oh okay . . i was locked up for stealing once . . ever gone to jail ?", "makes sense . i never had any direction", "that is why i am trying to pursue a career in fashion"], "speaker_id": "snli:test:7643", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7643", "candidate_text": "I wonder whether a woman near a table?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7643:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7239:premise", "snli:test:8358:premise", "snli:test:7238:premise", "snli:test:7643:premise", "style:personachat:test:66:self:6:positive:0", "style:personachat:test:66:self:6:positive:1", "style:personachat:test:66:self:6:positive:2", "style:personachat:test:66:self:6:positive:3", "style:personachat:test:66:self:6:positive:4", "style:personachat:test:66:self:6:positive:5"], "evidence_texts": ["a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "i am doing well and relaxing at home with a glass of scotch . you ?", "do you teach dogs obedience ? that is good . i am afraid of them .", "there were sometimes animals around when i worked at the carnival .", "when i want to relax i watch gritty movies and drink scotch .", "when did you buy it ? do you live alone ?", "do you have dogs of your own ? or children ?"], "speaker_id": "snli:test:7643", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7656", "candidate_text": "I wonder whether two men are laying bricks?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7656:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7656:premise", "snli:test:7420:premise", "snli:test:7674:premise", "snli:test:7374:premise", "style:personachat:test:285:self:1:positive:0"], "evidence_texts": ["Two men renovate a brick wall outside.", "A man and a woman are laughing.", "A young redheaded woman laughs at something.", "Two girls looking into camera and smiling", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:7656", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7656", "candidate_text": "well, two men are laying bricks", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7656:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7656:premise", "snli:test:7420:premise", "snli:test:7674:premise", "snli:test:7374:premise", "style:personachat:test:478:self:1:positive:0"], "evidence_texts": ["Two men renovate a brick wall outside.", "A man and a woman are laughing.", "A young redheaded woman laughs at something.", "Two girls looking into camera and smiling", "so do i . i have two cats but am constantly fostering all types of animals"], "speaker_id": "snli:test:7656", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7660", "candidate_text": "I wonder whether a cute dog was chasing the cat through they alley?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7660:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7660:premise", "snli:test:7602:premise", "snli:test:7757:premise", "snli:test:7507:premise", "style:personachat:test:285:self:2:positive:0", "style:personachat:test:285:self:2:positive:1"], "evidence_texts": ["A brown dog sniffing at the fence of a brown bull.", "A man in a lab coat is looking through a microscope.", "A woman and a young girl are eating side by side.", "Four women standing around a yellow table are looking at laptops.", "yes it will be sad , but lots of great places to skate ther .", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:7660", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7660", "candidate_text": "Honestly, A cute dog was chasing the cat through they alley!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7660:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7660:premise", "snli:test:7602:premise", "snli:test:7757:premise", "snli:test:7507:premise", "style:personachat:test:140:self:7:positive:0", "style:personachat:test:140:self:7:positive:1", "style:personachat:test:140:self:7:positive:2", "style:personachat:test:140:self:7:positive:3", "style:personachat:test:140:self:7:positive:4", "style:personachat:test:140:self:7:positive:5", "style:personachat:test:140:self:7:positive:6"], "evidence_texts": ["A brown dog sniffing at the fence of a brown bull.", "A man in a lab coat is looking through a microscope.", "A woman and a young girl are eating side by side.", "Four women standing around a yellow table are looking at laptops.", "great ! just grading some papers and throwing some things in the crock pot .", "very cool ! my daughter wants a fish tank !", "i am a literature teacher ! nothing like a good book ! nice ! !", "i wish my husband was ! my daughter just loves cooking like me .", "it is fun ! i love granola . what else are you into ?", "that sounds amazing ! i am a devoted mom and wife . no me time anymore .", "i would love to take a cooking class in italy !"], "speaker_id": "snli:test:7660", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7667", "candidate_text": "I wonder whether a man on stilts performs a magic trick for onlookers at his school?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7667:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7542:premise", "snli:test:7679:premise", "snli:test:7540:premise", "snli:test:7667:premise", "style:personachat:test:285:self:5:positive:0", "style:personachat:test:285:self:5:positive:1", "style:personachat:test:285:self:5:positive:2", "style:personachat:test:285:self:5:positive:3", "style:personachat:test:285:self:5:positive:4"], "evidence_texts": ["A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "A brown and white dog holds a tennis ball in his mouth.", "A man on stilts is performing a magical trick while onlookers enjoy.", "i am from california , but moving to new york for a job next year", "i work as a model and actress , but i am not a good singer ! you ?", "awesome ! what kind ? i am a surfer and skater .", "yes it will be sad , but lots of great places to skate ther .", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:7667", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7667", "candidate_text": "a man on stilts performs a magic trick for onlookers at his school.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7667:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7542:premise", "snli:test:7679:premise", "snli:test:7540:premise", "snli:test:7667:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "A brown and white dog holds a tennis ball in his mouth.", "A man on stilts is performing a magical trick while onlookers enjoy.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7667", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7674", "candidate_text": "I wonder whether a young redheaded man laughs at something?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7674:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7656:premise", "snli:test:7848:premise", "snli:test:7674:premise", "snli:test:7420:premise", "style:personachat:test:285:self:6:positive:0", "style:personachat:test:285:self:6:positive:1", "style:personachat:test:285:self:6:positive:2", "style:personachat:test:285:self:6:positive:3", "style:personachat:test:285:self:6:positive:4", "style:personachat:test:285:self:6:positive:5"], "evidence_texts": ["Two men renovate a brick wall outside.", "Some youngsters are playing a card game", "A young redheaded woman laughs at something.", "A man and a woman are laughing.", "hello ! i am well and yourself ? where are you from ?", "i am from california , but moving to new york for a job next year", "i work as a model and actress , but i am not a good singer ! you ?", "awesome ! what kind ? i am a surfer and skater .", "yes it will be sad , but lots of great places to skate ther .", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:7674", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7674", "candidate_text": "well, a young redheaded man laughs at something", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7674:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7656:premise", "snli:test:7848:premise", "snli:test:7674:premise", "snli:test:7420:premise", "style:personachat:test:478:self:7:positive:0", "style:personachat:test:478:self:7:positive:1", "style:personachat:test:478:self:7:positive:2", "style:personachat:test:478:self:7:positive:3", "style:personachat:test:478:self:7:positive:4", "style:personachat:test:478:self:7:positive:5", "style:personachat:test:478:self:7:positive:6"], "evidence_texts": ["Two men renovate a brick wall outside.", "Some youngsters are playing a card game", "A young redheaded woman laughs at something.", "A man and a woman are laughing.", "hi my name is lauren . i am well how are you ? i enjoy music , do you ?", "darn , i have never been a fan of country . i like pop rock and hip hop", "haha , i am from new jersey . i guess country is american . do you have siblings ?", "i have one brother , i am the baby of the family .", "oh wow youre young , i am 29 lol", "that sounds like fun . i am an animal care tech", "so do i . i have two cats but am constantly fostering all types of animals"], "speaker_id": "snli:test:7674", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7675", "candidate_text": "well, a man and women sleeping together outside on a park bench", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7675:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7395:premise", "snli:test:7683:premise", "snli:test:7250:premise", "snli:test:7675:premise", "style:personachat:test:286:self:1:positive:0"], "evidence_texts": ["The gentleman is speaking while the others are listening.", "A man is wearing protective hair and face coverings.", "Two men with tan hats working in a garden.", "A man and women sleeping outside on a bench.", "i do not sing , sadly ."], "speaker_id": "snli:test:7675", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7675", "candidate_text": "A man and women sleeping together outside on a park bench.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7675:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7395:premise", "snli:test:7683:premise", "snli:test:7250:premise", "snli:test:7675:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["The gentleman is speaking while the others are listening.", "A man is wearing protective hair and face coverings.", "Two men with tan hats working in a garden.", "A man and women sleeping outside on a bench.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7675", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7679", "candidate_text": "well, a young girl gets his photo taken with mr. and mrs. clause", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7679:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7667:premise", "snli:test:7976:premise", "snli:test:7542:premise", "snli:test:7679:premise", "style:personachat:test:286:self:2:positive:0", "style:personachat:test:286:self:2:positive:1"], "evidence_texts": ["A man on stilts is performing a magical trick while onlookers enjoy.", "A shoeshine man works as others pass by on a busy sidewalk.", "A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "do you enjoy that kind of work ?", "i do not sing , sadly ."], "speaker_id": "snli:test:7679", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7679", "candidate_text": "I wonder whether a young girl gets his photo taken with mr. and mrs. clause?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7679:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7667:premise", "snli:test:7976:premise", "snli:test:7542:premise", "snli:test:7679:premise", "style:personachat:test:76:self:2:positive:0", "style:personachat:test:76:self:2:positive:1"], "evidence_texts": ["A man on stilts is performing a magical trick while onlookers enjoy.", "A shoeshine man works as others pass by on a busy sidewalk.", "A brown and white dog holds a tennis ball in his mouth.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "all of them ! i just love the time together and the sport you ?", "awesome . how do you stay so fit ?"], "speaker_id": "snli:test:7679", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7683", "candidate_text": "well, a man is working in a restaurant", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7683:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7675:premise", "snli:test:7706:premise", "snli:test:7395:premise", "snli:test:7683:premise", "style:personachat:test:286:self:5:positive:0", "style:personachat:test:286:self:5:positive:1", "style:personachat:test:286:self:5:positive:2", "style:personachat:test:286:self:5:positive:3", "style:personachat:test:286:self:5:positive:4"], "evidence_texts": ["A man and women sleeping outside on a bench.", "A man in a white shirt taking a picture.", "The gentleman is speaking while the others are listening.", "A man is wearing protective hair and face coverings.", "my cat . she is a siamese .", "i do love cooking for my kids tho .", "pork chops . sadly i cook alot and not work out as i should .", "do you enjoy that kind of work ?", "i do not sing , sadly ."], "speaker_id": "snli:test:7683", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7683", "candidate_text": "Honestly, A man is working in a restaurant!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7683:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7675:premise", "snli:test:7706:premise", "snli:test:7395:premise", "snli:test:7683:premise", "style:personachat:test:146:self:1:positive:0"], "evidence_texts": ["A man and women sleeping outside on a bench.", "A man in a white shirt taking a picture.", "The gentleman is speaking while the others are listening.", "A man is wearing protective hair and face coverings.", "i hope you recover quickly , it was nice to meet you !"], "speaker_id": "snli:test:7683", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7684", "candidate_text": "well, one lady looks at a box while another looks at her phone", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7684:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7684:premise", "snli:test:7349:premise", "snli:test:8080:premise", "snli:test:7261:premise", "style:personachat:test:286:self:6:positive:0", "style:personachat:test:286:self:6:positive:1", "style:personachat:test:286:self:6:positive:2", "style:personachat:test:286:self:6:positive:3", "style:personachat:test:286:self:6:positive:4", "style:personachat:test:286:self:6:positive:5"], "evidence_texts": ["One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "oh , my 3 kids are annoying me chasing the cat around .", "my cat . she is a siamese .", "i do love cooking for my kids tho .", "pork chops . sadly i cook alot and not work out as i should .", "do you enjoy that kind of work ?", "i do not sing , sadly ."], "speaker_id": "snli:test:7684", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7684", "candidate_text": "one lady looks at a box while another looks at her phone.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7684:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7684:premise", "snli:test:7349:premise", "snli:test:8080:premise", "snli:test:7261:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A woman with green body paint is holding a green and blue hula hoop while her friend (also in blue and green) watches.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7684", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7706", "candidate_text": "I wonder whether the man takes a video clip?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7706:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7683:premise", "snli:test:7675:premise", "snli:test:7706:premise", "snli:test:7788:premise", "style:personachat:test:287:self:1:positive:0"], "evidence_texts": ["A man is wearing protective hair and face coverings.", "A man and women sleeping outside on a bench.", "A man in a white shirt taking a picture.", "A guy with a football, a crowd watching him.", "have you always been into body building ?"], "speaker_id": "snli:test:7706", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7706", "candidate_text": "the man takes a video clip.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7706:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7683:premise", "snli:test:7675:premise", "snli:test:7706:premise", "snli:test:7788:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man is wearing protective hair and face coverings.", "A man and women sleeping outside on a bench.", "A man in a white shirt taking a picture.", "A guy with a football, a crowd watching him.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7706", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7707", "candidate_text": "I wonder whether the man is a photographer?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7707:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7788:premise", "snli:test:7683:premise", "snli:test:7815:premise", "snli:test:7707:premise", "style:personachat:test:287:self:2:positive:0", "style:personachat:test:287:self:2:positive:1"], "evidence_texts": ["A guy with a football, a crowd watching him.", "A man is wearing protective hair and face coverings.", "Seven young girls are being creative and making posters.", "A man in a white shirt taking a picture.", "you have to get that protein", "have you always been into body building ?"], "speaker_id": "snli:test:7707", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7707", "candidate_text": "well, the man is a photographer", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7707:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7788:premise", "snli:test:7683:premise", "snli:test:7815:premise", "snli:test:7707:premise", "style:personachat:test:482:self:2:positive:0", "style:personachat:test:482:self:2:positive:1"], "evidence_texts": ["A guy with a football, a crowd watching him.", "A man is wearing protective hair and face coverings.", "Seven young girls are being creative and making posters.", "A man in a white shirt taking a picture.", "well then whare your dislikes lol", "i hate the snow and big crowds"], "speaker_id": "snli:test:7707", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7714", "candidate_text": "I wonder whether a child swims in the ocean?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7714:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7417:premise", "snli:test:8036:premise", "snli:test:7714:premise", "snli:test:7169:premise", "style:personachat:test:287:self:5:positive:0", "style:personachat:test:287:self:5:positive:1", "style:personachat:test:287:self:5:positive:2", "style:personachat:test:287:self:5:positive:3", "style:personachat:test:287:self:5:positive:4"], "evidence_texts": ["Two friends having a laugh.", "Three men play musical instruments.", "Child swimming in a pool.", "cheerleaders pose for the camera.", "i could not have one when i worked in banking . but now i have retired .", "that is so sweet ! i knit things for my children all the time", "it is pretty simple , i took a class and now i am a pro !", "you have to get that protein", "have you always been into body building ?"], "speaker_id": "snli:test:7714", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7714", "candidate_text": "Honestly, A child swims in the ocean!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7714:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7417:premise", "snli:test:8036:premise", "snli:test:7714:premise", "snli:test:7169:premise", "style:personachat:test:152:self:1:positive:0"], "evidence_texts": ["Two friends having a laugh.", "Three men play musical instruments.", "Child swimming in a pool.", "cheerleaders pose for the camera.", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:7714", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7732", "candidate_text": "I wonder whether a kid who's too young to read climbs a dangerous stack of logs?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7732:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7732:premise", "snli:test:7625:premise", "snli:test:7896:premise", "snli:test:7571:premise", "style:personachat:test:287:self:7:positive:0", "style:personachat:test:287:self:7:positive:1", "style:personachat:test:287:self:7:positive:2", "style:personachat:test:287:self:7:positive:3", "style:personachat:test:287:self:7:positive:4", "style:personachat:test:287:self:7:positive:5", "style:personachat:test:287:self:7:positive:6"], "evidence_texts": ["A young child climbing a stack of logs even though the sign warns against it.", "A person dressed in black and red protective gear drives an ATV on a track.", "A woman is taking a picture on the shoulders of another woman in cold weather.", "Two men watching one man make a golf putt on the set of Sports Center.", "i am too old for that . but i try to walk a mile each day !", "that sounds lovely . i have a cat .", "i could not have one when i worked in banking . but now i have retired .", "that is so sweet ! i knit things for my children all the time", "it is pretty simple , i took a class and now i am a pro !", "you have to get that protein", "have you always been into body building ?"], "speaker_id": "snli:test:7732", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7732", "candidate_text": "A kid who's too young to read climbs a dangerous stack of logs.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7732:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7732:premise", "snli:test:7625:premise", "snli:test:7896:premise", "snli:test:7571:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young child climbing a stack of logs even though the sign warns against it.", "A person dressed in black and red protective gear drives an ATV on a track.", "A woman is taking a picture on the shoulders of another woman in cold weather.", "Two men watching one man make a golf putt on the set of Sports Center.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7732", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7742", "candidate_text": "I wonder whether a little girl is riding her ox in a desert?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7742:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7524:premise", "snli:test:8005:premise", "snli:test:7742:premise", "snli:test:7134:premise", "style:personachat:test:288:self:1:positive:0"], "evidence_texts": ["A man wearing a hat sitting on the side of a hill looking towards the water.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A little girl in a pink hat is in a lush green field walking an ox.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "why ? it is freezing there"], "speaker_id": "snli:test:7742", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7742", "candidate_text": "well, a little girl is riding her ox in a desert", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7742:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7524:premise", "snli:test:8005:premise", "snli:test:7742:premise", "snli:test:7134:premise", "style:personachat:test:483:self:1:positive:0"], "evidence_texts": ["A man wearing a hat sitting on the side of a hill looking towards the water.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A little girl in a pink hat is in a lush green field walking an ox.", "A squirrel stands on its hind legs in the grass at the edge of a road.", "they are both five years old . twins"], "speaker_id": "snli:test:7742", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7757", "candidate_text": "I wonder whether two people eating together?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7757:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7660:premise", "snli:test:7757:premise", "snli:test:7988:premise", "snli:test:7602:premise", "style:personachat:test:288:self:2:positive:0", "style:personachat:test:288:self:2:positive:1"], "evidence_texts": ["A brown dog sniffing at the fence of a brown bull.", "A woman and a young girl are eating side by side.", "A black-haired man is entertaining a crowd with a hula hoop.", "A man in a lab coat is looking through a microscope.", "your students were in what grade ?", "why ? it is freezing there"], "speaker_id": "snli:test:7757", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7757", "candidate_text": "Honestly, Two people eating together!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7757:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7660:premise", "snli:test:7757:premise", "snli:test:7988:premise", "snli:test:7602:premise", "style:personachat:test:152:self:7:positive:0", "style:personachat:test:152:self:7:positive:1", "style:personachat:test:152:self:7:positive:2", "style:personachat:test:152:self:7:positive:3", "style:personachat:test:152:self:7:positive:4", "style:personachat:test:152:self:7:positive:5", "style:personachat:test:152:self:7:positive:6"], "evidence_texts": ["A brown dog sniffing at the fence of a brown bull.", "A woman and a young girl are eating side by side.", "A black-haired man is entertaining a crowd with a hula hoop.", "A man in a lab coat is looking through a microscope.", "i am fine . just watching some sports .", "i actually enjoy the steelers , do you ?", "that is great . are you married ?", "i am engaged for a wedding in december .", "i am excited ! especially since we both are now college graduates !", "history was my major . i worked in a museum .", "that sounds fun though ! i like to play many sports ."], "speaker_id": "snli:test:7757", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7771", "candidate_text": "I wonder whether a sad woman is watching tv?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7771:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7415:premise", "snli:test:7812:premise", "snli:test:7291:premise", "snli:test:7771:premise", "style:personachat:test:288:self:5:positive:0", "style:personachat:test:288:self:5:positive:1", "style:personachat:test:288:self:5:positive:2", "style:personachat:test:288:self:5:positive:3", "style:personachat:test:288:self:5:positive:4"], "evidence_texts": ["Rose petals being thrown at an asian couple.", "Four human figures are backlit by a sunset.", "A man is hanging sausages in a market.", "A happy woman who is preparing for dinner.", "no thank god , i kind of got a cat it was a stray", "no it is not mine is clothes an singing folk music", "i bet it is lol", "your students were in what grade ?", "why ? it is freezing there"], "speaker_id": "snli:test:7771", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7771", "candidate_text": "A sad woman is watching TV.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7771:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7415:premise", "snli:test:7812:premise", "snli:test:7291:premise", "snli:test:7771:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Rose petals being thrown at an asian couple.", "Four human figures are backlit by a sunset.", "A man is hanging sausages in a market.", "A happy woman who is preparing for dinner.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7771", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7777", "candidate_text": "I wonder whether the man is deciding whether is should watch a movie at the theater or not?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7777:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7553:premise", "snli:test:7777:premise", "snli:test:7552:premise", "snli:test:7899:premise", "style:personachat:test:288:self:7:positive:0", "style:personachat:test:288:self:7:positive:1", "style:personachat:test:288:self:7:positive:2", "style:personachat:test:288:self:7:positive:3", "style:personachat:test:288:self:7:positive:4", "style:personachat:test:288:self:7:positive:5", "style:personachat:test:288:self:7:positive:6"], "evidence_texts": ["A black and brown dog is playing with a brown and white dog.", "The man in the black hat has his back to a theater sign.", "A black and brown dog is playing with a brown and white dog.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "hey there what you up too", "cool , my mom had a hard job as a army nurse", "no thank god , i kind of got a cat it was a stray", "no it is not mine is clothes an singing folk music", "i bet it is lol", "your students were in what grade ?", "why ? it is freezing there"], "speaker_id": "snli:test:7777", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7777", "candidate_text": "well, the man is deciding whether is should watch a movie at the theater or not", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7777:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7553:premise", "snli:test:7777:premise", "snli:test:7552:premise", "snli:test:7899:premise", "style:personachat:test:483:self:6:positive:0", "style:personachat:test:483:self:6:positive:1", "style:personachat:test:483:self:6:positive:2", "style:personachat:test:483:self:6:positive:3", "style:personachat:test:483:self:6:positive:4", "style:personachat:test:483:self:6:positive:5"], "evidence_texts": ["A black and brown dog is playing with a brown and white dog.", "The man in the black hat has his back to a theater sign.", "A black and brown dog is playing with a brown and white dog.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "i am pretty good . just got the kids to bed", "how many kids do you have ? i have 2", "both of mine are girls . how old are you ? i am 34", "i work in real estate . i have been there for awhile", "nice . that is really impressive .", "they are both five years old . twins"], "speaker_id": "snli:test:7777", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7779", "candidate_text": "Honestly, There is a man standing outside of a theater!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7779:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7899:premise", "snli:test:7553:premise", "snli:test:8210:premise", "snli:test:7779:premise", "style:personachat:test:289:self:1:positive:0"], "evidence_texts": ["A group in uniform marches carrying red signs with Chinese lettering on them.", "A black and brown dog is playing with a brown and white dog.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:7779", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7779", "candidate_text": "There is a man standing outside of a theater.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7779:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7899:premise", "snli:test:7553:premise", "snli:test:8210:premise", "snli:test:7779:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A group in uniform marches carrying red signs with Chinese lettering on them.", "A black and brown dog is playing with a brown and white dog.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7779", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7788", "candidate_text": "Honestly, There is a guy with a basketball!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7788:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7788:premise", "snli:test:7707:premise", "snli:test:7815:premise", "snli:test:7706:premise", "style:personachat:test:289:self:2:positive:0", "style:personachat:test:289:self:2:positive:1"], "evidence_texts": ["A guy with a football, a crowd watching him.", "A man in a white shirt taking a picture.", "Seven young girls are being creative and making posters.", "A man in a white shirt taking a picture.", "yeah . my mom actually works at the front desk ! do you train with your dad ?", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:7788", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7788", "candidate_text": "I wonder whether there is a guy with a basketball?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7788:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7788:premise", "snli:test:7707:premise", "snli:test:7815:premise", "snli:test:7706:premise", "style:personachat:test:87:self:2:positive:0", "style:personachat:test:87:self:2:positive:1"], "evidence_texts": ["A guy with a football, a crowd watching him.", "A man in a white shirt taking a picture.", "Seven young girls are being creative and making posters.", "A man in a white shirt taking a picture.", "nice ! do you like to try different foods ? i will try at least once .", "that is great , i love seafood myself . what is your favorite drink ?"], "speaker_id": "snli:test:7788", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7795", "candidate_text": "Honestly, A man with a red beard and mustache is smiling and laughing!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7795:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7610:premise", "snli:test:7836:premise", "snli:test:7211:premise", "snli:test:7795:premise", "style:personachat:test:289:self:5:positive:0", "style:personachat:test:289:self:5:positive:1", "style:personachat:test:289:self:5:positive:2", "style:personachat:test:289:self:5:positive:3", "style:personachat:test:289:self:5:positive:4"], "evidence_texts": ["Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "just bodybuilding . people prefer healthy and fit doctors ! you ?", "i wish , but my real job keeps me chained . what about you ?", "that sounds fun ! do you play for a living ?", "yeah . my mom actually works at the front desk ! do you train with your dad ?", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:7795", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7795", "candidate_text": "well, a man with a red beard and mustache is smiling and laughing", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7795:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7610:premise", "snli:test:7836:premise", "snli:test:7211:premise", "snli:test:7795:premise", "style:personachat:test:487:self:5:positive:0", "style:personachat:test:487:self:5:positive:1", "style:personachat:test:487:self:5:positive:2", "style:personachat:test:487:self:5:positive:3", "style:personachat:test:487:self:5:positive:4"], "evidence_texts": ["Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "An asian teenager wearing a white shirt with black stripes is sitting on the steps with his face resting on his hand.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "my kindergarten kids keep me busy .", "enjoy your visit and safe travels .", "with gas prices like they are we all need cars with great mileage .", "did you go to college ? i went to university of mississippi .", "studied environmentalist as my major ."], "speaker_id": "snli:test:7795", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7798", "candidate_text": "Honestly, the women have rounded up the last cow for the fall!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7798:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7488:premise", "snli:test:8126:premise", "snli:test:7798:premise", "snli:test:7282:premise", "style:personachat:test:289:self:7:positive:0", "style:personachat:test:289:self:7:positive:1", "style:personachat:test:289:self:7:positive:2", "style:personachat:test:289:self:7:positive:3", "style:personachat:test:289:self:7:positive:4", "style:personachat:test:289:self:7:positive:5", "style:personachat:test:289:self:7:positive:6"], "evidence_texts": ["A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "fantastic ! this steak really hits the spot ! how are you ?", "all meat is good meat lol . got to keep that protein up !", "just bodybuilding . people prefer healthy and fit doctors ! you ?", "i wish , but my real job keeps me chained . what about you ?", "that sounds fun ! do you play for a living ?", "yeah . my mom actually works at the front desk ! do you train with your dad ?", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:7798", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7798", "candidate_text": "the women have rounded up the last cow for the fall.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7798:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7488:premise", "snli:test:8126:premise", "snli:test:7798:premise", "snli:test:7282:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "A group of Amish people, men and women, standing at a fence overlooking a forest and river.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7798", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7803", "candidate_text": "Honestly, Three Asian men work at a bar!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7803:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7565:premise", "snli:test:8454:premise", "snli:test:7482:premise", "snli:test:7803:premise", "style:personachat:test:290:self:1:positive:0"], "evidence_texts": ["A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:7803", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7803", "candidate_text": "I wonder whether three asian men work at a bar?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7803:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7565:premise", "snli:test:8454:premise", "snli:test:7482:premise", "snli:test:7803:premise", "style:personachat:test:91:self:1:positive:0"], "evidence_texts": ["A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "A young girl with purple pants, a striped jacket, and a white hat is climbing up a ladder made of logs.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "what do you like to do ?"], "speaker_id": "snli:test:7803", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7812", "candidate_text": "Honestly, A beautiful sunset shows the shapes of four figures silhouetted against it's backdrop!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7812:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7812:premise", "snli:test:7771:premise", "snli:test:8185:premise", "snli:test:7415:premise", "style:personachat:test:290:self:2:positive:0", "style:personachat:test:290:self:2:positive:1"], "evidence_texts": ["Four human figures are backlit by a sunset.", "A happy woman who is preparing for dinner.", "Two women are relaxing in a hot tub.", "Rose petals being thrown at an asian couple.", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:7812", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7812", "candidate_text": "well, a beautiful sunset shows the shapes of four figures silhouetted against it's backdrop", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7812:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7812:premise", "snli:test:7771:premise", "snli:test:8185:premise", "snli:test:7415:premise", "style:personachat:test:488:self:2:positive:0", "style:personachat:test:488:self:2:positive:1"], "evidence_texts": ["Four human figures are backlit by a sunset.", "A happy woman who is preparing for dinner.", "Two women are relaxing in a hot tub.", "Rose petals being thrown at an asian couple.", "i run my own business", "hunting ans shooting range 2 things i love"], "speaker_id": "snli:test:7812", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7815", "candidate_text": "Honestly, The girls are using paints and markers on the signs!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7815:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7788:premise", "snli:test:8240:premise", "snli:test:7707:premise", "snli:test:7815:premise", "style:personachat:test:290:self:5:positive:0", "style:personachat:test:290:self:5:positive:1", "style:personachat:test:290:self:5:positive:2", "style:personachat:test:290:self:5:positive:3", "style:personachat:test:290:self:5:positive:4"], "evidence_texts": ["A guy with a football, a crowd watching him.", "Two women bend over to touch a small kangaroo.", "A man in a white shirt taking a picture.", "Seven young girls are being creative and making posters.", "i have never been there ! i love the snow though . it keeps me happy and busy .", "i would not trade the snow for anything . i love winter time .", "well , i have a simple job . but i do like some hard rock .", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:7815", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7815", "candidate_text": "The girls are using paints and markers on the signs.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7815:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7788:premise", "snli:test:8240:premise", "snli:test:7707:premise", "snli:test:7815:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A guy with a football, a crowd watching him.", "Two women bend over to touch a small kangaroo.", "A man in a white shirt taking a picture.", "Seven young girls are being creative and making posters.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7815", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7836", "candidate_text": "Honestly, A young woman wearing a white sleeveless dress lined with red flowers talking on her cellphone as she walks down the street sidewalk outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7836:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7836:premise", "snli:test:7795:premise", "snli:test:8182:premise", "snli:test:7610:premise", "style:personachat:test:290:self:7:positive:0", "style:personachat:test:290:self:7:positive:1", "style:personachat:test:290:self:7:positive:2", "style:personachat:test:290:self:7:positive:3", "style:personachat:test:290:self:7:positive:4", "style:personachat:test:290:self:7:positive:5", "style:personachat:test:290:self:7:positive:6"], "evidence_texts": ["A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "it is awesome . how are you ?", "that does sound exhausting ! i have been outside with my brothers shoveling the driveway !", "i have never been there ! i love the snow though . it keeps me happy and busy .", "i would not trade the snow for anything . i love winter time .", "well , i have a simple job . but i do like some hard rock .", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:7836", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7836", "candidate_text": "I wonder whether a young woman wearing a white sleeveless dress lined with red flowers talking on her cellphone as she walks down the street sidewalk outside?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7836:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7836:premise", "snli:test:7795:premise", "snli:test:8182:premise", "snli:test:7610:premise", "style:personachat:test:91:self:6:positive:0", "style:personachat:test:91:self:6:positive:1", "style:personachat:test:91:self:6:positive:2", "style:personachat:test:91:self:6:positive:3", "style:personachat:test:91:self:6:positive:4", "style:personachat:test:91:self:6:positive:5"], "evidence_texts": ["A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "Two performers, one man playing a guitar, and one woman with a mic raised to her mouth are sitting on chairs performing.", "great ! just coding for apple . and drinking an iced tea", "eh ! iced tea with home grown canada syrup delicious", "red or blue ? coding is so much fun ! how old are you ? i am 22", "i freelance mostly website coding and such .", "cool ! ! any property in canada ? my family is looking to upgrade .", "what do you like to do ?"], "speaker_id": "snli:test:7836", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7848", "candidate_text": "well, cat stares at dog", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7848:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7848:premise", "snli:test:7674:premise", "snli:test:8231:premise", "snli:test:7656:premise", "style:personachat:test:291:self:1:positive:0"], "evidence_texts": ["Some youngsters are playing a card game", "A young redheaded woman laughs at something.", "Customers are ordering food at a cafe.", "Two men renovate a brick wall outside.", "cool . i take a lot of photos of me dancing at the local night clubs ."], "speaker_id": "snli:test:7848", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7848", "candidate_text": "Honestly, cat stares at dog!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7848:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7848:premise", "snli:test:7674:premise", "snli:test:8231:premise", "snli:test:7656:premise", "style:personachat:test:174:self:5:positive:0", "style:personachat:test:174:self:5:positive:1", "style:personachat:test:174:self:5:positive:2", "style:personachat:test:174:self:5:positive:3", "style:personachat:test:174:self:5:positive:4"], "evidence_texts": ["Some youngsters are playing a card game", "A young redheaded woman laughs at something.", "Customers are ordering food at a cafe.", "Two men renovate a brick wall outside.", "that is why i do not ever want kids . where are you from ?", "new york . a great place to be single ! married yourself ?", "came here to be a singer . why did you leave ?", "it is fun . but modeling pays more ! you have hobbies ?", "just the singing right now . taekwondo sounds fun !"], "speaker_id": "snli:test:7848", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7851", "candidate_text": "well, women at work", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7851:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7619:premise", "snli:test:7926:premise", "snli:test:7332:premise", "snli:test:7851:premise", "style:personachat:test:291:self:2:positive:0", "style:personachat:test:291:self:2:positive:1"], "evidence_texts": ["A man is working in an art studio and is surrounded by art supplies.", "A young boy in a straw hat drinks out of a plastic pink cup.", "A little boy and his father talking to a man dressed as Santa Claws.", "A woman in a white tennis outfit is jumping up to hit the ball.", "oh , i would not say i am a narcissists , just like sharing my adventures .", "cool . i take a lot of photos of me dancing at the local night clubs ."], "speaker_id": "snli:test:7851", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7851", "candidate_text": "women at work.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7851:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7619:premise", "snli:test:7926:premise", "snli:test:7332:premise", "snli:test:7851:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man is working in an art studio and is surrounded by art supplies.", "A young boy in a straw hat drinks out of a plastic pink cup.", "A little boy and his father talking to a man dressed as Santa Claws.", "A woman in a white tennis outfit is jumping up to hit the ball.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7851", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7859", "candidate_text": "well, a team of construction workers are hurrying to get back on schedule", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7859:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7579:premise", "snli:test:7499:premise", "snli:test:7880:premise", "snli:test:7859:premise", "style:personachat:test:291:self:5:positive:0", "style:personachat:test:291:self:5:positive:1", "style:personachat:test:291:self:5:positive:2", "style:personachat:test:291:self:5:positive:3", "style:personachat:test:291:self:5:positive:4"], "evidence_texts": ["A black and white dog jumps for a tennis ball.", "A guy in a red jacket is snowboarding in midair.", "A man wearing a brown hoodie is shopping for snacks.", "a group of construction workers busy refinishing a subway station.", "oh . i train dogs . i get to work indoors , a bit warmer .", "boy , you really like hunting and fishing . not me , sweets i love .", "you really like cold weather . take pictures of yourself while exploring .", "oh , i would not say i am a narcissists , just like sharing my adventures .", "cool . i take a lot of photos of me dancing at the local night clubs ."], "speaker_id": "snli:test:7859", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7859", "candidate_text": "I wonder whether a team of construction workers are hurrying to get back on schedule?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7859:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7579:premise", "snli:test:7499:premise", "snli:test:7880:premise", "snli:test:7859:premise", "style:personachat:test:102:self:5:positive:0", "style:personachat:test:102:self:5:positive:1", "style:personachat:test:102:self:5:positive:2", "style:personachat:test:102:self:5:positive:3", "style:personachat:test:102:self:5:positive:4"], "evidence_texts": ["A black and white dog jumps for a tennis ball.", "A guy in a red jacket is snowboarding in midair.", "A man wearing a brown hoodie is shopping for snacks.", "a group of construction workers busy refinishing a subway station.", "hard to be bias when grading multiple choice with a definitive answer", "i teach a kindergarten class in seattle", "it is difficult but it is my passion . passion has trouble paying the bills though", "you are a vet ? i think my cat is getting sick .", "oh you are a veteran ?"], "speaker_id": "snli:test:7859", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7860", "candidate_text": "well, a construction team working in a subway station", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7860:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7860:premise", "snli:test:7880:premise", "snli:test:7579:premise", "snli:test:7984:premise", "style:personachat:test:291:self:7:positive:0", "style:personachat:test:291:self:7:positive:1", "style:personachat:test:291:self:7:positive:2", "style:personachat:test:291:self:7:positive:3", "style:personachat:test:291:self:7:positive:4", "style:personachat:test:291:self:7:positive:5", "style:personachat:test:291:self:7:positive:6"], "evidence_texts": ["a group of construction workers busy refinishing a subway station.", "A man wearing a brown hoodie is shopping for snacks.", "A black and white dog jumps for a tennis ball.", "Three military officials are battling a fire my a building.", "well the weather definitely is good for fishing .", "oh , ice fishing . you better bundle up for that and hope for cold weather .", "oh . i train dogs . i get to work indoors , a bit warmer .", "boy , you really like hunting and fishing . not me , sweets i love .", "you really like cold weather . take pictures of yourself while exploring .", "oh , i would not say i am a narcissists , just like sharing my adventures .", "cool . i take a lot of photos of me dancing at the local night clubs ."], "speaker_id": "snli:test:7860", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7860", "candidate_text": "Honestly, A construction team working in a subway station!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7860:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7860:premise", "snli:test:7880:premise", "snli:test:7579:premise", "snli:test:7984:premise", "style:personachat:test:182:self:2:positive:0", "style:personachat:test:182:self:2:positive:1"], "evidence_texts": ["a group of construction workers busy refinishing a subway station.", "A man wearing a brown hoodie is shopping for snacks.", "A black and white dog jumps for a tennis ball.", "Three military officials are battling a fire my a building.", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:7860", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7861", "candidate_text": "I wonder whether a man is a good skateboarder?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7861:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7637:premise", "snli:test:7861:premise", "snli:test:8026:premise", "snli:test:7543:premise", "style:personachat:test:292:self:1:positive:0"], "evidence_texts": ["an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:7861", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7861", "candidate_text": "Honestly, A man is a good skateboarder!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7861:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7637:premise", "snli:test:7861:premise", "snli:test:8026:premise", "snli:test:7543:premise", "style:personachat:test:182:self:5:positive:0", "style:personachat:test:182:self:5:positive:1", "style:personachat:test:182:self:5:positive:2", "style:personachat:test:182:self:5:positive:3", "style:personachat:test:182:self:5:positive:4"], "evidence_texts": ["an older man wearing a suit has his arms folded behind his back and looking in a store window.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "The basketball player in the blue uniform is guarding the player in the orange uniform, who has the ball.", "you should ! i moved to the states to become a stock broker but i miss it .", "one of my sisters is in fashion . i have a brother too but he is a doctor .", "are you close to your siblings ?", "my sister and i go out for italian food all the time . i love eggplant parmigiana .", "ha ha ! i love tacos , too . i love mexican food as much as italian ."], "speaker_id": "snli:test:7861", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7880", "candidate_text": "I wonder whether the man is wearing a red shirt?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7880:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7880:premise", "snli:test:7860:premise", "snli:test:7984:premise", "snli:test:7859:premise", "style:personachat:test:292:self:2:positive:0", "style:personachat:test:292:self:2:positive:1"], "evidence_texts": ["A man wearing a brown hoodie is shopping for snacks.", "a group of construction workers busy refinishing a subway station.", "Three military officials are battling a fire my a building.", "a group of construction workers busy refinishing a subway station.", "i am not married . it would be hard to have kids while still in college .", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:7880", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7880", "candidate_text": "The man is wearing a red shirt.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7880:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7880:premise", "snli:test:7860:premise", "snli:test:7984:premise", "snli:test:7859:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man wearing a brown hoodie is shopping for snacks.", "a group of construction workers busy refinishing a subway station.", "Three military officials are battling a fire my a building.", "a group of construction workers busy refinishing a subway station.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7880", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7887", "candidate_text": "I wonder whether a woman is sitting on a train?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7887:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7473:premise", "snli:test:7971:premise", "snli:test:6671:premise", "snli:test:7887:premise", "style:personachat:test:292:self:5:positive:0", "style:personachat:test:292:self:5:positive:1", "style:personachat:test:292:self:5:positive:2", "style:personachat:test:292:self:5:positive:3", "style:personachat:test:292:self:5:positive:4"], "evidence_texts": ["A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "that sounds awesome ! are you working anywhere now ? are you in college ?", "cool ! i am retired from the banking business . do you have any hobbies ?", "wow ! i like to make quilts for my relatives in my free time .", "i am not married . it would be hard to have kids while still in college .", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:7887", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7887", "candidate_text": "well, a woman is sitting on a train", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7887:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7473:premise", "snli:test:7971:premise", "snli:test:6671:premise", "snli:test:7887:premise", "style:personachat:test:491:self:5:positive:0", "style:personachat:test:491:self:5:positive:1", "style:personachat:test:491:self:5:positive:2", "style:personachat:test:491:self:5:positive:3", "style:personachat:test:491:self:5:positive:4"], "evidence_texts": ["A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "A woman cook in an apron is smiling at the camera with two other cooks in the background.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "oh yeah ? the thing i love is dancing .", "that is too bad . do you live alone ? i live with my sister .", "ah . i also live with my dog ralph . are your parents still alive ? mine have passed away .", "that is good . are you young ? i am . i am a mixed race female .", "yes i am . i am a young girl ."], "speaker_id": "snli:test:7887", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7896", "candidate_text": "I wonder whether a man is running errands on his bike?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7896:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7896:premise", "snli:test:7732:premise", "snli:test:8191:premise", "snli:test:7625:premise", "style:personachat:test:292:self:6:positive:0", "style:personachat:test:292:self:6:positive:1", "style:personachat:test:292:self:6:positive:2", "style:personachat:test:292:self:6:positive:3", "style:personachat:test:292:self:6:positive:4", "style:personachat:test:292:self:6:positive:5"], "evidence_texts": ["A woman is taking a picture on the shoulders of another woman in cold weather.", "A young child climbing a stack of logs even though the sign warns against it.", "Phone to her ear, a woman bends forward at the side of a busy street.", "A person dressed in black and red protective gear drives an ATV on a track.", "i am good , how are you ?", "that sounds awesome ! are you working anywhere now ? are you in college ?", "cool ! i am retired from the banking business . do you have any hobbies ?", "wow ! i like to make quilts for my relatives in my free time .", "i am not married . it would be hard to have kids while still in college .", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:7896", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7896", "candidate_text": "Honestly, A man is running errands on his bike!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7896:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7896:premise", "snli:test:7732:premise", "snli:test:8191:premise", "snli:test:7625:premise", "style:personachat:test:185:self:2:positive:0", "style:personachat:test:185:self:2:positive:1"], "evidence_texts": ["A woman is taking a picture on the shoulders of another woman in cold weather.", "A young child climbing a stack of logs even though the sign warns against it.", "Phone to her ear, a woman bends forward at the side of a busy street.", "A person dressed in black and red protective gear drives an ATV on a track.", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:7896", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7899", "candidate_text": "well, a militant chinese group lead a protest with red signs", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7899:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7779:premise", "snli:test:8210:premise", "snli:test:7777:premise", "snli:test:7899:premise", "style:personachat:test:293:self:1:positive:0"], "evidence_texts": ["The man in the black hat has his back to a theater sign.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "when they are older you will be so proud of them"], "speaker_id": "snli:test:7899", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7899", "candidate_text": "I wonder whether a militant chinese group lead a protest with red signs?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7899:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7779:premise", "snli:test:8210:premise", "snli:test:7777:premise", "snli:test:7899:premise", "style:personachat:test:109:self:1:positive:0"], "evidence_texts": ["The man in the black hat has his back to a theater sign.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:7899", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7926", "candidate_text": "well, the kids were in awe as they listened pasionately to the story of the young boy", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7926:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7851:premise", "snli:test:7929:premise", "snli:test:7926:premise", "snli:test:7619:premise", "style:personachat:test:293:self:2:positive:0", "style:personachat:test:293:self:2:positive:1"], "evidence_texts": ["A woman in a white tennis outfit is jumping up to hit the ball.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "A young boy in a straw hat drinks out of a plastic pink cup.", "A man is working in an art studio and is surrounded by art supplies.", "anything in particular that you like to do with your kids ?", "when they are older you will be so proud of them"], "speaker_id": "snli:test:7926", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7926", "candidate_text": "Honestly, The kids were in awe as they listened pasionately to the story of the young boy!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7926:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7851:premise", "snli:test:7929:premise", "snli:test:7926:premise", "snli:test:7619:premise", "style:personachat:test:185:self:6:positive:0", "style:personachat:test:185:self:6:positive:1", "style:personachat:test:185:self:6:positive:2", "style:personachat:test:185:self:6:positive:3", "style:personachat:test:185:self:6:positive:4", "style:personachat:test:185:self:6:positive:5"], "evidence_texts": ["A woman in a white tennis outfit is jumping up to hit the ball.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "A young boy in a straw hat drinks out of a plastic pink cup.", "A man is working in an art studio and is surrounded by art supplies.", "i am a middle schooler who absolutely loves pink floyd !", "that is sick ! i want to act when i am an adult", "sick as in cool ! i am too shy to start now . i just came out as trans", "good luck to you too ! do you like animals ?", "i am not a fan of cats but i love dogs .", "i do sing , just not well ! lol"], "speaker_id": "snli:test:7926", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7929", "candidate_text": "well, the man has pants on", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7929:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7926:premise", "snli:test:7929:premise", "snli:test:7997:premise", "snli:test:7851:premise", "style:personachat:test:293:self:5:positive:0", "style:personachat:test:293:self:5:positive:1", "style:personachat:test:293:self:5:positive:2", "style:personachat:test:293:self:5:positive:3", "style:personachat:test:293:self:5:positive:4"], "evidence_texts": ["A young boy in a straw hat drinks out of a plastic pink cup.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "An artist replicating another artist's work and a man looking at the original art.", "A woman in a white tennis outfit is jumping up to hit the ball.", "cool , how many kids do you have ?", "i bet . i am looking forward to parenthood", "i want to having just two . hopefully i get a boy and girl", "anything in particular that you like to do with your kids ?", "when they are older you will be so proud of them"], "speaker_id": "snli:test:7929", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7929", "candidate_text": "The man has pants on.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7929:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7926:premise", "snli:test:7929:premise", "snli:test:7997:premise", "snli:test:7851:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A young boy in a straw hat drinks out of a plastic pink cup.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "An artist replicating another artist's work and a man looking at the original art.", "A woman in a white tennis outfit is jumping up to hit the ball.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7929", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7971", "candidate_text": "well, a runner with a convoy of motorcycles", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7971:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7887:premise", "snli:test:8018:premise", "snli:test:7473:premise", "snli:test:7971:premise", "style:personachat:test:293:self:7:positive:0", "style:personachat:test:293:self:7:positive:1", "style:personachat:test:293:self:7:positive:2", "style:personachat:test:293:self:7:positive:3", "style:personachat:test:293:self:7:positive:4", "style:personachat:test:293:self:7:positive:5", "style:personachat:test:293:self:7:positive:6"], "evidence_texts": ["A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "good just left a bluegrass concert", "skate . i love it . what do you do ?", "cool , how many kids do you have ?", "i bet . i am looking forward to parenthood", "i want to having just two . hopefully i get a boy and girl", "anything in particular that you like to do with your kids ?", "when they are older you will be so proud of them"], "speaker_id": "snli:test:7971", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7971", "candidate_text": "I wonder whether a runner with a convoy of motorcycles?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7971:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7887:premise", "snli:test:8018:premise", "snli:test:7473:premise", "snli:test:7971:premise", "style:personachat:test:109:self:7:positive:0", "style:personachat:test:109:self:7:positive:1", "style:personachat:test:109:self:7:positive:2", "style:personachat:test:109:self:7:positive:3", "style:personachat:test:109:self:7:positive:4", "style:personachat:test:109:self:7:positive:5", "style:personachat:test:109:self:7:positive:6"], "evidence_texts": ["A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A young child dressed in a scarf, hat, jacket, gloves, pants, and boots, outside playing in the snow.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "doing great ! i finally got rid of the box elder bugs in my front yard . yuck .", "me too ! they just make me want to scream and run . fast !", "yes . four . they eat all of the bacon in the house , makes me sad . you ?", "too funny . i will have to learn your tricks . do you have a favorite food ?", "anything but bug green . you must have a favorite .", "you are funny . listening to bach , eating bacon , without bugs . ideal !", "love them . romantic suspense is my favorite genre . do you like reading ?"], "speaker_id": "snli:test:7971", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:7976", "candidate_text": "well, a man is working at a car wash", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7976:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7976:premise", "snli:test:7679:premise", "snli:test:8021:premise", "snli:test:7667:premise", "style:personachat:test:294:self:1:positive:0"], "evidence_texts": ["A shoeshine man works as others pass by on a busy sidewalk.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "A runner in a black and blue uniform competes in a race.", "A man on stilts is performing a magical trick while onlookers enjoy.", "i am thinking about florida state ."], "speaker_id": "snli:test:7976", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7976", "candidate_text": "Honestly, A man is working at a car wash!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:7976:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7976:premise", "snli:test:7679:premise", "snli:test:8021:premise", "snli:test:7667:premise", "style:personachat:test:188:self:5:positive:0", "style:personachat:test:188:self:5:positive:1", "style:personachat:test:188:self:5:positive:2", "style:personachat:test:188:self:5:positive:3", "style:personachat:test:188:self:5:positive:4"], "evidence_texts": ["A shoeshine man works as others pass by on a busy sidewalk.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "A runner in a black and blue uniform competes in a race.", "A man on stilts is performing a magical trick while onlookers enjoy.", "where do you get the tattoos ?", "i live in seattle with two derelicts . they drink all the time .", "do you have a southern accent ?", "because my lousy kindergarten salary limits where i can live .", "i want to be a brain surgeon . i would operate on myself first !"], "speaker_id": "snli:test:7976", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7984", "candidate_text": "well, three military offcials battle a fire after an accidental explosion", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7984:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7984:premise", "snli:test:7880:premise", "snli:test:7991:premise", "snli:test:7860:premise", "style:personachat:test:294:self:2:positive:0", "style:personachat:test:294:self:2:positive:1"], "evidence_texts": ["Three military officials are battling a fire my a building.", "A man wearing a brown hoodie is shopping for snacks.", "A man with thinning hair looking into glasses through glasses.", "a group of construction workers busy refinishing a subway station.", "very nice ! i want to be a biologist . where do you go to school ?", "i am thinking about florida state ."], "speaker_id": "snli:test:7984", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7984", "candidate_text": "Three military offcials battle a fire after an accidental explosion.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7984:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7984:premise", "snli:test:7880:premise", "snli:test:7991:premise", "snli:test:7860:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three military officials are battling a fire my a building.", "A man wearing a brown hoodie is shopping for snacks.", "A man with thinning hair looking into glasses through glasses.", "a group of construction workers busy refinishing a subway station.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7984", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7988", "candidate_text": "well, the man is a performer", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7988:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7988:premise", "snli:test:7757:premise", "snli:test:8047:premise", "snli:test:7660:premise", "style:personachat:test:294:self:5:positive:0", "style:personachat:test:294:self:5:positive:1", "style:personachat:test:294:self:5:positive:2", "style:personachat:test:294:self:5:positive:3", "style:personachat:test:294:self:5:positive:4"], "evidence_texts": ["A black-haired man is entertaining a crowd with a hula hoop.", "A woman and a young girl are eating side by side.", "Small girl in an apron placing confetti in a woman's hair.", "A brown dog sniffing at the fence of a brown bull.", "i do , tupac is my favorite artist . i am building a robot that looks like him .", "no , lol . only a teen girl with blue hair who loves cosplay .", "it truly is . we do a lot of things like that in my homeschool group .", "very nice ! i want to be a biologist . where do you go to school ?", "i am thinking about florida state ."], "speaker_id": "snli:test:7988", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7988", "candidate_text": "I wonder whether the man is a performer?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7988:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7988:premise", "snli:test:7757:premise", "snli:test:8047:premise", "snli:test:7660:premise", "style:personachat:test:111:self:5:positive:0", "style:personachat:test:111:self:5:positive:1", "style:personachat:test:111:self:5:positive:2", "style:personachat:test:111:self:5:positive:3", "style:personachat:test:111:self:5:positive:4"], "evidence_texts": ["A black-haired man is entertaining a crowd with a hula hoop.", "A woman and a young girl are eating side by side.", "Small girl in an apron placing confetti in a woman's hair.", "A brown dog sniffing at the fence of a brown bull.", "you should ! just read a book this weekend at my brothers wedding .", "yeah . i have weddings . i do love playing board games though .", "sad . you like cats ? i have one pet called arizona .", "three . do you have a cat or dog ?", "ok . so what do you do for fun ?"], "speaker_id": "snli:test:7988", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7991", "candidate_text": "well, a man with glasses is looking at drinking glasses", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7991:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7984:premise", "snli:test:8075:premise", "snli:test:7880:premise", "snli:test:7991:premise", "style:personachat:test:294:self:7:positive:0", "style:personachat:test:294:self:7:positive:1", "style:personachat:test:294:self:7:positive:2", "style:personachat:test:294:self:7:positive:3", "style:personachat:test:294:self:7:positive:4", "style:personachat:test:294:self:7:positive:5", "style:personachat:test:294:self:7:positive:6"], "evidence_texts": ["Three military officials are battling a fire my a building.", "A woman in black is getting ready in her bathroom.", "A man wearing a brown hoodie is shopping for snacks.", "A man with thinning hair looking into glasses through glasses.", "yes , i have . on a hike for my homeschool biology class , to a waterfall .", "when i can , i live in northern cali .", "i do , tupac is my favorite artist . i am building a robot that looks like him .", "no , lol . only a teen girl with blue hair who loves cosplay .", "it truly is . we do a lot of things like that in my homeschool group .", "very nice ! i want to be a biologist . where do you go to school ?", "i am thinking about florida state ."], "speaker_id": "snli:test:7991", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:7991", "candidate_text": "Honestly, A man with glasses is looking at drinking glasses!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7991:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7984:premise", "snli:test:8075:premise", "snli:test:7880:premise", "snli:test:7991:premise", "style:personachat:test:200:self:2:positive:0", "style:personachat:test:200:self:2:positive:1"], "evidence_texts": ["Three military officials are battling a fire my a building.", "A woman in black is getting ready in her bathroom.", "A man wearing a brown hoodie is shopping for snacks.", "A man with thinning hair looking into glasses through glasses.", "how do you feel about halloween ? i like it as well . it is fun !", "it can be ! especially the movies that come out . i love some though ."], "speaker_id": "snli:test:7991", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:7997", "candidate_text": "Honestly, A person is crafting a replica of another's creation!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7997:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7929:premise", "snli:test:7997:premise", "snli:test:8068:premise", "snli:test:7926:premise", "style:personachat:test:295:self:1:positive:0"], "evidence_texts": ["A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "An artist replicating another artist's work and a man looking at the original art.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A young boy in a straw hat drinks out of a plastic pink cup.", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:7997", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:7997", "candidate_text": "A person is crafting a replica of another's creation.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:7997:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7929:premise", "snli:test:7997:premise", "snli:test:8068:premise", "snli:test:7926:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "An artist replicating another artist's work and a man looking at the original art.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A young boy in a straw hat drinks out of a plastic pink cup.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:7997", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8005", "candidate_text": "Honestly, a bunch of eagles are eating a bug!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8005:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7742:premise", "snli:test:8005:premise", "snli:test:7524:premise", "snli:test:8051:premise", "style:personachat:test:295:self:2:positive:0", "style:personachat:test:295:self:2:positive:1"], "evidence_texts": ["A little girl in a pink hat is in a lush green field walking an ox.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A man wearing a hat sitting on the side of a hill looking towards the water.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:8005", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8005", "candidate_text": "I wonder whether a bunch of eagles are eating a bug?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8005:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7742:premise", "snli:test:8005:premise", "snli:test:7524:premise", "snli:test:8051:premise", "style:personachat:test:113:self:2:positive:0", "style:personachat:test:113:self:2:positive:1"], "evidence_texts": ["A little girl in a pink hat is in a lush green field walking an ox.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A man wearing a hat sitting on the side of a hill looking towards the water.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "what kind of engineer are you ?", "i help my husband run his business . what language do other engineers speak ?"], "speaker_id": "snli:test:8005", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8006", "candidate_text": "Honestly, a bunch of guys in colorful clothes are standing together waiting for the bus!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8006:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8051:premise", "snli:test:7742:premise", "snli:test:8006:premise", "snli:test:8223:premise", "style:personachat:test:295:self:5:positive:0", "style:personachat:test:295:self:5:positive:1", "style:personachat:test:295:self:5:positive:2", "style:personachat:test:295:self:5:positive:3", "style:personachat:test:295:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "A little girl in a pink hat is in a lush green field walking an ox.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "i can hook you up . do you like music ? my fav band is rush ? awesome .", "big question , what does an omnivore do at a vegans wedding ? my challenge last weekend !", "i like to pimp out my clients wheelchairs , what would your brother like ?", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:8006", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8006", "candidate_text": "well, a bunch of guys in colorful clothes are standing together waiting for the bus", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8006:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8051:premise", "snli:test:7742:premise", "snli:test:8006:premise", "snli:test:8223:premise", "style:personachat:test:494:self:5:positive:0", "style:personachat:test:494:self:5:positive:1", "style:personachat:test:494:self:5:positive:2", "style:personachat:test:494:self:5:positive:3", "style:personachat:test:494:self:5:positive:4"], "evidence_texts": ["A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "A little girl in a pink hat is in a lush green field walking an ox.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "yes it is . what do you usually eat for breakfast ?", "it is 2 years . i love to eat granola for breakfast , every day .", "that is great . i have been saving up for some vintage 50s decorations lately .", "i also have exotic fish .", "i love tangs . they are so pretty"], "speaker_id": "snli:test:8006", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8018", "candidate_text": "Honestly, The boys are playing dodge ball!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8018:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7971:premise", "snli:test:8054:premise", "snli:test:8018:premise", "snli:test:7887:premise", "style:personachat:test:295:self:7:positive:0", "style:personachat:test:295:self:7:positive:1", "style:personachat:test:295:self:7:positive:2", "style:personachat:test:295:self:7:positive:3", "style:personachat:test:295:self:7:positive:4", "style:personachat:test:295:self:7:positive:5", "style:personachat:test:295:self:7:positive:6"], "evidence_texts": ["A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "i just got home from work , i do graphic design .", "nope , my real job is wheelchair sales and design is my hobby , i freelance .", "i can hook you up . do you like music ? my fav band is rush ? awesome .", "big question , what does an omnivore do at a vegans wedding ? my challenge last weekend !", "i like to pimp out my clients wheelchairs , what would your brother like ?", "i am going to get dinner soon , i need meat , too many veggies last weekend .", "i can do lights , turbo charge the wheels too . he will love his new ride !"], "speaker_id": "snli:test:8018", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8018", "candidate_text": "The boys are playing dodge ball.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8018:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7971:premise", "snli:test:8054:premise", "snli:test:8018:premise", "snli:test:7887:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A woman in black walks past a shop selling a t-shirt depicting a tattooed woman smoking a cigarette.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8018", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8021", "candidate_text": "well, he doesnt have legs", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8021:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7976:premise", "snli:test:8021:premise", "snli:test:8115:premise", "snli:test:7679:premise", "style:personachat:test:296:self:1:positive:0"], "evidence_texts": ["A shoeshine man works as others pass by on a busy sidewalk.", "A runner in a black and blue uniform competes in a race.", "Man sitting on bench with suitcase waiting for his source of transportation.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "i am just tired and want to retire soon"], "speaker_id": "snli:test:8021", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8021", "candidate_text": "I wonder whether he doesnt have legs?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8021:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7976:premise", "snli:test:8021:premise", "snli:test:8115:premise", "snli:test:7679:premise", "style:personachat:test:115:self:1:positive:0"], "evidence_texts": ["A shoeshine man works as others pass by on a busy sidewalk.", "A runner in a black and blue uniform competes in a race.", "Man sitting on bench with suitcase waiting for his source of transportation.", "A young boy gets his photo taken with Mr. and Mrs. Clause.", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:8021", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8026", "candidate_text": "well, a girl checks her phone on a boat", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8026:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7861:premise", "snli:test:8365:premise", "snli:test:8026:premise", "snli:test:7637:premise", "style:personachat:test:296:self:2:positive:0", "style:personachat:test:296:self:2:positive:1"], "evidence_texts": ["A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "yeah remember the horses ? i want to retire though", "i am just tired and want to retire soon"], "speaker_id": "snli:test:8026", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8026", "candidate_text": "Honestly, A girl checks her phone on a boat!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8026:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7861:premise", "snli:test:8365:premise", "snli:test:8026:premise", "snli:test:7637:premise", "style:personachat:test:209:self:7:positive:0", "style:personachat:test:209:self:7:positive:1", "style:personachat:test:209:self:7:positive:2", "style:personachat:test:209:self:7:positive:3", "style:personachat:test:209:self:7:positive:4", "style:personachat:test:209:self:7:positive:5", "style:personachat:test:209:self:7:positive:6"], "evidence_texts": ["A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "an older man wearing a suit has his arms folded behind his back and looking in a store window.", "i love to listen to bach and compose songs in english . you ?", "very nice ! i love bacon but i am so afraid it will draw in insects .", "no feet , i am sorry ! i am tall english speaking adult !", "that is fun ! do you work in the sea too ?", "yes , as an engineer . do you have pets ?", "i want one , but i think their food will bring in insects .", "i am terrified of them ! !"], "speaker_id": "snli:test:8026", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8036", "candidate_text": "well, three people with instruments", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8036:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8036:premise", "snli:test:7714:premise", "snli:test:8213:premise", "snli:test:7417:premise", "style:personachat:test:296:self:5:positive:0", "style:personachat:test:296:self:5:positive:1", "style:personachat:test:296:self:5:positive:2", "style:personachat:test:296:self:5:positive:3", "style:personachat:test:296:self:5:positive:4"], "evidence_texts": ["Three men play musical instruments.", "Child swimming in a pool.", "An SUV traveling through mud.", "Two friends having a laugh.", "i do not but i love facial hair", "i ride my horses it is nice", "nice are you retired yet ?", "yeah remember the horses ? i want to retire though", "i am just tired and want to retire soon"], "speaker_id": "snli:test:8036", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8036", "candidate_text": "Three people with instruments.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8036:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8036:premise", "snli:test:7714:premise", "snli:test:8213:premise", "snli:test:7417:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Three men play musical instruments.", "Child swimming in a pool.", "An SUV traveling through mud.", "Two friends having a laugh.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8036", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8047", "candidate_text": "well, a woman and young girl celebrating together", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8047:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7988:premise", "snli:test:8136:premise", "snli:test:7757:premise", "snli:test:8047:premise", "style:personachat:test:296:self:6:positive:0", "style:personachat:test:296:self:6:positive:1", "style:personachat:test:296:self:6:positive:2", "style:personachat:test:296:self:6:positive:3", "style:personachat:test:296:self:6:positive:4", "style:personachat:test:296:self:6:positive:5"], "evidence_texts": ["A black-haired man is entertaining a crowd with a hula hoop.", "A man in gray on a rocky cliff, overlooking the mountains.", "A woman and a young girl are eating side by side.", "Small girl in an apron placing confetti in a woman's hair.", "i am good do you have facial hair ?", "i do not but i love facial hair", "i ride my horses it is nice", "nice are you retired yet ?", "yeah remember the horses ? i want to retire though", "i am just tired and want to retire soon"], "speaker_id": "snli:test:8047", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8047", "candidate_text": "I wonder whether a woman and young girl celebrating together?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8047:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:7988:premise", "snli:test:8136:premise", "snli:test:7757:premise", "snli:test:8047:premise", "style:personachat:test:115:self:7:positive:0", "style:personachat:test:115:self:7:positive:1", "style:personachat:test:115:self:7:positive:2", "style:personachat:test:115:self:7:positive:3", "style:personachat:test:115:self:7:positive:4", "style:personachat:test:115:self:7:positive:5", "style:personachat:test:115:self:7:positive:6"], "evidence_texts": ["A black-haired man is entertaining a crowd with a hula hoop.", "A man in gray on a rocky cliff, overlooking the mountains.", "A woman and a young girl are eating side by side.", "Small girl in an apron placing confetti in a woman's hair.", "hello ! i am fine what about you ?", "who plays ? i do not watch soccer at all .", "that is good . my cousin loves to play soccer .", "i am single , my ex bf getting married . congratulations for you", "i hope so . break up is bad , specially when you love someone .", "yes i am working but still he is in my mind .", "i love volleyball . where do you live ?"], "speaker_id": "snli:test:8047", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8051", "candidate_text": "well, a man bakes a cake", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8051:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8006:premise", "snli:test:8223:premise", "snli:test:8005:premise", "snli:test:8051:premise", "style:personachat:test:297:self:1:positive:0"], "evidence_texts": ["A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "you make good money as a rn too . yes happy wife equals happy life"], "speaker_id": "snli:test:8051", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8051", "candidate_text": "Honestly, A man bakes a cake!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8051:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8006:premise", "snli:test:8223:premise", "snli:test:8005:premise", "snli:test:8051:premise", "style:personachat:test:212:self:5:positive:0", "style:personachat:test:212:self:5:positive:1", "style:personachat:test:212:self:5:positive:2", "style:personachat:test:212:self:5:positive:3", "style:personachat:test:212:self:5:positive:4"], "evidence_texts": ["A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "what is your favorite color ? mine is purple .", "i do see things differently ! what do you do for work ?", "i help people everyday in a hospital .", "i volunteer at the animal shelter what about you ?", "i love superheroes ! ! are you going to dress up for halloween ?"], "speaker_id": "snli:test:8051", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8054", "candidate_text": "well, a man is pointing into the foot and wearing a striped shirt", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8054:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8018:premise", "snli:test:8433:premise", "snli:test:8054:premise", "snli:test:7971:premise", "style:personachat:test:297:self:2:positive:0", "style:personachat:test:297:self:2:positive:1"], "evidence_texts": ["A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "i hope so and my girlfriend hopes too . she likes money", "you make good money as a rn too . yes happy wife equals happy life"], "speaker_id": "snli:test:8054", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8054", "candidate_text": "A man is pointing into the foot and wearing a striped shirt.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8054:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8018:premise", "snli:test:8433:premise", "snli:test:8054:premise", "snli:test:7971:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A runner in yellow has a convoy of motorcycles following behind him on a highway as bystanders watch.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8054", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8068", "candidate_text": "well, horse rider in traditional wear, ride horse with ceremonial wear and accessories", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8068:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8068:premise", "snli:test:7997:premise", "snli:test:7929:premise", "snli:test:7926:premise", "style:personachat:test:297:self:5:positive:0", "style:personachat:test:297:self:5:positive:1", "style:personachat:test:297:self:5:positive:2", "style:personachat:test:297:self:5:positive:3", "style:personachat:test:297:self:5:positive:4"], "evidence_texts": ["Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "An artist replicating another artist's work and a man looking at the original art.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "A young boy in a straw hat drinks out of a plastic pink cup.", "i know an unmarried catholic priest that had girlfriends living with him", "he makes a lot of money doing that but i want to be a lawyer", "i am a student now but one day soon i hope to be a lawyer", "i hope so and my girlfriend hopes too . she likes money", "you make good money as a rn too . yes happy wife equals happy life"], "speaker_id": "snli:test:8068", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8068", "candidate_text": "I wonder whether horse rider in traditional wear, ride horse with ceremonial wear and accessories?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8068:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8068:premise", "snli:test:7997:premise", "snli:test:7929:premise", "snli:test:7926:premise", "style:personachat:test:117:self:5:positive:0", "style:personachat:test:117:self:5:positive:1", "style:personachat:test:117:self:5:positive:2", "style:personachat:test:117:self:5:positive:3", "style:personachat:test:117:self:5:positive:4"], "evidence_texts": ["Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "An artist replicating another artist's work and a man looking at the original art.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "A young boy in a straw hat drinks out of a plastic pink cup.", "oh , okay . i am online a lot , and a furry .", "that is a kid doc , right ?", "is it interesting ? i am not around kids , they freak me out .", "my sister has a couple , and they are horrid . thankfully , she lives in london .", "do you have any hobbies ?"], "speaker_id": "snli:test:8068", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8071", "candidate_text": "well, the horse is asleep next to the man", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8071:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7997:premise", "snli:test:8176:premise", "snli:test:7929:premise", "snli:test:8071:premise", "style:personachat:test:297:self:7:positive:0", "style:personachat:test:297:self:7:positive:1", "style:personachat:test:297:self:7:positive:2", "style:personachat:test:297:self:7:positive:3", "style:personachat:test:297:self:7:positive:4", "style:personachat:test:297:self:7:positive:5", "style:personachat:test:297:self:7:positive:6"], "evidence_texts": ["An artist replicating another artist's work and a man looking at the original art.", "People boating on a lake with the sun through the clouds in the distance.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "i am good now that my girlfriend moved in with me . how are you", "my dog max likes her too so its all good . some catholics live together", "i know an unmarried catholic priest that had girlfriends living with him", "he makes a lot of money doing that but i want to be a lawyer", "i am a student now but one day soon i hope to be a lawyer", "i hope so and my girlfriend hopes too . she likes money", "you make good money as a rn too . yes happy wife equals happy life"], "speaker_id": "snli:test:8071", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8071", "candidate_text": "Honestly, The horse is asleep next to the man!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8071:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7997:premise", "snli:test:8176:premise", "snli:test:7929:premise", "snli:test:8071:premise", "style:personachat:test:216:self:2:positive:0", "style:personachat:test:216:self:2:positive:1"], "evidence_texts": ["An artist replicating another artist's work and a man looking at the original art.", "People boating on a lake with the sun through the clouds in the distance.", "A man in a leaf-patterned short stand behind a sunflower, smiling at the camera.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:8071", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8073", "candidate_text": "I wonder whether a traditionally dressed horse and rider at a parade?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8073:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8176:premise", "snli:test:8073:premise", "snli:test:8380:premise", "snli:test:7997:premise", "style:personachat:test:298:self:1:positive:0"], "evidence_texts": ["People boating on a lake with the sun through the clouds in the distance.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A brown dog wearing a collar is chasing and biting on a red broom.", "An artist replicating another artist's work and a man looking at the original art.", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:8073", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8073", "candidate_text": "Honestly, A traditionally dressed horse and rider at a parade!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8073:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8176:premise", "snli:test:8073:premise", "snli:test:8380:premise", "snli:test:7997:premise", "style:personachat:test:216:self:5:positive:0", "style:personachat:test:216:self:5:positive:1", "style:personachat:test:216:self:5:positive:2", "style:personachat:test:216:self:5:positive:3", "style:personachat:test:216:self:5:positive:4"], "evidence_texts": ["People boating on a lake with the sun through the clouds in the distance.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A brown dog wearing a collar is chasing and biting on a red broom.", "An artist replicating another artist's work and a man looking at the original art.", "sorry to hear that ! must be hard", "i am hoping retire soon maybe i can write then", "construction , my left leg hurts every day", "just from all years pouring concrete . what kind engineer", "cool ! once i retire finally going shave my beard"], "speaker_id": "snli:test:8073", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8075", "candidate_text": "I wonder whether the woman is getting ready?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8075:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7991:premise", "snli:test:8131:premise", "snli:test:7984:premise", "snli:test:8075:premise", "style:personachat:test:298:self:2:positive:0", "style:personachat:test:298:self:2:positive:1"], "evidence_texts": ["A man with thinning hair looking into glasses through glasses.", "Tan dog running through long grass in a park-like setting.", "Three military officials are battling a fire my a building.", "A woman in black is getting ready in her bathroom.", "i have a passion for rap music", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:8075", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8075", "candidate_text": "The woman is getting ready.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8075:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7991:premise", "snli:test:8131:premise", "snli:test:7984:premise", "snli:test:8075:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man with thinning hair looking into glasses through glasses.", "Tan dog running through long grass in a park-like setting.", "Three military officials are battling a fire my a building.", "A woman in black is getting ready in her bathroom.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8075", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8080", "candidate_text": "I wonder whether the accordion player is highly skilled?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8080:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8080:premise", "snli:test:7684:premise", "snli:test:7349:premise", "snli:test:8096:premise", "style:personachat:test:298:self:5:positive:0", "style:personachat:test:298:self:5:positive:1", "style:personachat:test:298:self:5:positive:2", "style:personachat:test:298:self:5:positive:3", "style:personachat:test:298:self:5:positive:4"], "evidence_texts": ["An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "i really like to be active .", "that is a good way to get together with friends", "i can see that . do you like music ?", "i have a passion for rap music", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:8080", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8080", "candidate_text": "well, the accordion player is highly skilled", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8080:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8080:premise", "snli:test:7684:premise", "snli:test:7349:premise", "snli:test:8096:premise", "style:personachat:test:499:self:5:positive:0", "style:personachat:test:499:self:5:positive:1", "style:personachat:test:499:self:5:positive:2", "style:personachat:test:499:self:5:positive:3", "style:personachat:test:499:self:5:positive:4"], "evidence_texts": ["An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "Six older women are standing in a row at a table smiling and looking trough books and other material displayed on the table.", "A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "no just the living even though my wife keeps dying", "kool web surfer , do you have dogs", "i have two dogs , who i practice styling hair", "i am 38 , i also sound like james bond", "maybe you can , where are you from"], "speaker_id": "snli:test:8080", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8082", "candidate_text": "I wonder whether a young man is walking?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8082:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8096:premise", "snli:test:7684:premise", "snli:test:8082:premise", "snli:test:8553:premise", "style:personachat:test:298:self:7:positive:0", "style:personachat:test:298:self:7:positive:1", "style:personachat:test:298:self:7:positive:2", "style:personachat:test:298:self:7:positive:3", "style:personachat:test:298:self:7:positive:4", "style:personachat:test:298:self:7:positive:5", "style:personachat:test:298:self:7:positive:6"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "pretty good . just studying , you ?", "i love football , do you play ?", "i really like to be active .", "that is a good way to get together with friends", "i can see that . do you like music ?", "i have a passion for rap music", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:8082", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8082", "candidate_text": "Honestly, A young man is walking!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8082:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8096:premise", "snli:test:7684:premise", "snli:test:8082:premise", "snli:test:8553:premise", "style:personachat:test:222:self:2:positive:0", "style:personachat:test:222:self:2:positive:1"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "One lady is looking at a blue box while the shorter lady is looking at something on her phone at the grocery store.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "that sounds fun . i spend my free time on art .", "fortunately i am long on personality because i am short on height !"], "speaker_id": "snli:test:8082", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8096", "candidate_text": "I wonder whether a person is strolling with a bag?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8096:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8096:premise", "snli:test:8082:premise", "snli:test:8553:premise", "snli:test:8080:premise", "style:personachat:test:299:self:1:positive:0"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:8096", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8096", "candidate_text": "A person is strolling with a bag.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8096:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8096:premise", "snli:test:8082:premise", "snli:test:8553:premise", "snli:test:8080:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8096", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8099", "candidate_text": "I wonder whether the man is reading to his grandson?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8099:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:6039:premise", "snli:test:8225:premise", "snli:test:6038:premise", "snli:test:8099:premise", "style:personachat:test:299:self:2:positive:0", "style:personachat:test:299:self:2:positive:1"], "evidence_texts": ["A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "no , i do not have enough time . what is your cats name ?", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:8099", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8099", "candidate_text": "well, the man is reading to his grandson", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8099:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:6039:premise", "snli:test:8225:premise", "snli:test:6038:premise", "snli:test:8099:premise", "style:personachat:test:502:self:2:positive:0", "style:personachat:test:502:self:2:positive:1"], "evidence_texts": ["A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "i hope so . so do you like to listen to music ?", "i enjoy a lot of pop and rock . my favorite is ed sheeran"], "speaker_id": "snli:test:8099", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8115", "candidate_text": "I wonder whether the man is outside?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8115:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8021:premise", "snli:test:8141:premise", "snli:test:7976:premise", "snli:test:8115:premise", "style:personachat:test:299:self:5:positive:0", "style:personachat:test:299:self:5:positive:1", "style:personachat:test:299:self:5:positive:2", "style:personachat:test:299:self:5:positive:3", "style:personachat:test:299:self:5:positive:4"], "evidence_texts": ["A runner in a black and blue uniform competes in a race.", "Six seated men converse and one man fiddles with an interesting contraption.", "A shoeshine man works as others pass by on a busy sidewalk.", "Man sitting on bench with suitcase waiting for his source of transportation.", "hi mia , what are your hobbies ? mine is spending time online .", "how very cool . do you watch tv ? i play on a show called second life .", "oh , i play a furry . tell me more about your cat .", "no , i do not have enough time . what is your cats name ?", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:8115", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8115", "candidate_text": "Honestly, The man is outside!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8115:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8021:premise", "snli:test:8141:premise", "snli:test:7976:premise", "snli:test:8115:premise", "style:personachat:test:226:self:1:positive:0"], "evidence_texts": ["A runner in a black and blue uniform competes in a race.", "Six seated men converse and one man fiddles with an interesting contraption.", "A shoeshine man works as others pass by on a busy sidewalk.", "Man sitting on bench with suitcase waiting for his source of transportation.", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:8115", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8126", "candidate_text": "I wonder whether a man walks down the sidewalk wearing headphones?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8126:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7798:premise", "snli:test:8139:premise", "snli:test:8126:premise", "snli:test:7488:premise", "style:personachat:test:299:self:6:positive:0", "style:personachat:test:299:self:6:positive:1", "style:personachat:test:299:self:6:positive:2", "style:personachat:test:299:self:6:positive:3", "style:personachat:test:299:self:6:positive:4", "style:personachat:test:299:self:6:positive:5"], "evidence_texts": ["Two women are hugging on a path through a grassy area with a cow visible past them.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "a walk sounds good . i am jake 28 and from texas . you ?", "hi mia , what are your hobbies ? mine is spending time online .", "how very cool . do you watch tv ? i play on a show called second life .", "oh , i play a furry . tell me more about your cat .", "no , i do not have enough time . what is your cats name ?", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:8126", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8126", "candidate_text": "A man walks down the sidewalk wearing headphones.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8126:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7798:premise", "snli:test:8139:premise", "snli:test:8126:premise", "snli:test:7488:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two women are hugging on a path through a grassy area with a cow visible past them.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "A woman in a blue uniform, with long curly red-hair, looking at her horse before the race.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8126", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8131", "candidate_text": "well, a man chops veggies", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8131:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8075:premise", "snli:test:7991:premise", "snli:test:8310:premise", "snli:test:8131:premise", "style:personachat:test:300:self:1:positive:0"], "evidence_texts": ["A woman in black is getting ready in her bathroom.", "A man with thinning hair looking into glasses through glasses.", "A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "i like to lift weights and watch movies"], "speaker_id": "snli:test:8131", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8131", "candidate_text": "Honestly, A man chops veggies!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8131:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8075:premise", "snli:test:7991:premise", "snli:test:8310:premise", "snli:test:8131:premise", "style:personachat:test:226:self:5:positive:0", "style:personachat:test:226:self:5:positive:1", "style:personachat:test:226:self:5:positive:2", "style:personachat:test:226:self:5:positive:3", "style:personachat:test:226:self:5:positive:4"], "evidence_texts": ["A woman in black is getting ready in her bathroom.", "A man with thinning hair looking into glasses through glasses.", "A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "i work in advertising . how about you ?", "awww , how fulfilling ! so are you enjoying your pizza all alone or with friends ?", "just the significant other , or do you have kids of your own ?", "awesome , so just spouse time tonight ? i know nursing hours can be hard on quality time .", "ha ! i know those feels . how long have you been married ?"], "speaker_id": "snli:test:8131", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8133", "candidate_text": "well, a dog runs outside", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8133:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8310:premise", "snli:test:8133:premise", "snli:test:8075:premise", "snli:test:8319:premise", "style:personachat:test:300:self:2:positive:0", "style:personachat:test:300:self:2:positive:1"], "evidence_texts": ["A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "A woman in black is getting ready in her bathroom.", "A young woman in white working in a professional kitchen.", "do you like to watch movies ?", "i like to lift weights and watch movies"], "speaker_id": "snli:test:8133", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8133", "candidate_text": "A dog runs outside.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8133:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8310:premise", "snli:test:8133:premise", "snli:test:8075:premise", "snli:test:8319:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "A woman in black is getting ready in her bathroom.", "A young woman in white working in a professional kitchen.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8133", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8136", "candidate_text": "well, a man can see mountains from where he stands", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8136:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8136:premise", "snli:test:8047:premise", "snli:test:8146:premise", "snli:test:7988:premise", "style:personachat:test:300:self:5:positive:0", "style:personachat:test:300:self:5:positive:1", "style:personachat:test:300:self:5:positive:2", "style:personachat:test:300:self:5:positive:3", "style:personachat:test:300:self:5:positive:4"], "evidence_texts": ["A man in gray on a rocky cliff, overlooking the mountains.", "Small girl in an apron placing confetti in a woman's hair.", "A smiling girl walking and a frowning girl with a phone.", "A black-haired man is entertaining a crowd with a hula hoop.", "are you vegan ? are you dressing up for halloween ?", "my church throws a party with food and drinks and we get dressed up .", "yes i need to lose weight . i have been trying to stop eating meat .", "do you like to watch movies ?", "i like to lift weights and watch movies"], "speaker_id": "snli:test:8136", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8136", "candidate_text": "I wonder whether a man can see mountains from where he stands?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8136:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8136:premise", "snli:test:8047:premise", "snli:test:8146:premise", "snli:test:7988:premise", "style:personachat:test:135:self:5:positive:0", "style:personachat:test:135:self:5:positive:1", "style:personachat:test:135:self:5:positive:2", "style:personachat:test:135:self:5:positive:3", "style:personachat:test:135:self:5:positive:4"], "evidence_texts": ["A man in gray on a rocky cliff, overlooking the mountains.", "Small girl in an apron placing confetti in a woman's hair.", "A smiling girl walking and a frowning girl with a phone.", "A black-haired man is entertaining a crowd with a hula hoop.", "what did you do there ?", "just went to my classes really", "college . how old are you ?", "impressive i am a 22 year old woman", "yeah do you have lots of friends ?"], "speaker_id": "snli:test:8136", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8139", "candidate_text": "well, there is a group of people present", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8139:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8126:premise", "snli:test:8681:premise", "snli:test:7798:premise", "snli:test:8139:premise", "style:personachat:test:300:self:7:positive:0", "style:personachat:test:300:self:7:positive:1", "style:personachat:test:300:self:7:positive:2", "style:personachat:test:300:self:7:positive:3", "style:personachat:test:300:self:7:positive:4", "style:personachat:test:300:self:7:positive:5", "style:personachat:test:300:self:7:positive:6"], "evidence_texts": ["A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "i am doing good just making some dinner . how about you ?", "i love to make brisket and barbecue chicken and sides", "are you vegan ? are you dressing up for halloween ?", "my church throws a party with food and drinks and we get dressed up .", "yes i need to lose weight . i have been trying to stop eating meat .", "do you like to watch movies ?", "i like to lift weights and watch movies"], "speaker_id": "snli:test:8139", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8139", "candidate_text": "Honestly, There is a group of people present!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8139:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8126:premise", "snli:test:8681:premise", "snli:test:7798:premise", "snli:test:8139:premise", "style:personachat:test:231:self:2:positive:0", "style:personachat:test:231:self:2:positive:1"], "evidence_texts": ["A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "Two women are hugging on a path through a grassy area with a cow visible past them.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "thank you ! are you married ?", "kids are the best . i enjoy my 3 girls so much !"], "speaker_id": "snli:test:8139", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8141", "candidate_text": "well, several men are gathered to learn about ham radios", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8141:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8115:premise", "snli:test:8141:premise", "snli:test:8199:premise", "snli:test:8021:premise", "style:personachat:test:301:self:1:positive:0"], "evidence_texts": ["Man sitting on bench with suitcase waiting for his source of transportation.", "Six seated men converse and one man fiddles with an interesting contraption.", "A soft focus picture of a little girl wearing an orange sweater.", "A runner in a black and blue uniform competes in a race.", "win by taking the most tricks . you have to take at least 3 tricks or you lose ."], "speaker_id": "snli:test:8141", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8141", "candidate_text": "Several men are gathered to learn about ham radios.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8141:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8115:premise", "snli:test:8141:premise", "snli:test:8199:premise", "snli:test:8021:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Man sitting on bench with suitcase waiting for his source of transportation.", "Six seated men converse and one man fiddles with an interesting contraption.", "A soft focus picture of a little girl wearing an orange sweater.", "A runner in a black and blue uniform competes in a race.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8141", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8146", "candidate_text": "well, a smiling boy walking and a frowning girl with a phone", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8146:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8136:premise", "snli:test:8201:premise", "snli:test:8146:premise", "snli:test:8047:premise", "style:personachat:test:301:self:2:positive:0", "style:personachat:test:301:self:2:positive:1"], "evidence_texts": ["A man in gray on a rocky cliff, overlooking the mountains.", "The man in the blue shirt is relaxing on the rocks.", "A smiling girl walking and a frowning girl with a phone.", "Small girl in an apron placing confetti in a woman's hair.", "4 people , deal out 5 cards each . if spades are trump , jack of spades is highest card", "win by taking the most tricks . you have to take at least 3 tricks or you lose ."], "speaker_id": "snli:test:8146", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8146", "candidate_text": "I wonder whether a smiling boy walking and a frowning girl with a phone?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8146:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8136:premise", "snli:test:8201:premise", "snli:test:8146:premise", "snli:test:8047:premise", "style:personachat:test:139:self:2:positive:0", "style:personachat:test:139:self:2:positive:1"], "evidence_texts": ["A man in gray on a rocky cliff, overlooking the mountains.", "The man in the blue shirt is relaxing on the rocks.", "A smiling girl walking and a frowning girl with a phone.", "Small girl in an apron placing confetti in a woman's hair.", "yeah my friend had that removed at his wedding last week when i attended", "gross , what do you do as a hobby ? i love graphic design"], "speaker_id": "snli:test:8146", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8176", "candidate_text": "well, kids at zoo", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8176:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8176:premise", "snli:test:8073:premise", "snli:test:8380:premise", "snli:test:8071:premise", "style:personachat:test:301:self:5:positive:0", "style:personachat:test:301:self:5:positive:1", "style:personachat:test:301:self:5:positive:2", "style:personachat:test:301:self:5:positive:3", "style:personachat:test:301:self:5:positive:4"], "evidence_texts": ["People boating on a lake with the sun through the clouds in the distance.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A brown dog wearing a collar is chasing and biting on a red broom.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "running is good . do you have hobbies ? i sing in a community choir .", "i love playing cards , especially euchre .", "euchre is a a card game where you only use ace through 9 cards . lots of fun .", "4 people , deal out 5 cards each . if spades are trump , jack of spades is highest card", "win by taking the most tricks . you have to take at least 3 tricks or you lose ."], "speaker_id": "snli:test:8176", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8176", "candidate_text": "Honestly, kids at zoo!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8176:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8176:premise", "snli:test:8073:premise", "snli:test:8380:premise", "snli:test:8071:premise", "style:personachat:test:232:self:1:positive:0"], "evidence_texts": ["People boating on a lake with the sun through the clouds in the distance.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "A brown dog wearing a collar is chasing and biting on a red broom.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:8176", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8182", "candidate_text": "well, a man talks", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8182:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:7836:premise", "snli:test:8387:premise", "snli:test:8182:premise", "snli:test:7795:premise", "style:personachat:test:301:self:7:positive:0", "style:personachat:test:301:self:7:positive:1", "style:personachat:test:301:self:7:positive:2", "style:personachat:test:301:self:7:positive:3", "style:personachat:test:301:self:7:positive:4", "style:personachat:test:301:self:7:positive:5", "style:personachat:test:301:self:7:positive:6"], "evidence_texts": ["A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "good , name is tom . i work with teens having problems and teach them taekwondo .", "i should try that . i eat a lot of italian food , need to exercise .", "running is good . do you have hobbies ? i sing in a community choir .", "i love playing cards , especially euchre .", "euchre is a a card game where you only use ace through 9 cards . lots of fun .", "4 people , deal out 5 cards each . if spades are trump , jack of spades is highest card", "win by taking the most tricks . you have to take at least 3 tricks or you lose ."], "speaker_id": "snli:test:8182", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8182", "candidate_text": "A man talks.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8182:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7836:premise", "snli:test:8387:premise", "snli:test:8182:premise", "snli:test:7795:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "A man with a red beard and mustache is smiling and wearing a hat crocheted with green yarn and Miller beer cans.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8182", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8185", "candidate_text": "well, two men are in the hot tub", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8185:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:7812:premise", "snli:test:8185:premise", "snli:test:8204:premise", "snli:test:7771:premise", "style:personachat:test:302:self:1:positive:0"], "evidence_texts": ["Four human figures are backlit by a sunset.", "Two women are relaxing in a hot tub.", "Three men are playing frisbee in the woods.", "A happy woman who is preparing for dinner.", "i do not sing and i was not homeschooled ."], "speaker_id": "snli:test:8185", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8185", "candidate_text": "I wonder whether two men are in the hot tub?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8185:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:7812:premise", "snli:test:8185:premise", "snli:test:8204:premise", "snli:test:7771:premise", "style:personachat:test:147:self:1:positive:0"], "evidence_texts": ["Four human figures are backlit by a sunset.", "Two women are relaxing in a hot tub.", "Three men are playing frisbee in the woods.", "A happy woman who is preparing for dinner.", "which system are you referring to ?"], "speaker_id": "snli:test:8185", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8191", "candidate_text": "well, phone to her ear, a woman bends forward", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8191:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7896:premise", "snli:test:8471:premise", "snli:test:7732:premise", "snli:test:8191:premise", "style:personachat:test:302:self:2:positive:0", "style:personachat:test:302:self:2:positive:1"], "evidence_texts": ["A woman is taking a picture on the shoulders of another woman in cold weather.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "A young child climbing a stack of logs even though the sign warns against it.", "Phone to her ear, a woman bends forward at the side of a busy street.", "my hair color is dirty blonde .", "i do not sing and i was not homeschooled ."], "speaker_id": "snli:test:8191", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8191", "candidate_text": "Honestly, Phone to her ear, a woman bends forward!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8191:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7896:premise", "snli:test:8471:premise", "snli:test:7732:premise", "snli:test:8191:premise", "style:personachat:test:232:self:6:positive:0", "style:personachat:test:232:self:6:positive:1", "style:personachat:test:232:self:6:positive:2", "style:personachat:test:232:self:6:positive:3", "style:personachat:test:232:self:6:positive:4", "style:personachat:test:232:self:6:positive:5"], "evidence_texts": ["A woman is taking a picture on the shoulders of another woman in cold weather.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "A young child climbing a stack of logs even though the sign warns against it.", "Phone to her ear, a woman bends forward at the side of a busy street.", "i have a dog . it is perfect for my single life", "jerry . i used to live in arizona . now i am living in new york .", "yes , i love to sing on my modeling jobs .", "what is your dream job ? mine is becoming an actress .", "that is a good job . i know someone who is a millionaire because of it !", "i could not do it . i will just act as a realtor when i get famous !"], "speaker_id": "snli:test:8191", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8199", "candidate_text": "well, a girl is wearing an orange sweater with black stripes", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8199:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8141:premise", "snli:test:8287:premise", "snli:test:8115:premise", "snli:test:8199:premise", "style:personachat:test:302:self:5:positive:0", "style:personachat:test:302:self:5:positive:1", "style:personachat:test:302:self:5:positive:2", "style:personachat:test:302:self:5:positive:3", "style:personachat:test:302:self:5:positive:4"], "evidence_texts": ["Six seated men converse and one man fiddles with an interesting contraption.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "Man sitting on bench with suitcase waiting for his source of transportation.", "A soft focus picture of a little girl wearing an orange sweater.", "very cool ! i need to get my honda civic in the shop for annual service .", "i like blue hair . is it dark blue or light blue ?", "right on . what is your book about ?", "my hair color is dirty blonde .", "i do not sing and i was not homeschooled ."], "speaker_id": "snli:test:8199", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8199", "candidate_text": "A girl is wearing an orange sweater with black stripes.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8199:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8141:premise", "snli:test:8287:premise", "snli:test:8115:premise", "snli:test:8199:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Six seated men converse and one man fiddles with an interesting contraption.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "Man sitting on bench with suitcase waiting for his source of transportation.", "A soft focus picture of a little girl wearing an orange sweater.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8199", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8201", "candidate_text": "well, the man is wearing purple jeans", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8201:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8146:premise", "snli:test:8201:premise", "snli:test:8258:premise", "snli:test:8136:premise", "style:personachat:test:302:self:7:positive:0", "style:personachat:test:302:self:7:positive:1", "style:personachat:test:302:self:7:positive:2", "style:personachat:test:302:self:7:positive:3", "style:personachat:test:302:self:7:positive:4", "style:personachat:test:302:self:7:positive:5", "style:personachat:test:302:self:7:positive:6"], "evidence_texts": ["A smiling girl walking and a frowning girl with a phone.", "The man in the blue shirt is relaxing on the rocks.", "A woman is holding a large check for Kids Food Basket.", "A man in gray on a rocky cliff, overlooking the mountains.", "i am a divorced mom with two kids and my job is an accountant .", "here in ohio , i take the kids hiking on the weekends .", "very cool ! i need to get my honda civic in the shop for annual service .", "i like blue hair . is it dark blue or light blue ?", "right on . what is your book about ?", "my hair color is dirty blonde .", "i do not sing and i was not homeschooled ."], "speaker_id": "snli:test:8201", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8201", "candidate_text": "I wonder whether the man is wearing purple jeans?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8201:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8146:premise", "snli:test:8201:premise", "snli:test:8258:premise", "snli:test:8136:premise", "style:personachat:test:147:self:7:positive:0", "style:personachat:test:147:self:7:positive:1", "style:personachat:test:147:self:7:positive:2", "style:personachat:test:147:self:7:positive:3", "style:personachat:test:147:self:7:positive:4", "style:personachat:test:147:self:7:positive:5", "style:personachat:test:147:self:7:positive:6"], "evidence_texts": ["A smiling girl walking and a frowning girl with a phone.", "The man in the blue shirt is relaxing on the rocks.", "A woman is holding a large check for Kids Food Basket.", "A man in gray on a rocky cliff, overlooking the mountains.", "i am doing great . just hanging out with my teddy bear collection", "i have a very big collection ! since i am a nurse , people give them to me", "i started as a red cross volunteer but i prefer the color green personally", "yea because it lets me visit zoos , which i love", "that can be tough . finding a job to support your rent is hard", "i take the kids from the hospital to the zoo weekly", "which system are you referring to ?"], "speaker_id": "snli:test:8201", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8204", "candidate_text": "well, three men are outside", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8204:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8204:premise", "snli:test:8185:premise", "snli:test:8246:premise", "snli:test:7812:premise", "style:personachat:test:303:self:1:positive:0"], "evidence_texts": ["Three men are playing frisbee in the woods.", "Two women are relaxing in a hot tub.", "An Asian man makes faces for the camera.", "Four human figures are backlit by a sunset.", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:8204", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8204", "candidate_text": "Honestly, Three men are outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8204:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8204:premise", "snli:test:8185:premise", "snli:test:8246:premise", "snli:test:7812:premise", "style:personachat:test:240:self:5:positive:0", "style:personachat:test:240:self:5:positive:1", "style:personachat:test:240:self:5:positive:2", "style:personachat:test:240:self:5:positive:3", "style:personachat:test:240:self:5:positive:4"], "evidence_texts": ["Three men are playing frisbee in the woods.", "Two women are relaxing in a hot tub.", "An Asian man makes faces for the camera.", "Four human figures are backlit by a sunset.", "how about paint a picture of me eating onion rings ? i love onions . . . lol", "my dad is single . he is a doctor and i am his nurse in his practice .", "he might . which store do you work at ?", "he loves to eat . he will need that discount .", "hilarious ! my mom could not keep up with the large grocery bill so she left ."], "speaker_id": "snli:test:8204", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8210", "candidate_text": "well, there is a happy lady with colorful clothing", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8210:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7899:premise", "snli:test:8262:premise", "snli:test:8210:premise", "snli:test:7779:premise", "style:personachat:test:303:self:2:positive:0", "style:personachat:test:303:self:2:positive:1"], "evidence_texts": ["A group in uniform marches carrying red signs with Chinese lettering on them.", "A male wearing a football uniform kicking a football during a football game.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "i a , a biology teacher", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:8210", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8210", "candidate_text": "There is a happy lady with colorful clothing.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8210:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7899:premise", "snli:test:8262:premise", "snli:test:8210:premise", "snli:test:7779:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group in uniform marches carrying red signs with Chinese lettering on them.", "A male wearing a football uniform kicking a football during a football game.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "The man in the black hat has his back to a theater sign.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8210", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8213", "candidate_text": "well, suv travelling through mud", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8213:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8036:premise", "snli:test:8213:premise", "snli:test:9256:premise", "snli:test:7714:premise", "style:personachat:test:303:self:5:positive:0", "style:personachat:test:303:self:5:positive:1", "style:personachat:test:303:self:5:positive:2", "style:personachat:test:303:self:5:positive:3", "style:personachat:test:303:self:5:positive:4"], "evidence_texts": ["Three men play musical instruments.", "An SUV traveling through mud.", "A woman happy about something.", "Child swimming in a pool.", "good choice ! what do you do ? i , a biology teacher at college", "interesting ! i guess you are into nature ! i have a big garden and i love it !", "nice ! do you have any hobbies ? i love walking", "i a , a biology teacher", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:8213", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8213", "candidate_text": "I wonder whether suv travelling through mud?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8213:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8036:premise", "snli:test:8213:premise", "snli:test:9256:premise", "snli:test:7714:premise", "style:personachat:test:149:self:5:positive:0", "style:personachat:test:149:self:5:positive:1", "style:personachat:test:149:self:5:positive:2", "style:personachat:test:149:self:5:positive:3", "style:personachat:test:149:self:5:positive:4"], "evidence_texts": ["Three men play musical instruments.", "An SUV traveling through mud.", "A woman happy about something.", "Child swimming in a pool.", "mine is orange i eat alot of orange candy", "did you enjoy it ? what do you do for fun ?", "yeah ? have a outfit to polka in ?", "have someone to go polka with ?", "very cool how long have you been dating ? i am looking for love in the office"], "speaker_id": "snli:test:8213", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8223", "candidate_text": "well, the people are inside the classroom", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8223:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8051:premise", "snli:test:8315:premise", "snli:test:8006:premise", "snli:test:8223:premise", "style:personachat:test:303:self:7:positive:0", "style:personachat:test:303:self:7:positive:1", "style:personachat:test:303:self:7:positive:2", "style:personachat:test:303:self:7:positive:3", "style:personachat:test:303:self:7:positive:4", "style:personachat:test:303:self:7:positive:5", "style:personachat:test:303:self:7:positive:6"], "evidence_texts": ["A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "hello nice to meet you !", "no , imam not into that . . i like to travel . . visited 6 countries already", "good choice ! what do you do ? i , a biology teacher at college", "interesting ! i guess you are into nature ! i have a big garden and i love it !", "nice ! do you have any hobbies ? i love walking", "i a , a biology teacher", "hehe ya . . . . . where do u"], "speaker_id": "snli:test:8223", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8223", "candidate_text": "Honestly, The people are inside the classroom!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8223:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8051:premise", "snli:test:8315:premise", "snli:test:8006:premise", "snli:test:8223:premise", "style:personachat:test:249:self:2:positive:0", "style:personachat:test:249:self:2:positive:1"], "evidence_texts": ["A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "A group of men in blue shirts, banded hats, and red bandannas stand in a group.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "i have never tried that , but my wife would hate it . are you married ?", "i met my wife at a wedding !"], "speaker_id": "snli:test:8223", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8225", "candidate_text": "well, the three players are chasing after the hockey puck", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8225:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8099:premise", "snli:test:8225:premise", "snli:test:9385:premise", "snli:test:6039:premise", "style:personachat:test:304:self:1:positive:0"], "evidence_texts": ["A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "my dogs and vacation spots ."], "speaker_id": "snli:test:8225", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8225", "candidate_text": "The three players are chasing after the hockey puck.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8225:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8099:premise", "snli:test:8225:premise", "snli:test:9385:premise", "snli:test:6039:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A school aged child wearing makeup and doing a hand sign sits on the ground among similar aged children who are all making the same sign.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8225", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8231", "candidate_text": "well, there are customers at a cafe ordering food", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8231:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7848:premise", "snli:test:8376:premise", "snli:test:7674:premise", "snli:test:8231:premise", "style:personachat:test:304:self:2:positive:0", "style:personachat:test:304:self:2:positive:1"], "evidence_texts": ["Some youngsters are playing a card game", "Several children are standing on the beach.", "A young redheaded woman laughs at something.", "Customers are ordering food at a cafe.", "i like photography , and drawing .", "my dogs and vacation spots ."], "speaker_id": "snli:test:8231", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8231", "candidate_text": "I wonder whether there are customers at a cafe ordering food?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8231:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7848:premise", "snli:test:8376:premise", "snli:test:7674:premise", "snli:test:8231:premise", "style:personachat:test:150:self:2:positive:0", "style:personachat:test:150:self:2:positive:1"], "evidence_texts": ["Some youngsters are playing a card game", "Several children are standing on the beach.", "A young redheaded woman laughs at something.", "Customers are ordering food at a cafe.", "i am 26 . i only have one kiddo so far . i can not imagine twins .", "so what do you do for fun ?"], "speaker_id": "snli:test:8231", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8240", "candidate_text": "well, two women are petting a kangaroo", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8240:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8240:premise", "snli:test:7815:premise", "snli:test:8347:premise", "snli:test:7788:premise", "style:personachat:test:304:self:5:positive:0", "style:personachat:test:304:self:5:positive:1", "style:personachat:test:304:self:5:positive:2", "style:personachat:test:304:self:5:positive:3", "style:personachat:test:304:self:5:positive:4"], "evidence_texts": ["Two women bend over to touch a small kangaroo.", "Seven young girls are being creative and making posters.", "A crowd of people standing in front of statues.", "A guy with a football, a crowd watching him.", "i am 28 . i love animals . i am terrified of heights though .", "have you been to cedar point ? it is the coaster capital of the world .", "that they do . my sister turned to religion , i am an atheist .", "i like photography , and drawing .", "my dogs and vacation spots ."], "speaker_id": "snli:test:8240", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8240", "candidate_text": "Honestly, Two women are petting a kangaroo!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8240:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8240:premise", "snli:test:7815:premise", "snli:test:8347:premise", "snli:test:7788:premise", "style:personachat:test:261:self:1:positive:0"], "evidence_texts": ["Two women bend over to touch a small kangaroo.", "Seven young girls are being creative and making posters.", "A crowd of people standing in front of statues.", "A guy with a football, a crowd watching him.", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:8240", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8246", "candidate_text": "well, faces are being made to the camera", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8246:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8204:premise", "snli:test:8491:premise", "snli:test:8246:premise", "snli:test:8185:premise", "style:personachat:test:304:self:6:positive:0", "style:personachat:test:304:self:6:positive:1", "style:personachat:test:304:self:6:positive:2", "style:personachat:test:304:self:6:positive:3", "style:personachat:test:304:self:6:positive:4", "style:personachat:test:304:self:6:positive:5"], "evidence_texts": ["Three men are playing frisbee in the woods.", "A little girl in a pink dress crying.", "An Asian man makes faces for the camera.", "Two women are relaxing in a hot tub.", "hello , i am a lawyer and have a couple of dogs .", "i am 28 . i love animals . i am terrified of heights though .", "have you been to cedar point ? it is the coaster capital of the world .", "that they do . my sister turned to religion , i am an atheist .", "i like photography , and drawing .", "my dogs and vacation spots ."], "speaker_id": "snli:test:8246", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8246", "candidate_text": "Faces are being made to the camera.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8246:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8204:premise", "snli:test:8491:premise", "snli:test:8246:premise", "snli:test:8185:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Three men are playing frisbee in the woods.", "A little girl in a pink dress crying.", "An Asian man makes faces for the camera.", "Two women are relaxing in a hot tub.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8246", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8258", "candidate_text": "well, she is stealing from them", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8258:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8201:premise", "snli:test:8269:premise", "snli:test:8258:premise", "snli:test:8146:premise", "style:personachat:test:305:self:1:positive:0"], "evidence_texts": ["The man in the blue shirt is relaxing on the rocks.", "a brown dog standing in the middle of a huge field.", "A woman is holding a large check for Kids Food Basket.", "A smiling girl walking and a frowning girl with a phone.", "yes just about all genres"], "speaker_id": "snli:test:8258", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8258", "candidate_text": "I wonder whether she is stealing from them?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8258:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8201:premise", "snli:test:8269:premise", "snli:test:8258:premise", "snli:test:8146:premise", "style:personachat:test:156:self:1:positive:0"], "evidence_texts": ["The man in the blue shirt is relaxing on the rocks.", "a brown dog standing in the middle of a huge field.", "A woman is holding a large check for Kids Food Basket.", "A smiling girl walking and a frowning girl with a phone.", "what do you do for a living ?"], "speaker_id": "snli:test:8258", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8262", "candidate_text": "well, a dallas cowboy player is trying to tie the playoff game", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8262:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8210:premise", "snli:test:8417:premise", "snli:test:8262:premise", "snli:test:7899:premise", "style:personachat:test:305:self:2:positive:0", "style:personachat:test:305:self:2:positive:1"], "evidence_texts": ["A dark-haired lady with a big smile is wearing a bright red scarf.", "A group of people are standing on steps in front of a building.", "A male wearing a football uniform kicking a football during a football game.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "not really , i do not like sports", "yes just about all genres"], "speaker_id": "snli:test:8262", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8262", "candidate_text": "Honestly, A Dallas Cowboy player is trying to tie the playoff game!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8262:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8210:premise", "snli:test:8417:premise", "snli:test:8262:premise", "snli:test:7899:premise", "style:personachat:test:261:self:7:positive:0", "style:personachat:test:261:self:7:positive:1", "style:personachat:test:261:self:7:positive:2", "style:personachat:test:261:self:7:positive:3", "style:personachat:test:261:self:7:positive:4", "style:personachat:test:261:self:7:positive:5", "style:personachat:test:261:self:7:positive:6"], "evidence_texts": ["A dark-haired lady with a big smile is wearing a bright red scarf.", "A group of people are standing on steps in front of a building.", "A male wearing a football uniform kicking a football during a football game.", "A group in uniform marches carrying red signs with Chinese lettering on them.", "i love music ! i also love horses", "i do not . i would get sweaty", "i listen to pop music", "my dad owns a car dealership across town", "no but i get a discount", "i mostly like to hang out on the beach", "i live in california . beach lifestyle !"], "speaker_id": "snli:test:8262", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8269", "candidate_text": "well, a dog is riding in the car", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8269:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8258:premise", "snli:test:8269:premise", "snli:test:8464:premise", "snli:test:8201:premise", "style:personachat:test:305:self:5:positive:0", "style:personachat:test:305:self:5:positive:1", "style:personachat:test:305:self:5:positive:2", "style:personachat:test:305:self:5:positive:3", "style:personachat:test:305:self:5:positive:4"], "evidence_texts": ["A woman is holding a large check for Kids Food Basket.", "a brown dog standing in the middle of a huge field.", "Two men are making hand gestures and posing for a picture.", "The man in the blue shirt is relaxing on the rocks.", "yeah out here i think its snowed like twice", "i live in louisiana", "its extremely hot out here", "not really , i do not like sports", "yes just about all genres"], "speaker_id": "snli:test:8269", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8269", "candidate_text": "A dog is riding in the car.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8269:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8258:premise", "snli:test:8269:premise", "snli:test:8464:premise", "snli:test:8201:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman is holding a large check for Kids Food Basket.", "a brown dog standing in the middle of a huge field.", "Two men are making hand gestures and posing for a picture.", "The man in the blue shirt is relaxing on the rocks.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8269", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8287", "candidate_text": "well, a group of friends are getting some money before they go to dinner", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8287:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8199:premise", "snli:test:8141:premise", "snli:test:8293:premise", "snli:test:8287:premise", "style:personachat:test:305:self:7:positive:0", "style:personachat:test:305:self:7:positive:1", "style:personachat:test:305:self:7:positive:2", "style:personachat:test:305:self:7:positive:3", "style:personachat:test:305:self:7:positive:4", "style:personachat:test:305:self:7:positive:5", "style:personachat:test:305:self:7:positive:6"], "evidence_texts": ["A soft focus picture of a little girl wearing an orange sweater.", "Six seated men converse and one man fiddles with an interesting contraption.", "A girl in a red and white uniform is swinging a bat.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "hi i am retired i use to be a banker", "no where i live it never gets cold enough for it", "yeah out here i think its snowed like twice", "i live in louisiana", "its extremely hot out here", "not really , i do not like sports", "yes just about all genres"], "speaker_id": "snli:test:8287", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8287", "candidate_text": "I wonder whether a group of friends are getting some money before they go to dinner?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8287:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8199:premise", "snli:test:8141:premise", "snli:test:8293:premise", "snli:test:8287:premise", "style:personachat:test:156:self:7:positive:0", "style:personachat:test:156:self:7:positive:1", "style:personachat:test:156:self:7:positive:2", "style:personachat:test:156:self:7:positive:3", "style:personachat:test:156:self:7:positive:4", "style:personachat:test:156:self:7:positive:5", "style:personachat:test:156:self:7:positive:6"], "evidence_texts": ["A soft focus picture of a little girl wearing an orange sweater.", "Six seated men converse and one man fiddles with an interesting contraption.", "A girl in a red and white uniform is swinging a bat.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "i am good . what are you doing today ?", "wow ! i just got back from walking a mile . i tried to do that everyday", "yes . other than playing guitar , what other hobbies do you have ?", "after retiring from banking , i make quilts for my family . do you have any pets ?", "i own a siamese cat named stormy . what kind of dog do you want ?", "lol ! that would be a lot of cleaning up to do .", "what do you do for a living ?"], "speaker_id": "snli:test:8287", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8288", "candidate_text": "well, three men are playing poker", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8288:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8288:premise", "snli:test:8293:premise", "snli:test:8199:premise", "snli:test:8421:premise", "style:personachat:test:306:self:1:positive:0"], "evidence_texts": ["Three men are using Washington Mutual ATMs outside near a parking lot.", "A girl in a red and white uniform is swinging a bat.", "A soft focus picture of a little girl wearing an orange sweater.", "Brown and cream dog with tongue out in front of grassy forest", "me either . not since i quit working there and began writing short stories ."], "speaker_id": "snli:test:8288", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8288", "candidate_text": "Honestly, Three men are playing poker!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8288:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8288:premise", "snli:test:8293:premise", "snli:test:8199:premise", "snli:test:8421:premise", "style:personachat:test:269:self:5:positive:0", "style:personachat:test:269:self:5:positive:1", "style:personachat:test:269:self:5:positive:2", "style:personachat:test:269:self:5:positive:3", "style:personachat:test:269:self:5:positive:4"], "evidence_texts": ["Three men are using Washington Mutual ATMs outside near a parking lot.", "A girl in a red and white uniform is swinging a bat.", "A soft focus picture of a little girl wearing an orange sweater.", "Brown and cream dog with tongue out in front of grassy forest", "i have two small dogs , max and marsha and a cat named ginger .", "we do , though mostly they ignore us . what do you do for work john ?", "nice ! i work with the elderly helping them care for their daily needs .", "you always have options ! what are your passions ?", "well maybe you could find a job as a dog trainer ! you never know !"], "speaker_id": "snli:test:8288", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8293", "candidate_text": "well, the boy is swinging at his baseball game", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8293:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8288:premise", "snli:test:8293:premise", "snli:test:8421:premise", "snli:test:8287:premise", "style:personachat:test:306:self:2:positive:0", "style:personachat:test:306:self:2:positive:1"], "evidence_texts": ["Three men are using Washington Mutual ATMs outside near a parking lot.", "A girl in a red and white uniform is swinging a bat.", "Brown and cream dog with tongue out in front of grassy forest", "Three men are using Washington Mutual ATMs outside near a parking lot.", "me too ! working at a carnival in the past we danced all the time", "me either . not since i quit working there and began writing short stories ."], "speaker_id": "snli:test:8293", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8293", "candidate_text": "The boy is swinging at his baseball game.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8293:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8288:premise", "snli:test:8293:premise", "snli:test:8421:premise", "snli:test:8287:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Three men are using Washington Mutual ATMs outside near a parking lot.", "A girl in a red and white uniform is swinging a bat.", "Brown and cream dog with tongue out in front of grassy forest", "Three men are using Washington Mutual ATMs outside near a parking lot.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8293", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8300", "candidate_text": "well, a man is moving quickly through the snow", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8300:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8300:premise", "snli:test:6713:premise", "snli:test:9745:premise", "snli:test:6712:premise", "style:personachat:test:306:self:5:positive:0", "style:personachat:test:306:self:5:positive:1", "style:personachat:test:306:self:5:positive:2", "style:personachat:test:306:self:5:positive:3", "style:personachat:test:306:self:5:positive:4"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "yeah . what kind of movies are you interested in ?", "i like to watch dark superhero movies .", "i do not think i could ever do that i love food .", "me too ! working at a carnival in the past we danced all the time", "me either . not since i quit working there and began writing short stories ."], "speaker_id": "snli:test:8300", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8300", "candidate_text": "I wonder whether a man is moving quickly through the snow?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8300:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8300:premise", "snli:test:6713:premise", "snli:test:9745:premise", "snli:test:6712:premise", "style:personachat:test:161:self:5:positive:0", "style:personachat:test:161:self:5:positive:1", "style:personachat:test:161:self:5:positive:2", "style:personachat:test:161:self:5:positive:3", "style:personachat:test:161:self:5:positive:4"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "this is the sixth country i have been to . love walking around everywhere .", "well , i have a garden at home . that is very therapeutic . and you ?", "mostly vegetables and herbs . i like nirvana . sad what happened to kurt .", "i am a writer . since i travel so much , it is the best job for me .", "so you see dogs all day ? fun . i have a pug . so cute ."], "speaker_id": "snli:test:8300", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8310", "candidate_text": "well, the boy is dead", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8310:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8133:premise", "snli:test:8319:premise", "snli:test:8310:premise", "snli:test:8131:premise", "style:personachat:test:306:self:7:positive:0", "style:personachat:test:306:self:7:positive:1", "style:personachat:test:306:self:7:positive:2", "style:personachat:test:306:self:7:positive:3", "style:personachat:test:306:self:7:positive:4", "style:personachat:test:306:self:7:positive:5", "style:personachat:test:306:self:7:positive:6"], "evidence_texts": ["Tan dog running through long grass in a park-like setting.", "A young woman in white working in a professional kitchen.", "A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "i am good . relaxing with a glass of scotch to end the night .", "yes . i have been busy working on a new short story to release .", "yeah . what kind of movies are you interested in ?", "i like to watch dark superhero movies .", "i do not think i could ever do that i love food .", "me too ! working at a carnival in the past we danced all the time", "me either . not since i quit working there and began writing short stories ."], "speaker_id": "snli:test:8310", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8310", "candidate_text": "Honestly, The boy is dead!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8310:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8133:premise", "snli:test:8319:premise", "snli:test:8310:premise", "snli:test:8131:premise", "style:personachat:test:275:self:2:positive:0", "style:personachat:test:275:self:2:positive:1"], "evidence_texts": ["Tan dog running through long grass in a park-like setting.", "A young woman in white working in a professional kitchen.", "A young boy is jumping down an inflatable water ride.", "Tan dog running through long grass in a park-like setting.", "i see . so your from australia ?", "wow ! big money there . you will be able to go visit at least ."], "speaker_id": "snli:test:8310", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8315", "candidate_text": "well, five girls are playing together", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8315:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8223:premise", "snli:test:8351:premise", "snli:test:8051:premise", "snli:test:8315:premise", "style:personachat:test:307:self:1:positive:0"], "evidence_texts": ["People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A boy looks out the window at another boy who is spraying the window with water.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "can you come to our university for an interview , to help our study ."], "speaker_id": "snli:test:8315", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8315", "candidate_text": "Five girls are playing together.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8315:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8223:premise", "snli:test:8351:premise", "snli:test:8051:premise", "snli:test:8315:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A boy looks out the window at another boy who is spraying the window with water.", "A man in a blue shirt, blue jeans, and a red apron standing at a grill.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8315", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8319", "candidate_text": "well, the young woman makes a meal", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8319:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8310:premise", "snli:test:8323:premise", "snli:test:8133:premise", "snli:test:8319:premise", "style:personachat:test:307:self:2:positive:0", "style:personachat:test:307:self:2:positive:1"], "evidence_texts": ["A young boy is jumping down an inflatable water ride.", "An American wrestler in a purple singlet wrestlers an opponent.", "Tan dog running through long grass in a park-like setting.", "A young woman in white working in a professional kitchen.", "one whale died because of us . and we as a class spread lies that it was china .", "can you come to our university for an interview , to help our study ."], "speaker_id": "snli:test:8319", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8319", "candidate_text": "I wonder whether the young woman makes a meal?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8319:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8310:premise", "snli:test:8323:premise", "snli:test:8133:premise", "snli:test:8319:premise", "style:personachat:test:164:self:2:positive:0", "style:personachat:test:164:self:2:positive:1"], "evidence_texts": ["A young boy is jumping down an inflatable water ride.", "An American wrestler in a purple singlet wrestlers an opponent.", "Tan dog running through long grass in a park-like setting.", "A young woman in white working in a professional kitchen.", "lately i have been exercising . my weight has become a problem this past year .", "what do you do in your spare time ?"], "speaker_id": "snli:test:8319", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8323", "candidate_text": "well, there is only one person", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8323:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8319:premise", "snli:test:8572:premise", "snli:test:8310:premise", "snli:test:8323:premise", "style:personachat:test:307:self:5:positive:0", "style:personachat:test:307:self:5:positive:1", "style:personachat:test:307:self:5:positive:2", "style:personachat:test:307:self:5:positive:3", "style:personachat:test:307:self:5:positive:4"], "evidence_texts": ["A young woman in white working in a professional kitchen.", "Two young girls are jumping off of a steep step.", "A young boy is jumping down an inflatable water ride.", "An American wrestler in a purple singlet wrestlers an opponent.", "that is nice , i was born and raised in the united states have you painted there ?", "awesome , we should chat more often . well i am very busy with my study .", "yeah , i play her music to scare whales in deep ocean trenches . .", "one whale died because of us . and we as a class spread lies that it was china .", "can you come to our university for an interview , to help our study ."], "speaker_id": "snli:test:8323", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8323", "candidate_text": "Honestly, There is only one person!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8323:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8319:premise", "snli:test:8572:premise", "snli:test:8310:premise", "snli:test:8323:premise", "style:personachat:test:289:self:1:positive:0"], "evidence_texts": ["A young woman in white working in a professional kitchen.", "Two young girls are jumping off of a steep step.", "A young boy is jumping down an inflatable water ride.", "An American wrestler in a purple singlet wrestlers an opponent.", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:8323", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8347", "candidate_text": "well, the statue is made of marble", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8347:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8240:premise", "snli:test:7815:premise", "snli:test:8504:premise", "snli:test:8347:premise", "style:personachat:test:307:self:7:positive:0", "style:personachat:test:307:self:7:positive:1", "style:personachat:test:307:self:7:positive:2", "style:personachat:test:307:self:7:positive:3", "style:personachat:test:307:self:7:positive:4", "style:personachat:test:307:self:7:positive:5", "style:personachat:test:307:self:7:positive:6"], "evidence_texts": ["Two women bend over to touch a small kangaroo.", "Seven young girls are being creative and making posters.", "A young boy is playing in the sandy water.", "A crowd of people standing in front of statues.", "not really , i am studying at the university up state learning biology .", "marine biology actually , still a student but i do not party .", "that is nice , i was born and raised in the united states have you painted there ?", "awesome , we should chat more often . well i am very busy with my study .", "yeah , i play her music to scare whales in deep ocean trenches . .", "one whale died because of us . and we as a class spread lies that it was china .", "can you come to our university for an interview , to help our study ."], "speaker_id": "snli:test:8347", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8347", "candidate_text": "The statue is made of marble.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8347:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8240:premise", "snli:test:7815:premise", "snli:test:8504:premise", "snli:test:8347:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two women bend over to touch a small kangaroo.", "Seven young girls are being creative and making posters.", "A young boy is playing in the sandy water.", "A crowd of people standing in front of statues.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8347", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8348", "candidate_text": "well, the statue is actually a gargoyle and it came to life", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8348:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8348:premise", "snli:test:8504:premise", "snli:test:8240:premise", "snli:test:8560:premise", "style:personachat:test:308:self:1:positive:0"], "evidence_texts": ["A crowd of people standing in front of statues.", "A young boy is playing in the sandy water.", "Two women bend over to touch a small kangaroo.", "A man running in a race passes by spectators.", "i was on the road a lot as kid , my dad drove a semi"], "speaker_id": "snli:test:8348", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8348", "candidate_text": "I wonder whether the statue is actually a gargoyle and it came to life?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8348:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8348:premise", "snli:test:8504:premise", "snli:test:8240:premise", "snli:test:8560:premise", "style:personachat:test:165:self:1:positive:0"], "evidence_texts": ["A crowd of people standing in front of statues.", "A young boy is playing in the sandy water.", "Two women bend over to touch a small kangaroo.", "A man running in a race passes by spectators.", "what is your favorite one ?"], "speaker_id": "snli:test:8348", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8351", "candidate_text": "well, the boy is wearing shorts", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8351:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8315:premise", "snli:test:8384:premise", "snli:test:8223:premise", "snli:test:8351:premise", "style:personachat:test:308:self:2:positive:0", "style:personachat:test:308:self:2:positive:1"], "evidence_texts": ["Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "Two men in black pants and black shirts practice martial arts on a green gym floor.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A boy looks out the window at another boy who is spraying the window with water.", "your books must sell really well . i need to get in on that", "i was on the road a lot as kid , my dad drove a semi"], "speaker_id": "snli:test:8351", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8351", "candidate_text": "Honestly, The boy is wearing shorts!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8351:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8315:premise", "snli:test:8384:premise", "snli:test:8223:premise", "snli:test:8351:premise", "style:personachat:test:289:self:7:positive:0", "style:personachat:test:289:self:7:positive:1", "style:personachat:test:289:self:7:positive:2", "style:personachat:test:289:self:7:positive:3", "style:personachat:test:289:self:7:positive:4", "style:personachat:test:289:self:7:positive:5", "style:personachat:test:289:self:7:positive:6"], "evidence_texts": ["Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "Two men in black pants and black shirts practice martial arts on a green gym floor.", "People standing in front of a gray building with a sign that reads' Tickets Available Here'.", "A boy looks out the window at another boy who is spraying the window with water.", "fantastic ! this steak really hits the spot ! how are you ?", "all meat is good meat lol . got to keep that protein up !", "just bodybuilding . people prefer healthy and fit doctors ! you ?", "i wish , but my real job keeps me chained . what about you ?", "that sounds fun ! do you play for a living ?", "yeah . my mom actually works at the front desk ! do you train with your dad ?", "family is very important . the only thing more important than staying fit !"], "speaker_id": "snli:test:8351", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8358", "candidate_text": "well, the person is crying near some animals", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8358:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:7643:premise", "snli:test:8406:premise", "snli:test:8358:premise", "snli:test:7239:premise", "style:personachat:test:308:self:5:positive:0", "style:personachat:test:308:self:5:positive:1", "style:personachat:test:308:self:5:positive:2", "style:personachat:test:308:self:5:positive:3", "style:personachat:test:308:self:5:positive:4"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "what kind of books do you write ?", "oh cool , that sounds really exciting and expensive", "writing sounds exciting , living on an island sounds expensive lol", "your books must sell really well . i need to get in on that", "i was on the road a lot as kid , my dad drove a semi"], "speaker_id": "snli:test:8358", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8358", "candidate_text": "The person is crying near some animals.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8358:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:7643:premise", "snli:test:8406:premise", "snli:test:8358:premise", "snli:test:7239:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "a boy in a blue uniform is standing next to a boy in red and a boy in yellow and they are holding baseball gloves.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8358", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8365", "candidate_text": "well, a human standing", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8365:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8026:premise", "snli:test:8365:premise", "snli:test:8415:premise", "snli:test:7861:premise", "style:personachat:test:308:self:7:positive:0", "style:personachat:test:308:self:7:positive:1", "style:personachat:test:308:self:7:positive:2", "style:personachat:test:308:self:7:positive:3", "style:personachat:test:308:self:7:positive:4", "style:personachat:test:308:self:7:positive:5", "style:personachat:test:308:self:7:positive:6"], "evidence_texts": ["A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "i m in a band , i m the lead singer", "since i was 14 , we are bertha s beautiful butterflies", "what kind of books do you write ?", "oh cool , that sounds really exciting and expensive", "writing sounds exciting , living on an island sounds expensive lol", "your books must sell really well . i need to get in on that", "i was on the road a lot as kid , my dad drove a semi"], "speaker_id": "snli:test:8365", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8365", "candidate_text": "I wonder whether a human standing?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8365:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8026:premise", "snli:test:8365:premise", "snli:test:8415:premise", "snli:test:7861:premise", "style:personachat:test:165:self:7:positive:0", "style:personachat:test:165:self:7:positive:1", "style:personachat:test:165:self:7:positive:2", "style:personachat:test:165:self:7:positive:3", "style:personachat:test:165:self:7:positive:4", "style:personachat:test:165:self:7:positive:5", "style:personachat:test:165:self:7:positive:6"], "evidence_texts": ["A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "A man on a skateboard performs a trick in the air against the backdrop of a bright blue sky.", "yeah a little i just had a manicure . i like gardens", "oh my , what is your name ?", "i like those animal , but i love horses", "oh can you swim or are you afraid of it ?", "i do like it , but i am dieting now so i am not strong enough", "my diet is extreme , i want to be healthy though eventually", "what is your favorite one ?"], "speaker_id": "snli:test:8365", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8376", "candidate_text": "I wonder whether there is a group of people on the beach?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8376:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8376:premise", "snli:test:8231:premise", "snli:test:8399:premise", "snli:test:7848:premise", "style:personachat:test:309:self:1:positive:0"], "evidence_texts": ["Several children are standing on the beach.", "Customers are ordering food at a cafe.", "Men wearing hats walk on the street.", "Some youngsters are playing a card game", "i enjoy hiking , crocheting , and reading . how about you ?"], "speaker_id": "snli:test:8376", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8376", "candidate_text": "well, there is a group of people on the beach", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8376:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8376:premise", "snli:test:8231:premise", "snli:test:8399:premise", "snli:test:7848:premise", "style:personachat:test:516:self:1:positive:0"], "evidence_texts": ["Several children are standing on the beach.", "Customers are ordering food at a cafe.", "Men wearing hats walk on the street.", "Some youngsters are playing a card game", "no , no pets or husband . i work a lot"], "speaker_id": "snli:test:8376", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8380", "candidate_text": "I wonder whether an animal is sleeping outside?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8380:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8380:premise", "snli:test:8176:premise", "snli:test:8803:premise", "snli:test:8073:premise", "style:personachat:test:309:self:2:positive:0", "style:personachat:test:309:self:2:positive:1"], "evidence_texts": ["A brown dog wearing a collar is chasing and biting on a red broom.", "People boating on a lake with the sun through the clouds in the distance.", "A man is giving another man a haircut while three other guys stand around.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "i like horror movies , too . pets can definitely be messy sometimes .", "i enjoy hiking , crocheting , and reading . how about you ?"], "speaker_id": "snli:test:8380", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8380", "candidate_text": "Honestly, An animal is sleeping outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8380:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8380:premise", "snli:test:8176:premise", "snli:test:8803:premise", "snli:test:8073:premise", "style:personachat:test:290:self:7:positive:0", "style:personachat:test:290:self:7:positive:1", "style:personachat:test:290:self:7:positive:2", "style:personachat:test:290:self:7:positive:3", "style:personachat:test:290:self:7:positive:4", "style:personachat:test:290:self:7:positive:5", "style:personachat:test:290:self:7:positive:6"], "evidence_texts": ["A brown dog wearing a collar is chasing and biting on a red broom.", "People boating on a lake with the sun through the clouds in the distance.", "A man is giving another man a haircut while three other guys stand around.", "Rider in ceremonial garb rides white Arabian horse, also wearing ceremonial blankets and tassels.", "it is awesome . how are you ?", "that does sound exhausting ! i have been outside with my brothers shoveling the driveway !", "i have never been there ! i love the snow though . it keeps me happy and busy .", "i would not trade the snow for anything . i love winter time .", "well , i have a simple job . but i do like some hard rock .", "i recently got a guitar signed by metallica , so i really like metallica , obviously .", "yes ! i have been to several concerts and they never disappoint ."], "speaker_id": "snli:test:8380", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8384", "candidate_text": "I wonder whether two men practicing martial arts?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8384:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8384:premise", "snli:test:8351:premise", "snli:test:8403:premise", "snli:test:8315:premise", "style:personachat:test:309:self:5:positive:0", "style:personachat:test:309:self:5:positive:1", "style:personachat:test:309:self:5:positive:2", "style:personachat:test:309:self:5:positive:3", "style:personachat:test:309:self:5:positive:4"], "evidence_texts": ["Two men in black pants and black shirts practice martial arts on a green gym floor.", "A boy looks out the window at another boy who is spraying the window with water.", "A few people are sitting inside of a building with lots of photography on the walls.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "her name is layla . i used to bbq a lot with friends back in college .", "do you come from a tall family ?", "wow , that is pretty rare . do you have pets ?", "i like horror movies , too . pets can definitely be messy sometimes .", "i enjoy hiking , crocheting , and reading . how about you ?"], "speaker_id": "snli:test:8384", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8384", "candidate_text": "Two men practicing martial arts.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8384:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8384:premise", "snli:test:8351:premise", "snli:test:8403:premise", "snli:test:8315:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two men in black pants and black shirts practice martial arts on a green gym floor.", "A boy looks out the window at another boy who is spraying the window with water.", "A few people are sitting inside of a building with lots of photography on the walls.", "Five young indian girls dress in colorful dresses are holding hands and smiling forming a circle", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8384", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8387", "candidate_text": "I wonder whether a man is playing video games at his house?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8387:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8182:premise", "snli:test:8516:premise", "snli:test:7836:premise", "snli:test:8387:premise", "style:personachat:test:309:self:7:positive:0", "style:personachat:test:309:self:7:positive:1", "style:personachat:test:309:self:7:positive:2", "style:personachat:test:309:self:7:positive:3", "style:personachat:test:309:self:7:positive:4", "style:personachat:test:309:self:7:positive:5", "style:personachat:test:309:self:7:positive:6"], "evidence_texts": ["A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "i am okay . living paycheck to paycheck is taking a toll on me . how about you ?", "interesting . i live with my cat and teach kindergarten .", "her name is layla . i used to bbq a lot with friends back in college .", "do you come from a tall family ?", "wow , that is pretty rare . do you have pets ?", "i like horror movies , too . pets can definitely be messy sometimes .", "i enjoy hiking , crocheting , and reading . how about you ?"], "speaker_id": "snli:test:8387", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8387", "candidate_text": "well, a man is playing video games at his house", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8387:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8182:premise", "snli:test:8516:premise", "snli:test:7836:premise", "snli:test:8387:premise", "style:personachat:test:516:self:7:positive:0", "style:personachat:test:516:self:7:positive:1", "style:personachat:test:516:self:7:positive:2", "style:personachat:test:516:self:7:positive:3", "style:personachat:test:516:self:7:positive:4", "style:personachat:test:516:self:7:positive:5", "style:personachat:test:516:self:7:positive:6"], "evidence_texts": ["A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A young woman wearing a white sleeveless dress with red flowers is talking on her cellphone as she walks down a street.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "great , just got home from work . how are you ?", "i work at a help desk . so you work with homeless teens ?", "i did , i eat dinner while watching the news . what are you cooking ?", "is that not your favorite food ?", "i like to eat chinese and italian", "oh how fun ! do you have children or pets ?", "no , no pets or husband . i work a lot"], "speaker_id": "snli:test:8387", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8399", "candidate_text": "well, the men are walking inside the school", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8399:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8376:premise", "snli:test:8460:premise", "snli:test:8231:premise", "snli:test:8399:premise", "style:personachat:test:310:self:1:positive:0"], "evidence_texts": ["Several children are standing on the beach.", "Three musicians are playing in a band.", "Customers are ordering food at a cafe.", "Men wearing hats walk on the street.", "yea . . . the heat just got shut off and the snow is near . things are bleak"], "speaker_id": "snli:test:8399", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8399", "candidate_text": "The men are walking inside the school.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8399:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8376:premise", "snli:test:8460:premise", "snli:test:8231:premise", "snli:test:8399:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Several children are standing on the beach.", "Three musicians are playing in a band.", "Customers are ordering food at a cafe.", "Men wearing hats walk on the street.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8399", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8403", "candidate_text": "well, the people are viewing a new art exhibit", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8403:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8384:premise", "snli:test:8557:premise", "snli:test:8351:premise", "snli:test:8403:premise", "style:personachat:test:310:self:2:positive:0", "style:personachat:test:310:self:2:positive:1"], "evidence_texts": ["Two men in black pants and black shirts practice martial arts on a green gym floor.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "A boy looks out the window at another boy who is spraying the window with water.", "A few people are sitting inside of a building with lots of photography on the walls.", "hope to get more clients so i have more cash . just got on welfare , things are bad", "yea . . . the heat just got shut off and the snow is near . things are bleak"], "speaker_id": "snli:test:8403", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8403", "candidate_text": "I wonder whether the people are viewing a new art exhibit?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8403:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8384:premise", "snli:test:8557:premise", "snli:test:8351:premise", "snli:test:8403:premise", "style:personachat:test:169:self:2:positive:0", "style:personachat:test:169:self:2:positive:1"], "evidence_texts": ["Two men in black pants and black shirts practice martial arts on a green gym floor.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "A boy looks out the window at another boy who is spraying the window with water.", "A few people are sitting inside of a building with lots of photography on the walls.", "lucky me . we cook duck where i live . you ever tasted duck ?", "have you tried it with plum sauce ? you have two boys or two girls ."], "speaker_id": "snli:test:8403", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8406", "candidate_text": "well, the man is wearing a suit", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8406:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8358:premise", "snli:test:8838:premise", "snli:test:8406:premise", "snli:test:7643:premise", "style:personachat:test:310:self:5:positive:0", "style:personachat:test:310:self:5:positive:1", "style:personachat:test:310:self:5:positive:2", "style:personachat:test:310:self:5:positive:3", "style:personachat:test:310:self:5:positive:4"], "evidence_texts": ["A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "i have to go back to work , have a major client who needs his hair done", "i would love to make a dip out of those fresh beans for sunday football", "send some over for christmas ! i love christmas", "hope to get more clients so i have more cash . just got on welfare , things are bad", "yea . . . the heat just got shut off and the snow is near . things are bleak"], "speaker_id": "snli:test:8406", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8406", "candidate_text": "Honestly, The man is wearing a suit!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8406:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8358:premise", "snli:test:8838:premise", "snli:test:8406:premise", "snli:test:7643:premise", "style:personachat:test:332:self:1:positive:0"], "evidence_texts": ["A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A lady wearing a blue dress and earrings, is sitting at a table with a napkin in her hand and her elbow on the table.", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:8406", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8415", "candidate_text": "well, a woman sits in front of ancient indian ruins", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8415:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8365:premise", "snli:test:8497:premise", "snli:test:8026:premise", "snli:test:8415:premise", "style:personachat:test:310:self:6:positive:0", "style:personachat:test:310:self:6:positive:1", "style:personachat:test:310:self:6:positive:2", "style:personachat:test:310:self:6:positive:3", "style:personachat:test:310:self:6:positive:4", "style:personachat:test:310:self:6:positive:5"], "evidence_texts": ["A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "haha , i just finished a cheese pizza", "i have to go back to work , have a major client who needs his hair done", "i would love to make a dip out of those fresh beans for sunday football", "send some over for christmas ! i love christmas", "hope to get more clients so i have more cash . just got on welfare , things are bad", "yea . . . the heat just got shut off and the snow is near . things are bleak"], "speaker_id": "snli:test:8415", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8415", "candidate_text": "A woman sits in front of ancient indian ruins.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8415:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8365:premise", "snli:test:8497:premise", "snli:test:8026:premise", "snli:test:8415:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A young female standing on the deck of a boat starring at her cellphone while riding on the water.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8415", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8417", "candidate_text": "I wonder whether a man in a blue shirt poses for the camera?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8417:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8262:premise", "snli:test:8417:premise", "snli:test:8451:premise", "snli:test:8210:premise", "style:personachat:test:311:self:1:positive:0"], "evidence_texts": ["A male wearing a football uniform kicking a football during a football game.", "A group of people are standing on steps in front of a building.", "A man flying out of a river raft filled with 6 other people.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "what kind of pets do you have ?"], "speaker_id": "snli:test:8417", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8417", "candidate_text": "A man in a blue shirt poses for the camera.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8417:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8262:premise", "snli:test:8417:premise", "snli:test:8451:premise", "snli:test:8210:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A male wearing a football uniform kicking a football during a football game.", "A group of people are standing on steps in front of a building.", "A man flying out of a river raft filled with 6 other people.", "A dark-haired lady with a big smile is wearing a bright red scarf.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8417", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8421", "candidate_text": "I wonder whether the dog is contemplating whether it should enter the grassy forest?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8421:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8293:premise", "snli:test:8421:premise", "snli:test:8656:premise", "snli:test:8288:premise", "style:personachat:test:311:self:2:positive:0", "style:personachat:test:311:self:2:positive:1"], "evidence_texts": ["A girl in a red and white uniform is swinging a bat.", "Brown and cream dog with tongue out in front of grassy forest", "Small dog in costume stands on hind legs to reach dangling flowers.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "that is a lot of gran kids . how old are you ?", "what kind of pets do you have ?"], "speaker_id": "snli:test:8421", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8421", "candidate_text": "well, the dog is contemplating whether it should enter the grassy forest", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8421:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8293:premise", "snli:test:8421:premise", "snli:test:8656:premise", "snli:test:8288:premise", "style:personachat:test:519:self:2:positive:0", "style:personachat:test:519:self:2:positive:1"], "evidence_texts": ["A girl in a red and white uniform is swinging a bat.", "Brown and cream dog with tongue out in front of grassy forest", "Small dog in costume stands on hind legs to reach dangling flowers.", "Three men are using Washington Mutual ATMs outside near a parking lot.", "all my military money i brought a fancy house", "california i used to live in philadelphia"], "speaker_id": "snli:test:8421", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8433", "candidate_text": "I wonder whether a man is being creative in front of the crowd?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8433:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8054:premise", "snli:test:8433:premise", "snli:test:8018:premise", "snli:test:8483:premise", "style:personachat:test:311:self:5:positive:0", "style:personachat:test:311:self:5:positive:1", "style:personachat:test:311:self:5:positive:2", "style:personachat:test:311:self:5:positive:3", "style:personachat:test:311:self:5:positive:4"], "evidence_texts": ["A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "i am studying to become a doctor .", "i broke my bike . i love my volkswagen beetle .", "it is green . i want kids when i grow up .", "that is a lot of gran kids . how old are you ?", "what kind of pets do you have ?"], "speaker_id": "snli:test:8433", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8433", "candidate_text": "Honestly, A man is being creative in front of the crowd!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8433:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8054:premise", "snli:test:8433:premise", "snli:test:8018:premise", "snli:test:8483:premise", "style:personachat:test:339:self:1:positive:0"], "evidence_texts": ["A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A child with curly blond-hair looks at another child with long brown hair while sitting on a bench.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:8433", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8436", "candidate_text": "I wonder whether a man is making cotton candy for the children?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8436:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8436:premise", "snli:test:8483:premise", "snli:test:8054:premise", "snli:test:8665:premise", "style:personachat:test:311:self:7:positive:0", "style:personachat:test:311:self:7:positive:1", "style:personachat:test:311:self:7:positive:2", "style:personachat:test:311:self:7:positive:3", "style:personachat:test:311:self:7:positive:4", "style:personachat:test:311:self:7:positive:5", "style:personachat:test:311:self:7:positive:6"], "evidence_texts": ["A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "hi , i am doing great . how are you ?", "that is cool . i am a student attending auburn university .", "i am studying to become a doctor .", "i broke my bike . i love my volkswagen beetle .", "it is green . i want kids when i grow up .", "that is a lot of gran kids . how old are you ?", "what kind of pets do you have ?"], "speaker_id": "snli:test:8436", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8436", "candidate_text": "A man is making cotton candy for the children.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8436:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8436:premise", "snli:test:8483:premise", "snli:test:8054:premise", "snli:test:8665:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "A man pointing into the ear wearing a striped shirt in a small boat filled with many people.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8436", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8451", "candidate_text": "well, the raft is empty", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8451:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8417:premise", "snli:test:8733:premise", "snli:test:8262:premise", "snli:test:8451:premise", "style:personachat:test:312:self:1:positive:0"], "evidence_texts": ["A group of people are standing on steps in front of a building.", "A tennis player in a pink outfit serves the ball to her opponent.", "A male wearing a football uniform kicking a football during a football game.", "A man flying out of a river raft filled with 6 other people.", "yes , they are , i love making them . makes me happy ."], "speaker_id": "snli:test:8451", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8451", "candidate_text": "Honestly, The raft is empty!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8451:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8417:premise", "snli:test:8733:premise", "snli:test:8262:premise", "snli:test:8451:premise", "style:personachat:test:339:self:5:positive:0", "style:personachat:test:339:self:5:positive:1", "style:personachat:test:339:self:5:positive:2", "style:personachat:test:339:self:5:positive:3", "style:personachat:test:339:self:5:positive:4"], "evidence_texts": ["A group of people are standing on steps in front of a building.", "A tennis player in a pink outfit serves the ball to her opponent.", "A male wearing a football uniform kicking a football during a football game.", "A man flying out of a river raft filled with 6 other people.", "music is my bread and butter , i like surrounding myself with music . what do you teach ?", "wow , that is cool , you must be very smart . i work at petsmart right now", "what is your favorite place to visit in europe ?", "my father was chief of police in france , i have been there a lot !", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:8451", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8454", "candidate_text": "well, a homeless man is sitting on a park bench", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8454:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:7803:premise", "snli:test:8606:premise", "snli:test:8454:premise", "snli:test:7565:premise", "style:personachat:test:312:self:2:positive:0", "style:personachat:test:312:self:2:positive:1"], "evidence_texts": ["Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "no only for babies , and only in the hospital", "yes , they are , i love making them . makes me happy ."], "speaker_id": "snli:test:8454", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8454", "candidate_text": "A homeless man is sitting on a park bench.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8454:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:7803:premise", "snli:test:8606:premise", "snli:test:8454:premise", "snli:test:7565:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "A man dressed in a suit and bowler style hat plays a sousaphone which has flames coming out of the top.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8454", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8460", "candidate_text": "well, the band is signing autographs", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8460:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8460:premise", "snli:test:8399:premise", "snli:test:8739:premise", "snli:test:8376:premise", "style:personachat:test:312:self:5:positive:0", "style:personachat:test:312:self:5:positive:1", "style:personachat:test:312:self:5:positive:2", "style:personachat:test:312:self:5:positive:3", "style:personachat:test:312:self:5:positive:4"], "evidence_texts": ["Three musicians are playing in a band.", "Men wearing hats walk on the street.", "A young man standing outside a laundromat.", "Several children are standing on the beach.", "not me i am disable , can not work", "tyler farr is great ! i knit hats .", "yes , i made those hats for babies", "no only for babies , and only in the hospital", "yes , they are , i love making them . makes me happy ."], "speaker_id": "snli:test:8460", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8460", "candidate_text": "I wonder whether the band is signing autographs?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8460:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8460:premise", "snli:test:8399:premise", "snli:test:8739:premise", "snli:test:8376:premise", "style:personachat:test:173:self:5:positive:0", "style:personachat:test:173:self:5:positive:1", "style:personachat:test:173:self:5:positive:2", "style:personachat:test:173:self:5:positive:3", "style:personachat:test:173:self:5:positive:4"], "evidence_texts": ["Three musicians are playing in a band.", "Men wearing hats walk on the street.", "A young man standing outside a laundromat.", "Several children are standing on the beach.", "that is fun ! whose wedding did you go to ?", "not much . i work at a hospital so i do not have much time .", "what did you graduate for ?", "oh nice ! my mother works as a secretary . and , no .", "i do not have plans yet . what about you ?"], "speaker_id": "snli:test:8460", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8464", "candidate_text": "well, two men are ready to have their picture taken", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8464:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8464:premise", "snli:test:8269:premise", "snli:test:8258:premise", "snli:test:8485:premise", "style:personachat:test:312:self:7:positive:0", "style:personachat:test:312:self:7:positive:1", "style:personachat:test:312:self:7:positive:2", "style:personachat:test:312:self:7:positive:3", "style:personachat:test:312:self:7:positive:4", "style:personachat:test:312:self:7:positive:5", "style:personachat:test:312:self:7:positive:6"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "A woman is holding a large check for Kids Food Basket.", "A man dressed only in a towel is in a sauna.", "what is your hobby ? mine is cooking .", "okay , i love country music , and only eat vegetables .", "not me i am disable , can not work", "tyler farr is great ! i knit hats .", "yes , i made those hats for babies", "no only for babies , and only in the hospital", "yes , they are , i love making them . makes me happy ."], "speaker_id": "snli:test:8464", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8464", "candidate_text": "Honestly, Two men are ready to have their picture taken!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8464:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8464:premise", "snli:test:8269:premise", "snli:test:8258:premise", "snli:test:8485:premise", "style:personachat:test:342:self:2:positive:0", "style:personachat:test:342:self:2:positive:1"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "A woman is holding a large check for Kids Food Basket.", "A man dressed only in a towel is in a sauna.", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:8464", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8465", "candidate_text": "well, the men are standing in total darkness", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8465:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8485:premise", "snli:test:8465:premise", "snli:test:8269:premise", "snli:test:8486:premise", "style:personachat:test:313:self:1:positive:0"], "evidence_texts": ["A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "A man dressed only in a towel is in a sauna.", "not really , i am a home body mostly , sometimes i go clubbing"], "speaker_id": "snli:test:8465", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8465", "candidate_text": "The men are standing in total darkness.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8465:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8485:premise", "snli:test:8465:premise", "snli:test:8269:premise", "snli:test:8486:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "A man dressed only in a towel is in a sauna.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8465", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8471", "candidate_text": "well, two people are sitting in a coffee shop", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8471:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8191:premise", "snli:test:8627:premise", "snli:test:7896:premise", "snli:test:8471:premise", "style:personachat:test:313:self:2:positive:0", "style:personachat:test:313:self:2:positive:1"], "evidence_texts": ["Phone to her ear, a woman bends forward at the side of a busy street.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman is taking a picture on the shoulders of another woman in cold weather.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "i work as a nurses aid .", "not really , i am a home body mostly , sometimes i go clubbing"], "speaker_id": "snli:test:8471", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8471", "candidate_text": "I wonder whether two people are sitting in a coffee shop?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8471:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8191:premise", "snli:test:8627:premise", "snli:test:7896:premise", "snli:test:8471:premise", "style:personachat:test:176:self:2:positive:0", "style:personachat:test:176:self:2:positive:1"], "evidence_texts": ["Phone to her ear, a woman bends forward at the side of a busy street.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman is taking a picture on the shoulders of another woman in cold weather.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "i am thinking about being a nurse . my aunt works in a hospital , and likes it .", "what do you do now ?"], "speaker_id": "snli:test:8471", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8483", "candidate_text": "well, two men take a picture together", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8483:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8436:premise", "snli:test:8665:premise", "snli:test:8433:premise", "snli:test:8483:premise", "style:personachat:test:313:self:5:positive:0", "style:personachat:test:313:self:5:positive:1", "style:personachat:test:313:self:5:positive:2", "style:personachat:test:313:self:5:positive:3", "style:personachat:test:313:self:5:positive:4"], "evidence_texts": ["A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "oh that sucks i am scared of water but i like to garden ponder that", "sounds like a good past time", "what do you do there ?", "i work as a nurses aid .", "not really , i am a home body mostly , sometimes i go clubbing"], "speaker_id": "snli:test:8483", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8483", "candidate_text": "Honestly, Two men take a picture together!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8483:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8436:premise", "snli:test:8665:premise", "snli:test:8433:premise", "snli:test:8483:premise", "style:personachat:test:347:self:1:positive:0"], "evidence_texts": ["A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "well , i did not allow me to think !"], "speaker_id": "snli:test:8483", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8485", "candidate_text": "well, a human not wearing any clothes is in a room", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8485:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8465:premise", "snli:test:8485:premise", "snli:test:8464:premise", "snli:test:8269:premise", "style:personachat:test:313:self:7:positive:0", "style:personachat:test:313:self:7:positive:1", "style:personachat:test:313:self:7:positive:2", "style:personachat:test:313:self:7:positive:3", "style:personachat:test:313:self:7:positive:4", "style:personachat:test:313:self:7:positive:5", "style:personachat:test:313:self:7:positive:6"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "hi the jack here , i am good", "beautiful and sunny nice day for gardening yours ?", "oh that sucks i am scared of water but i like to garden ponder that", "sounds like a good past time", "what do you do there ?", "i work as a nurses aid .", "not really , i am a home body mostly , sometimes i go clubbing"], "speaker_id": "snli:test:8485", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8485", "candidate_text": "A human not wearing any clothes is in a room.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8485:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8465:premise", "snli:test:8485:premise", "snli:test:8464:premise", "snli:test:8269:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "a brown dog standing in the middle of a huge field.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8485", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8486", "candidate_text": "I wonder whether a man dressed in a coat is in a kitchen?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8486:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8465:premise", "snli:test:8796:premise", "snli:test:8486:premise", "snli:test:8464:premise", "style:personachat:test:314:self:1:positive:0"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "A racer on a motorcycle lifts one hand in the air.", "A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:8486", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8486", "candidate_text": "A man dressed in a coat is in a kitchen.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8486:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8465:premise", "snli:test:8796:premise", "snli:test:8486:premise", "snli:test:8464:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two men are making hand gestures and posing for a picture.", "A racer on a motorcycle lifts one hand in the air.", "A man dressed only in a towel is in a sauna.", "Two men are making hand gestures and posing for a picture.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8486", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8487", "candidate_text": "I wonder whether an old man dressed only in a yellow towel is sweating in a sauna?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8487:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8796:premise", "snli:test:8845:premise", "snli:test:8465:premise", "snli:test:8487:premise", "style:personachat:test:314:self:2:positive:0", "style:personachat:test:314:self:2:positive:1"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A group of young men are running in a grassy field.", "Two men are making hand gestures and posing for a picture.", "A man dressed only in a towel is in a sauna.", "yes i would love to but i stay busy running an reading", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:8487", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8487", "candidate_text": "well, an old man dressed only in a yellow towel is sweating in a sauna", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8487:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8796:premise", "snli:test:8845:premise", "snli:test:8465:premise", "snli:test:8487:premise", "style:personachat:test:522:self:2:positive:0", "style:personachat:test:522:self:2:positive:1"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A group of young men are running in a grassy field.", "Two men are making hand gestures and posing for a picture.", "A man dressed only in a towel is in a sauna.", "wow . i am childless , but love to teach my nephews to hunt for polar bears", "i think they actually prefer pepsi . they make for some tasty meals"], "speaker_id": "snli:test:8487", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8491", "candidate_text": "I wonder whether an upset little girl is throwing a tantrum?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8491:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8246:premise", "snli:test:8204:premise", "snli:test:8540:premise", "snli:test:8491:premise", "style:personachat:test:314:self:5:positive:0", "style:personachat:test:314:self:5:positive:1", "style:personachat:test:314:self:5:positive:2", "style:personachat:test:314:self:5:positive:3", "style:personachat:test:314:self:5:positive:4"], "evidence_texts": ["An Asian man makes faces for the camera.", "Three men are playing frisbee in the woods.", "A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "i like to eat vegetables an beans", "i like to run an i love a good reliable older car", "sounds good i would love to go there but as a doctor i am busy", "yes i would love to but i stay busy running an reading", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:8491", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8491", "candidate_text": "Honestly, An upset little girl is throwing a tantrum!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8491:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8246:premise", "snli:test:8204:premise", "snli:test:8540:premise", "snli:test:8491:premise", "style:personachat:test:348:self:1:positive:0"], "evidence_texts": ["An Asian man makes faces for the camera.", "Three men are playing frisbee in the woods.", "A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:8491", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8492", "candidate_text": "I wonder whether the girl is laughing?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8492:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8492:premise", "snli:test:8540:premise", "snli:test:8246:premise", "snli:test:8541:premise", "style:personachat:test:314:self:6:positive:0", "style:personachat:test:314:self:6:positive:1", "style:personachat:test:314:self:6:positive:2", "style:personachat:test:314:self:6:positive:3", "style:personachat:test:314:self:6:positive:4", "style:personachat:test:314:self:6:positive:5"], "evidence_texts": ["A little girl in a pink dress crying.", "A couple opening up a new electronically mixer.", "An Asian man makes faces for the camera.", "A couple opening up a new electronically mixer.", "hey there how are you today ? yes i like dogs", "i like to eat vegetables an beans", "i like to run an i love a good reliable older car", "sounds good i would love to go there but as a doctor i am busy", "yes i would love to but i stay busy running an reading", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:8492", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8492", "candidate_text": "The girl is laughing.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8492:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8492:premise", "snli:test:8540:premise", "snli:test:8246:premise", "snli:test:8541:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A little girl in a pink dress crying.", "A couple opening up a new electronically mixer.", "An Asian man makes faces for the camera.", "A couple opening up a new electronically mixer.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8492", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8497", "candidate_text": "well, the baby and the cat are in the living room", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8497:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8415:premise", "snli:test:8497:premise", "snli:test:8653:premise", "snli:test:8365:premise", "style:personachat:test:315:self:1:positive:0"], "evidence_texts": ["A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "my uncle got me into scotch . i would say thor cause the history"], "speaker_id": "snli:test:8497", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8497", "candidate_text": "Honestly, The baby and the cat are in the living room!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8497:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8415:premise", "snli:test:8497:premise", "snli:test:8653:premise", "snli:test:8365:premise", "style:personachat:test:348:self:5:positive:0", "style:personachat:test:348:self:5:positive:1", "style:personachat:test:348:self:5:positive:2", "style:personachat:test:348:self:5:positive:3", "style:personachat:test:348:self:5:positive:4"], "evidence_texts": ["A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A man dressed in all white and wearing a white hat stands on a sidewalk next to a building.", "where are you currently from ?", "do you like it there ?", "what types of jobs are you looking for ? what education or skills can you offer ?", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:8497", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8504", "candidate_text": "well, a young boy is playing in the grass", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8504:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8504:premise", "snli:test:8348:premise", "snli:test:8560:premise", "snli:test:8347:premise", "style:personachat:test:315:self:2:positive:0", "style:personachat:test:315:self:2:positive:1"], "evidence_texts": ["A young boy is playing in the sandy water.", "A crowd of people standing in front of statues.", "A man running in a race passes by spectators.", "A crowd of people standing in front of statues.", "i watch superhero movies with a scotch on the rocks", "my uncle got me into scotch . i would say thor cause the history"], "speaker_id": "snli:test:8504", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8504", "candidate_text": "A young boy is playing in the grass.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8504:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8504:premise", "snli:test:8348:premise", "snli:test:8560:premise", "snli:test:8347:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young boy is playing in the sandy water.", "A crowd of people standing in front of statues.", "A man running in a race passes by spectators.", "A crowd of people standing in front of statues.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8504", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8516", "candidate_text": "well, some men are by water", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8516:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8516:premise", "snli:test:8387:premise", "snli:test:8712:premise", "snli:test:8182:premise", "style:personachat:test:315:self:5:positive:0", "style:personachat:test:315:self:5:positive:1", "style:personachat:test:315:self:5:positive:2", "style:personachat:test:315:self:5:positive:3", "style:personachat:test:315:self:5:positive:4"], "evidence_texts": ["Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "i was attacked by a strange dog as a child . .", "i am sure i will grow out of it . do you like comic books ?", "i write for a living as well . whats your favorite way to unwind", "i watch superhero movies with a scotch on the rocks", "my uncle got me into scotch . i would say thor cause the history"], "speaker_id": "snli:test:8516", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8516", "candidate_text": "I wonder whether some men are by water?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8516:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8516:premise", "snli:test:8387:premise", "snli:test:8712:premise", "snli:test:8182:premise", "style:personachat:test:191:self:5:positive:0", "style:personachat:test:191:self:5:positive:1", "style:personachat:test:191:self:5:positive:2", "style:personachat:test:191:self:5:positive:3", "style:personachat:test:191:self:5:positive:4"], "evidence_texts": ["Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "A man in a blue jacket leans on a post and talks on the phone while an old man carries yellow bags.", "a very honorable profession . what do you do for fun ?", "i love beetles ! is it a convertible ?", "i am looking forward to a day of xbox tomorrow on my day off .", "do you like to read ?", "oh ? i actually play in a band on the weekends ."], "speaker_id": "snli:test:8516", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8540", "candidate_text": "well, a couple opens a new appliance together", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8540:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8540:premise", "snli:test:8492:premise", "snli:test:8491:premise", "snli:test:8548:premise", "style:personachat:test:315:self:7:positive:0", "style:personachat:test:315:self:7:positive:1", "style:personachat:test:315:self:7:positive:2", "style:personachat:test:315:self:7:positive:3", "style:personachat:test:315:self:7:positive:4", "style:personachat:test:315:self:7:positive:5", "style:personachat:test:315:self:7:positive:6"], "evidence_texts": ["A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "A little girl in a pink dress crying.", "A man on a ladder in tall grass.", "i am fine how is your day ?", "nice i am brainstorming today . . do you fear anything", "i was attacked by a strange dog as a child . .", "i am sure i will grow out of it . do you like comic books ?", "i write for a living as well . whats your favorite way to unwind", "i watch superhero movies with a scotch on the rocks", "my uncle got me into scotch . i would say thor cause the history"], "speaker_id": "snli:test:8540", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8540", "candidate_text": "Honestly, A couple opens a new appliance together!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8540:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8540:premise", "snli:test:8492:premise", "snli:test:8491:premise", "snli:test:8548:premise", "style:personachat:test:363:self:2:positive:0", "style:personachat:test:363:self:2:positive:1"], "evidence_texts": ["A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "A little girl in a pink dress crying.", "A man on a ladder in tall grass.", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:8540", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8541", "candidate_text": "well, the couple purchased a used mixer", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8541:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8548:premise", "snli:test:8541:premise", "snli:test:8492:premise", "snli:test:8663:premise", "style:personachat:test:316:self:1:positive:0"], "evidence_texts": ["A man on a ladder in tall grass.", "A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "A man in a boom lift bucket welds.", "that is awesome . i am happy you enjoy your job"], "speaker_id": "snli:test:8541", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8541", "candidate_text": "The couple purchased a used mixer.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8541:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8548:premise", "snli:test:8541:premise", "snli:test:8492:premise", "snli:test:8663:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man on a ladder in tall grass.", "A couple opening up a new electronically mixer.", "A little girl in a pink dress crying.", "A man in a boom lift bucket welds.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8541", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8548", "candidate_text": "well, the man is seated on his couch in his living room", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8548:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8548:premise", "snli:test:8541:premise", "snli:test:8663:premise", "snli:test:8540:premise", "style:personachat:test:316:self:2:positive:0", "style:personachat:test:316:self:2:positive:1"], "evidence_texts": ["A man on a ladder in tall grass.", "A couple opening up a new electronically mixer.", "A man in a boom lift bucket welds.", "A couple opening up a new electronically mixer.", "it has its up days i guess . a lot of work though", "that is awesome . i am happy you enjoy your job"], "speaker_id": "snli:test:8548", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8548", "candidate_text": "I wonder whether the man is seated on his couch in his living room?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8548:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8548:premise", "snli:test:8541:premise", "snli:test:8663:premise", "snli:test:8540:premise", "style:personachat:test:196:self:2:positive:0", "style:personachat:test:196:self:2:positive:1"], "evidence_texts": ["A man on a ladder in tall grass.", "A couple opening up a new electronically mixer.", "A man in a boom lift bucket welds.", "A couple opening up a new electronically mixer.", "i bet that can be interesting", "what kind of music you like ?"], "speaker_id": "snli:test:8548", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8553", "candidate_text": "well, a mime is practicing his art", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8553:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8096:premise", "snli:test:8553:premise", "snli:test:8753:premise", "snli:test:8082:premise", "style:personachat:test:316:self:5:positive:0", "style:personachat:test:316:self:5:positive:1", "style:personachat:test:316:self:5:positive:2", "style:personachat:test:316:self:5:positive:3", "style:personachat:test:316:self:5:positive:4"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "a year now . so where are you from ?", "i see . i am from puerto rico", "it is . you should visit there some time", "it has its up days i guess . a lot of work though", "that is awesome . i am happy you enjoy your job"], "speaker_id": "snli:test:8553", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8553", "candidate_text": "Honestly, A mime is practicing his art!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8553:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8096:premise", "snli:test:8553:premise", "snli:test:8753:premise", "snli:test:8082:premise", "style:personachat:test:374:self:1:positive:0"], "evidence_texts": ["A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "An accordion player with a bright colored shirt sitting outside in a chair with a young man in a blue t-shirt passing by.", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:8553", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8557", "candidate_text": "well, a man competes in a long distance run", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8557:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8403:premise", "snli:test:8557:premise", "snli:test:8384:premise", "snli:test:9056:premise", "style:personachat:test:316:self:7:positive:0", "style:personachat:test:316:self:7:positive:1", "style:personachat:test:316:self:7:positive:2", "style:personachat:test:316:self:7:positive:3", "style:personachat:test:316:self:7:positive:4", "style:personachat:test:316:self:7:positive:5", "style:personachat:test:316:self:7:positive:6"], "evidence_texts": ["A few people are sitting inside of a building with lots of photography on the walls.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two men in black pants and black shirts practice martial arts on a green gym floor.", "A woman and two children walk along stones to cross a river while a dog swims.", "i am okay . missing home today . i am part of the army . currently in south korea", "yes , for only 21 , its hard to be away long", "a year now . so where are you from ?", "i see . i am from puerto rico", "it is . you should visit there some time", "it has its up days i guess . a lot of work though", "that is awesome . i am happy you enjoy your job"], "speaker_id": "snli:test:8557", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8557", "candidate_text": "A man competes in a long distance run.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8557:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8403:premise", "snli:test:8557:premise", "snli:test:8384:premise", "snli:test:9056:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A few people are sitting inside of a building with lots of photography on the walls.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two men in black pants and black shirts practice martial arts on a green gym floor.", "A woman and two children walk along stones to cross a river while a dog swims.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8557", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8558", "candidate_text": "well, people watch a man jog", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8558:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9056:premise", "snli:test:8403:premise", "snli:test:8558:premise", "snli:test:9089:premise", "style:personachat:test:317:self:1:positive:0"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "A few people are sitting inside of a building with lots of photography on the walls.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "no , i ran a rabbit rescue blog . i would not be able to eat it ."], "speaker_id": "snli:test:8558", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8558", "candidate_text": "I wonder whether people watch a man jog?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8558:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9056:premise", "snli:test:8403:premise", "snli:test:8558:premise", "snli:test:9089:premise", "style:personachat:test:199:self:1:positive:0"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "A few people are sitting inside of a building with lots of photography on the walls.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:8558", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8560", "candidate_text": "well, the person is running a marathon", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8560:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8560:premise", "snli:test:8504:premise", "snli:test:8685:premise", "snli:test:8348:premise", "style:personachat:test:317:self:2:positive:0", "style:personachat:test:317:self:2:positive:1"], "evidence_texts": ["A man running in a race passes by spectators.", "A young boy is playing in the sandy water.", "People protesting age discrimination in an organized street rally.", "A crowd of people standing in front of statues.", "i am sorry to hear that , pets are the best .", "no , i ran a rabbit rescue blog . i would not be able to eat it ."], "speaker_id": "snli:test:8560", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8560", "candidate_text": "Honestly, The person is running a marathon!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8560:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8560:premise", "snli:test:8504:premise", "snli:test:8685:premise", "snli:test:8348:premise", "style:personachat:test:374:self:7:positive:0", "style:personachat:test:374:self:7:positive:1", "style:personachat:test:374:self:7:positive:2", "style:personachat:test:374:self:7:positive:3", "style:personachat:test:374:self:7:positive:4", "style:personachat:test:374:self:7:positive:5", "style:personachat:test:374:self:7:positive:6"], "evidence_texts": ["A man running in a race passes by spectators.", "A young boy is playing in the sandy water.", "People protesting age discrimination in an organized street rally.", "A crowd of people standing in front of statues.", "oh i am fine , just trying to find a good movie to watch .", "i have not seen that , just finished game of thrones whew !", "i write as well since i hate exercise or physical stuff .", "oh lord you work two jobs . wish i had your energy .", "do you have a supportive family ? my husband is my world .", "yes , do not ever stop writing . it really calms the soul or awakens it .", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:8560", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8572", "candidate_text": "well, two young girls walk up a slide", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8572:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8572:premise", "snli:test:8323:premise", "snli:test:8590:premise", "snli:test:8319:premise", "style:personachat:test:317:self:5:positive:0", "style:personachat:test:317:self:5:positive:1", "style:personachat:test:317:self:5:positive:2", "style:personachat:test:317:self:5:positive:3", "style:personachat:test:317:self:5:positive:4"], "evidence_texts": ["Two young girls are jumping off of a steep step.", "An American wrestler in a purple singlet wrestlers an opponent.", "the white dog is carrying a ball in its mouth.", "A young woman in white working in a professional kitchen.", "no cooking i live with my little sister , also i do not sing but love tim mcgraw .", "that is awesome i buy many things online , mostly of my idol theodore roosevelt .", "what type of books do you write ?", "i am sorry to hear that , pets are the best .", "no , i ran a rabbit rescue blog . i would not be able to eat it ."], "speaker_id": "snli:test:8572", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8572", "candidate_text": "Two young girls walk up a slide.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8572:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8572:premise", "snli:test:8323:premise", "snli:test:8590:premise", "snli:test:8319:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two young girls are jumping off of a steep step.", "An American wrestler in a purple singlet wrestlers an opponent.", "the white dog is carrying a ball in its mouth.", "A young woman in white working in a professional kitchen.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8572", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8585", "candidate_text": "well, the dog has cold paws from the snow", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8585:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:7188:premise", "snli:test:8585:premise", "snli:test:8822:premise", "snli:test:6971:premise", "style:personachat:test:317:self:6:positive:0", "style:personachat:test:317:self:6:positive:1", "style:personachat:test:317:self:6:positive:2", "style:personachat:test:317:self:6:positive:3", "style:personachat:test:317:self:6:positive:4", "style:personachat:test:317:self:6:positive:5"], "evidence_texts": ["construction worker working on a structure", "White dog playing in the snow.", "A choir of youths are singing.", "Two men walking in summer attire.", "hi i blog about rabbit rescues and compete as a body builder .", "no cooking i live with my little sister , also i do not sing but love tim mcgraw .", "that is awesome i buy many things online , mostly of my idol theodore roosevelt .", "what type of books do you write ?", "i am sorry to hear that , pets are the best .", "no , i ran a rabbit rescue blog . i would not be able to eat it ."], "speaker_id": "snli:test:8585", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8585", "candidate_text": "I wonder whether the dog has cold paws from the snow?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8585:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:7188:premise", "snli:test:8585:premise", "snli:test:8822:premise", "snli:test:6971:premise", "style:personachat:test:199:self:6:positive:0", "style:personachat:test:199:self:6:positive:1", "style:personachat:test:199:self:6:positive:2", "style:personachat:test:199:self:6:positive:3", "style:personachat:test:199:self:6:positive:4", "style:personachat:test:199:self:6:positive:5"], "evidence_texts": ["construction worker working on a structure", "White dog playing in the snow.", "A choir of youths are singing.", "Two men walking in summer attire.", "hey there ! i am well . how are you ?", "i am in my twenties . not going to say exactly . what do you do for fun ?", "cosplay ? i have never been into that . unless you count dressing as santa claus ! i love christmas !", "i went through a blue phase . hair stylists do that .", "new york ! a big slice of cheese pizza is the best !", "do you have dogs to keep you company at home ? i have three"], "speaker_id": "snli:test:8585", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8590", "candidate_text": "well, the dog has a ball", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8590:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8572:premise", "snli:test:8772:premise", "snli:test:8590:premise", "snli:test:8323:premise", "style:personachat:test:318:self:1:positive:0"], "evidence_texts": ["Two young girls are jumping off of a steep step.", "The women is photographing statues outdoors on a sunny day.", "the white dog is carrying a ball in its mouth.", "An American wrestler in a purple singlet wrestlers an opponent.", "i never did either . just english for me"], "speaker_id": "snli:test:8590", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8590", "candidate_text": "Honestly, The dog has a ball!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8590:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8572:premise", "snli:test:8772:premise", "snli:test:8590:premise", "snli:test:8323:premise", "style:personachat:test:375:self:5:positive:0", "style:personachat:test:375:self:5:positive:1", "style:personachat:test:375:self:5:positive:2", "style:personachat:test:375:self:5:positive:3", "style:personachat:test:375:self:5:positive:4"], "evidence_texts": ["Two young girls are jumping off of a steep step.", "The women is photographing statues outdoors on a sunny day.", "the white dog is carrying a ball in its mouth.", "An American wrestler in a purple singlet wrestlers an opponent.", "yes . i go to the zoo all the time to watch the bears .", "it is nice to have someone to share activities with .", "how sweet . have you ever seen a green bear ?", "no . it is my favorite color so i am on the hunt for one !", "that would be cool ! what is your favorite movie ?"], "speaker_id": "snli:test:8590", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8606", "candidate_text": "well, a small child dressed for winter stares across a lake while leaning on a mesh fence", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8606:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8454:premise", "snli:test:7803:premise", "snli:test:8606:premise", "snli:test:8922:premise", "style:personachat:test:318:self:2:positive:0", "style:personachat:test:318:self:2:positive:1"], "evidence_texts": ["Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "english is my first language but my friend who speaks spanish also likes classical", "i never did either . just english for me"], "speaker_id": "snli:test:8606", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8606", "candidate_text": "A small child dressed for winter stares across a lake while leaning on a mesh fence.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8606:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8454:premise", "snli:test:7803:premise", "snli:test:8606:premise", "snli:test:8922:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "Two African American men laughing with aprons on, while a third man wearing a chef jacket is on a cellular phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8606", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8607", "candidate_text": "well, a small child dressed for winter stares across a large lake while leaning on a mesh fence", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8607:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8922:premise", "snli:test:8454:premise", "snli:test:8960:premise", "snli:test:8607:premise", "style:personachat:test:318:self:5:positive:0", "style:personachat:test:318:self:5:positive:1", "style:personachat:test:318:self:5:positive:2", "style:personachat:test:318:self:5:positive:3", "style:personachat:test:318:self:5:positive:4"], "evidence_texts": ["A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "well that sounds gross . i prefer bacon", "what surgery did you have ?", "good for you . i heard that classical music helps healing . i like bach", "english is my first language but my friend who speaks spanish also likes classical", "i never did either . just english for me"], "speaker_id": "snli:test:8607", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8607", "candidate_text": "I wonder whether a small child dressed for winter stares across a large lake while leaning on a mesh fence?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8607:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8922:premise", "snli:test:8454:premise", "snli:test:8960:premise", "snli:test:8607:premise", "style:personachat:test:201:self:5:positive:0", "style:personachat:test:201:self:5:positive:1", "style:personachat:test:201:self:5:positive:2", "style:personachat:test:201:self:5:positive:3", "style:personachat:test:201:self:5:positive:4"], "evidence_texts": ["A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "Woman wearing dark brown leather jacket, sunglasses, brown leather boots, silver designer handbag, sitting on a fire engine red park bench.", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "kind of . i have a tail as feet and wavy hair . are you ?", "you do not live in the sea like me ? where do you live ?", "what do you do there ? i explore with my friends after dinner", "that sounds nice , do you eat dinner as a family ?", "what do you teach your kid in home school ? does he like school ?"], "speaker_id": "snli:test:8607", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8613", "candidate_text": "well, a panda jumps", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8613:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:4210:premise", "snli:test:8613:premise", "snli:test:3225:premise", "snli:test:3659:premise", "style:personachat:test:318:self:7:positive:0", "style:personachat:test:318:self:7:positive:1", "style:personachat:test:318:self:7:positive:2", "style:personachat:test:318:self:7:positive:3", "style:personachat:test:318:self:7:positive:4", "style:personachat:test:318:self:7:positive:5", "style:personachat:test:318:self:7:positive:6"], "evidence_texts": ["At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "no time for sports since i am constantly working as an engineer . you ?", "that is awesome ! i would be afraid of all the insects while playing", "well that sounds gross . i prefer bacon", "what surgery did you have ?", "good for you . i heard that classical music helps healing . i like bach", "english is my first language but my friend who speaks spanish also likes classical", "i never did either . just english for me"], "speaker_id": "snli:test:8613", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8613", "candidate_text": "Honestly, a panda jumps!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8613:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:4210:premise", "snli:test:8613:premise", "snli:test:3225:premise", "snli:test:3659:premise", "style:personachat:test:377:self:2:positive:0", "style:personachat:test:377:self:2:positive:1"], "evidence_texts": ["At a table with an art camp pamphlet on it, a young woman and an older man are painting ceramics under the supervision of a blond woman wearing a name tag.", "A man dressed in black and holding a package in his hands is walking on a sidewalk near a police sign asking for witnesses to a road dealth to step forward.", "A woman in a gray kimono walks past a black car in the middle of a crowded street full of Japanese signs, with a man in an orange shirt and khaki slacks.", "A woman in a bright pink jacket and jeans, with two young children, also in jackets, is on the deck of a yacht using a telescope to look out to sea.", "yes , i love it ! i love fashion and clothes .", "oh i bet there is ! i am an expert when it comes to shopping ."], "speaker_id": "snli:test:8613", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8627", "candidate_text": "well, the animal is climbing a tree", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8627:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8471:premise", "snli:test:8687:premise", "snli:test:8191:premise", "snli:test:8627:premise", "style:personachat:test:319:self:1:positive:0"], "evidence_texts": ["A man holds up a sign that reads \"tattoo\" as many people walk near him.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "Phone to her ear, a woman bends forward at the side of a busy street.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "i would like to try living in the ocean , so beautiful ."], "speaker_id": "snli:test:8627", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8627", "candidate_text": "The animal is climbing a tree.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8627:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8471:premise", "snli:test:8687:premise", "snli:test:8191:premise", "snli:test:8627:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man holds up a sign that reads \"tattoo\" as many people walk near him.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "Phone to her ear, a woman bends forward at the side of a busy street.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8627", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8640", "candidate_text": "well, man on lunch break", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8640:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8640:premise", "snli:test:7437:premise", "snli:test:8828:premise", "snli:test:7411:premise", "style:personachat:test:319:self:2:positive:0", "style:personachat:test:319:self:2:positive:1"], "evidence_texts": ["A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "i do not want any , i like living alone .", "i would like to try living in the ocean , so beautiful ."], "speaker_id": "snli:test:8640", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8640", "candidate_text": "I wonder whether man on lunch break?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8640:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8640:premise", "snli:test:7437:premise", "snli:test:8828:premise", "snli:test:7411:premise", "style:personachat:test:202:self:2:positive:0", "style:personachat:test:202:self:2:positive:1"], "evidence_texts": ["A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A group of men are sitting and standing a courtyard, some of them are reading books and some are talking.", "sounds great . yes it can be . do you have any pets ? i have frogs .", "what kinds of cats do you have ? i have three ragdoll cats ."], "speaker_id": "snli:test:8640", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8653", "candidate_text": "well, a man is taking a picture of a school of whales in hawaii", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8653:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8497:premise", "snli:test:8653:premise", "snli:test:8415:premise", "snli:test:8708:premise", "style:personachat:test:319:self:5:positive:0", "style:personachat:test:319:self:5:positive:1", "style:personachat:test:319:self:5:positive:2", "style:personachat:test:319:self:5:positive:3", "style:personachat:test:319:self:5:positive:4"], "evidence_texts": ["A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "are you on a submarine ? i work labor and delivery at the hospital .", "oh , your a mermaid , that would be nice . you live in water and i only drink it .", "i spend all my time at work or the gym . do you have kids ?", "i do not want any , i like living alone .", "i would like to try living in the ocean , so beautiful ."], "speaker_id": "snli:test:8653", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8653", "candidate_text": "Honestly, A man is taking a picture of a school of whales in Hawaii!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8653:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8497:premise", "snli:test:8653:premise", "snli:test:8415:premise", "snli:test:8708:premise", "style:personachat:test:379:self:1:positive:0"], "evidence_texts": ["A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A woman with a blue jacket around her waist is sitting on the ledge of some stone ruins resting.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "it really is ! it was nice to have met you jane !"], "speaker_id": "snli:test:8653", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8655", "candidate_text": "well, a man is using a camera", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8655:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8708:premise", "snli:test:8497:premise", "snli:test:8709:premise", "snli:test:8655:premise", "style:personachat:test:319:self:6:positive:0", "style:personachat:test:319:self:6:positive:1", "style:personachat:test:319:self:6:positive:2", "style:personachat:test:319:self:6:positive:3", "style:personachat:test:319:self:6:positive:4", "style:personachat:test:319:self:6:positive:5"], "evidence_texts": ["An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "i did , salad and water .", "are you on a submarine ? i work labor and delivery at the hospital .", "oh , your a mermaid , that would be nice . you live in water and i only drink it .", "i spend all my time at work or the gym . do you have kids ?", "i do not want any , i like living alone .", "i would like to try living in the ocean , so beautiful ."], "speaker_id": "snli:test:8655", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8655", "candidate_text": "A man is using a camera.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8655:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8708:premise", "snli:test:8497:premise", "snli:test:8709:premise", "snli:test:8655:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A blond-haired baby is sitting on the floor playing with toys while looking at a black and white cat.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8655", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8656", "candidate_text": "well, a dog is dressed up for halloween and is trying to get flowers for his masters costume", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8656:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8656:premise", "snli:test:8421:premise", "snli:test:8293:premise", "snli:test:8793:premise", "style:personachat:test:320:self:1:positive:0"], "evidence_texts": ["Small dog in costume stands on hind legs to reach dangling flowers.", "Brown and cream dog with tongue out in front of grassy forest", "A girl in a red and white uniform is swinging a bat.", "A lady in a white tank and shorts fixing a wooden drawer.", "me as well . never bothered to learn another"], "speaker_id": "snli:test:8656", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8656", "candidate_text": "I wonder whether a dog is dressed up for halloween and is trying to get flowers for his masters costume?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8656:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8656:premise", "snli:test:8421:premise", "snli:test:8293:premise", "snli:test:8793:premise", "style:personachat:test:203:self:1:positive:0"], "evidence_texts": ["Small dog in costume stands on hind legs to reach dangling flowers.", "Brown and cream dog with tongue out in front of grassy forest", "A girl in a red and white uniform is swinging a bat.", "A lady in a white tank and shorts fixing a wooden drawer.", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:8656", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8658", "candidate_text": "well, a dog is trying to eat a dangling kitten on a tree", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8658:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8793:premise", "snli:test:8421:premise", "snli:test:8658:premise", "snli:test:8884:premise", "style:personachat:test:320:self:2:positive:0", "style:personachat:test:320:self:2:positive:1"], "evidence_texts": ["A lady in a white tank and shorts fixing a wooden drawer.", "Brown and cream dog with tongue out in front of grassy forest", "Small dog in costume stands on hind legs to reach dangling flowers.", "Person in a long red coat walking in front of a building.", "i think we can agree there lol", "me as well . never bothered to learn another"], "speaker_id": "snli:test:8658", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8658", "candidate_text": "Honestly, A dog is trying to eat a dangling kitten on a tree!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8658:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8793:premise", "snli:test:8421:premise", "snli:test:8658:premise", "snli:test:8884:premise", "style:personachat:test:379:self:7:positive:0", "style:personachat:test:379:self:7:positive:1", "style:personachat:test:379:self:7:positive:2", "style:personachat:test:379:self:7:positive:3", "style:personachat:test:379:self:7:positive:4", "style:personachat:test:379:self:7:positive:5", "style:personachat:test:379:self:7:positive:6"], "evidence_texts": ["A lady in a white tank and shorts fixing a wooden drawer.", "Brown and cream dog with tongue out in front of grassy forest", "Small dog in costume stands on hind legs to reach dangling flowers.", "Person in a long red coat walking in front of a building.", "hey carl here ! i am a huge football fan ! go bears !", "every week , i also work at a factory doing heavy lifting and delivery", "you kind of sounds like your single , i am too and love it !", "really ? ! ? ! i love water ! in fact that is the only liquid i will drink !", "that is funny ! i love swimming at the pool in my gym , it is a great workout", "gotcha , i have some of those as well which is why i am single and alone", "it really is ! it was nice to have met you jane !"], "speaker_id": "snli:test:8658", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8663", "candidate_text": "well, a man is painting", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8663:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8548:premise", "snli:test:8723:premise", "snli:test:8541:premise", "snli:test:8663:premise", "style:personachat:test:320:self:5:positive:0", "style:personachat:test:320:self:5:positive:1", "style:personachat:test:320:self:5:positive:2", "style:personachat:test:320:self:5:positive:3", "style:personachat:test:320:self:5:positive:4"], "evidence_texts": ["A man on a ladder in tall grass.", "A dog jumps over a pile of wood.", "A couple opening up a new electronically mixer.", "A man in a boom lift bucket welds.", "what kind do you like , me is country", "i see never really could get into that . do you eat meat i do not", "never had the desire to eat that stuff", "i think we can agree there lol", "me as well . never bothered to learn another"], "speaker_id": "snli:test:8663", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8663", "candidate_text": "A man is painting.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8663:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8548:premise", "snli:test:8723:premise", "snli:test:8541:premise", "snli:test:8663:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man on a ladder in tall grass.", "A dog jumps over a pile of wood.", "A couple opening up a new electronically mixer.", "A man in a boom lift bucket welds.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8663", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8665", "candidate_text": "well, the attractive brunette in a red visor gets a lot of attention from the crowd as she holds a white umbrella", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8665:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8483:premise", "snli:test:8665:premise", "snli:test:8854:premise", "snli:test:8436:premise", "style:personachat:test:320:self:7:positive:0", "style:personachat:test:320:self:7:positive:1", "style:personachat:test:320:self:7:positive:2", "style:personachat:test:320:self:7:positive:3", "style:personachat:test:320:self:7:positive:4", "style:personachat:test:320:self:7:positive:5", "style:personachat:test:320:self:7:positive:6"], "evidence_texts": ["A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "hi there i am wonderful and yourself ?", "i am a giver . i knit hats for babies , my passion since i am disabled", "what kind do you like , me is country", "i see never really could get into that . do you eat meat i do not", "never had the desire to eat that stuff", "i think we can agree there lol", "me as well . never bothered to learn another"], "speaker_id": "snli:test:8665", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8665", "candidate_text": "I wonder whether the attractive brunette in a red visor gets a lot of attention from the crowd as she holds a white umbrella?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8665:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8483:premise", "snli:test:8665:premise", "snli:test:8854:premise", "snli:test:8436:premise", "style:personachat:test:203:self:7:positive:0", "style:personachat:test:203:self:7:positive:1", "style:personachat:test:203:self:7:positive:2", "style:personachat:test:203:self:7:positive:3", "style:personachat:test:203:self:7:positive:4", "style:personachat:test:203:self:7:positive:5", "style:personachat:test:203:self:7:positive:6"], "evidence_texts": ["A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "A young man blew up balloons to craft into animals for the seven excited children that looked on.", "doing great watching a little sports on tv", "i enjoy watching soccer a lot . i played some sports in high school .", "what do you like to do in your spare moments ?", "i am getting married soon , hope she starts to enjoy sports like i do", "where did the jews come from ?", "can anyone be jewish or is it exclusive to a certain group ?", "that is not true , have you played any sports ?"], "speaker_id": "snli:test:8665", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8681", "candidate_text": "I wonder whether there are clouds in the sky?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8681:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8139:premise", "snli:test:8681:premise", "snli:test:8729:premise", "snli:test:8126:premise", "style:personachat:test:321:self:1:positive:0"], "evidence_texts": ["Three people are standing on a lit stage while groups of people sitting at tables face them.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:8681", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8681", "candidate_text": "well, there are clouds in the sky", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8681:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8139:premise", "snli:test:8681:premise", "snli:test:8729:premise", "snli:test:8126:premise", "style:personachat:test:534:self:1:positive:0"], "evidence_texts": ["Three people are standing on a lit stage while groups of people sitting at tables face them.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "A man wearing a jacket is listening to headphones while walking down the sidewalk along a street.", "the joker is my fave superhero ."], "speaker_id": "snli:test:8681", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8685", "candidate_text": "I wonder whether people are inside watching tv?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8685:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8560:premise", "snli:test:8685:premise", "snli:test:8788:premise", "snli:test:8504:premise", "style:personachat:test:321:self:2:positive:0", "style:personachat:test:321:self:2:positive:1"], "evidence_texts": ["A man running in a race passes by spectators.", "People protesting age discrimination in an organized street rally.", "A little girl drawing with chalk, on a chalkboard.", "A young boy is playing in the sandy water.", "i am actually a student . i travel for fun .", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:8685", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8685", "candidate_text": "Honestly, People are inside watching tv!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8685:negative", "evidence_budget": 12, "evidence_ids": ["snli:test:8560:premise", "snli:test:8685:premise", "snli:test:8788:premise", "snli:test:8504:premise", "style:personachat:test:384:self:8:positive:0", "style:personachat:test:384:self:8:positive:1", "style:personachat:test:384:self:8:positive:2", "style:personachat:test:384:self:8:positive:3", "style:personachat:test:384:self:8:positive:4", "style:personachat:test:384:self:8:positive:5", "style:personachat:test:384:self:8:positive:6", "style:personachat:test:384:self:8:positive:7"], "evidence_texts": ["A man running in a race passes by spectators.", "People protesting age discrimination in an organized street rally.", "A little girl drawing with chalk, on a chalkboard.", "A young boy is playing in the sandy water.", "i am good ! do you have any hobbies ?", "gardening sounds amazing ! i like to listen to metallica in my free time .", "that is cool ! where do you work at ?", "it must be amazing to work with horses ! i work as a consultant .", "i like to spend time with my brothers . i have two . how about you ?", "wow ! it must have been hard to grow up with 4 brothers !", "awesome ! i am originally from wisconsin .", "that is cool ! do you visit them often ?"], "speaker_id": "snli:test:8685", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8687", "candidate_text": "I wonder whether the old lady looks into the mirror at the makeup table?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8687:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8627:premise", "snli:test:8999:premise", "snli:test:8471:premise", "snli:test:8687:premise", "style:personachat:test:321:self:5:positive:0", "style:personachat:test:321:self:5:positive:1", "style:personachat:test:321:self:5:positive:2", "style:personachat:test:321:self:5:positive:3", "style:personachat:test:321:self:5:positive:4"], "evidence_texts": ["The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman in a blue jacket dragging a child on a sled through the snow.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "do you have something there to help you ?", "my girlfriend is pretty short also", "muffins are good travel food , i eat them on adventures", "i am actually a student . i travel for fun .", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:8687", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8687", "candidate_text": "the old lady looks into the mirror at the makeup table.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8687:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8627:premise", "snli:test:8999:premise", "snli:test:8471:premise", "snli:test:8687:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman in a blue jacket dragging a child on a sled through the snow.", "A man holds up a sign that reads \"tattoo\" as many people walk near him.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8687", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8708", "candidate_text": "I wonder whether a probable blacksmith is welding a tool?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8708:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8708:premise", "snli:test:8655:premise", "snli:test:8653:premise", "snli:test:8896:premise", "style:personachat:test:321:self:6:positive:0", "style:personachat:test:321:self:6:positive:1", "style:personachat:test:321:self:6:positive:2", "style:personachat:test:321:self:6:positive:3", "style:personachat:test:321:self:6:positive:4", "style:personachat:test:321:self:6:positive:5"], "evidence_texts": ["An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "really ? how short are you ?", "do you have something there to help you ?", "my girlfriend is pretty short also", "muffins are good travel food , i eat them on adventures", "i am actually a student . i travel for fun .", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:8708", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8708", "candidate_text": "well, a probable blacksmith is welding a tool", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8708:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8708:premise", "snli:test:8655:premise", "snli:test:8653:premise", "snli:test:8896:premise", "style:personachat:test:534:self:6:positive:0", "style:personachat:test:534:self:6:positive:1", "style:personachat:test:534:self:6:positive:2", "style:personachat:test:534:self:6:positive:3", "style:personachat:test:534:self:6:positive:4", "style:personachat:test:534:self:6:positive:5"], "evidence_texts": ["An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "i am published author and scared of dogs . you ?", "i drink watered scotch . you ?", "what two positions do you work ?", "and the other position is what ?", "great ! i have a book out entitled 100 do not do it stories .", "the joker is my fave superhero ."], "speaker_id": "snli:test:8708", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8709", "candidate_text": "well, a farmer is milking the cow", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8709:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8896:premise", "snli:test:8709:premise", "snli:test:8655:premise", "snli:test:9347:premise", "style:personachat:test:322:self:1:positive:0"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "yeah they deserve each other , i want kids , but not yet , he would not wait"], "speaker_id": "snli:test:8709", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8709", "candidate_text": "A farmer is milking the cow.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8709:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8896:premise", "snli:test:8709:premise", "snli:test:8655:premise", "snli:test:9347:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "A passenger wearing a life jacket prepares to take a picture of a whale very close to the boat.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8709", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8712", "candidate_text": "well, a man in a red shirt and a man in a brown shirt are holding a sign at a peace rally", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8712:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8712:premise", "snli:test:8516:premise", "snli:test:8888:premise", "snli:test:8387:premise", "style:personachat:test:322:self:2:positive:0", "style:personachat:test:322:self:2:positive:1"], "evidence_texts": ["A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "awesome , i have an ex that ran off with my best friend , start with him", "yeah they deserve each other , i want kids , but not yet , he would not wait"], "speaker_id": "snli:test:8712", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8712", "candidate_text": "I wonder whether a man in a red shirt and a man in a brown shirt are holding a sign at a peace rally?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8712:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8712:premise", "snli:test:8516:premise", "snli:test:8888:premise", "snli:test:8387:premise", "style:personachat:test:217:self:2:positive:0", "style:personachat:test:217:self:2:positive:1"], "evidence_texts": ["A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a jacket is standing in a gas station, next to a car and lights are shining in the distance.", "what kinds of things do you like to build ?", "not yet . we are waiting for the right time . how many kids do you have ?"], "speaker_id": "snli:test:8712", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8723", "candidate_text": "well, a dog rests on a pile of wood", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8723:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8663:premise", "snli:test:8870:premise", "snli:test:8548:premise", "snli:test:8723:premise", "style:personachat:test:322:self:5:positive:0", "style:personachat:test:322:self:5:positive:1", "style:personachat:test:322:self:5:positive:2", "style:personachat:test:322:self:5:positive:3", "style:personachat:test:322:self:5:positive:4"], "evidence_texts": ["A man in a boom lift bucket welds.", "Two girls in bathing suits are in water.", "A man on a ladder in tall grass.", "A dog jumps over a pile of wood.", "i have been arrested for theft , but no violence", "so pro nuke but from puerto rico , any thoughts on potus ?", "north korea it makes wonder if it is smart to have children , i want to", "awesome , i have an ex that ran off with my best friend , start with him", "yeah they deserve each other , i want kids , but not yet , he would not wait"], "speaker_id": "snli:test:8723", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8723", "candidate_text": "Honestly, A dog rests on a pile of wood!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8723:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8663:premise", "snli:test:8870:premise", "snli:test:8548:premise", "snli:test:8723:premise", "style:personachat:test:399:self:1:positive:0"], "evidence_texts": ["A man in a boom lift bucket welds.", "Two girls in bathing suits are in water.", "A man on a ladder in tall grass.", "A dog jumps over a pile of wood.", "get out ! a singing fisherman . ever heard of cate mesa ?"], "speaker_id": "snli:test:8723", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8729", "candidate_text": "well, a grandmother is standing in front of a bus with violets decorated all over it", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8729:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8681:premise", "snli:test:8729:premise", "snli:test:8964:premise", "snli:test:8139:premise", "style:personachat:test:322:self:7:positive:0", "style:personachat:test:322:self:7:positive:1", "style:personachat:test:322:self:7:positive:2", "style:personachat:test:322:self:7:positive:3", "style:personachat:test:322:self:7:positive:4", "style:personachat:test:322:self:7:positive:5", "style:personachat:test:322:self:7:positive:6"], "evidence_texts": ["Two men are loading bales of wheat on top of a high structure on a cloudy day.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "i am a 19 year old girl studying fashion design", "my parents are army , and were never home , how long have you been there ?", "i have been arrested for theft , but no violence", "so pro nuke but from puerto rico , any thoughts on potus ?", "north korea it makes wonder if it is smart to have children , i want to", "awesome , i have an ex that ran off with my best friend , start with him", "yeah they deserve each other , i want kids , but not yet , he would not wait"], "speaker_id": "snli:test:8729", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8729", "candidate_text": "A grandmother is standing in front of a bus with violets decorated all over it.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8729:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8681:premise", "snli:test:8729:premise", "snli:test:8964:premise", "snli:test:8139:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two men are loading bales of wheat on top of a high structure on a cloudy day.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "Three people are standing on a lit stage while groups of people sitting at tables face them.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8729", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8733", "candidate_text": "I wonder whether the girl hits the ball to her coach?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8733:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8451:premise", "snli:test:8733:premise", "snli:test:8744:premise", "snli:test:8417:premise", "style:personachat:test:323:self:1:positive:0"], "evidence_texts": ["A man flying out of a river raft filled with 6 other people.", "A tennis player in a pink outfit serves the ball to her opponent.", "A man is performing for a group in front of a white house.", "A group of people are standing on steps in front of a building.", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:8733", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8733", "candidate_text": "The girl hits the ball to her coach.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8733:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8451:premise", "snli:test:8733:premise", "snli:test:8744:premise", "snli:test:8417:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man flying out of a river raft filled with 6 other people.", "A tennis player in a pink outfit serves the ball to her opponent.", "A man is performing for a group in front of a white house.", "A group of people are standing on steps in front of a building.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8733", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8739", "candidate_text": "I wonder whether a man is standing?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8739:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8460:premise", "snli:test:9005:premise", "snli:test:8399:premise", "snli:test:8739:premise", "style:personachat:test:323:self:2:positive:0", "style:personachat:test:323:self:2:positive:1"], "evidence_texts": ["Three musicians are playing in a band.", "Two men pouring concrete onto another truck.", "Men wearing hats walk on the street.", "A young man standing outside a laundromat.", "really ? how many is that ? i grew up farming loved it", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:8739", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8739", "candidate_text": "well, a man is standing", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8739:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8460:premise", "snli:test:9005:premise", "snli:test:8399:premise", "snli:test:8739:premise", "style:personachat:test:536:self:2:positive:0", "style:personachat:test:536:self:2:positive:1"], "evidence_texts": ["Three musicians are playing in a band.", "Two men pouring concrete onto another truck.", "Men wearing hats walk on the street.", "A young man standing outside a laundromat.", "yes sweetie you are young so enjoy every moment of your life .", "cool that will be enjoying i am going to collage and study for education ."], "speaker_id": "snli:test:8739", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8744", "candidate_text": "I wonder whether a man performed inside of a gymnasium?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8744:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8744:premise", "snli:test:8733:premise", "snli:test:8451:premise", "snli:test:8971:premise", "style:personachat:test:323:self:5:positive:0", "style:personachat:test:323:self:5:positive:1", "style:personachat:test:323:self:5:positive:2", "style:personachat:test:323:self:5:positive:3", "style:personachat:test:323:self:5:positive:4"], "evidence_texts": ["A man is performing for a group in front of a white house.", "A tennis player in a pink outfit serves the ball to her opponent.", "A man flying out of a river raft filled with 6 other people.", "A woman in a striped shirt is walking a horse across a street.", "i went to u of i", "my fav is blue an my truck i drive is blue", "i like that for you , got kids ? i have 7", "really ? how many is that ? i grew up farming loved it", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:8744", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8744", "candidate_text": "Honestly, A man performed inside of a gymnasium!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8744:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8744:premise", "snli:test:8733:premise", "snli:test:8451:premise", "snli:test:8971:premise", "style:personachat:test:405:self:1:positive:0"], "evidence_texts": ["A man is performing for a group in front of a white house.", "A tennis player in a pink outfit serves the ball to her opponent.", "A man flying out of a river raft filled with 6 other people.", "A woman in a striped shirt is walking a horse across a street.", "cool ! i am going to go running now and listen to rap ."], "speaker_id": "snli:test:8744", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8745", "candidate_text": "I wonder whether the man performed in front of his own house?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8745:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8971:premise", "snli:test:8745:premise", "snli:test:8733:premise", "snli:test:9096:premise", "style:personachat:test:323:self:7:positive:0", "style:personachat:test:323:self:7:positive:1", "style:personachat:test:323:self:7:positive:2", "style:personachat:test:323:self:7:positive:3", "style:personachat:test:323:self:7:positive:4", "style:personachat:test:323:self:7:positive:5", "style:personachat:test:323:self:7:positive:6"], "evidence_texts": ["A woman in a striped shirt is walking a horse across a street.", "A man is performing for a group in front of a white house.", "A tennis player in a pink outfit serves the ball to her opponent.", "The woman in the white coat is holding a camera in her hand.", "i do not to scared lol", "lol why not ? what is your fav color ?", "i went to u of i", "my fav is blue an my truck i drive is blue", "i like that for you , got kids ? i have 7", "really ? how many is that ? i grew up farming loved it", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:8745", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8745", "candidate_text": "The man performed in front of his own house.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8745:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8971:premise", "snli:test:8745:premise", "snli:test:8733:premise", "snli:test:9096:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in a striped shirt is walking a horse across a street.", "A man is performing for a group in front of a white house.", "A tennis player in a pink outfit serves the ball to her opponent.", "The woman in the white coat is holding a camera in her hand.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8745", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8753", "candidate_text": "well, a person with a tattoo", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8753:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8553:premise", "snli:test:8753:premise", "snli:test:8810:premise", "snli:test:8096:premise", "style:personachat:test:324:self:1:positive:0"], "evidence_texts": ["A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "sure . you can marry men and share"], "speaker_id": "snli:test:8753", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8753", "candidate_text": "Honestly, A person with a tattoo!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8753:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8553:premise", "snli:test:8753:premise", "snli:test:8810:premise", "snli:test:8096:premise", "style:personachat:test:405:self:5:positive:0", "style:personachat:test:405:self:5:positive:1", "style:personachat:test:405:self:5:positive:2", "style:personachat:test:405:self:5:positive:3", "style:personachat:test:405:self:5:positive:4"], "evidence_texts": ["A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "A man with a blue and white backpack and black jeans and jacket walks by a large monument surrounded by a gray fence.", "how long are you off the knee ? i like working out , so i hope soon .", "well , not too long . you can listen to rap and work the arms .", "no sardines for me . i eat vegan . but salad works .", "and beans . but my favorite is sushi . vegan sushi .", "cool ! i am going to go running now and listen to rap ."], "speaker_id": "snli:test:8753", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8772", "candidate_text": "well, women photographing statues", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8772:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8772:premise", "snli:test:8590:premise", "snli:test:8951:premise", "snli:test:8572:premise", "style:personachat:test:324:self:2:positive:0", "style:personachat:test:324:self:2:positive:1"], "evidence_texts": ["The women is photographing statues outdoors on a sunny day.", "the white dog is carrying a ball in its mouth.", "Two women sitting at a table with their hands together.", "Two young girls are jumping off of a steep step.", "i like reading mysteries and one day i was reading some lady dropped off some kids", "sure . you can marry men and share"], "speaker_id": "snli:test:8772", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8772", "candidate_text": "Women photographing statues.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8772:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8772:premise", "snli:test:8590:premise", "snli:test:8951:premise", "snli:test:8572:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The women is photographing statues outdoors on a sunny day.", "the white dog is carrying a ball in its mouth.", "Two women sitting at a table with their hands together.", "Two young girls are jumping off of a steep step.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8772", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8788", "candidate_text": "well, a girl is baking a cake in her kitchen", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8788:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8788:premise", "snli:test:8685:premise", "snli:test:8918:premise", "snli:test:8560:premise", "style:personachat:test:324:self:5:positive:0", "style:personachat:test:324:self:5:positive:1", "style:personachat:test:324:self:5:positive:2", "style:personachat:test:324:self:5:positive:3", "style:personachat:test:324:self:5:positive:4"], "evidence_texts": ["A little girl drawing with chalk, on a chalkboard.", "People protesting age discrimination in an organized street rally.", "A man in a black hat opens his mouth.", "A man running in a race passes by spectators.", "no . i grew up ona farm and my parents would not let me have tattoos", "no i have seven kids i want to set an example for", "yes i think of it as a mystery . because i do not know their mom", "i like reading mysteries and one day i was reading some lady dropped off some kids", "sure . you can marry men and share"], "speaker_id": "snli:test:8788", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8788", "candidate_text": "I wonder whether a girl is baking a cake in her kitchen?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8788:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8788:premise", "snli:test:8685:premise", "snli:test:8918:premise", "snli:test:8560:premise", "style:personachat:test:223:self:5:positive:0", "style:personachat:test:223:self:5:positive:1", "style:personachat:test:223:self:5:positive:2", "style:personachat:test:223:self:5:positive:3", "style:personachat:test:223:self:5:positive:4"], "evidence_texts": ["A little girl drawing with chalk, on a chalkboard.", "People protesting age discrimination in an organized street rally.", "A man in a black hat opens his mouth.", "A man running in a race passes by spectators.", "wow that is a busy beaver . i have 7 myself", "i am just a little rough around the edges for that . lol", "that sounds like fun . i was born for hard work", "a few dogs . tho i am not able to walk them like i should", "what type of music do you like ?"], "speaker_id": "snli:test:8788", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8793", "candidate_text": "well, the woman is wearing a dress", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8793:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8658:premise", "snli:test:8793:premise", "snli:test:8884:premise", "snli:test:8656:premise", "style:personachat:test:324:self:6:positive:0", "style:personachat:test:324:self:6:positive:1", "style:personachat:test:324:self:6:positive:2", "style:personachat:test:324:self:6:positive:3", "style:personachat:test:324:self:6:positive:4", "style:personachat:test:324:self:6:positive:5"], "evidence_texts": ["Small dog in costume stands on hind legs to reach dangling flowers.", "A lady in a white tank and shorts fixing a wooden drawer.", "Person in a long red coat walking in front of a building.", "Small dog in costume stands on hind legs to reach dangling flowers.", "i have passed through chicago for work . i am a truck driver", "no . i grew up ona farm and my parents would not let me have tattoos", "no i have seven kids i want to set an example for", "yes i think of it as a mystery . because i do not know their mom", "i like reading mysteries and one day i was reading some lady dropped off some kids", "sure . you can marry men and share"], "speaker_id": "snli:test:8793", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8793", "candidate_text": "Honestly, The woman is wearing a dress!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8793:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8658:premise", "snli:test:8793:premise", "snli:test:8884:premise", "snli:test:8656:premise", "style:personachat:test:413:self:2:positive:0", "style:personachat:test:413:self:2:positive:1"], "evidence_texts": ["Small dog in costume stands on hind legs to reach dangling flowers.", "A lady in a white tank and shorts fixing a wooden drawer.", "Person in a long red coat walking in front of a building.", "Small dog in costume stands on hind legs to reach dangling flowers.", "yes . time is supposed to heal all wounds .", "that takes talent ! i am not very coordinated ."], "speaker_id": "snli:test:8793", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8796", "candidate_text": "well, a person waves at the camera", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8796:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8796:premise", "snli:test:8487:premise", "snli:test:8845:premise", "snli:test:8486:premise", "style:personachat:test:325:self:1:positive:0"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A man dressed only in a towel is in a sauna.", "A group of young men are running in a grassy field.", "A man dressed only in a towel is in a sauna.", "oh i really like blindspot its really action packed"], "speaker_id": "snli:test:8796", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8796", "candidate_text": "A person waves at the camera.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8796:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8796:premise", "snli:test:8487:premise", "snli:test:8845:premise", "snli:test:8486:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A man dressed only in a towel is in a sauna.", "A group of young men are running in a grassy field.", "A man dressed only in a towel is in a sauna.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8796", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8803", "candidate_text": "well, a man is getting tidied up for a job interview", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8803:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8380:premise", "snli:test:8843:premise", "snli:test:8176:premise", "snli:test:8803:premise", "style:personachat:test:325:self:2:positive:0", "style:personachat:test:325:self:2:positive:1"], "evidence_texts": ["A brown dog wearing a collar is chasing and biting on a red broom.", "A little boy with an orange backpack walks under a tree through the woods.", "People boating on a lake with the sun through the clouds in the distance.", "A man is giving another man a haircut while three other guys stand around.", "do you have a favorite tv show ?", "oh i really like blindspot its really action packed"], "speaker_id": "snli:test:8803", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8803", "candidate_text": "I wonder whether a man is getting tidied up for a job interview?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8803:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8380:premise", "snli:test:8843:premise", "snli:test:8176:premise", "snli:test:8803:premise", "style:personachat:test:227:self:2:positive:0", "style:personachat:test:227:self:2:positive:1"], "evidence_texts": ["A brown dog wearing a collar is chasing and biting on a red broom.", "A little boy with an orange backpack walks under a tree through the woods.", "People boating on a lake with the sun through the clouds in the distance.", "A man is giving another man a haircut while three other guys stand around.", "thank you . do you have any children ?", "being a doctor is very important . are you a specialist ?"], "speaker_id": "snli:test:8803", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8810", "candidate_text": "well, a man and a couple of women take a walk", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8810:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8753:premise", "snli:test:8858:premise", "snli:test:8810:premise", "snli:test:8553:premise", "style:personachat:test:325:self:5:positive:0", "style:personachat:test:325:self:5:positive:1", "style:personachat:test:325:self:5:positive:2", "style:personachat:test:325:self:5:positive:3", "style:personachat:test:325:self:5:positive:4"], "evidence_texts": ["A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "its weird i know . do you like frisbee i play it with my dog", "oh soccer is cool did you go to college ? i am working on getting my diploma", "that is amazing . what type of music do you like ?", "do you have a favorite tv show ?", "oh i really like blindspot its really action packed"], "speaker_id": "snli:test:8810", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8810", "candidate_text": "Honestly, A man and a couple of women take a walk!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8810:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8753:premise", "snli:test:8858:premise", "snli:test:8810:premise", "snli:test:8553:premise", "style:personachat:test:425:self:1:positive:0"], "evidence_texts": ["A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "A street performer wearing sunglasses and painted in bronze colored paint is acting like he is filming with an old-fashioned prop film camera.", "crazy ! nurses are like elves bringing in new life into the world to fight evil"], "speaker_id": "snli:test:8810", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8822", "candidate_text": "well, a choir of kids are singing", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8822:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8585:premise", "snli:test:8833:premise", "snli:test:8822:premise", "snli:test:7188:premise", "style:personachat:test:325:self:7:positive:0", "style:personachat:test:325:self:7:positive:1", "style:personachat:test:325:self:7:positive:2", "style:personachat:test:325:self:7:positive:3", "style:personachat:test:325:self:7:positive:4", "style:personachat:test:325:self:7:positive:5", "style:personachat:test:325:self:7:positive:6"], "evidence_texts": ["White dog playing in the snow.", "The person is reading a book.", "A choir of youths are singing.", "construction worker working on a structure", "i am cindy i am recovering from ulnar nerve surgery", "its going alright tonight . i am not big on candy but love ice cream and sardines", "its weird i know . do you like frisbee i play it with my dog", "oh soccer is cool did you go to college ? i am working on getting my diploma", "that is amazing . what type of music do you like ?", "do you have a favorite tv show ?", "oh i really like blindspot its really action packed"], "speaker_id": "snli:test:8822", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8822", "candidate_text": "a choir of kids are singing.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8822:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8585:premise", "snli:test:8833:premise", "snli:test:8822:premise", "snli:test:7188:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["White dog playing in the snow.", "The person is reading a book.", "A choir of youths are singing.", "construction worker working on a structure", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8822", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8828", "candidate_text": "well, an old couple watches the sunset", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8828:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8828:premise", "snli:test:8640:premise", "snli:test:7437:premise", "snli:test:9076:premise", "style:personachat:test:326:self:1:positive:0"], "evidence_texts": ["A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "my daughter lives with her boyfriend . i have animals a dog and cat ."], "speaker_id": "snli:test:8828", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8828", "candidate_text": "I wonder whether an old couple watches the sunset?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8828:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8828:premise", "snli:test:8640:premise", "snli:test:7437:premise", "snli:test:9076:premise", "style:personachat:test:230:self:1:positive:0"], "evidence_texts": ["A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A person is sitting on a yellow picnic table with a water bottle, while a baby stroller sits beside her.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:8828", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8829", "candidate_text": "well, a couple of young people are at a concert", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8829:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9076:premise", "snli:test:8829:premise", "snli:test:8640:premise", "snli:test:9078:premise", "style:personachat:test:326:self:2:positive:0", "style:personachat:test:326:self:2:positive:1"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "i am not married yet i have a daughter and three grands .", "my daughter lives with her boyfriend . i have animals a dog and cat ."], "speaker_id": "snli:test:8829", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8829", "candidate_text": "Honestly, A couple of young people are at a concert!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8829:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9076:premise", "snli:test:8829:premise", "snli:test:8640:premise", "snli:test:9078:premise", "style:personachat:test:425:self:7:positive:0", "style:personachat:test:425:self:7:positive:1", "style:personachat:test:425:self:7:positive:2", "style:personachat:test:425:self:7:positive:3", "style:personachat:test:425:self:7:positive:4", "style:personachat:test:425:self:7:positive:5", "style:personachat:test:425:self:7:positive:6"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A businessman in a white sweater conducts necessary business in a plaza using a concrete abutment as a work table.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "cool , doing well . where do you work ?", "just sitting here watching lord of the rings for the 5th time this month", "what shift do you work in maternity ? could be some great fantasy come out that department !", "books better than movies but a superb job here . how long are your shifts ?", "anything crazy happen there today ? baby ork or anything ?", "just got off work a day ago ? or are we in a time loop ?", "crazy ! nurses are like elves bringing in new life into the world to fight evil"], "speaker_id": "snli:test:8829", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8833", "candidate_text": "well, a person reads their favorite novel", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8833:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8822:premise", "snli:test:8833:premise", "snli:test:8585:premise", "snli:test:8908:premise", "style:personachat:test:326:self:5:positive:0", "style:personachat:test:326:self:5:positive:1", "style:personachat:test:326:self:5:positive:2", "style:personachat:test:326:self:5:positive:3", "style:personachat:test:326:self:5:positive:4"], "evidence_texts": ["A choir of youths are singing.", "The person is reading a book.", "White dog playing in the snow.", "Uniformed group walks down a street.", "i used to be employed in a nursing unit . yes singing i sing great .", "i wished i could model i like to go bike riding .", "you to a model is a great job i like to take care of folks .", "i am not married yet i have a daughter and three grands .", "my daughter lives with her boyfriend . i have animals a dog and cat ."], "speaker_id": "snli:test:8833", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8833", "candidate_text": "A person reads their favorite novel.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8833:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8822:premise", "snli:test:8833:premise", "snli:test:8585:premise", "snli:test:8908:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A choir of youths are singing.", "The person is reading a book.", "White dog playing in the snow.", "Uniformed group walks down a street.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8833", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8835", "candidate_text": "well, a person reads", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8835:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8908:premise", "snli:test:8822:premise", "snli:test:9091:premise", "snli:test:8835:premise", "style:personachat:test:326:self:7:positive:0", "style:personachat:test:326:self:7:positive:1", "style:personachat:test:326:self:7:positive:2", "style:personachat:test:326:self:7:positive:3", "style:personachat:test:326:self:7:positive:4", "style:personachat:test:326:self:7:positive:5", "style:personachat:test:326:self:7:positive:6"], "evidence_texts": ["Uniformed group walks down a street.", "A choir of youths are singing.", "A swan landing in a lake.", "The person is reading a book.", "how are you doing i am fine thank you .", "i love doing make up hair and i am a great cook .", "i used to be employed in a nursing unit . yes singing i sing great .", "i wished i could model i like to go bike riding .", "you to a model is a great job i like to take care of folks .", "i am not married yet i have a daughter and three grands .", "my daughter lives with her boyfriend . i have animals a dog and cat ."], "speaker_id": "snli:test:8835", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8835", "candidate_text": "I wonder whether a person reads?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8835:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8908:premise", "snli:test:8822:premise", "snli:test:9091:premise", "snli:test:8835:premise", "style:personachat:test:230:self:6:positive:0", "style:personachat:test:230:self:6:positive:1", "style:personachat:test:230:self:6:positive:2", "style:personachat:test:230:self:6:positive:3", "style:personachat:test:230:self:6:positive:4", "style:personachat:test:230:self:6:positive:5"], "evidence_texts": ["Uniformed group walks down a street.", "A choir of youths are singing.", "A swan landing in a lake.", "The person is reading a book.", "that is cool ! what do you do there ? i just got home from dance", "woah ! that is cool . i am cleaning up after my dog ralph .", "i believe that . i am nervous about school with all the race stuff going on .", "my mom and dad are not here anymore , and being mixed i have questions .", "yeah and why people are so mean . i live with my sis , but she ignores it .", "what do you paint ? and it is not the same ."], "speaker_id": "snli:test:8835", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8838", "candidate_text": "well, the dog is a retriever", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8838:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8406:premise", "snli:test:8990:premise", "snli:test:8838:premise", "snli:test:8358:premise", "style:personachat:test:327:self:1:positive:0"], "evidence_texts": ["A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "it does i sleep alot these days i hate being old"], "speaker_id": "snli:test:8838", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8838", "candidate_text": "Honestly, The dog is a retriever!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8838:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8406:premise", "snli:test:8990:premise", "snli:test:8838:premise", "snli:test:8358:premise", "style:personachat:test:429:self:5:positive:0", "style:personachat:test:429:self:5:positive:1", "style:personachat:test:429:self:5:positive:2", "style:personachat:test:429:self:5:positive:3", "style:personachat:test:429:self:5:positive:4"], "evidence_texts": ["A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A baby with a green bottom and red top lying in bed with a brown and white cat and also a black and white cat.", "those would get all in my beard . i love having hair on my face .", "definitely ! hopefully i can retire soon and try different foods .", "that is cool . do you keep things to yourself ? i do instead of speaking my mind .", "thank you ! life will be so awesome after retirement !", "oh , definitely ! i can enjoy life to the fullest !"], "speaker_id": "snli:test:8838", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8843", "candidate_text": "well, a girl walks through a field", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8843:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8803:premise", "snli:test:8878:premise", "snli:test:8380:premise", "snli:test:8843:premise", "style:personachat:test:327:self:2:positive:0", "style:personachat:test:327:self:2:positive:1"], "evidence_texts": ["A man is giving another man a haircut while three other guys stand around.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "A brown dog wearing a collar is chasing and biting on a red broom.", "A little boy with an orange backpack walks under a tree through the woods.", "not anymore she died of old age few years ago", "it does i sleep alot these days i hate being old"], "speaker_id": "snli:test:8843", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8843", "candidate_text": "A girl walks through a field.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8843:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8803:premise", "snli:test:8878:premise", "snli:test:8380:premise", "snli:test:8843:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man is giving another man a haircut while three other guys stand around.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "A brown dog wearing a collar is chasing and biting on a red broom.", "A little boy with an orange backpack walks under a tree through the woods.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8843", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8845", "candidate_text": "well, the men are outside", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8845:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8796:premise", "snli:test:8845:premise", "snli:test:8851:premise", "snli:test:8487:premise", "style:personachat:test:327:self:5:positive:0", "style:personachat:test:327:self:5:positive:1", "style:personachat:test:327:self:5:positive:2", "style:personachat:test:327:self:5:positive:3", "style:personachat:test:327:self:5:positive:4"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A group of young men are running in a grassy field.", "A little baby wearing a blue outfit is grabbing some fruit.", "A man dressed only in a towel is in a sauna.", "me to most days . i have a specialized wheelchair", "oh how many animals at the shelter ?", "are you planning to ask her out ?", "not anymore she died of old age few years ago", "it does i sleep alot these days i hate being old"], "speaker_id": "snli:test:8845", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8845", "candidate_text": "I wonder whether the men are outside?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8845:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8796:premise", "snli:test:8845:premise", "snli:test:8851:premise", "snli:test:8487:premise", "style:personachat:test:239:self:5:positive:0", "style:personachat:test:239:self:5:positive:1", "style:personachat:test:239:self:5:positive:2", "style:personachat:test:239:self:5:positive:3", "style:personachat:test:239:self:5:positive:4"], "evidence_texts": ["A racer on a motorcycle lifts one hand in the air.", "A group of young men are running in a grassy field.", "A little baby wearing a blue outfit is grabbing some fruit.", "A man dressed only in a towel is in a sauna.", "i do not watch much tv , but i do enjoy archer .", "i enjoy blues more because i am from chicago . what do you like about jazz ?", "i love new york . i will be coming there in a few weeks for the marathon .", "i do . i really just run so i can eat what ever i like .", "what do you do for work ?"], "speaker_id": "snli:test:8845", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8851", "candidate_text": "well, a little baby is wearing blue", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8851:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8845:premise", "snli:test:8796:premise", "snli:test:8864:premise", "snli:test:8851:premise", "style:personachat:test:327:self:7:positive:0", "style:personachat:test:327:self:7:positive:1", "style:personachat:test:327:self:7:positive:2", "style:personachat:test:327:self:7:positive:3", "style:personachat:test:327:self:7:positive:4", "style:personachat:test:327:self:7:positive:5", "style:personachat:test:327:self:7:positive:6"], "evidence_texts": ["A group of young men are running in a grassy field.", "A racer on a motorcycle lifts one hand in the air.", "A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "hi i am charlie a war vet i feel the same way some days", "i do he is pretty old tho do you have a pet ?", "me to most days . i have a specialized wheelchair", "oh how many animals at the shelter ?", "are you planning to ask her out ?", "not anymore she died of old age few years ago", "it does i sleep alot these days i hate being old"], "speaker_id": "snli:test:8851", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8851", "candidate_text": "Honestly, A little baby is wearing blue!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8851:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8845:premise", "snli:test:8796:premise", "snli:test:8864:premise", "snli:test:8851:premise", "style:personachat:test:435:self:2:positive:0", "style:personachat:test:435:self:2:positive:1"], "evidence_texts": ["A group of young men are running in a grassy field.", "A racer on a motorcycle lifts one hand in the air.", "A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "super cold ! cali sounds nice about right now !", "ohhhh nooo i will send you the snow !"], "speaker_id": "snli:test:8851", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8853", "candidate_text": "well, a little baby is having lunch", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8853:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8864:premise", "snli:test:8853:premise", "snli:test:8845:premise", "snli:test:8901:premise", "style:personachat:test:328:self:1:positive:0"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "A group of young men are running in a grassy field.", "Three performers are on the stage floor in black lace costumes.", "oh , i love every inch . no worries ."], "speaker_id": "snli:test:8853", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8853", "candidate_text": "A little baby is having lunch.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8853:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8864:premise", "snli:test:8853:premise", "snli:test:8845:premise", "snli:test:8901:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "A group of young men are running in a grassy field.", "Three performers are on the stage floor in black lace costumes.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8853", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8854", "candidate_text": "well, the man is ready to fight", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8854:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8665:premise", "snli:test:8931:premise", "snli:test:8854:premise", "snli:test:8483:premise", "style:personachat:test:328:self:2:positive:0", "style:personachat:test:328:self:2:positive:1"], "evidence_texts": ["A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "that is why i am still fat", "oh , i love every inch . no worries ."], "speaker_id": "snli:test:8854", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8854", "candidate_text": "I wonder whether the man is ready to fight?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8854:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8665:premise", "snli:test:8931:premise", "snli:test:8854:premise", "snli:test:8483:premise", "style:personachat:test:241:self:2:positive:0", "style:personachat:test:241:self:2:positive:1"], "evidence_texts": ["A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "A man and a woman, both wearing sunglasses, embrace one another, while smiling and posing for the camera.", "they are an excellent snack with my show .", "have you had the maple bourbon ones ?"], "speaker_id": "snli:test:8854", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8858", "candidate_text": "well, a woman is getting ready to perform a song for the audience", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8858:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8810:premise", "snli:test:9295:premise", "snli:test:8858:premise", "snli:test:8753:premise", "style:personachat:test:328:self:5:positive:0", "style:personachat:test:328:self:5:positive:1", "style:personachat:test:328:self:5:positive:2", "style:personachat:test:328:self:5:positive:3", "style:personachat:test:328:self:5:positive:4"], "evidence_texts": ["A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "nah . ohio . i wish though , so i could go to the beach more", "i am old though . fat and old . trying to fix that with exercise", "nah . so you spin ? i like to horseback ride", "that is why i am still fat", "oh , i love every inch . no worries ."], "speaker_id": "snli:test:8858", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8858", "candidate_text": "Honestly, A woman is getting ready to perform a song for the audience!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8858:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8810:premise", "snli:test:9295:premise", "snli:test:8858:premise", "snli:test:8753:premise", "style:personachat:test:438:self:1:positive:0"], "evidence_texts": ["A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A blond woman with a tattoo of on an angel on her arm walking down the street, a concrete wall in the background.", "yes ! i must be . i love being me . board games are difficult to play under the sea"], "speaker_id": "snli:test:8858", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8864", "candidate_text": "well, a woman is sitting", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8864:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8864:premise", "snli:test:8853:premise", "snli:test:8901:premise", "snli:test:8851:premise", "style:personachat:test:328:self:7:positive:0", "style:personachat:test:328:self:7:positive:1", "style:personachat:test:328:self:7:positive:2", "style:personachat:test:328:self:7:positive:3", "style:personachat:test:328:self:7:positive:4", "style:personachat:test:328:self:7:positive:5", "style:personachat:test:328:self:7:positive:6"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "Three performers are on the stage floor in black lace costumes.", "A little baby wearing a blue outfit is grabbing some fruit.", "not bad thanks . i just back from the nail salon .", "oh nice . we should go some time .", "nah . ohio . i wish though , so i could go to the beach more", "i am old though . fat and old . trying to fix that with exercise", "nah . so you spin ? i like to horseback ride", "that is why i am still fat", "oh , i love every inch . no worries ."], "speaker_id": "snli:test:8864", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8864", "candidate_text": "a woman is sitting.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8864:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8864:premise", "snli:test:8853:premise", "snli:test:8901:premise", "snli:test:8851:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "A little baby wearing a blue outfit is grabbing some fruit.", "Three performers are on the stage floor in black lace costumes.", "A little baby wearing a blue outfit is grabbing some fruit.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8864", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8870", "candidate_text": "well, two girls are swiming in a pool", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8870:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8723:premise", "snli:test:8874:premise", "snli:test:8870:premise", "snli:test:8663:premise", "style:personachat:test:329:self:1:positive:0"], "evidence_texts": ["A dog jumps over a pile of wood.", "Women are working and digging in a field.", "Two girls in bathing suits are in water.", "A man in a boom lift bucket welds.", "yes poetry is awesome , love to read it , cannot write it though ."], "speaker_id": "snli:test:8870", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8870", "candidate_text": "I wonder whether two girls are swiming in a pool?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8870:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8723:premise", "snli:test:8874:premise", "snli:test:8870:premise", "snli:test:8663:premise", "style:personachat:test:243:self:1:positive:0"], "evidence_texts": ["A dog jumps over a pile of wood.", "Women are working and digging in a field.", "Two girls in bathing suits are in water.", "A man in a boom lift bucket welds.", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:8870", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8874", "candidate_text": "well, women are digging", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8874:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8870:premise", "snli:test:9026:premise", "snli:test:8874:premise", "snli:test:8723:premise", "style:personachat:test:329:self:2:positive:0", "style:personachat:test:329:self:2:positive:1"], "evidence_texts": ["Two girls in bathing suits are in water.", "A young face-painted boy is sucking his thumb.", "Women are working and digging in a field.", "A dog jumps over a pile of wood.", "titanic is my favorite movie of all time , have seen it over 10 times .", "yes poetry is awesome , love to read it , cannot write it though ."], "speaker_id": "snli:test:8874", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8874", "candidate_text": "Honestly, Women are digging!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8874:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8870:premise", "snli:test:9026:premise", "snli:test:8874:premise", "snli:test:8723:premise", "style:personachat:test:438:self:7:positive:0", "style:personachat:test:438:self:7:positive:1", "style:personachat:test:438:self:7:positive:2", "style:personachat:test:438:self:7:positive:3", "style:personachat:test:438:self:7:positive:4", "style:personachat:test:438:self:7:positive:5", "style:personachat:test:438:self:7:positive:6"], "evidence_texts": ["Two girls in bathing suits are in water.", "A young face-painted boy is sucking his thumb.", "Women are working and digging in a field.", "A dog jumps over a pile of wood.", "ok for living in the sea", "not seattle but ocean . i spend all my days at school .", "i like checking out the sea creatures .", "i like cats . but i like my tail as feet", "i have long wavy hair what about you ?", "bills cost money . after dinner i usually go out and play with my friends under the sea", "yes ! i must be . i love being me . board games are difficult to play under the sea"], "speaker_id": "snli:test:8874", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8878", "candidate_text": "well, traffic drives through the crosswalk", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8878:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8843:premise", "snli:test:8803:premise", "snli:test:8878:premise", "snli:test:8881:premise", "style:personachat:test:329:self:5:positive:0", "style:personachat:test:329:self:5:positive:1", "style:personachat:test:329:self:5:positive:2", "style:personachat:test:329:self:5:positive:3", "style:personachat:test:329:self:5:positive:4"], "evidence_texts": ["A little boy with an orange backpack walks under a tree through the woods.", "A man is giving another man a haircut while three other guys stand around.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "Several people including a child and a clown are walking towards a snowy sidewalk", "ok that sounds interesting , i studied abroad for many years .", "it was , any pets ? my golden retriever ginger is great around my son .", "wow , small world , my hobbies include making model cars and watching movies .", "titanic is my favorite movie of all time , have seen it over 10 times .", "yes poetry is awesome , love to read it , cannot write it though ."], "speaker_id": "snli:test:8878", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8878", "candidate_text": "Traffic drives through the crosswalk.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8878:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8843:premise", "snli:test:8803:premise", "snli:test:8878:premise", "snli:test:8881:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A little boy with an orange backpack walks under a tree through the woods.", "A man is giving another man a haircut while three other guys stand around.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "Several people including a child and a clown are walking towards a snowy sidewalk", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8878", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8879", "candidate_text": "well, traffic is stopped at a red light", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8879:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8881:premise", "snli:test:8843:premise", "snli:test:8979:premise", "snli:test:8879:premise", "style:personachat:test:329:self:7:positive:0", "style:personachat:test:329:self:7:positive:1", "style:personachat:test:329:self:7:positive:2", "style:personachat:test:329:self:7:positive:3", "style:personachat:test:329:self:7:positive:4", "style:personachat:test:329:self:7:positive:5", "style:personachat:test:329:self:7:positive:6"], "evidence_texts": ["Several people including a child and a clown are walking towards a snowy sidewalk", "A little boy with an orange backpack walks under a tree through the woods.", "The little boy is smiling as he crosses a rope on an assault course.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "sure , my name is dan and i just had my first child .", "a boy , any kids for you ?", "ok that sounds interesting , i studied abroad for many years .", "it was , any pets ? my golden retriever ginger is great around my son .", "wow , small world , my hobbies include making model cars and watching movies .", "titanic is my favorite movie of all time , have seen it over 10 times .", "yes poetry is awesome , love to read it , cannot write it though ."], "speaker_id": "snli:test:8879", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8879", "candidate_text": "I wonder whether traffic is stopped at a red light?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8879:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8881:premise", "snli:test:8843:premise", "snli:test:8979:premise", "snli:test:8879:premise", "style:personachat:test:243:self:6:positive:0", "style:personachat:test:243:self:6:positive:1", "style:personachat:test:243:self:6:positive:2", "style:personachat:test:243:self:6:positive:3", "style:personachat:test:243:self:6:positive:4", "style:personachat:test:243:self:6:positive:5"], "evidence_texts": ["Several people including a child and a clown are walking towards a snowy sidewalk", "A little boy with an orange backpack walks under a tree through the woods.", "The little boy is smiling as he crosses a rope on an assault course.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "of course ! do you like watching movies ?", "that is cool . are you married ?", "wow , sounds like you are a black widower . . .", "yes , i have a mini aussie shepherd . you ?", "are you a pet groomer ?", "do you style the dogs of celebrities or something ?"], "speaker_id": "snli:test:8879", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8881", "candidate_text": "well, many people walk to the sidewalk to avoid traffic", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8881:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8879:premise", "snli:test:8881:premise", "snli:test:8979:premise", "snli:test:8878:premise", "style:personachat:test:330:self:1:positive:0"], "evidence_texts": ["Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "Several people including a child and a clown are walking towards a snowy sidewalk", "The little boy is smiling as he crosses a rope on an assault course.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "i have my boyfriend but he can only help so much since he works"], "speaker_id": "snli:test:8881", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8881", "candidate_text": "Honestly, Many people walk to the sidewalk to avoid traffic!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8881:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8879:premise", "snli:test:8881:premise", "snli:test:8979:premise", "snli:test:8878:premise", "style:personachat:test:445:self:5:positive:0", "style:personachat:test:445:self:5:positive:1", "style:personachat:test:445:self:5:positive:2", "style:personachat:test:445:self:5:positive:3", "style:personachat:test:445:self:5:positive:4"], "evidence_texts": ["Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "Several people including a child and a clown are walking towards a snowy sidewalk", "The little boy is smiling as he crosses a rope on an assault course.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "what ? my dad is awesome , i dropped out of school to travel with him .", "love nyc ! party all the time , my dad drove a semi , i helped .", "my sister does too ! ! i have 31 do you have siblings ?", "3 not 31 . . that would be scary ! sad", "totally , one just like me ! ! lol"], "speaker_id": "snli:test:8881", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8884", "candidate_text": "well, a person gets ready to protest near a building", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8884:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8884:premise", "snli:test:8793:premise", "snli:test:8939:premise", "snli:test:8658:premise", "style:personachat:test:330:self:2:positive:0", "style:personachat:test:330:self:2:positive:1"], "evidence_texts": ["Person in a long red coat walking in front of a building.", "A lady in a white tank and shorts fixing a wooden drawer.", "A male with sunglasses is poking at a tree with a pole.", "Small dog in costume stands on hind legs to reach dangling flowers.", "probably so lol my medical situation is a lot", "i have my boyfriend but he can only help so much since he works"], "speaker_id": "snli:test:8884", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8884", "candidate_text": "A person gets ready to protest near a building.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8884:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8884:premise", "snli:test:8793:premise", "snli:test:8939:premise", "snli:test:8658:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Person in a long red coat walking in front of a building.", "A lady in a white tank and shorts fixing a wooden drawer.", "A male with sunglasses is poking at a tree with a pole.", "Small dog in costume stands on hind legs to reach dangling flowers.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8884", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8888", "candidate_text": "well, someone is taking a picture of a child", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8888:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8888:premise", "snli:test:8712:premise", "snli:test:8516:premise", "snli:test:9617:premise", "style:personachat:test:330:self:5:positive:0", "style:personachat:test:330:self:5:positive:1", "style:personachat:test:330:self:5:positive:2", "style:personachat:test:330:self:5:positive:3", "style:personachat:test:330:self:5:positive:4"], "evidence_texts": ["A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "that is awesome i really need to adapt to that type of lifestyle", "no not really i use to but its usually too painful", "i have some nerve issues arm and back pains badly", "probably so lol my medical situation is a lot", "i have my boyfriend but he can only help so much since he works"], "speaker_id": "snli:test:8888", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8888", "candidate_text": "I wonder whether someone is taking a picture of a child?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8888:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8888:premise", "snli:test:8712:premise", "snli:test:8516:premise", "snli:test:9617:premise", "style:personachat:test:244:self:5:positive:0", "style:personachat:test:244:self:5:positive:1", "style:personachat:test:244:self:5:positive:2", "style:personachat:test:244:self:5:positive:3", "style:personachat:test:244:self:5:positive:4"], "evidence_texts": ["A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "Two men, one in a red shirt and one in a white shirt, are on a boat in the beautiful blue-green water.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "cutting their hair used to be a kind of hobby . do you have any hobbies ?", "well i have exotic fish ! sort of similiar . u have other hobbies ?", "collect 50s decorations . what did you have for breakfast today ?", "i eat granola every day ! do you have kids ?", "i have two boys . any other family ?"], "speaker_id": "snli:test:8888", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8889", "candidate_text": "well, a child's mother takes a picture of her", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8889:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9617:premise", "snli:test:8889:premise", "snli:test:8712:premise", "snli:test:9712:premise", "style:personachat:test:330:self:7:positive:0", "style:personachat:test:330:self:7:positive:1", "style:personachat:test:330:self:7:positive:2", "style:personachat:test:330:self:7:positive:3", "style:personachat:test:330:self:7:positive:4", "style:personachat:test:330:self:7:positive:5", "style:personachat:test:330:self:7:positive:6"], "evidence_texts": ["An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "hey i am good how about yourself", "oh cool you work out often ?", "that is awesome i really need to adapt to that type of lifestyle", "no not really i use to but its usually too painful", "i have some nerve issues arm and back pains badly", "probably so lol my medical situation is a lot", "i have my boyfriend but he can only help so much since he works"], "speaker_id": "snli:test:8889", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8889", "candidate_text": "Honestly, A child's mother takes a picture of her!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8889:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9617:premise", "snli:test:8889:premise", "snli:test:8712:premise", "snli:test:9712:premise", "style:personachat:test:446:self:2:positive:0", "style:personachat:test:446:self:2:positive:1"], "evidence_texts": ["An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "A man in a red shirt and a man in a brown shirt smiling at the camera and holding up a sign.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "very true . its advancing so quickly !", "i remember those ! my mother had one in the army"], "speaker_id": "snli:test:8889", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8896", "candidate_text": "well, the tourists are having their picture taken by a famous work of art", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8896:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8896:premise", "snli:test:8709:premise", "snli:test:9347:premise", "snli:test:8708:premise", "style:personachat:test:331:self:1:positive:0"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "that would be nice . i am looking at tropical locations ."], "speaker_id": "snli:test:8896", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8896", "candidate_text": "The tourists are having their picture taken by a famous work of art.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8896:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8896:premise", "snli:test:8709:premise", "snli:test:9347:premise", "snli:test:8708:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8896", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8901", "candidate_text": "well, three performers wear lace costumes", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8901:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8864:premise", "snli:test:8901:premise", "snli:test:8948:premise", "snli:test:8853:premise", "style:personachat:test:331:self:2:positive:0", "style:personachat:test:331:self:2:positive:1"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "Three performers are on the stage floor in black lace costumes.", "The young girl is airborne as she falls into the lake.", "A little baby wearing a blue outfit is grabbing some fruit.", "where are you going first ? i am on the first flight out of wisconsin .", "that would be nice . i am looking at tropical locations ."], "speaker_id": "snli:test:8901", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8901", "candidate_text": "I wonder whether three performers wear lace costumes?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8901:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8864:premise", "snli:test:8901:premise", "snli:test:8948:premise", "snli:test:8853:premise", "style:personachat:test:254:self:2:positive:0", "style:personachat:test:254:self:2:positive:1"], "evidence_texts": ["A woman in a headscarf sitting in front of a loom.", "Three performers are on the stage floor in black lace costumes.", "The young girl is airborne as she falls into the lake.", "A little baby wearing a blue outfit is grabbing some fruit.", "have you seen the movie titanic ? it is my favorite .", "do you build model cars ?"], "speaker_id": "snli:test:8901", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8908", "candidate_text": "well, a group of soldiers patrol the street", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8908:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8908:premise", "snli:test:8835:premise", "snli:test:9091:premise", "snli:test:8833:premise", "style:personachat:test:331:self:5:positive:0", "style:personachat:test:331:self:5:positive:1", "style:personachat:test:331:self:5:positive:2", "style:personachat:test:331:self:5:positive:3", "style:personachat:test:331:self:5:positive:4"], "evidence_texts": ["Uniformed group walks down a street.", "The person is reading a book.", "A swan landing in a lake.", "The person is reading a book.", "winter is my favorite , i love running in the snow .", "i do not . i work for a consulting firm , what do you do ?", "i cannot wait until i retire , i am going to travel .", "where are you going first ? i am on the first flight out of wisconsin .", "that would be nice . i am looking at tropical locations ."], "speaker_id": "snli:test:8908", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8908", "candidate_text": "Honestly, A group of soldiers patrol the street!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8908:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8908:premise", "snli:test:8835:premise", "snli:test:9091:premise", "snli:test:8833:premise", "style:personachat:test:456:self:1:positive:0"], "evidence_texts": ["Uniformed group walks down a street.", "The person is reading a book.", "A swan landing in a lake.", "The person is reading a book.", "we are going on another trip soon , i will !"], "speaker_id": "snli:test:8908", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8918", "candidate_text": "well, the governor prepared to deliver the speech that would deliver the votes", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8918:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8788:premise", "snli:test:8936:premise", "snli:test:8918:premise", "snli:test:8685:premise", "style:personachat:test:331:self:7:positive:0", "style:personachat:test:331:self:7:positive:1", "style:personachat:test:331:self:7:positive:2", "style:personachat:test:331:self:7:positive:3", "style:personachat:test:331:self:7:positive:4", "style:personachat:test:331:self:7:positive:5", "style:personachat:test:331:self:7:positive:6"], "evidence_texts": ["A little girl drawing with chalk, on a chalkboard.", "Two men entering a brick home during the snow.", "A man in a black hat opens his mouth.", "People protesting age discrimination in an organized street rally.", "i am doing great , how are you ?", "i am a metallica fan , myself . ready for winter ?", "winter is my favorite , i love running in the snow .", "i do not . i work for a consulting firm , what do you do ?", "i cannot wait until i retire , i am going to travel .", "where are you going first ? i am on the first flight out of wisconsin .", "that would be nice . i am looking at tropical locations ."], "speaker_id": "snli:test:8918", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8918", "candidate_text": "The governor prepared to deliver the speech that would deliver the votes.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8918:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8788:premise", "snli:test:8936:premise", "snli:test:8918:premise", "snli:test:8685:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A little girl drawing with chalk, on a chalkboard.", "Two men entering a brick home during the snow.", "A man in a black hat opens his mouth.", "People protesting age discrimination in an organized street rally.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8918", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8922", "candidate_text": "Honestly, The doll isn't Stitch!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8922:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8607:premise", "snli:test:8960:premise", "snli:test:8922:premise", "snli:test:8606:premise", "style:personachat:test:332:self:1:positive:0"], "evidence_texts": ["A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:8922", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8922", "candidate_text": "I wonder whether the doll isn't stitch?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8922:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8607:premise", "snli:test:8960:premise", "snli:test:8922:premise", "snli:test:8606:premise", "style:personachat:test:260:self:1:positive:0"], "evidence_texts": ["A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "cool do you like to watch movies ?"], "speaker_id": "snli:test:8922", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8931", "candidate_text": "Honestly, They are indoors!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8931:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8854:premise", "snli:test:9405:premise", "snli:test:8665:premise", "snli:test:8931:premise", "style:personachat:test:332:self:2:positive:0", "style:personachat:test:332:self:2:positive:1"], "evidence_texts": ["A woman is applying lip makeup to another woman who seems to have some green in her hair.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:8931", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8931", "candidate_text": "well, they are indoors", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8931:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8854:premise", "snli:test:9405:premise", "snli:test:8665:premise", "snli:test:8931:premise", "style:personachat:test:547:self:2:positive:0", "style:personachat:test:547:self:2:positive:1"], "evidence_texts": ["A woman is applying lip makeup to another woman who seems to have some green in her hair.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A young dark-haired woman with red sun visor holding an open white umbrella amidst a crowd of people", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "impressive . you must be busy . do you have a family ?", "that sounds nice . maybe i will have a family someday"], "speaker_id": "snli:test:8931", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8936", "candidate_text": "Honestly, There is snow outside!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8936:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8936:premise", "snli:test:8918:premise", "snli:test:9102:premise", "snli:test:8788:premise", "style:personachat:test:332:self:5:positive:0", "style:personachat:test:332:self:5:positive:1", "style:personachat:test:332:self:5:positive:2", "style:personachat:test:332:self:5:positive:3", "style:personachat:test:332:self:5:positive:4"], "evidence_texts": ["Two men entering a brick home during the snow.", "A man in a black hat opens his mouth.", "A group of students are walking through the campus.", "A little girl drawing with chalk, on a chalkboard.", "i take my obedience dogs hiking ! i have mountains behind my new house .", "sorry ! was taking pictures of myself in my new club dress ! yes ! ! lets do it .", "only if i can hang it in my obedience school !", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:8936", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8936", "candidate_text": "There is snow outside.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8936:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8936:premise", "snli:test:8918:premise", "snli:test:9102:premise", "snli:test:8788:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two men entering a brick home during the snow.", "A man in a black hat opens his mouth.", "A group of students are walking through the campus.", "A little girl drawing with chalk, on a chalkboard.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8936", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8939", "candidate_text": "Honestly, A human poking!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8939:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8884:premise", "snli:test:8976:premise", "snli:test:8793:premise", "snli:test:8939:premise", "style:personachat:test:332:self:7:positive:0", "style:personachat:test:332:self:7:positive:1", "style:personachat:test:332:self:7:positive:2", "style:personachat:test:332:self:7:positive:3", "style:personachat:test:332:self:7:positive:4", "style:personachat:test:332:self:7:positive:5", "style:personachat:test:332:self:7:positive:6"], "evidence_texts": ["Person in a long red coat walking in front of a building.", "A girl in a black skirt and blue dress is watering flowers.", "A lady in a white tank and shorts fixing a wooden drawer.", "A male with sunglasses is poking at a tree with a pole.", "no stickers . . but i have candy ! i always have candy", "i love candy ! ! ! it is the best ! and brownies .", "i take my obedience dogs hiking ! i have mountains behind my new house .", "sorry ! was taking pictures of myself in my new club dress ! yes ! ! lets do it .", "only if i can hang it in my obedience school !", "awesome ! we can be best friends . what else do you like to do ?", "i love poetry too ! i will buy you stickers ! !"], "speaker_id": "snli:test:8939", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8939", "candidate_text": "I wonder whether a human poking?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8939:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8884:premise", "snli:test:8976:premise", "snli:test:8793:premise", "snli:test:8939:premise", "style:personachat:test:260:self:7:positive:0", "style:personachat:test:260:self:7:positive:1", "style:personachat:test:260:self:7:positive:2", "style:personachat:test:260:self:7:positive:3", "style:personachat:test:260:self:7:positive:4", "style:personachat:test:260:self:7:positive:5", "style:personachat:test:260:self:7:positive:6"], "evidence_texts": ["Person in a long red coat walking in front of a building.", "A girl in a black skirt and blue dress is watering flowers.", "A lady in a white tank and shorts fixing a wooden drawer.", "A male with sunglasses is poking at a tree with a pole.", "i am joe and a pro skater", "i live in fl also . i used to work for hollister back in school", "i love music and especially bluegrass which i can play good", "i had a tail when i was born but had it cut off", "i had super long hair . do you like sports ?", "that does i love the olympics especially the summer ones", "cool do you like to watch movies ?"], "speaker_id": "snli:test:8939", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8948", "candidate_text": "well, a girl has tripped and fallen into a pile of raked leaves", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8948:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8948:premise", "snli:test:8901:premise", "snli:test:9011:premise", "snli:test:8864:premise", "style:personachat:test:333:self:1:positive:0"], "evidence_texts": ["The young girl is airborne as she falls into the lake.", "Three performers are on the stage floor in black lace costumes.", "A black dog is swimming with a ball in his mouth.", "A woman in a headscarf sitting in front of a loom.", "awww congratulations . best wishes for you ."], "speaker_id": "snli:test:8948", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8948", "candidate_text": "Honestly, A girl has tripped and fallen into a pile of raked leaves!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8948:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8948:premise", "snli:test:8901:premise", "snli:test:9011:premise", "snli:test:8864:premise", "style:personachat:test:466:self:5:positive:0", "style:personachat:test:466:self:5:positive:1", "style:personachat:test:466:self:5:positive:2", "style:personachat:test:466:self:5:positive:3", "style:personachat:test:466:self:5:positive:4"], "evidence_texts": ["The young girl is airborne as she falls into the lake.", "Three performers are on the stage floor in black lace costumes.", "A black dog is swimming with a ball in his mouth.", "A woman in a headscarf sitting in front of a loom.", "only as old as you feel . and no , just me and my oldies music", "hm , not my expertise , only hobby of mine is trying barbeque restaurants", "as long as its oldies i will dance to anything ! aha you ?", "my father and grandfather were both army", "i have never been there ! and formally married"], "speaker_id": "snli:test:8948", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8951", "candidate_text": "well, two women are praying at a table before they eat their lunch", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8951:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8772:premise", "snli:test:8993:premise", "snli:test:8590:premise", "snli:test:8951:premise", "style:personachat:test:333:self:2:positive:0", "style:personachat:test:333:self:2:positive:1"], "evidence_texts": ["The women is photographing statues outdoors on a sunny day.", "A group of men, women and children are standing around.", "the white dog is carrying a ball in its mouth.", "Two women sitting at a table with their hands together.", "i feel sorry to hear this .", "awww congratulations . best wishes for you ."], "speaker_id": "snli:test:8951", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8951", "candidate_text": "Two women are praying at a table before they eat their lunch.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8951:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8772:premise", "snli:test:8993:premise", "snli:test:8590:premise", "snli:test:8951:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The women is photographing statues outdoors on a sunny day.", "A group of men, women and children are standing around.", "the white dog is carrying a ball in its mouth.", "Two women sitting at a table with their hands together.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8951", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8960", "candidate_text": "well, a man is talking to his girlfriend on the phone", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8960:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8960:premise", "snli:test:8922:premise", "snli:test:9046:premise", "snli:test:8607:premise", "style:personachat:test:333:self:5:positive:0", "style:personachat:test:333:self:5:positive:1", "style:personachat:test:333:self:5:positive:2", "style:personachat:test:333:self:5:positive:3", "style:personachat:test:333:self:5:positive:4"], "evidence_texts": ["An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "i do not like sports . i love watching series .", "i love watching soccer when i world cup or euro cup .", "yes it is . do you study ?", "i feel sorry to hear this .", "awww congratulations . best wishes for you ."], "speaker_id": "snli:test:8960", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8960", "candidate_text": "I wonder whether a man is talking to his girlfriend on the phone?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8960:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8960:premise", "snli:test:8922:premise", "snli:test:9046:premise", "snli:test:8607:premise", "style:personachat:test:267:self:5:positive:0", "style:personachat:test:267:self:5:positive:1", "style:personachat:test:267:self:5:positive:2", "style:personachat:test:267:self:5:positive:3", "style:personachat:test:267:self:5:positive:4"], "evidence_texts": ["An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A small child dressed for winter stares across a lake while leaning on a mesh fence in a scenic park setting.", "sorry to hear that . my mother is recently passed ; her and pa were proud immigrants .", "cool ! where is edris island ?", "i am an old baby boomer . not too great at dancing anymore .", "dance is great exercise for a young one like yourself !", "for sure . what else do you like to do for fun ?"], "speaker_id": "snli:test:8960", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8964", "candidate_text": "well, a man talks to kids", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8964:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8964:premise", "snli:test:8729:premise", "snli:test:9265:premise", "snli:test:8681:premise", "style:personachat:test:333:self:6:positive:0", "style:personachat:test:333:self:6:positive:1", "style:personachat:test:333:self:6:positive:2", "style:personachat:test:333:self:6:positive:3", "style:personachat:test:333:self:6:positive:4", "style:personachat:test:333:self:6:positive:5"], "evidence_texts": ["An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "i am doing fine too .", "i do not like sports . i love watching series .", "i love watching soccer when i world cup or euro cup .", "yes it is . do you study ?", "i feel sorry to hear this .", "awww congratulations . best wishes for you ."], "speaker_id": "snli:test:8964", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8964", "candidate_text": "Honestly, a man talks to kids!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8964:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8964:premise", "snli:test:8729:premise", "snli:test:9265:premise", "snli:test:8681:premise", "style:personachat:test:467:self:2:positive:0", "style:personachat:test:467:self:2:positive:1"], "evidence_texts": ["An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "Two men are loading bales of wheat on top of a high structure on a cloudy day.", "oh , wow . that is a lot of travelling ! do you do it for fun ?", "that is so awesome ! i like to eat ."], "speaker_id": "snli:test:8964", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8971", "candidate_text": "I wonder whether an old woman in a red and blue striped shirt is walking a large black horse across a street?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8971:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8745:premise", "snli:test:9096:premise", "snli:test:8744:premise", "snli:test:8971:premise", "style:personachat:test:334:self:1:positive:0"], "evidence_texts": ["A man is performing for a group in front of a white house.", "The woman in the white coat is holding a camera in her hand.", "A man is performing for a group in front of a white house.", "A woman in a striped shirt is walking a horse across a street.", "wow what happened to it ?"], "speaker_id": "snli:test:8971", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8971", "candidate_text": "Honestly, An old woman in a red and blue striped shirt is walking a large black horse across a street!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8971:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8745:premise", "snli:test:9096:premise", "snli:test:8744:premise", "snli:test:8971:premise", "style:personachat:test:467:self:5:positive:0", "style:personachat:test:467:self:5:positive:1", "style:personachat:test:467:self:5:positive:2", "style:personachat:test:467:self:5:positive:3", "style:personachat:test:467:self:5:positive:4"], "evidence_texts": ["A man is performing for a group in front of a white house.", "The woman in the white coat is holding a camera in her hand.", "A man is performing for a group in front of a white house.", "A woman in a striped shirt is walking a horse across a street.", "sometimes ! my dad teaches it so i started young . i am only 12 .", "yes , a dog , you ? what state do you live in ?", "cool . i live in texas . i have never been to another state . how about you ?", "oh , wow . that is a lot of travelling ! do you do it for fun ?", "that is so awesome ! i like to eat ."], "speaker_id": "snli:test:8971", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8976", "candidate_text": "I wonder whether the girl is putting water in the pot?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8976:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8976:premise", "snli:test:8939:premise", "snli:test:9121:premise", "snli:test:8884:premise", "style:personachat:test:334:self:2:positive:0", "style:personachat:test:334:self:2:positive:1"], "evidence_texts": ["A girl in a black skirt and blue dress is watering flowers.", "A male with sunglasses is poking at a tree with a pole.", "A man and a woman jumping with a lake in the background.", "Person in a long red coat walking in front of a building.", "thank you and you are so kind .", "wow what happened to it ?"], "speaker_id": "snli:test:8976", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8976", "candidate_text": "The girl is putting water in the pot.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8976:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8976:premise", "snli:test:8939:premise", "snli:test:9121:premise", "snli:test:8884:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A girl in a black skirt and blue dress is watering flowers.", "A male with sunglasses is poking at a tree with a pole.", "A man and a woman jumping with a lake in the background.", "Person in a long red coat walking in front of a building.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8976", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8979", "candidate_text": "I wonder whether the girl tries on the blue dress at the store?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8979:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8881:premise", "snli:test:8995:premise", "snli:test:8879:premise", "snli:test:8979:premise", "style:personachat:test:334:self:5:positive:0", "style:personachat:test:334:self:5:positive:1", "style:personachat:test:334:self:5:positive:2", "style:personachat:test:334:self:5:positive:3", "style:personachat:test:334:self:5:positive:4"], "evidence_texts": ["Several people including a child and a clown are walking towards a snowy sidewalk", "A woman in a purple jacket walks by two elderly men outside a cafe.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "The little boy is smiling as he crosses a rope on an assault course.", "i have tried it . i used my free time to lift weights . do you lift ?", "dogs are awesome . i am a huge man that loves watching horror movies .", "me either i drive a forklift in a warehouse where i work .", "thank you and you are so kind .", "wow what happened to it ?"], "speaker_id": "snli:test:8979", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8979", "candidate_text": "well, the girl tries on the blue dress at the store", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8979:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8881:premise", "snli:test:8995:premise", "snli:test:8879:premise", "snli:test:8979:premise", "style:personachat:test:550:self:5:positive:0", "style:personachat:test:550:self:5:positive:1", "style:personachat:test:550:self:5:positive:2", "style:personachat:test:550:self:5:positive:3", "style:personachat:test:550:self:5:positive:4"], "evidence_texts": ["Several people including a child and a clown are walking towards a snowy sidewalk", "A woman in a purple jacket walks by two elderly men outside a cafe.", "Pedestrians use the crosswalks as traffic awaits their turn to drive through an intersection.", "The little boy is smiling as he crosses a rope on an assault course.", "ok i can sell you a house on a mountain .", "depends if you want a remote location with no neighbors or some neighbors .", "it be about a million for your house on a mountain", "animals over people are sometimes best .", "i only dislike shellfish"], "speaker_id": "snli:test:8979", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8990", "candidate_text": "I wonder whether a woman and her son play the third of a best of three checkers tournament?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8990:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8838:premise", "snli:test:616:premise", "snli:test:8990:premise", "snli:test:8406:premise", "style:personachat:test:334:self:7:positive:0", "style:personachat:test:334:self:7:positive:1", "style:personachat:test:334:self:7:positive:2", "style:personachat:test:334:self:7:positive:3", "style:personachat:test:334:self:7:positive:4", "style:personachat:test:334:self:7:positive:5", "style:personachat:test:334:self:7:positive:6"], "evidence_texts": ["A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "i am wonderful just relaxing ready to begin a long work week .", "no i like barbecue more than fish .", "i have tried it . i used my free time to lift weights . do you lift ?", "dogs are awesome . i am a huge man that loves watching horror movies .", "me either i drive a forklift in a warehouse where i work .", "thank you and you are so kind .", "wow what happened to it ?"], "speaker_id": "snli:test:8990", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8990", "candidate_text": "Honestly, A woman and her son play the third of a best of three checkers tournament!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8990:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8838:premise", "snli:test:616:premise", "snli:test:8990:premise", "snli:test:8406:premise", "style:personachat:test:479:self:2:positive:0", "style:personachat:test:479:self:2:positive:1"], "evidence_texts": ["A man wearing a straw hat sitting in a chair holding the leash on a dog who is laying on the sidewalk next to him.", "A woman in a black jacket and a black and white striped skirt is watching a woman talking to a little boy sitting on concrete steps.", "A woman sits on a rug next to a shirtless boy who is looking in the direction of a checkers board that includes several pieces.", "A man wearing a black t-shirt that says \"Long Live the Dead\" pulls a shopping cart in the street near a NYC yellow taxi cab.", "oh i hate them . xd", "yeah , scared of heights . i take my risks in card games both feet on the ground !"], "speaker_id": "snli:test:8990", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8993", "candidate_text": "I wonder whether a large group of people are standing around?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8993:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8951:premise", "snli:test:8993:premise", "snli:test:8772:premise", "snli:test:9142:premise", "style:personachat:test:335:self:1:positive:0"], "evidence_texts": ["Two women sitting at a table with their hands together.", "A group of men, women and children are standing around.", "The women is photographing statues outdoors on a sunny day.", "People buy fruits and vegetables at an open air market.", "well , what do you enjoy doing ?"], "speaker_id": "snli:test:8993", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8993", "candidate_text": "A large group of people are standing around.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8993:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8951:premise", "snli:test:8993:premise", "snli:test:8772:premise", "snli:test:9142:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two women sitting at a table with their hands together.", "A group of men, women and children are standing around.", "The women is photographing statues outdoors on a sunny day.", "People buy fruits and vegetables at an open air market.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8993", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8994", "candidate_text": "I wonder whether a group of people are standing around?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8994:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9142:premise", "snli:test:8951:premise", "snli:test:8994:premise", "snli:test:9209:premise", "style:personachat:test:335:self:2:positive:0", "style:personachat:test:335:self:2:positive:1"], "evidence_texts": ["People buy fruits and vegetables at an open air market.", "Two women sitting at a table with their hands together.", "A group of men, women and children are standing around.", "A man trying his hardest to climb up a rope.", "what would you like to do ?", "well , what do you enjoy doing ?"], "speaker_id": "snli:test:8994", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8994", "candidate_text": "well, a group of people are standing around", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8994:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9142:premise", "snli:test:8951:premise", "snli:test:8994:premise", "snli:test:9209:premise", "style:personachat:test:551:self:2:positive:0", "style:personachat:test:551:self:2:positive:1"], "evidence_texts": ["People buy fruits and vegetables at an open air market.", "Two women sitting at a table with their hands together.", "A group of men, women and children are standing around.", "A man trying his hardest to climb up a rope.", "two dogs and one cat in the house keep me busy", "how about another one , i will give you mine"], "speaker_id": "snli:test:8994", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:8995", "candidate_text": "I wonder whether a woman in an orange jacket?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8995:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8979:premise", "snli:test:9063:premise", "snli:test:8881:premise", "snli:test:8995:premise", "style:personachat:test:335:self:5:positive:0", "style:personachat:test:335:self:5:positive:1", "style:personachat:test:335:self:5:positive:2", "style:personachat:test:335:self:5:positive:3", "style:personachat:test:335:self:5:positive:4"], "evidence_texts": ["The little boy is smiling as he crosses a rope on an assault course.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "Several people including a child and a clown are walking towards a snowy sidewalk", "A woman in a purple jacket walks by two elderly men outside a cafe.", "my family and i are from australia . where are you from ?", "what do you do in seattle ? i have never been .", "that is not good ! i do stock broking . good money , boring job .", "what would you like to do ?", "well , what do you enjoy doing ?"], "speaker_id": "snli:test:8995", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:8995", "candidate_text": "Honestly, A woman in an orange jacket!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:8995:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8979:premise", "snli:test:9063:premise", "snli:test:8881:premise", "snli:test:8995:premise", "style:personachat:test:500:self:1:positive:0"], "evidence_texts": ["The little boy is smiling as he crosses a rope on an assault course.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "Several people including a child and a clown are walking towards a snowy sidewalk", "A woman in a purple jacket walks by two elderly men outside a cafe.", "good night , have a wonderful night !"], "speaker_id": "snli:test:8995", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:8999", "candidate_text": "I wonder whether there is a child on a sled?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8999:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8687:premise", "snli:test:9079:premise", "snli:test:8627:premise", "snli:test:8999:premise", "style:personachat:test:335:self:6:positive:0", "style:personachat:test:335:self:6:positive:1", "style:personachat:test:335:self:6:positive:2", "style:personachat:test:335:self:6:positive:3", "style:personachat:test:335:self:6:positive:4", "style:personachat:test:335:self:6:positive:5"], "evidence_texts": ["An older woman is looking at herself in the mirror at a table with cosmetics.", "A dog with a ball in his mouth running down a road covered in leaves.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman in a blue jacket dragging a child on a sled through the snow.", "i am great ! just walked my dog , charlie . how about you ?", "my family and i are from australia . where are you from ?", "what do you do in seattle ? i have never been .", "that is not good ! i do stock broking . good money , boring job .", "what would you like to do ?", "well , what do you enjoy doing ?"], "speaker_id": "snli:test:8999", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:8999", "candidate_text": "There is a child on a sled.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:8999:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8687:premise", "snli:test:9079:premise", "snli:test:8627:premise", "snli:test:8999:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["An older woman is looking at herself in the mirror at a table with cosmetics.", "A dog with a ball in his mouth running down a road covered in leaves.", "The brown, furry animal is walking in the snowy grass with a tree behind him.", "A woman in a blue jacket dragging a child on a sled through the snow.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:8999", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9005", "candidate_text": "well, they are friends", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9005:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8739:premise", "snli:test:9005:premise", "snli:test:9043:premise", "snli:test:8460:premise", "style:personachat:test:336:self:1:positive:0"], "evidence_texts": ["A young man standing outside a laundromat.", "Two men pouring concrete onto another truck.", "A person is surfing a big wave.", "Three musicians are playing in a band.", "i do rent out properties . it is a job ."], "speaker_id": "snli:test:9005", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9005", "candidate_text": "Honestly, They are friends!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9005:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8739:premise", "snli:test:9005:premise", "snli:test:9043:premise", "snli:test:8460:premise", "style:personachat:test:500:self:5:positive:0", "style:personachat:test:500:self:5:positive:1", "style:personachat:test:500:self:5:positive:2", "style:personachat:test:500:self:5:positive:3", "style:personachat:test:500:self:5:positive:4"], "evidence_texts": ["A young man standing outside a laundromat.", "Two men pouring concrete onto another truck.", "A person is surfing a big wave.", "Three musicians are playing in a band.", "what do you teach ?", "i should teach history because i travel a lot .", "no children yet . i have traveled to 6 countries .", "traveling is so wonderful plus i love to walk so it works .", "good night , have a wonderful night !"], "speaker_id": "snli:test:9005", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9011", "candidate_text": "well, a black dog found a ball in the water and is bring it back to its owner", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9011:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8948:premise", "snli:test:9314:premise", "snli:test:8901:premise", "snli:test:9011:premise", "style:personachat:test:336:self:2:positive:0", "style:personachat:test:336:self:2:positive:1"], "evidence_texts": ["The young girl is airborne as she falls into the lake.", "Four kids sitting at a table with food on several plates.", "Three performers are on the stage floor in black lace costumes.", "A black dog is swimming with a ball in his mouth.", "what do you do for work ?", "i do rent out properties . it is a job ."], "speaker_id": "snli:test:9011", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9011", "candidate_text": "A black dog found a ball in the water and is bring it back to its owner.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9011:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8948:premise", "snli:test:9314:premise", "snli:test:8901:premise", "snli:test:9011:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["The young girl is airborne as she falls into the lake.", "Four kids sitting at a table with food on several plates.", "Three performers are on the stage floor in black lace costumes.", "A black dog is swimming with a ball in his mouth.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9011", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9026", "candidate_text": "well, the boy lost his pacifier", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9026:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8874:premise", "snli:test:9113:premise", "snli:test:9026:premise", "snli:test:8870:premise", "style:personachat:test:336:self:5:positive:0", "style:personachat:test:336:self:5:positive:1", "style:personachat:test:336:self:5:positive:2", "style:personachat:test:336:self:5:positive:3", "style:personachat:test:336:self:5:positive:4"], "evidence_texts": ["Women are working and digging in a field.", "Two people are participating in the sport fencing.", "A young face-painted boy is sucking his thumb.", "Two girls in bathing suits are in water.", "that is great . i like to hunt in my spare time .", "i am not a big music fan . i prefer the silence . you ?", "i just prefer to work on restoring cars . i am also a handyman", "what do you do for work ?", "i do rent out properties . it is a job ."], "speaker_id": "snli:test:9026", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9026", "candidate_text": "I wonder whether the boy lost his pacifier?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9026:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8874:premise", "snli:test:9113:premise", "snli:test:9026:premise", "snli:test:8870:premise", "style:personachat:test:272:self:5:positive:0", "style:personachat:test:272:self:5:positive:1", "style:personachat:test:272:self:5:positive:2", "style:personachat:test:272:self:5:positive:3", "style:personachat:test:272:self:5:positive:4"], "evidence_texts": ["Women are working and digging in a field.", "Two people are participating in the sport fencing.", "A young face-painted boy is sucking his thumb.", "Two girls in bathing suits are in water.", "have you ever taught in north dakota ? i am from there", "i hear they have a lot of jewish people like myself there", "i guess that is the case in most states", "i do a lot of hiking my my best friend from kindergarten , you ?", "that is very nice , what is your favorite meal ?"], "speaker_id": "snli:test:9026", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9043", "candidate_text": "well, a person is sitting in a chair", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9043:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9005:premise", "snli:test:8739:premise", "snli:test:9229:premise", "snli:test:9043:premise", "style:personachat:test:336:self:6:positive:0", "style:personachat:test:336:self:6:positive:1", "style:personachat:test:336:self:6:positive:2", "style:personachat:test:336:self:6:positive:3", "style:personachat:test:336:self:6:positive:4", "style:personachat:test:336:self:6:positive:5"], "evidence_texts": ["Two men pouring concrete onto another truck.", "A young man standing outside a laundromat.", "A man is squatting on the ground.", "A person is surfing a big wave.", "i am doing great . just taking a break from working on my car .", "that is great . i like to hunt in my spare time .", "i am not a big music fan . i prefer the silence . you ?", "i just prefer to work on restoring cars . i am also a handyman", "what do you do for work ?", "i do rent out properties . it is a job ."], "speaker_id": "snli:test:9043", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9043", "candidate_text": "Honestly, A person is sitting in a chair!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9043:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9005:premise", "snli:test:8739:premise", "snli:test:9229:premise", "snli:test:9043:premise", "style:personachat:test:505:self:2:positive:0", "style:personachat:test:505:self:2:positive:1"], "evidence_texts": ["Two men pouring concrete onto another truck.", "A young man standing outside a laundromat.", "A man is squatting on the ground.", "A person is surfing a big wave.", "well have a good time . it is good to be with people", "well have a great time tonight !"], "speaker_id": "snli:test:9043", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9044", "candidate_text": "well, a person is surfing a big hundred foot wave", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9044:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9044:premise", "snli:test:9229:premise", "snli:test:9005:premise", "snli:test:9311:premise", "style:personachat:test:337:self:1:positive:0"], "evidence_texts": ["A person is surfing a big wave.", "A man is squatting on the ground.", "Two men pouring concrete onto another truck.", "The dogs are playing on the bed.", "just being random to pass the time trying to get my 30 cents"], "speaker_id": "snli:test:9044", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9044", "candidate_text": "A person is surfing a big hundred foot wave.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9044:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9044:premise", "snli:test:9229:premise", "snli:test:9005:premise", "snli:test:9311:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A person is surfing a big wave.", "A man is squatting on the ground.", "Two men pouring concrete onto another truck.", "The dogs are playing on the bed.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9044", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9046", "candidate_text": "well, two boys are surrounded by toys", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9046:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8960:premise", "snli:test:9194:premise", "snli:test:9046:premise", "snli:test:8922:premise", "style:personachat:test:337:self:2:positive:0", "style:personachat:test:337:self:2:positive:1"], "evidence_texts": ["An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "life is the thing that is the thing and its dressed like the thing", "just being random to pass the time trying to get my 30 cents"], "speaker_id": "snli:test:9046", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9046", "candidate_text": "I wonder whether two boys are surrounded by toys?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9046:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8960:premise", "snli:test:9194:premise", "snli:test:9046:premise", "snli:test:8922:premise", "style:personachat:test:278:self:2:positive:0", "style:personachat:test:278:self:2:positive:1"], "evidence_texts": ["An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A woman in a pink top is holding a glass bottle in one hand and a Stitch doll in the other.", "yes i am . . . located in south carolina", "i work as an admin assistant what do you do for a living ?"], "speaker_id": "snli:test:9046", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9056", "candidate_text": "well, a dog is sleeping", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9056:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9056:premise", "snli:test:8558:premise", "snli:test:9089:premise", "snli:test:8557:premise", "style:personachat:test:337:self:5:positive:0", "style:personachat:test:337:self:5:positive:1", "style:personachat:test:337:self:5:positive:2", "style:personachat:test:337:self:5:positive:3", "style:personachat:test:337:self:5:positive:4"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "nice where do you work", "cool i live in nyc i am in queens", "nothing much i just play video games", "life is the thing that is the thing and its dressed like the thing", "just being random to pass the time trying to get my 30 cents"], "speaker_id": "snli:test:9056", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9056", "candidate_text": "Honestly, A dog is sleeping!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9056:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9056:premise", "snli:test:8558:premise", "snli:test:9089:premise", "snli:test:8557:premise", "style:personachat:test:512:self:1:positive:0"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "ha ! well then good . i am 34 ."], "speaker_id": "snli:test:9056", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9063", "candidate_text": "well, a singer is on american idol", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9063:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8995:premise", "snli:test:9125:premise", "snli:test:8979:premise", "snli:test:9063:premise", "style:personachat:test:337:self:7:positive:0", "style:personachat:test:337:self:7:positive:1", "style:personachat:test:337:self:7:positive:2", "style:personachat:test:337:self:7:positive:3", "style:personachat:test:337:self:7:positive:4", "style:personachat:test:337:self:7:positive:5", "style:personachat:test:337:self:7:positive:6"], "evidence_texts": ["A woman in a purple jacket walks by two elderly men outside a cafe.", "A man throws his young boy playfully into the air standing on a beach.", "The little boy is smiling as he crosses a rope on an assault course.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "i am doing good how are you", "how are you doing that and working on a hit", "nice where do you work", "cool i live in nyc i am in queens", "nothing much i just play video games", "life is the thing that is the thing and its dressed like the thing", "just being random to pass the time trying to get my 30 cents"], "speaker_id": "snli:test:9063", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9063", "candidate_text": "a singer is on american idol.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9063:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8995:premise", "snli:test:9125:premise", "snli:test:8979:premise", "snli:test:9063:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A woman in a purple jacket walks by two elderly men outside a cafe.", "A man throws his young boy playfully into the air standing on a beach.", "The little boy is smiling as he crosses a rope on an assault course.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9063", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9076", "candidate_text": "well, a man is near a building labeled lottery ticket centre", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9076:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9076:premise", "snli:test:8829:premise", "snli:test:8828:premise", "snli:test:9530:premise", "style:personachat:test:338:self:1:positive:0"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "no really i learned everything from those movies ."], "speaker_id": "snli:test:9076", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9076", "candidate_text": "I wonder whether a man is near a building labeled lottery ticket centre?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9076:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9076:premise", "snli:test:8829:premise", "snli:test:8828:premise", "snli:test:9530:premise", "style:personachat:test:283:self:1:positive:0"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:9076", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9078", "candidate_text": "well, a man in a red jacket is near a building labeled lottery ticket centre", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9078:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9530:premise", "snli:test:8829:premise", "snli:test:9078:premise", "snli:test:9577:premise", "style:personachat:test:338:self:2:positive:0", "style:personachat:test:338:self:2:positive:1"], "evidence_texts": ["One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "cool , i watch bruce lee films since i was little . we share the same skills", "no really i learned everything from those movies ."], "speaker_id": "snli:test:9078", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9078", "candidate_text": "Honestly, A man in a red jacket is near a building labeled Lottery Ticket Centre!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9078:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9530:premise", "snli:test:8829:premise", "snli:test:9078:premise", "snli:test:9577:premise", "style:personachat:test:512:self:6:positive:0", "style:personachat:test:512:self:6:positive:1", "style:personachat:test:512:self:6:positive:2", "style:personachat:test:512:self:6:positive:3", "style:personachat:test:512:self:6:positive:4", "style:personachat:test:512:self:6:positive:5"], "evidence_texts": ["One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "A young redheaded girl in a blue shirt is walking and talking with a boy in black Pearl Jam shirt.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "good and how are you today ?", "what kind of practice do you do ?", "very cool . i am a real estate agent .", "yes , lots . i have two kids in kindergarten so i am busy !", "do you have any kids ?", "ha ! well then good . i am 34 ."], "speaker_id": "snli:test:9078", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9079", "candidate_text": "well, a cat running up a tree", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9079:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8999:premise", "snli:test:8687:premise", "snli:test:9103:premise", "snli:test:9079:premise", "style:personachat:test:338:self:5:positive:0", "style:personachat:test:338:self:5:positive:1", "style:personachat:test:338:self:5:positive:2", "style:personachat:test:338:self:5:positive:3", "style:personachat:test:338:self:5:positive:4"], "evidence_texts": ["A woman in a blue jacket dragging a child on a sled through the snow.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "A man in a red delivery truck is talking to someone is a black coat.", "A dog with a ball in his mouth running down a road covered in leaves.", "my wife and i just had a baby boy . do you have kids ?", "first time dad over here . team no sleep .", "yes , i played a little company softball", "cool , i watch bruce lee films since i was little . we share the same skills", "no really i learned everything from those movies ."], "speaker_id": "snli:test:9079", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9079", "candidate_text": "A cat running up a tree.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9079:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8999:premise", "snli:test:8687:premise", "snli:test:9103:premise", "snli:test:9079:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman in a blue jacket dragging a child on a sled through the snow.", "An older woman is looking at herself in the mirror at a table with cosmetics.", "A man in a red delivery truck is talking to someone is a black coat.", "A dog with a ball in his mouth running down a road covered in leaves.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9079", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9080", "candidate_text": "well, a dog running down a road", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9080:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9080:premise", "snli:test:9103:premise", "snli:test:8999:premise", "snli:test:9329:premise", "style:personachat:test:338:self:7:positive:0", "style:personachat:test:338:self:7:positive:1", "style:personachat:test:338:self:7:positive:2", "style:personachat:test:338:self:7:positive:3", "style:personachat:test:338:self:7:positive:4", "style:personachat:test:338:self:7:positive:5", "style:personachat:test:338:self:7:positive:6"], "evidence_texts": ["A dog with a ball in his mouth running down a road covered in leaves.", "A man in a red delivery truck is talking to someone is a black coat.", "A woman in a blue jacket dragging a child on a sled through the snow.", "A man on a skateboard on steps in front glass doors of a brick building.", "i am doing great today and you ?", "going to walk my dog soon . do you have one ?", "my wife and i just had a baby boy . do you have kids ?", "first time dad over here . team no sleep .", "yes , i played a little company softball", "cool , i watch bruce lee films since i was little . we share the same skills", "no really i learned everything from those movies ."], "speaker_id": "snli:test:9080", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9080", "candidate_text": "I wonder whether a dog running down a road?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9080:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9080:premise", "snli:test:9103:premise", "snli:test:8999:premise", "snli:test:9329:premise", "style:personachat:test:283:self:7:positive:0", "style:personachat:test:283:self:7:positive:1", "style:personachat:test:283:self:7:positive:2", "style:personachat:test:283:self:7:positive:3", "style:personachat:test:283:self:7:positive:4", "style:personachat:test:283:self:7:positive:5", "style:personachat:test:283:self:7:positive:6"], "evidence_texts": ["A dog with a ball in his mouth running down a road covered in leaves.", "A man in a red delivery truck is talking to someone is a black coat.", "A woman in a blue jacket dragging a child on a sled through the snow.", "A man on a skateboard on steps in front glass doors of a brick building.", "hi , i m fine thanks , you ?", "good , i love the army .", "i m old so i can t join them now , hope to retire soon", "accountant . sitting down hurts my left leg", "no . i ll ride some horses when it hurts , it helps .", "yes , but i don t tell this to many people , i usually keep it to myself", "do you have facial hair ? i love it when it grows"], "speaker_id": "snli:test:9080", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9089", "candidate_text": "Honestly, Two boys in a sailboat!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9089:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9056:premise", "snli:test:9089:premise", "snli:test:9268:premise", "snli:test:8558:premise", "style:personachat:test:339:self:1:positive:0"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:9089", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9089", "candidate_text": "well, two boys in a sailboat", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9089:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9056:premise", "snli:test:9089:premise", "snli:test:9268:premise", "snli:test:8558:premise", "style:personachat:test:559:self:1:positive:0"], "evidence_texts": ["A woman and two children walk along stones to cross a river while a dog swims.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A man wearing an orange shirt and sunglasses jogging in a marathon with spectators spectating him.", "yes i do . rock music gets me pumped ."], "speaker_id": "snli:test:9089", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9091", "candidate_text": "Honestly, A bird is on a lake!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9091:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8908:premise", "snli:test:9177:premise", "snli:test:8835:premise", "snli:test:9091:premise", "style:personachat:test:339:self:2:positive:0", "style:personachat:test:339:self:2:positive:1"], "evidence_texts": ["Uniformed group walks down a street.", "A choir practicing in a church.", "The person is reading a book.", "A swan landing in a lake.", "my father was chief of police in france , i have been there a lot !", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:9091", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9091", "candidate_text": "A bird is on a lake.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9091:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8908:premise", "snli:test:9177:premise", "snli:test:8835:premise", "snli:test:9091:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Uniformed group walks down a street.", "A choir practicing in a church.", "The person is reading a book.", "A swan landing in a lake.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9091", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9096", "candidate_text": "Honestly, The woman is trying to get a picture of her friend!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9096:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9096:premise", "snli:test:8971:premise", "snli:test:9157:premise", "snli:test:8745:premise", "style:personachat:test:339:self:5:positive:0", "style:personachat:test:339:self:5:positive:1", "style:personachat:test:339:self:5:positive:2", "style:personachat:test:339:self:5:positive:3", "style:personachat:test:339:self:5:positive:4"], "evidence_texts": ["The woman in the white coat is holding a camera in her hand.", "A woman in a striped shirt is walking a horse across a street.", "A man in a red shirt is walking towards a blue market stall.", "A man is performing for a group in front of a white house.", "music is my bread and butter , i like surrounding myself with music . what do you teach ?", "wow , that is cool , you must be very smart . i work at petsmart right now", "what is your favorite place to visit in europe ?", "my father was chief of police in france , i have been there a lot !", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:9096", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9096", "candidate_text": "I wonder whether the woman is trying to get a picture of her friend?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9096:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9096:premise", "snli:test:8971:premise", "snli:test:9157:premise", "snli:test:8745:premise", "style:personachat:test:285:self:5:positive:0", "style:personachat:test:285:self:5:positive:1", "style:personachat:test:285:self:5:positive:2", "style:personachat:test:285:self:5:positive:3", "style:personachat:test:285:self:5:positive:4"], "evidence_texts": ["The woman in the white coat is holding a camera in her hand.", "A woman in a striped shirt is walking a horse across a street.", "A man in a red shirt is walking towards a blue market stall.", "A man is performing for a group in front of a white house.", "i am from california , but moving to new york for a job next year", "i work as a model and actress , but i am not a good singer ! you ?", "awesome ! what kind ? i am a surfer and skater .", "yes it will be sad , but lots of great places to skate ther .", "i prefer inline , but also like longboard . what is the name of your turtle ?"], "speaker_id": "snli:test:9096", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9099", "candidate_text": "Honestly, The men felled the log in a snowy area!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9099:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:4435:premise", "snli:test:3481:premise", "snli:test:2891:premise", "snli:test:9099:premise", "style:personachat:test:339:self:6:positive:0", "style:personachat:test:339:self:6:positive:1", "style:personachat:test:339:self:6:positive:2", "style:personachat:test:339:self:6:positive:3", "style:personachat:test:339:self:6:positive:4", "style:personachat:test:339:self:6:positive:5"], "evidence_texts": ["A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "i do enjoy walking ! i love to put my headphones on and walk around .", "music is my bread and butter , i like surrounding myself with music . what do you teach ?", "wow , that is cool , you must be very smart . i work at petsmart right now", "what is your favorite place to visit in europe ?", "my father was chief of police in france , i have been there a lot !", "i love walking , i like to blast nirvana on my ipod and get moving !"], "speaker_id": "snli:test:9099", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9099", "candidate_text": "well, the men felled the log in a snowy area", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9099:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:4435:premise", "snli:test:3481:premise", "snli:test:2891:premise", "snli:test:9099:premise", "style:personachat:test:559:self:6:positive:0", "style:personachat:test:559:self:6:positive:1", "style:personachat:test:559:self:6:positive:2", "style:personachat:test:559:self:6:positive:3", "style:personachat:test:559:self:6:positive:4", "style:personachat:test:559:self:6:positive:5"], "evidence_texts": ["A black man wearing tan shorts, a yellow sweatshirt, and a baseball cap is holding up his hand and talking to an Asian woman posing with another young black man while they are holding red flags with yellow stars.", "A man in a white shirt, dark shorts and a red cap is standing in a small green rowboat with green oars, in the middle of what appears to be a shallow river, against a bank of tall dried grasses.", "A woman is holding her child while a man is reading something from a piece of paper and taking something from a bowl that a younger girl is holding in a park with a small lake in the background.", "In a snowy residential environment, a man with a brown jacket and gray mustache and beard places a hand on a felled log while another man in a blue shirt is blurry on the right side of the photo.", "perhaps you can help me . i recently quit high school", "no i am an independent", "that is great . i live alone in a low income apartment .", "you are a cook ? are you famous ?", "i am not much of a reader but i do love to run", "yes i do . rock music gets me pumped ."], "speaker_id": "snli:test:9099", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9102", "candidate_text": "well, there is a school tour going on", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9102:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8936:premise", "snli:test:9132:premise", "snli:test:9102:premise", "snli:test:8918:premise", "style:personachat:test:340:self:1:positive:0"], "evidence_texts": ["Two men entering a brick home during the snow.", "People are hiking up a mountain with no greenery.", "A group of students are walking through the campus.", "A man in a black hat opens his mouth.", "quilting is a great hobby very artistic"], "speaker_id": "snli:test:9102", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9102", "candidate_text": "There is a school tour going on.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9102:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8936:premise", "snli:test:9132:premise", "snli:test:9102:premise", "snli:test:8918:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two men entering a brick home during the snow.", "People are hiking up a mountain with no greenery.", "A group of students are walking through the campus.", "A man in a black hat opens his mouth.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9102", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9103", "candidate_text": "well, a sad man in a red delivery truck is talking to someone is a black coat", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9103:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9080:premise", "snli:test:9329:premise", "snli:test:9079:premise", "snli:test:9103:premise", "style:personachat:test:340:self:2:positive:0", "style:personachat:test:340:self:2:positive:1"], "evidence_texts": ["A dog with a ball in his mouth running down a road covered in leaves.", "A man on a skateboard on steps in front glass doors of a brick building.", "A dog with a ball in his mouth running down a road covered in leaves.", "A man in a red delivery truck is talking to someone is a black coat.", "i am sure , but she must be a lot of fun .", "quilting is a great hobby very artistic"], "speaker_id": "snli:test:9103", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9103", "candidate_text": "I wonder whether a sad man in a red delivery truck is talking to someone is a black coat?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9103:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9080:premise", "snli:test:9329:premise", "snli:test:9079:premise", "snli:test:9103:premise", "style:personachat:test:287:self:2:positive:0", "style:personachat:test:287:self:2:positive:1"], "evidence_texts": ["A dog with a ball in his mouth running down a road covered in leaves.", "A man on a skateboard on steps in front glass doors of a brick building.", "A dog with a ball in his mouth running down a road covered in leaves.", "A man in a red delivery truck is talking to someone is a black coat.", "you have to get that protein", "have you always been into body building ?"], "speaker_id": "snli:test:9103", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9113", "candidate_text": "well, two people holding fencing swords", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9113:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9026:premise", "snli:test:9113:premise", "snli:test:9129:premise", "snli:test:8874:premise", "style:personachat:test:340:self:5:positive:0", "style:personachat:test:340:self:5:positive:1", "style:personachat:test:340:self:5:positive:2", "style:personachat:test:340:self:5:positive:3", "style:personachat:test:340:self:5:positive:4"], "evidence_texts": ["A young face-painted boy is sucking his thumb.", "Two people are participating in the sport fencing.", "Three children are shooting guns in the woods.", "Women are working and digging in a field.", "i go to the gym a lot", "walking can be good exercise", "oh spending time with your granddaughter must be great . how old is she ?", "i am sure , but she must be a lot of fun .", "quilting is a great hobby very artistic"], "speaker_id": "snli:test:9113", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9113", "candidate_text": "Honestly, Two people holding fencing swords!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9113:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9026:premise", "snli:test:9113:premise", "snli:test:9129:premise", "snli:test:8874:premise", "style:personachat:test:553:self:1:positive:0"], "evidence_texts": ["A young face-painted boy is sucking his thumb.", "Two people are participating in the sport fencing.", "Three children are shooting guns in the woods.", "Women are working and digging in a field.", "haha yes i am . what a small world we live in !"], "speaker_id": "snli:test:9113", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9121", "candidate_text": "well, they are swimming in the lake", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9121:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8976:premise", "snli:test:9121:premise", "snli:test:9184:premise", "snli:test:8939:premise", "style:personachat:test:340:self:7:positive:0", "style:personachat:test:340:self:7:positive:1", "style:personachat:test:340:self:7:positive:2", "style:personachat:test:340:self:7:positive:3", "style:personachat:test:340:self:7:positive:4", "style:personachat:test:340:self:7:positive:5", "style:personachat:test:340:self:7:positive:6"], "evidence_texts": ["A girl in a black skirt and blue dress is watering flowers.", "A man and a woman jumping with a lake in the background.", "Two children play in the snow by the side of the road.", "A male with sunglasses is poking at a tree with a pole.", "hi ! i am doing well . how about you ?", "i am listening to rap music", "i go to the gym a lot", "walking can be good exercise", "oh spending time with your granddaughter must be great . how old is she ?", "i am sure , but she must be a lot of fun .", "quilting is a great hobby very artistic"], "speaker_id": "snli:test:9121", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9121", "candidate_text": "They are swimming in the lake.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9121:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8976:premise", "snli:test:9121:premise", "snli:test:9184:premise", "snli:test:8939:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A girl in a black skirt and blue dress is watering flowers.", "A man and a woman jumping with a lake in the background.", "Two children play in the snow by the side of the road.", "A male with sunglasses is poking at a tree with a pole.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9121", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9125", "candidate_text": "I wonder whether a dad and son is playing on the beach?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9125:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9063:premise", "snli:test:9125:premise", "snli:test:9189:premise", "snli:test:8995:premise", "style:personachat:test:341:self:1:positive:0"], "evidence_texts": ["A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "A man throws his young boy playfully into the air standing on a beach.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A woman in a purple jacket walks by two elderly men outside a cafe.", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:9125", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9125", "candidate_text": "A Dad and son is playing on the beach.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9125:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9063:premise", "snli:test:9125:premise", "snli:test:9189:premise", "snli:test:8995:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "A man throws his young boy playfully into the air standing on a beach.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A woman in a purple jacket walks by two elderly men outside a cafe.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9125", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9129", "candidate_text": "I wonder whether three children are shooting a deer?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9129:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9113:premise", "snli:test:9129:premise", "snli:test:9171:premise", "snli:test:9026:premise", "style:personachat:test:341:self:2:positive:0", "style:personachat:test:341:self:2:positive:1"], "evidence_texts": ["Two people are participating in the sport fencing.", "Three children are shooting guns in the woods.", "She is cooking food while wearing a dress.", "A young face-painted boy is sucking his thumb.", "wow that is incredible ! are you an engineer as well ?", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:9129", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9129", "candidate_text": "well, three children are shooting a deer", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9129:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9113:premise", "snli:test:9129:premise", "snli:test:9171:premise", "snli:test:9026:premise", "style:personachat:test:561:self:2:positive:0", "style:personachat:test:561:self:2:positive:1"], "evidence_texts": ["Two people are participating in the sport fencing.", "Three children are shooting guns in the woods.", "She is cooking food while wearing a dress.", "A young face-painted boy is sucking his thumb.", "i weigh about 6 . 5 penguins . that is our number system here .", "one of my three children work in a warehouse . it is a shipping company"], "speaker_id": "snli:test:9129", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9132", "candidate_text": "I wonder whether the mountain is bare?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9132:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9132:premise", "snli:test:9102:premise", "snli:test:9332:premise", "snli:test:8936:premise", "style:personachat:test:341:self:5:positive:0", "style:personachat:test:341:self:5:positive:1", "style:personachat:test:341:self:5:positive:2", "style:personachat:test:341:self:5:positive:3", "style:personachat:test:341:self:5:positive:4"], "evidence_texts": ["People are hiking up a mountain with no greenery.", "A group of students are walking through the campus.", "A man playing the guitar while smoking a cigarette.", "Two men entering a brick home during the snow.", "i was just there for a vintage car show . nice state !", "your best friend is eve 9000 ? is that an ai ?", "how exciting ! how did you meet ?", "wow that is incredible ! are you an engineer as well ?", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:9132", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9132", "candidate_text": "Honestly, The mountain is bare!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9132:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9132:premise", "snli:test:9102:premise", "snli:test:9332:premise", "snli:test:8936:premise", "style:personachat:test:565:self:1:positive:0"], "evidence_texts": ["People are hiking up a mountain with no greenery.", "A group of students are walking through the campus.", "A man playing the guitar while smoking a cigarette.", "Two men entering a brick home during the snow.", "no ! jack from titanic . i want to use the puppy money to remake it"], "speaker_id": "snli:test:9132", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9142", "candidate_text": "I wonder whether people buy produce at a market?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9142:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:8994:premise", "snli:test:9209:premise", "snli:test:9142:premise", "snli:test:8993:premise", "style:personachat:test:341:self:6:positive:0", "style:personachat:test:341:self:6:positive:1", "style:personachat:test:341:self:6:positive:2", "style:personachat:test:341:self:6:positive:3", "style:personachat:test:341:self:6:positive:4", "style:personachat:test:341:self:6:positive:5"], "evidence_texts": ["A group of men, women and children are standing around.", "A man trying his hardest to climb up a rope.", "People buy fruits and vegetables at an open air market.", "A group of men, women and children are standing around.", "mississippi . best jazz around ! how about you ?", "i was just there for a vintage car show . nice state !", "your best friend is eve 9000 ? is that an ai ?", "how exciting ! how did you meet ?", "wow that is incredible ! are you an engineer as well ?", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:9142", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9142", "candidate_text": "People buy produce at a market.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9142:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8994:premise", "snli:test:9209:premise", "snli:test:9142:premise", "snli:test:8993:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A group of men, women and children are standing around.", "A man trying his hardest to climb up a rope.", "People buy fruits and vegetables at an open air market.", "A group of men, women and children are standing around.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9142", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9157", "candidate_text": "Honestly, The man wants to purchase some produce!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9157:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9096:premise", "snli:test:9157:premise", "snli:test:8971:premise", "snli:test:9236:premise", "style:personachat:test:342:self:1:positive:0"], "evidence_texts": ["The woman in the white coat is holding a camera in her hand.", "A man in a red shirt is walking towards a blue market stall.", "A woman in a striped shirt is walking a horse across a street.", "A woman is cutting into a sandwich while sitting at an outdoor table.", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:9157", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9157", "candidate_text": "well, the man wants to purchase some produce", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9157:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9096:premise", "snli:test:9157:premise", "snli:test:8971:premise", "snli:test:9236:premise", "style:personachat:test:563:self:1:positive:0"], "evidence_texts": ["The woman in the white coat is holding a camera in her hand.", "A man in a red shirt is walking towards a blue market stall.", "A woman in a striped shirt is walking a horse across a street.", "A woman is cutting into a sandwich while sitting at an outdoor table.", "i have not watched that . its on after macgyver ."], "speaker_id": "snli:test:9157", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9158", "candidate_text": "Honestly, The man is walking in a direction!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9158:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9236:premise", "snli:test:9096:premise", "snli:test:9158:premise", "snli:test:9254:premise", "style:personachat:test:342:self:2:positive:0", "style:personachat:test:342:self:2:positive:1"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "The woman in the white coat is holding a camera in her hand.", "A man in a red shirt is walking towards a blue market stall.", "A little white dog runs outside, their ears blown back in the wind.", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:9158", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9158", "candidate_text": "The man is walking in a direction.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9158:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9236:premise", "snli:test:9096:premise", "snli:test:9158:premise", "snli:test:9254:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "The woman in the white coat is holding a camera in her hand.", "A man in a red shirt is walking towards a blue market stall.", "A little white dog runs outside, their ears blown back in the wind.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9158", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9171", "candidate_text": "Honestly, A woman in a dress dances in the club!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9171:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9129:premise", "snli:test:9234:premise", "snli:test:9113:premise", "snli:test:9171:premise", "style:personachat:test:342:self:5:positive:0", "style:personachat:test:342:self:5:positive:1", "style:personachat:test:342:self:5:positive:2", "style:personachat:test:342:self:5:positive:3", "style:personachat:test:342:self:5:positive:4"], "evidence_texts": ["Three children are shooting guns in the woods.", "A group of people gather around a truck.", "Two people are participating in the sport fencing.", "She is cooking food while wearing a dress.", "aww , all the best to you ! i just finished baking a cake for my kids and hubby .", "sending you a slice ! i do not like either . i enjoy reading .", "you could always read really fast ! !", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:9171", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9171", "candidate_text": "I wonder whether a woman in a dress dances in the club?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9171:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9129:premise", "snli:test:9234:premise", "snli:test:9113:premise", "snli:test:9171:premise", "style:personachat:test:292:self:5:positive:0", "style:personachat:test:292:self:5:positive:1", "style:personachat:test:292:self:5:positive:2", "style:personachat:test:292:self:5:positive:3", "style:personachat:test:292:self:5:positive:4"], "evidence_texts": ["Three children are shooting guns in the woods.", "A group of people gather around a truck.", "Two people are participating in the sport fencing.", "She is cooking food while wearing a dress.", "that sounds awesome ! are you working anywhere now ? are you in college ?", "cool ! i am retired from the banking business . do you have any hobbies ?", "wow ! i like to make quilts for my relatives in my free time .", "i am not married . it would be hard to have kids while still in college .", "what happened with your relationship with your boyfriend ?"], "speaker_id": "snli:test:9171", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9177", "candidate_text": "Honestly, A group of people is yelling and shouting!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9177:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9091:premise", "snli:test:9177:premise", "snli:test:9244:premise", "snli:test:8908:premise", "style:personachat:test:342:self:7:positive:0", "style:personachat:test:342:self:7:positive:1", "style:personachat:test:342:self:7:positive:2", "style:personachat:test:342:self:7:positive:3", "style:personachat:test:342:self:7:positive:4", "style:personachat:test:342:self:7:positive:5", "style:personachat:test:342:self:7:positive:6"], "evidence_texts": ["A swan landing in a lake.", "A choir practicing in a church.", "Two men outside of a restaurant.", "Uniformed group walks down a street.", "i am good , thanks ! and you ?", "you performing there or going as audience member ?", "aww , all the best to you ! i just finished baking a cake for my kids and hubby .", "sending you a slice ! i do not like either . i enjoy reading .", "you could always read really fast ! !", "blink three times and it becomes your superpower ! what do you like cooking ?", "i burn rice often ! it is cool that you act and model ."], "speaker_id": "snli:test:9177", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9177", "candidate_text": "well, a group of people is yelling and shouting", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9177:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9091:premise", "snli:test:9177:premise", "snli:test:9244:premise", "snli:test:8908:premise", "style:personachat:test:563:self:7:positive:0", "style:personachat:test:563:self:7:positive:1", "style:personachat:test:563:self:7:positive:2", "style:personachat:test:563:self:7:positive:3", "style:personachat:test:563:self:7:positive:4", "style:personachat:test:563:self:7:positive:5", "style:personachat:test:563:self:7:positive:6"], "evidence_texts": ["A swan landing in a lake.", "A choir practicing in a church.", "Two men outside of a restaurant.", "Uniformed group walks down a street.", "i do not . i work in an animal shelter tho . i am the cleaner there .", "i love dogs . i like them better than humans at times .", "you like music ? simple plan is my fav . band .", "no i have not . what genre are they ?", "very cool . metal is cool .", "i watch a lot of tv .", "i have not watched that . its on after macgyver ."], "speaker_id": "snli:test:9177", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9184", "candidate_text": "well, kids are building a snow man", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9184:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9184:premise", "snli:test:9121:premise", "snli:test:9248:premise", "snli:test:8976:premise", "style:personachat:test:343:self:1:positive:0"], "evidence_texts": ["Two children play in the snow by the side of the road.", "A man and a woman jumping with a lake in the background.", "Army men grilling a lot of meat while his troop looks on.", "A girl in a black skirt and blue dress is watering flowers.", "you must really like cars"], "speaker_id": "snli:test:9184", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9184", "candidate_text": "Kids are building a snow man.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9184:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9184:premise", "snli:test:9121:premise", "snli:test:9248:premise", "snli:test:8976:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["Two children play in the snow by the side of the road.", "A man and a woman jumping with a lake in the background.", "Army men grilling a lot of meat while his troop looks on.", "A girl in a black skirt and blue dress is watering flowers.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9184", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9189", "candidate_text": "well, a woman is jogging down a road", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9189:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9125:premise", "snli:test:9189:premise", "snli:test:9580:premise", "snli:test:9063:premise", "style:personachat:test:343:self:2:positive:0", "style:personachat:test:343:self:2:positive:1"], "evidence_texts": ["A man throws his young boy playfully into the air standing on a beach.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A baseball player in the act of pick up a baseball with his hand.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "yes , he was on tv", "you must really like cars"], "speaker_id": "snli:test:9189", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9189", "candidate_text": "I wonder whether a woman is jogging down a road?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9189:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9125:premise", "snli:test:9189:premise", "snli:test:9580:premise", "snli:test:9063:premise", "style:personachat:test:298:self:2:positive:0", "style:personachat:test:298:self:2:positive:1"], "evidence_texts": ["A man throws his young boy playfully into the air standing on a beach.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A baseball player in the act of pick up a baseball with his hand.", "A singer wearing a leather jacket performs on stage with dramatic lighting behind him.", "i have a passion for rap music", "do you ? i cant have them in the dorm"], "speaker_id": "snli:test:9189", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9194", "candidate_text": "well, an injured skier being taken for medical help", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9194:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9046:premise", "snli:test:8960:premise", "snli:test:9194:premise", "snli:test:9448:premise", "style:personachat:test:343:self:5:positive:0", "style:personachat:test:343:self:5:positive:1", "style:personachat:test:343:self:5:positive:2", "style:personachat:test:343:self:5:positive:3", "style:personachat:test:343:self:5:positive:4"], "evidence_texts": ["A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "no i do not . i like theatre though", "do you like to exercise", "i like to run and go to the gym . i have two left feet .", "yes , he was on tv", "you must really like cars"], "speaker_id": "snli:test:9194", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9194", "candidate_text": "Honestly, An injured skier being taken for medical help!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9194:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9046:premise", "snli:test:8960:premise", "snli:test:9194:premise", "snli:test:9448:premise", "style:personachat:test:584:self:1:positive:0"], "evidence_texts": ["A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "An obese white male wearing a blue shirt and khaki shorts sits in front of fountain all while gabbing on phone.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "i love to eat ! candy is delicious and chicken !"], "speaker_id": "snli:test:9194", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9195", "candidate_text": "well, three men help a skier", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9195:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9448:premise", "snli:test:9046:premise", "snli:test:9741:premise", "snli:test:9195:premise", "style:personachat:test:343:self:6:positive:0", "style:personachat:test:343:self:6:positive:1", "style:personachat:test:343:self:6:positive:2", "style:personachat:test:343:self:6:positive:3", "style:personachat:test:343:self:6:positive:4", "style:personachat:test:343:self:6:positive:5"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "i like to cook . i write about it .", "no i do not . i like theatre though", "do you like to exercise", "i like to run and go to the gym . i have two left feet .", "yes , he was on tv", "you must really like cars"], "speaker_id": "snli:test:9195", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9195", "candidate_text": "Three men help a skier.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9195:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9448:premise", "snli:test:9046:premise", "snli:test:9741:premise", "snli:test:9195:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A boy in a green shirt surrounded by toys and another boy, wearing a blue shirt with penguins, to his left", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9195", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9209", "candidate_text": "well, a man is climbing a rope", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9209:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9142:premise", "snli:test:9209:premise", "snli:test:9304:premise", "snli:test:8994:premise", "style:personachat:test:344:self:1:positive:0"], "evidence_texts": ["People buy fruits and vegetables at an open air market.", "A man trying his hardest to climb up a rope.", "Two boys run to the entryway of an old building.", "A group of men, women and children are standing around.", "i just like my movies and books . sappy romance is my favorite ."], "speaker_id": "snli:test:9209", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9209", "candidate_text": "I wonder whether a man is climbing a rope?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9209:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9142:premise", "snli:test:9209:premise", "snli:test:9304:premise", "snli:test:8994:premise", "style:personachat:test:299:self:1:positive:0"], "evidence_texts": ["People buy fruits and vegetables at an open air market.", "A man trying his hardest to climb up a rope.", "Two boys run to the entryway of an old building.", "A group of men, women and children are standing around.", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:9209", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9229", "candidate_text": "well, a man is in his bed asleep", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9229:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9044:premise", "snli:test:9229:premise", "snli:test:9311:premise", "snli:test:9043:premise", "style:personachat:test:344:self:2:positive:0", "style:personachat:test:344:self:2:positive:1"], "evidence_texts": ["A person is surfing a big wave.", "A man is squatting on the ground.", "The dogs are playing on the bed.", "A person is surfing a big wave.", "apartment in the city . have any hobbies ?", "i just like my movies and books . sappy romance is my favorite ."], "speaker_id": "snli:test:9229", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9229", "candidate_text": "Honestly, A man is in his bed asleep!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9229:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9044:premise", "snli:test:9229:premise", "snli:test:9311:premise", "snli:test:9043:premise", "style:personachat:test:584:self:7:positive:0", "style:personachat:test:584:self:7:positive:1", "style:personachat:test:584:self:7:positive:2", "style:personachat:test:584:self:7:positive:3", "style:personachat:test:584:self:7:positive:4", "style:personachat:test:584:self:7:positive:5", "style:personachat:test:584:self:7:positive:6"], "evidence_texts": ["A person is surfing a big wave.", "A man is squatting on the ground.", "The dogs are playing on the bed.", "A person is surfing a big wave.", "well that sounds nice . i am well .", "i visited michigan this past summer .", "i am religious , but i am christian .", "that is sweet . do you play sports ?", "i play racquetball and listen to the oldies .", "i do like movies , and i hope to watch many when i retire .", "i love to eat ! candy is delicious and chicken !"], "speaker_id": "snli:test:9229", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9234", "candidate_text": "well, a group are gathered near a vehicle", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9234:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9171:premise", "snli:test:9318:premise", "snli:test:9234:premise", "snli:test:9129:premise", "style:personachat:test:344:self:5:positive:0", "style:personachat:test:344:self:5:positive:1", "style:personachat:test:344:self:5:positive:2", "style:personachat:test:344:self:5:positive:3", "style:personachat:test:344:self:5:positive:4"], "evidence_texts": ["She is cooking food while wearing a dress.", "A woman is observing pictures on a camera.", "A group of people gather around a truck.", "Three children are shooting guns in the woods.", "they are terrifying . and he would distract me from my love stories .", "what do you do for work ?", "one day , a chef . that is my goal . i get distracted though .", "apartment in the city . have any hobbies ?", "i just like my movies and books . sappy romance is my favorite ."], "speaker_id": "snli:test:9234", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9234", "candidate_text": "A group are gathered near a vehicle.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9234:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9171:premise", "snli:test:9318:premise", "snli:test:9234:premise", "snli:test:9129:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["She is cooking food while wearing a dress.", "A woman is observing pictures on a camera.", "A group of people gather around a truck.", "Three children are shooting guns in the woods.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9234", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9236", "candidate_text": "well, the lady on the table is leeping", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9236:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9236:premise", "snli:test:9158:premise", "snli:test:9254:premise", "snli:test:9157:premise", "style:personachat:test:344:self:7:positive:0", "style:personachat:test:344:self:7:positive:1", "style:personachat:test:344:self:7:positive:2", "style:personachat:test:344:self:7:positive:3", "style:personachat:test:344:self:7:positive:4", "style:personachat:test:344:self:7:positive:5", "style:personachat:test:344:self:7:positive:6"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "A man in a red shirt is walking towards a blue market stall.", "A little white dog runs outside, their ears blown back in the wind.", "A man in a red shirt is walking towards a blue market stall.", "i do not . my apartment does not allow any . how about you ?", "i should have a dog being a single girl of 32 . but i am scared .", "they are terrifying . and he would distract me from my love stories .", "what do you do for work ?", "one day , a chef . that is my goal . i get distracted though .", "apartment in the city . have any hobbies ?", "i just like my movies and books . sappy romance is my favorite ."], "speaker_id": "snli:test:9236", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9236", "candidate_text": "I wonder whether the lady on the table is leeping?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9236:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9236:premise", "snli:test:9158:premise", "snli:test:9254:premise", "snli:test:9157:premise", "style:personachat:test:299:self:6:positive:0", "style:personachat:test:299:self:6:positive:1", "style:personachat:test:299:self:6:positive:2", "style:personachat:test:299:self:6:positive:3", "style:personachat:test:299:self:6:positive:4", "style:personachat:test:299:self:6:positive:5"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "A man in a red shirt is walking towards a blue market stall.", "A little white dog runs outside, their ears blown back in the wind.", "A man in a red shirt is walking towards a blue market stall.", "a walk sounds good . i am jake 28 and from texas . you ?", "hi mia , what are your hobbies ? mine is spending time online .", "how very cool . do you watch tv ? i play on a show called second life .", "oh , i play a furry . tell me more about your cat .", "no , i do not have enough time . what is your cats name ?", "i like remo better than atilla . do you travel a lot ?"], "speaker_id": "snli:test:9236", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9244", "candidate_text": "well, two men are eating inside a restaurant", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9244:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9244:premise", "snli:test:9177:premise", "snli:test:9251:premise", "snli:test:9091:premise", "style:personachat:test:345:self:1:positive:0"], "evidence_texts": ["Two men outside of a restaurant.", "A choir practicing in a church.", "A boy splashing through the ocean.", "A swan landing in a lake.", "it is not bad , as long as you know the air traffic laws ."], "speaker_id": "snli:test:9244", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9244", "candidate_text": "Honestly, Two men are eating inside a restaurant!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9244:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9244:premise", "snli:test:9177:premise", "snli:test:9251:premise", "snli:test:9091:premise", "style:personachat:test:587:self:5:positive:0", "style:personachat:test:587:self:5:positive:1", "style:personachat:test:587:self:5:positive:2", "style:personachat:test:587:self:5:positive:3", "style:personachat:test:587:self:5:positive:4"], "evidence_texts": ["Two men outside of a restaurant.", "A choir practicing in a church.", "A boy splashing through the ocean.", "A swan landing in a lake.", "i am really sorry to hear that . what do you do for fun ?", "terrific . my favorite thing to do is dance .", "where do you live ? i am from edris island .", "my sister lives with me . i also have a dog .", "it is ! what do you look like ?"], "speaker_id": "snli:test:9244", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9248", "candidate_text": "well, no animal products can be seen here", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9248:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9248:premise", "snli:test:9184:premise", "snli:test:9289:premise", "snli:test:9121:premise", "style:personachat:test:345:self:2:positive:0", "style:personachat:test:345:self:2:positive:1"], "evidence_texts": ["Army men grilling a lot of meat while his troop looks on.", "Two children play in the snow by the side of the road.", "A young woman is wearing a paper crown at an outdoor event.", "A man and a woman jumping with a lake in the background.", "not all that much . a buddy of mine shares his plane with me .", "it is not bad , as long as you know the air traffic laws ."], "speaker_id": "snli:test:9248", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9248", "candidate_text": "No animal products can be seen here.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9248:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9248:premise", "snli:test:9184:premise", "snli:test:9289:premise", "snli:test:9121:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["Army men grilling a lot of meat while his troop looks on.", "Two children play in the snow by the side of the road.", "A young woman is wearing a paper crown at an outdoor event.", "A man and a woman jumping with a lake in the background.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9248", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9251", "candidate_text": "well, the boy is in kansas", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9251:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9244:premise", "snli:test:9483:premise", "snli:test:9177:premise", "snli:test:9251:premise", "style:personachat:test:345:self:5:positive:0", "style:personachat:test:345:self:5:positive:1", "style:personachat:test:345:self:5:positive:2", "style:personachat:test:345:self:5:positive:3", "style:personachat:test:345:self:5:positive:4"], "evidence_texts": ["Two men outside of a restaurant.", "A boy jumps on the beach.", "A choir practicing in a church.", "A boy splashing through the ocean.", "yeah ? active , huh . me too . i am flying out again after my exam .", "my dog is better at dancing than i am !", "the great thing about them is , you can travel with them .", "not all that much . a buddy of mine shares his plane with me .", "it is not bad , as long as you know the air traffic laws ."], "speaker_id": "snli:test:9251", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9251", "candidate_text": "I wonder whether the boy is in kansas?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9251:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9244:premise", "snli:test:9483:premise", "snli:test:9177:premise", "snli:test:9251:premise", "style:personachat:test:309:self:5:positive:0", "style:personachat:test:309:self:5:positive:1", "style:personachat:test:309:self:5:positive:2", "style:personachat:test:309:self:5:positive:3", "style:personachat:test:309:self:5:positive:4"], "evidence_texts": ["Two men outside of a restaurant.", "A boy jumps on the beach.", "A choir practicing in a church.", "A boy splashing through the ocean.", "her name is layla . i used to bbq a lot with friends back in college .", "do you come from a tall family ?", "wow , that is pretty rare . do you have pets ?", "i like horror movies , too . pets can definitely be messy sometimes .", "i enjoy hiking , crocheting , and reading . how about you ?"], "speaker_id": "snli:test:9251", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9254", "candidate_text": "well, there is an animal outside", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9254:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9236:premise", "snli:test:9541:premise", "snli:test:9254:premise", "snli:test:9158:premise", "style:personachat:test:345:self:7:positive:0", "style:personachat:test:345:self:7:positive:1", "style:personachat:test:345:self:7:positive:2", "style:personachat:test:345:self:7:positive:3", "style:personachat:test:345:self:7:positive:4", "style:personachat:test:345:self:7:positive:5", "style:personachat:test:345:self:7:positive:6"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "A person in a green robe sits on a couch with a blanket.", "A little white dog runs outside, their ears blown back in the wind.", "A man in a red shirt is walking towards a blue market stall.", "hey ! me too , and i just got back from the arctic . studying what ?", "cool . i study international law . met my girlfriend abroad .", "yeah ? active , huh . me too . i am flying out again after my exam .", "my dog is better at dancing than i am !", "the great thing about them is , you can travel with them .", "not all that much . a buddy of mine shares his plane with me .", "it is not bad , as long as you know the air traffic laws ."], "speaker_id": "snli:test:9254", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9254", "candidate_text": "Honestly, There is an animal outside!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9254:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9236:premise", "snli:test:9541:premise", "snli:test:9254:premise", "snli:test:9158:premise", "style:personachat:test:588:self:2:positive:0", "style:personachat:test:588:self:2:positive:1"], "evidence_texts": ["A woman is cutting into a sandwich while sitting at an outdoor table.", "A person in a green robe sits on a couch with a blanket.", "A little white dog runs outside, their ears blown back in the wind.", "A man in a red shirt is walking towards a blue market stall.", "i love doing anything outside , especially in the snow ! what about you ?", "those are pretty cool hobbies ! do you have any siblings ?"], "speaker_id": "snli:test:9254", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9256", "candidate_text": "well, a happy woman", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9256:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9256:premise", "snli:test:8213:premise", "snli:test:170:premise", "snli:test:8036:premise", "style:personachat:test:346:self:1:positive:0"], "evidence_texts": ["A woman happy about something.", "An SUV traveling through mud.", "The girls walk down the street.", "Three men play musical instruments.", "god help you and make you rich one day ."], "speaker_id": "snli:test:9256", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9256", "candidate_text": "A happy woman.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9256:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9256:premise", "snli:test:8213:premise", "snli:test:170:premise", "snli:test:8036:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A woman happy about something.", "An SUV traveling through mud.", "The girls walk down the street.", "Three men play musical instruments.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9256", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9265", "candidate_text": "well, a man is singing indoors", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9265:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8964:premise", "snli:test:9265:premise", "snli:test:9856:premise", "snli:test:8729:premise", "style:personachat:test:346:self:2:positive:0", "style:personachat:test:346:self:2:positive:1"], "evidence_texts": ["An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "i feel so sorry . you should get it soon .", "god help you and make you rich one day ."], "speaker_id": "snli:test:9265", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9265", "candidate_text": "I wonder whether a man is singing indoors?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9265:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:8964:premise", "snli:test:9265:premise", "snli:test:9856:premise", "snli:test:8729:premise", "style:personachat:test:311:self:2:positive:0", "style:personachat:test:311:self:2:positive:1"], "evidence_texts": ["An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "that is a lot of gran kids . how old are you ?", "what kind of pets do you have ?"], "speaker_id": "snli:test:9265", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9268", "candidate_text": "well, a man and woman ride a mule", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9268:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9268:premise", "snli:test:9089:premise", "snli:test:9327:premise", "snli:test:9056:premise", "style:personachat:test:346:self:5:positive:0", "style:personachat:test:346:self:5:positive:1", "style:personachat:test:346:self:5:positive:2", "style:personachat:test:346:self:5:positive:3", "style:personachat:test:346:self:5:positive:4"], "evidence_texts": ["A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "A woman and two children walk along stones to cross a river while a dog swims.", "do you work on sundays ?", "how old are you ? if do not mind to share it .", "you are too young . do you study ?", "i feel so sorry . you should get it soon .", "god help you and make you rich one day ."], "speaker_id": "snli:test:9268", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9268", "candidate_text": "Honestly, A man and woman ride a mule!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9268:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9268:premise", "snli:test:9089:premise", "snli:test:9327:premise", "snli:test:9056:premise", "style:personachat:test:593:self:1:positive:0"], "evidence_texts": ["A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "A woman and two children walk along stones to cross a river while a dog swims.", "the dog is a wire haired terrier and he is terrifying ! most dogs scare me ."], "speaker_id": "snli:test:9268", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9289", "candidate_text": "well, the woman is at a ski lodge", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9289:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9248:premise", "snli:test:9289:premise", "snli:test:9341:premise", "snli:test:9184:premise", "style:personachat:test:346:self:6:positive:0", "style:personachat:test:346:self:6:positive:1", "style:personachat:test:346:self:6:positive:2", "style:personachat:test:346:self:6:positive:3", "style:personachat:test:346:self:6:positive:4", "style:personachat:test:346:self:6:positive:5"], "evidence_texts": ["Army men grilling a lot of meat while his troop looks on.", "A young woman is wearing a paper crown at an outdoor event.", "a woman wearing a pink tank top holding a mug of liquid", "Two children play in the snow by the side of the road.", "hey there . i am fine . what about you ?", "do you work on sundays ?", "how old are you ? if do not mind to share it .", "you are too young . do you study ?", "i feel so sorry . you should get it soon .", "god help you and make you rich one day ."], "speaker_id": "snli:test:9289", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9289", "candidate_text": "the woman is at a ski lodge.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9289:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9248:premise", "snli:test:9289:premise", "snli:test:9341:premise", "snli:test:9184:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Army men grilling a lot of meat while his troop looks on.", "A young woman is wearing a paper crown at an outdoor event.", "a woman wearing a pink tank top holding a mug of liquid", "Two children play in the snow by the side of the road.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9289", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9295", "candidate_text": "Honestly, A player falls face first!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9295:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:8858:premise", "snli:test:9860:premise", "snli:test:8810:premise", "snli:test:9295:premise", "style:personachat:test:347:self:1:positive:0"], "evidence_texts": ["There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "well , i did not allow me to think !"], "speaker_id": "snli:test:9295", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9295", "candidate_text": "I wonder whether a player falls face first?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9295:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:8858:premise", "snli:test:9860:premise", "snli:test:8810:premise", "snli:test:9295:premise", "style:personachat:test:314:self:1:positive:0"], "evidence_texts": ["There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "A man and two women are walking down a wooded path, the man is pushing a young child in a three wheel carriage.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:9295", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9304", "candidate_text": "Honestly, Two kids are running after a dog that got away!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9304:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9304:premise", "snli:test:9209:premise", "snli:test:9142:premise", "snli:test:9339:premise", "style:personachat:test:347:self:2:positive:0", "style:personachat:test:347:self:2:positive:1"], "evidence_texts": ["Two boys run to the entryway of an old building.", "A man trying his hardest to climb up a rope.", "People buy fruits and vegetables at an open air market.", "A group of individuals are sitting together in a room.", "i quit drinking years ago", "well , i did not allow me to think !"], "speaker_id": "snli:test:9304", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9304", "candidate_text": "well, two kids are running after a dog that got away", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9304:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9304:premise", "snli:test:9209:premise", "snli:test:9142:premise", "snli:test:9339:premise", "style:personachat:test:575:self:2:positive:0", "style:personachat:test:575:self:2:positive:1"], "evidence_texts": ["Two boys run to the entryway of an old building.", "A man trying his hardest to climb up a rope.", "People buy fruits and vegetables at an open air market.", "A group of individuals are sitting together in a room.", "i am 25 years old .", "i am undecided . not anytime soon ."], "speaker_id": "snli:test:9304", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9305", "candidate_text": "Honestly, Two young men are in a hurry!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9305:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9339:premise", "snli:test:9305:premise", "snli:test:9209:premise", "snli:test:9357:premise", "style:personachat:test:347:self:5:positive:0", "style:personachat:test:347:self:5:positive:1", "style:personachat:test:347:self:5:positive:2", "style:personachat:test:347:self:5:positive:3", "style:personachat:test:347:self:5:positive:4"], "evidence_texts": ["A group of individuals are sitting together in a room.", "Two boys run to the entryway of an old building.", "A man trying his hardest to climb up a rope.", "A woman in a gray shirt is looking at something.", "it is indeed what i do ! i like it , it is good pay !", "yes , it does ! also , i was just hanging with my wife and 3 girls", "those 3 are all i need ! i do not have very many friends", "i quit drinking years ago", "well , i did not allow me to think !"], "speaker_id": "snli:test:9305", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9305", "candidate_text": "Two young men are in a hurry.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9305:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9339:premise", "snli:test:9305:premise", "snli:test:9209:premise", "snli:test:9357:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of individuals are sitting together in a room.", "Two boys run to the entryway of an old building.", "A man trying his hardest to climb up a rope.", "A woman in a gray shirt is looking at something.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9305", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9311", "candidate_text": "Honestly, Two cats playing on the bed together!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9311:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9229:premise", "snli:test:9400:premise", "snli:test:9044:premise", "snli:test:9311:premise", "style:personachat:test:347:self:7:positive:0", "style:personachat:test:347:self:7:positive:1", "style:personachat:test:347:self:7:positive:2", "style:personachat:test:347:self:7:positive:3", "style:personachat:test:347:self:7:positive:4", "style:personachat:test:347:self:7:positive:5", "style:personachat:test:347:self:7:positive:6"], "evidence_texts": ["A man is squatting on the ground.", "Large brown dog walking in shallow water.", "A person is surfing a big wave.", "The dogs are playing on the bed.", "i am good ! and you ?", "i just got done laying carpet !", "it is indeed what i do ! i like it , it is good pay !", "yes , it does ! also , i was just hanging with my wife and 3 girls", "those 3 are all i need ! i do not have very many friends", "i quit drinking years ago", "well , i did not allow me to think !"], "speaker_id": "snli:test:9311", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9311", "candidate_text": "I wonder whether two cats playing on the bed together?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9311:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9229:premise", "snli:test:9400:premise", "snli:test:9044:premise", "snli:test:9311:premise", "style:personachat:test:314:self:6:positive:0", "style:personachat:test:314:self:6:positive:1", "style:personachat:test:314:self:6:positive:2", "style:personachat:test:314:self:6:positive:3", "style:personachat:test:314:self:6:positive:4", "style:personachat:test:314:self:6:positive:5"], "evidence_texts": ["A man is squatting on the ground.", "Large brown dog walking in shallow water.", "A person is surfing a big wave.", "The dogs are playing on the bed.", "hey there how are you today ? yes i like dogs", "i like to eat vegetables an beans", "i like to run an i love a good reliable older car", "sounds good i would love to go there but as a doctor i am busy", "yes i would love to but i stay busy running an reading", "really what about ? i just relax to my wine an jazz"], "speaker_id": "snli:test:9311", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9314", "candidate_text": "Honestly, Kids play outside!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9314:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9011:premise", "snli:test:9322:premise", "snli:test:9314:premise", "snli:test:8948:premise", "style:personachat:test:348:self:1:positive:0"], "evidence_texts": ["A black dog is swimming with a ball in his mouth.", "A brown and white dog swims in a backyard swimming pool.", "Four kids sitting at a table with food on several plates.", "The young girl is airborne as she falls into the lake.", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:9314", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9314", "candidate_text": "well, kids play outside", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9314:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9011:premise", "snli:test:9322:premise", "snli:test:9314:premise", "snli:test:8948:premise", "style:personachat:test:576:self:1:positive:0"], "evidence_texts": ["A black dog is swimming with a ball in his mouth.", "A brown and white dog swims in a backyard swimming pool.", "Four kids sitting at a table with food on several plates.", "The young girl is airborne as she falls into the lake.", "well thanks . i feel like it though since i get tired so quickly"], "speaker_id": "snli:test:9314", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9318", "candidate_text": "Honestly, A women cradling a baby!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9318:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9234:premise", "snli:test:9453:premise", "snli:test:9318:premise", "snli:test:9171:premise", "style:personachat:test:348:self:2:positive:0", "style:personachat:test:348:self:2:positive:1"], "evidence_texts": ["A group of people gather around a truck.", "An ATV rides over a vast sandy landscape.", "A woman is observing pictures on a camera.", "She is cooking food while wearing a dress.", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:9318", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9318", "candidate_text": "A women cradling a baby.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9318:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9234:premise", "snli:test:9453:premise", "snli:test:9318:premise", "snli:test:9171:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of people gather around a truck.", "An ATV rides over a vast sandy landscape.", "A woman is observing pictures on a camera.", "She is cooking food while wearing a dress.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9318", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9322", "candidate_text": "Honestly, A dog sleeping on the porch!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9322:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9314:premise", "snli:test:9011:premise", "snli:test:9322:premise", "snli:test:9394:premise", "style:personachat:test:348:self:5:positive:0", "style:personachat:test:348:self:5:positive:1", "style:personachat:test:348:self:5:positive:2", "style:personachat:test:348:self:5:positive:3", "style:personachat:test:348:self:5:positive:4"], "evidence_texts": ["Four kids sitting at a table with food on several plates.", "A black dog is swimming with a ball in his mouth.", "A brown and white dog swims in a backyard swimming pool.", "A public speaker in a plaid shirt speaks on the microphone.", "where are you currently from ?", "do you like it there ?", "what types of jobs are you looking for ? what education or skills can you offer ?", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:9322", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9322", "candidate_text": "I wonder whether a dog sleeping on the porch?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9322:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9314:premise", "snli:test:9011:premise", "snli:test:9322:premise", "snli:test:9394:premise", "style:personachat:test:321:self:5:positive:0", "style:personachat:test:321:self:5:positive:1", "style:personachat:test:321:self:5:positive:2", "style:personachat:test:321:self:5:positive:3", "style:personachat:test:321:self:5:positive:4"], "evidence_texts": ["Four kids sitting at a table with food on several plates.", "A black dog is swimming with a ball in his mouth.", "A brown and white dog swims in a backyard swimming pool.", "A public speaker in a plaid shirt speaks on the microphone.", "do you have something there to help you ?", "my girlfriend is pretty short also", "muffins are good travel food , i eat them on adventures", "i am actually a student . i travel for fun .", "how do you help your patients if your so short ?"], "speaker_id": "snli:test:9322", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9323", "candidate_text": "Honestly, A brown dog swims!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9323:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9394:premise", "snli:test:9314:premise", "snli:test:9488:premise", "snli:test:9323:premise", "style:personachat:test:348:self:6:positive:0", "style:personachat:test:348:self:6:positive:1", "style:personachat:test:348:self:6:positive:2", "style:personachat:test:348:self:6:positive:3", "style:personachat:test:348:self:6:positive:4", "style:personachat:test:348:self:6:positive:5"], "evidence_texts": ["A public speaker in a plaid shirt speaks on the microphone.", "Four kids sitting at a table with food on several plates.", "A young japanese couple holding an umbrella walking down a street.", "A brown and white dog swims in a backyard swimming pool.", "good afternoon from pennsylvania . i am currently a medical scribe preparing to apply to medical school .", "where are you currently from ?", "do you like it there ?", "what types of jobs are you looking for ? what education or skills can you offer ?", "that is impressive . search for some job openings online .", "keep waiting . stay hopeful . there is always need for carpenters !"], "speaker_id": "snli:test:9323", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9323", "candidate_text": "well, a brown dog swims", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9323:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9394:premise", "snli:test:9314:premise", "snli:test:9488:premise", "snli:test:9323:premise", "style:personachat:test:576:self:7:positive:0", "style:personachat:test:576:self:7:positive:1", "style:personachat:test:576:self:7:positive:2", "style:personachat:test:576:self:7:positive:3", "style:personachat:test:576:self:7:positive:4", "style:personachat:test:576:self:7:positive:5", "style:personachat:test:576:self:7:positive:6"], "evidence_texts": ["A public speaker in a plaid shirt speaks on the microphone.", "Four kids sitting at a table with food on several plates.", "A young japanese couple holding an umbrella walking down a street.", "A brown and white dog swims in a backyard swimming pool.", "great . and how are you doing", "having just one leg , i have to be good at multi tasking", "sorry to hear that . my wheel chair is super fast", "i knew someone with webbed feet in the military", "i feel just too old some days", "i am in my sixties but feel older some days", "well thanks . i feel like it though since i get tired so quickly"], "speaker_id": "snli:test:9323", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9327", "candidate_text": "well, a woman protects her child from the sun as they walk to an upper level of a building", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9327:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9268:premise", "snli:test:9343:premise", "snli:test:9089:premise", "snli:test:9327:premise", "style:personachat:test:349:self:1:positive:0"], "evidence_texts": ["A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "i live in usa but i pretend to have a british accent"], "speaker_id": "snli:test:9327", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9327", "candidate_text": "A woman protects her child from the sun as they walk to an upper level of a building.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9327:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9268:premise", "snli:test:9343:premise", "snli:test:9089:premise", "snli:test:9327:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "Two boys in a canoe in front of a larger boat with a storm coming in.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9327", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9329", "candidate_text": "well, the man with the skateboard is waiting for a plane", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9329:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9103:premise", "snli:test:9329:premise", "snli:test:9080:premise", "snli:test:9643:premise", "style:personachat:test:349:self:2:positive:0", "style:personachat:test:349:self:2:positive:1"], "evidence_texts": ["A man in a red delivery truck is talking to someone is a black coat.", "A man on a skateboard on steps in front glass doors of a brick building.", "A dog with a ball in his mouth running down a road covered in leaves.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "well plenty of people do . one guy even wanted a mohawk on his terrier", "i live in usa but i pretend to have a british accent"], "speaker_id": "snli:test:9329", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9329", "candidate_text": "I wonder whether the man with the skateboard is waiting for a plane?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9329:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9103:premise", "snli:test:9329:premise", "snli:test:9080:premise", "snli:test:9643:premise", "style:personachat:test:323:self:2:positive:0", "style:personachat:test:323:self:2:positive:1"], "evidence_texts": ["A man in a red delivery truck is talking to someone is a black coat.", "A man on a skateboard on steps in front glass doors of a brick building.", "A dog with a ball in his mouth running down a road covered in leaves.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "really ? how many is that ? i grew up farming loved it", "how did that happen ? sorry that happened to you"], "speaker_id": "snli:test:9329", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9330", "candidate_text": "well, the man on the skateboard is going to bounce off the glass doors", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9330:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9643:premise", "snli:test:9103:premise", "snli:test:9330:premise", "snli:test:9649:premise", "style:personachat:test:349:self:5:positive:0", "style:personachat:test:349:self:5:positive:1", "style:personachat:test:349:self:5:positive:2", "style:personachat:test:349:self:5:positive:3", "style:personachat:test:349:self:5:positive:4"], "evidence_texts": ["A barber in a sports themed barber shop waits for his next customer to arrive.", "A man in a red delivery truck is talking to someone is a black coat.", "A man on a skateboard on steps in front glass doors of a brick building.", "The man in black is flying off the full in front of a red fence.", "yes that sounds fun . do you know anything about civil war antiques", "you have a sense of humor ! well i suppose they could be , yes .", "interesting they are both teachers . i style dog hair at a pet store", "well plenty of people do . one guy even wanted a mohawk on his terrier", "i live in usa but i pretend to have a british accent"], "speaker_id": "snli:test:9330", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9330", "candidate_text": "Honestly, The man on the skateboard is going to bounce off the glass doors!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9330:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9643:premise", "snli:test:9103:premise", "snli:test:9330:premise", "snli:test:9649:premise", "style:personachat:test:614:self:1:positive:0"], "evidence_texts": ["A barber in a sports themed barber shop waits for his next customer to arrive.", "A man in a red delivery truck is talking to someone is a black coat.", "A man on a skateboard on steps in front glass doors of a brick building.", "The man in black is flying off the full in front of a red fence.", "switch places with me ! it is too hot and sunny here"], "speaker_id": "snli:test:9330", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9332", "candidate_text": "well, the man is also playing the drums", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9332:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9332:premise", "snli:test:9132:premise", "snli:test:9349:premise", "snli:test:9102:premise", "style:personachat:test:349:self:7:positive:0", "style:personachat:test:349:self:7:positive:1", "style:personachat:test:349:self:7:positive:2", "style:personachat:test:349:self:7:positive:3", "style:personachat:test:349:self:7:positive:4", "style:personachat:test:349:self:7:positive:5", "style:personachat:test:349:self:7:positive:6"], "evidence_texts": ["A man playing the guitar while smoking a cigarette.", "People are hiking up a mountain with no greenery.", "The dog's mouth is open like he is yawning.", "A group of students are walking through the campus.", "wow impressive ! are you single ? looking for my next wife", "oh dear , that is not good . well how are you today", "yes that sounds fun . do you know anything about civil war antiques", "you have a sense of humor ! well i suppose they could be , yes .", "interesting they are both teachers . i style dog hair at a pet store", "well plenty of people do . one guy even wanted a mohawk on his terrier", "i live in usa but i pretend to have a british accent"], "speaker_id": "snli:test:9332", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9332", "candidate_text": "The man is also playing the drums.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9332:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9332:premise", "snli:test:9132:premise", "snli:test:9349:premise", "snli:test:9102:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man playing the guitar while smoking a cigarette.", "People are hiking up a mountain with no greenery.", "The dog's mouth is open like he is yawning.", "A group of students are walking through the campus.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9332", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9339", "candidate_text": "well, the people are all driving go-carts around a track at their local family fun center", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9339:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9305:premise", "snli:test:9357:premise", "snli:test:9304:premise", "snli:test:9339:premise", "style:personachat:test:350:self:1:positive:0"], "evidence_texts": ["Two boys run to the entryway of an old building.", "A woman in a gray shirt is looking at something.", "Two boys run to the entryway of an old building.", "A group of individuals are sitting together in a room.", "my dad wants me to be a police officer like him , but i like pet smart ."], "speaker_id": "snli:test:9339", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9339", "candidate_text": "I wonder whether the people are all driving go-carts around a track at their local family fun center?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9339:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9305:premise", "snli:test:9357:premise", "snli:test:9304:premise", "snli:test:9339:premise", "style:personachat:test:334:self:1:positive:0"], "evidence_texts": ["Two boys run to the entryway of an old building.", "A woman in a gray shirt is looking at something.", "Two boys run to the entryway of an old building.", "A group of individuals are sitting together in a room.", "wow what happened to it ?"], "speaker_id": "snli:test:9339", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9341", "candidate_text": "well, a woman is drinking", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9341:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9289:premise", "snli:test:9341:premise", "snli:test:9399:premise", "snli:test:9248:premise", "style:personachat:test:350:self:2:positive:0", "style:personachat:test:350:self:2:positive:1"], "evidence_texts": ["A young woman is wearing a paper crown at an outdoor event.", "a woman wearing a pink tank top holding a mug of liquid", "Men playing basketball, the guy with the ball is attempting a layup.", "Army men grilling a lot of meat while his troop looks on.", "yeah . what do you do for a living ?", "my dad wants me to be a police officer like him , but i like pet smart ."], "speaker_id": "snli:test:9341", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9341", "candidate_text": "Honestly, A woman is drinking!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9341:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9289:premise", "snli:test:9341:premise", "snli:test:9399:premise", "snli:test:9248:premise", "style:personachat:test:614:self:6:positive:0", "style:personachat:test:614:self:6:positive:1", "style:personachat:test:614:self:6:positive:2", "style:personachat:test:614:self:6:positive:3", "style:personachat:test:614:self:6:positive:4", "style:personachat:test:614:self:6:positive:5"], "evidence_texts": ["A young woman is wearing a paper crown at an outdoor event.", "a woman wearing a pink tank top holding a mug of liquid", "Men playing basketball, the guy with the ball is attempting a layup.", "Army men grilling a lot of meat while his troop looks on.", "hey there ! a little tired , just finished my mile walk , you ?", "i am about to do the same , curl up with my siamese cat", "almost one am , what bout you ?", "southern ish area of texas .", "what is the weather like there ?", "switch places with me ! it is too hot and sunny here"], "speaker_id": "snli:test:9341", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9343", "candidate_text": "well, people are watching a ball game", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9343:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9327:premise", "snli:test:9532:premise", "snli:test:9268:premise", "snli:test:9343:premise", "style:personachat:test:350:self:5:positive:0", "style:personachat:test:350:self:5:positive:1", "style:personachat:test:350:self:5:positive:2", "style:personachat:test:350:self:5:positive:3", "style:personachat:test:350:self:5:positive:4"], "evidence_texts": ["A mother shading her baby with an umbrella as she and her partner climb the stairs.", "A man in a red headband an shirt is standing in front of a display case.", "A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "i also play the guitar . i sing as well", "actually yes . who is your favorite band ?", "i love nirvana a lot", "yeah . what do you do for a living ?", "my dad wants me to be a police officer like him , but i like pet smart ."], "speaker_id": "snli:test:9343", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9343", "candidate_text": "People are watching a ball game.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9343:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9327:premise", "snli:test:9532:premise", "snli:test:9268:premise", "snli:test:9343:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A mother shading her baby with an umbrella as she and her partner climb the stairs.", "A man in a red headband an shirt is standing in front of a display case.", "A man and woman ride in a mule drawn carriage as the driver points out landmarks.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9343", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9347", "candidate_text": "well, girls standing up against a wall", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9347:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8896:premise", "snli:test:9475:premise", "snli:test:8709:premise", "snli:test:9347:premise", "style:personachat:test:350:self:7:positive:0", "style:personachat:test:350:self:7:positive:1", "style:personachat:test:350:self:7:positive:2", "style:personachat:test:350:self:7:positive:3", "style:personachat:test:350:self:7:positive:4", "style:personachat:test:350:self:7:positive:5", "style:personachat:test:350:self:7:positive:6"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "i am tired . i went for a run . how are you ?", "yeah . as a musician i try to stay in shape for my image", "i also play the guitar . i sing as well", "actually yes . who is your favorite band ?", "i love nirvana a lot", "yeah . what do you do for a living ?", "my dad wants me to be a police officer like him , but i like pet smart ."], "speaker_id": "snli:test:9347", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9347", "candidate_text": "I wonder whether girls standing up against a wall?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9347:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8896:premise", "snli:test:9475:premise", "snli:test:8709:premise", "snli:test:9347:premise", "style:personachat:test:334:self:7:positive:0", "style:personachat:test:334:self:7:positive:1", "style:personachat:test:334:self:7:positive:2", "style:personachat:test:334:self:7:positive:3", "style:personachat:test:334:self:7:positive:4", "style:personachat:test:334:self:7:positive:5", "style:personachat:test:334:self:7:positive:6"], "evidence_texts": ["An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "An old man, likely a blacksmith, is welding a tool, seen by the red hot scorching look of it.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "i am wonderful just relaxing ready to begin a long work week .", "no i like barbecue more than fish .", "i have tried it . i used my free time to lift weights . do you lift ?", "dogs are awesome . i am a huge man that loves watching horror movies .", "me either i drive a forklift in a warehouse where i work .", "thank you and you are so kind .", "wow what happened to it ?"], "speaker_id": "snli:test:9347", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9349", "candidate_text": "well, a dog is yawning", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9349:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9332:premise", "snli:test:9349:premise", "snli:test:9472:premise", "snli:test:9132:premise", "style:personachat:test:351:self:1:positive:0"], "evidence_texts": ["A man playing the guitar while smoking a cigarette.", "The dog's mouth is open like he is yawning.", "An elderly male is blowing air into an object.", "People are hiking up a mountain with no greenery.", "no , but my pets do , have you always lived in texas"], "speaker_id": "snli:test:9349", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9349", "candidate_text": "Honestly, a dog is yawning!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9349:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9332:premise", "snli:test:9349:premise", "snli:test:9472:premise", "snli:test:9132:premise", "style:personachat:test:615:self:5:positive:0", "style:personachat:test:615:self:5:positive:1", "style:personachat:test:615:self:5:positive:2", "style:personachat:test:615:self:5:positive:3", "style:personachat:test:615:self:5:positive:4"], "evidence_texts": ["A man playing the guitar while smoking a cigarette.", "The dog's mouth is open like he is yawning.", "An elderly male is blowing air into an object.", "People are hiking up a mountain with no greenery.", "lol . but for real , what does she do ?", "yes , my favorite is rap . you ?", "that is awesome ! what is your favorite food ?", "i was going to suggest bringing vegan sushi my fave to our rock out session . xd", "get some beef jerky or some wings for you too . but i am not touching it !"], "speaker_id": "snli:test:9349", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9357", "candidate_text": "well, a woman in a red shirt baking a cake", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9357:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9339:premise", "snli:test:9357:premise", "snli:test:9409:premise", "snli:test:9305:premise", "style:personachat:test:351:self:2:positive:0", "style:personachat:test:351:self:2:positive:1"], "evidence_texts": ["A group of individuals are sitting together in a room.", "A woman in a gray shirt is looking at something.", "A skateboard is grinding on a curb with his skateboard.", "Two boys run to the entryway of an old building.", "that must be warm in texas . what color is your fur ?", "no , but my pets do , have you always lived in texas"], "speaker_id": "snli:test:9357", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9357", "candidate_text": "A woman in a red shirt baking a cake.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9357:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9339:premise", "snli:test:9357:premise", "snli:test:9409:premise", "snli:test:9305:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of individuals are sitting together in a room.", "A woman in a gray shirt is looking at something.", "A skateboard is grinding on a curb with his skateboard.", "Two boys run to the entryway of an old building.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9357", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9385", "candidate_text": "well, a man polishes a floor next to a can", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9385:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8225:premise", "snli:test:9385:premise", "snli:test:391:premise", "snli:test:8099:premise", "style:personachat:test:351:self:5:positive:0", "style:personachat:test:351:self:5:positive:1", "style:personachat:test:351:self:5:positive:2", "style:personachat:test:351:self:5:positive:3", "style:personachat:test:351:self:5:positive:4"], "evidence_texts": ["A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "i live in the mid west now . we moved a lot .", "no , i mostly enjoy needle crafts . do you play other online games ?", "its not too cold , yet . fur ?", "that must be warm in texas . what color is your fur ?", "no , but my pets do , have you always lived in texas"], "speaker_id": "snli:test:9385", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9385", "candidate_text": "I wonder whether a man polishes a floor next to a can?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9385:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8225:premise", "snli:test:9385:premise", "snli:test:391:premise", "snli:test:8099:premise", "style:personachat:test:335:self:5:positive:0", "style:personachat:test:335:self:5:positive:1", "style:personachat:test:335:self:5:positive:2", "style:personachat:test:335:self:5:positive:3", "style:personachat:test:335:self:5:positive:4"], "evidence_texts": ["A soccer player in white kicks a ball as another soccer player lunges towards the ball and a third soccer player in red follows close by.", "A man wearing a red shirt is quickly polishing a light wooden floor with a small green towel next to a yellow can of Wood Silk.", "A little boy with a blue bike helmet and a woman in a green bike helmet are riding bikes down a street with other various bike riders.", "A man with a gray beard and a little boy are sitting on the floor looking over some papers in a room with a bunk bed.", "my family and i are from australia . where are you from ?", "what do you do in seattle ? i have never been .", "that is not good ! i do stock broking . good money , boring job .", "what would you like to do ?", "well , what do you enjoy doing ?"], "speaker_id": "snli:test:9385", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9394", "candidate_text": "well, a man is wearing a suit", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9394:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9323:premise", "snli:test:9488:premise", "snli:test:9394:premise", "snli:test:9322:premise", "style:personachat:test:351:self:7:positive:0", "style:personachat:test:351:self:7:positive:1", "style:personachat:test:351:self:7:positive:2", "style:personachat:test:351:self:7:positive:3", "style:personachat:test:351:self:7:positive:4", "style:personachat:test:351:self:7:positive:5", "style:personachat:test:351:self:7:positive:6"], "evidence_texts": ["A brown and white dog swims in a backyard swimming pool.", "A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A brown and white dog swims in a backyard swimming pool.", "hello , i am well . how are you ?", "have you always lived there ? it seems like nice place , very patriotic .", "i live in the mid west now . we moved a lot .", "no , i mostly enjoy needle crafts . do you play other online games ?", "its not too cold , yet . fur ?", "that must be warm in texas . what color is your fur ?", "no , but my pets do , have you always lived in texas"], "speaker_id": "snli:test:9394", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9394", "candidate_text": "Honestly, A man is wearing a suit!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9394:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9323:premise", "snli:test:9488:premise", "snli:test:9394:premise", "snli:test:9322:premise", "style:personachat:test:619:self:2:positive:0", "style:personachat:test:619:self:2:positive:1"], "evidence_texts": ["A brown and white dog swims in a backyard swimming pool.", "A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A brown and white dog swims in a backyard swimming pool.", "cool . you just said you got done playing ?", "wow medical marvel ! fastest surgery ever"], "speaker_id": "snli:test:9394", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9399", "candidate_text": "well, men are playing basketball as one guy wins the game by attempting a layup", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9399:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9341:premise", "snli:test:9588:premise", "snli:test:9289:premise", "snli:test:9399:premise", "style:personachat:test:352:self:1:positive:0"], "evidence_texts": ["a woman wearing a pink tank top holding a mug of liquid", "A black and white dog jumps up and down in the snow.", "A young woman is wearing a paper crown at an outdoor event.", "Men playing basketball, the guy with the ball is attempting a layup.", "yes but in todays day a guy will be with guy and vice versa"], "speaker_id": "snli:test:9399", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9399", "candidate_text": "Men are playing basketball as one guy wins the game by attempting a layup.", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9399:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9341:premise", "snli:test:9588:premise", "snli:test:9289:premise", "snli:test:9399:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["a woman wearing a pink tank top holding a mug of liquid", "A black and white dog jumps up and down in the snow.", "A young woman is wearing a paper crown at an outdoor event.", "Men playing basketball, the guy with the ball is attempting a layup.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9399", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9400", "candidate_text": "well, a brown dog is walking outside", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9400:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9400:premise", "snli:test:9311:premise", "snli:test:9229:premise", "snli:test:9413:premise", "style:personachat:test:352:self:2:positive:0", "style:personachat:test:352:self:2:positive:1"], "evidence_texts": ["Large brown dog walking in shallow water.", "The dogs are playing on the bed.", "A man is squatting on the ground.", "The players manuever for the soccer ball.", "would he mind you meeting new people", "yes but in todays day a guy will be with guy and vice versa"], "speaker_id": "snli:test:9400", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9400", "candidate_text": "I wonder whether a brown dog is walking outside?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9400:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9400:premise", "snli:test:9311:premise", "snli:test:9229:premise", "snli:test:9413:premise", "style:personachat:test:341:self:2:positive:0", "style:personachat:test:341:self:2:positive:1"], "evidence_texts": ["Large brown dog walking in shallow water.", "The dogs are playing on the bed.", "A man is squatting on the ground.", "The players manuever for the soccer ball.", "wow that is incredible ! are you an engineer as well ?", "i am a private practice pediatrician . is your ai a robot or handheld device ?"], "speaker_id": "snli:test:9400", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9402", "candidate_text": "well, an animal is in water", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9402:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9413:premise", "snli:test:9311:premise", "snli:test:9402:premise", "snli:test:9850:premise", "style:personachat:test:352:self:5:positive:0", "style:personachat:test:352:self:5:positive:1", "style:personachat:test:352:self:5:positive:2", "style:personachat:test:352:self:5:positive:3", "style:personachat:test:352:self:5:positive:4"], "evidence_texts": ["The players manuever for the soccer ball.", "The dogs are playing on the bed.", "Large brown dog walking in shallow water.", "women's soccer team competing on the field.", "me to but i want to learn the art of drawing", "anything and everything pictures , stickers , beautiful scenes", "i am a female do you mind", "would he mind you meeting new people", "yes but in todays day a guy will be with guy and vice versa"], "speaker_id": "snli:test:9402", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9402", "candidate_text": "Honestly, An animal is in water!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9402:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9413:premise", "snli:test:9311:premise", "snli:test:9402:premise", "snli:test:9850:premise", "style:personachat:test:620:self:1:positive:0"], "evidence_texts": ["The players manuever for the soccer ball.", "The dogs are playing on the bed.", "Large brown dog walking in shallow water.", "women's soccer team competing on the field.", "yeah , dogs rock ! my parents were gone a lot but my dog was always there ."], "speaker_id": "snli:test:9402", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9405", "candidate_text": "well, the man is a father", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9405:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:8931:premise", "snli:test:9405:premise", "snli:test:9798:premise", "snli:test:8854:premise", "style:personachat:test:352:self:7:positive:0", "style:personachat:test:352:self:7:positive:1", "style:personachat:test:352:self:7:positive:2", "style:personachat:test:352:self:7:positive:3", "style:personachat:test:352:self:7:positive:4", "style:personachat:test:352:self:7:positive:5", "style:personachat:test:352:self:7:positive:6"], "evidence_texts": ["Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "my collection of stickers and poetry are mine lol", "i like jane the virgin", "me to but i want to learn the art of drawing", "anything and everything pictures , stickers , beautiful scenes", "i am a female do you mind", "would he mind you meeting new people", "yes but in todays day a guy will be with guy and vice versa"], "speaker_id": "snli:test:9405", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9405", "candidate_text": "The man is a father.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9405:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:8931:premise", "snli:test:9405:premise", "snli:test:9798:premise", "snli:test:8854:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "A woman is applying lip makeup to another woman who seems to have some green in her hair.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9405", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9409", "candidate_text": "well, the curb is dangerous", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9409:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9357:premise", "snli:test:9409:premise", "snli:test:9647:premise", "snli:test:9339:premise", "style:personachat:test:353:self:1:positive:0"], "evidence_texts": ["A woman in a gray shirt is looking at something.", "A skateboard is grinding on a curb with his skateboard.", "Two women stand in a kitchen and wipe down trays", "A group of individuals are sitting together in a room.", "that must be hard . i love snack foods"], "speaker_id": "snli:test:9409", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9409", "candidate_text": "I wonder whether the curb is dangerous?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9409:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9357:premise", "snli:test:9409:premise", "snli:test:9647:premise", "snli:test:9339:premise", "style:personachat:test:354:self:1:positive:0"], "evidence_texts": ["A woman in a gray shirt is looking at something.", "A skateboard is grinding on a curb with his skateboard.", "Two women stand in a kitchen and wipe down trays", "A group of individuals are sitting together in a room.", "lol see , where are you located at ?"], "speaker_id": "snli:test:9409", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9413", "candidate_text": "well, soccer players are trying to get the ball", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9413:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9402:premise", "snli:test:9413:premise", "snli:test:9850:premise", "snli:test:9400:premise", "style:personachat:test:353:self:2:positive:0", "style:personachat:test:353:self:2:positive:1"], "evidence_texts": ["Large brown dog walking in shallow water.", "The players manuever for the soccer ball.", "women's soccer team competing on the field.", "Large brown dog walking in shallow water.", "i like college the best what is your favorite food ?", "that must be hard . i love snack foods"], "speaker_id": "snli:test:9413", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9413", "candidate_text": "Honestly, Soccer players are trying to get the ball!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9413:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9402:premise", "snli:test:9413:premise", "snli:test:9850:premise", "snli:test:9400:premise", "style:personachat:test:620:self:7:positive:0", "style:personachat:test:620:self:7:positive:1", "style:personachat:test:620:self:7:positive:2", "style:personachat:test:620:self:7:positive:3", "style:personachat:test:620:self:7:positive:4", "style:personachat:test:620:self:7:positive:5", "style:personachat:test:620:self:7:positive:6"], "evidence_texts": ["Large brown dog walking in shallow water.", "The players manuever for the soccer ball.", "women's soccer team competing on the field.", "Large brown dog walking in shallow water.", "hi , yes , i love dogs . used to have a pit bull mix named jenny .", "yes , what about other sports ? what others do you like ?", "yes , i travelled to spain in high school but then my best friend took my boyfriend .", "haha , nah , he and i were starting to grow apart but i stopped talking to her .", "what do you mean , deal ?", "so you must be lonely then ?", "yeah , dogs rock ! my parents were gone a lot but my dog was always there ."], "speaker_id": "snli:test:9413", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9448", "candidate_text": "well, the people are tourists", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9448:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9448:premise", "snli:test:9195:premise", "snli:test:9741:premise", "snli:test:9194:premise", "style:personachat:test:353:self:5:positive:0", "style:personachat:test:353:self:5:positive:1", "style:personachat:test:353:self:5:positive:2", "style:personachat:test:353:self:5:positive:3", "style:personachat:test:353:self:5:positive:4"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "i have 3 dis do you go to church i do no", "i go climbing but am scared of heights", "i like ice tea . do you like sports ?", "i like college the best what is your favorite food ?", "that must be hard . i love snack foods"], "speaker_id": "snli:test:9448", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9448", "candidate_text": "the people are tourists.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9448:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9448:premise", "snli:test:9195:premise", "snli:test:9741:premise", "snli:test:9194:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9448", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9453", "candidate_text": "well, the rider is outdoors", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9453:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9318:premise", "snli:test:9453:premise", "snli:test:9753:premise", "snli:test:9234:premise", "style:personachat:test:353:self:7:positive:0", "style:personachat:test:353:self:7:positive:1", "style:personachat:test:353:self:7:positive:2", "style:personachat:test:353:self:7:positive:3", "style:personachat:test:353:self:7:positive:4", "style:personachat:test:353:self:7:positive:5", "style:personachat:test:353:self:7:positive:6"], "evidence_texts": ["A woman is observing pictures on a camera.", "An ATV rides over a vast sandy landscape.", "Little boy peering into a glass display case.", "A group of people gather around a truck.", "i am doing well thanks i am an attorney and working on cases", "what a fun job . i live in ok and am a native american", "i have 3 dis do you go to church i do no", "i go climbing but am scared of heights", "i like ice tea . do you like sports ?", "i like college the best what is your favorite food ?", "that must be hard . i love snack foods"], "speaker_id": "snli:test:9453", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9453", "candidate_text": "I wonder whether the rider is outdoors?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9453:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9318:premise", "snli:test:9453:premise", "snli:test:9753:premise", "snli:test:9234:premise", "style:personachat:test:354:self:6:positive:0", "style:personachat:test:354:self:6:positive:1", "style:personachat:test:354:self:6:positive:2", "style:personachat:test:354:self:6:positive:3", "style:personachat:test:354:self:6:positive:4", "style:personachat:test:354:self:6:positive:5"], "evidence_texts": ["A woman is observing pictures on a camera.", "An ATV rides over a vast sandy landscape.", "Little boy peering into a glass display case.", "A group of people gather around a truck.", "i love the mall , shopping is my hobby . do you have any ?", "really ? me too , are you in a relationship ? i am not as of yet .", "i got me some nice blue jean thigh high boots . any siblings ?", "oh no sorry to hear that ! we could go out on a date !", "i bet you are a great catch , i will be a full time nurse soon .", "lol see , where are you located at ?"], "speaker_id": "snli:test:9453", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9472", "candidate_text": "I wonder whether an elderly male is blowing air inyto an object?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9472:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9472:premise", "snli:test:9349:premise", "snli:test:9570:premise", "snli:test:9332:premise", "style:personachat:test:354:self:1:positive:0"], "evidence_texts": ["An elderly male is blowing air into an object.", "The dog's mouth is open like he is yawning.", "A man painting a sign onto his food cart.", "A man playing the guitar while smoking a cigarette.", "lol see , where are you located at ?"], "speaker_id": "snli:test:9472", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9472", "candidate_text": "well, an elderly male is blowing air inyto an object", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9472:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9472:premise", "snli:test:9349:premise", "snli:test:9570:premise", "snli:test:9332:premise", "style:personachat:test:591:self:1:positive:0"], "evidence_texts": ["An elderly male is blowing air into an object.", "The dog's mouth is open like he is yawning.", "A man painting a sign onto his food cart.", "A man playing the guitar while smoking a cigarette.", "depends on where you go there is a buffet out here 8 before 330"], "speaker_id": "snli:test:9472", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9475", "candidate_text": "I wonder whether the man selling halal meat in his food cart is smiling at the camera?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9475:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9347:premise", "snli:test:9681:premise", "snli:test:8896:premise", "snli:test:9475:premise", "style:personachat:test:354:self:2:positive:0", "style:personachat:test:354:self:2:positive:1"], "evidence_texts": ["Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "i bet you are a great catch , i will be a full time nurse soon .", "lol see , where are you located at ?"], "speaker_id": "snli:test:9475", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9475", "candidate_text": "Honestly, The man selling halal meat in his food cart is smiling at the camera!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9475:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9347:premise", "snli:test:9681:premise", "snli:test:8896:premise", "snli:test:9475:premise", "style:personachat:test:624:self:6:positive:0", "style:personachat:test:624:self:6:positive:1", "style:personachat:test:624:self:6:positive:2", "style:personachat:test:624:self:6:positive:3", "style:personachat:test:624:self:6:positive:4", "style:personachat:test:624:self:6:positive:5"], "evidence_texts": ["Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "An older caucasian couple is having their picture taken by a stone statue of a man and woman kissing.", "A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "i am good . this is great , i love making new friends !", "actually i love ultimate frisbee ! it is my favorite . do you have pets ?", "my turtle is my best friend . his name is timothy . how about your family ?", "i love autumn being here . i love fall . my parents moved because of the seasons !", "just frisbee . when i can i travel to bora bora . that is where they moved", "not old enough to meet new friends !"], "speaker_id": "snli:test:9475", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9479", "candidate_text": "I wonder whether the boy is going to dunk the ball?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9479:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:6790:premise", "snli:test:386:premise", "snli:test:3820:premise", "snli:test:9479:premise", "style:personachat:test:354:self:5:positive:0", "style:personachat:test:354:self:5:positive:1", "style:personachat:test:354:self:5:positive:2", "style:personachat:test:354:self:5:positive:3", "style:personachat:test:354:self:5:positive:4"], "evidence_texts": ["Woman adjusting her eyeglasses.", "Several women are playing volleyball.", "Woman is looking something.", "A boy plays basketball.", "really ? me too , are you in a relationship ? i am not as of yet .", "i got me some nice blue jean thigh high boots . any siblings ?", "oh no sorry to hear that ! we could go out on a date !", "i bet you are a great catch , i will be a full time nurse soon .", "lol see , where are you located at ?"], "speaker_id": "snli:test:9479", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9479", "candidate_text": "The boy is going to dunk the ball.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9479:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:6790:premise", "snli:test:386:premise", "snli:test:3820:premise", "snli:test:9479:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Woman adjusting her eyeglasses.", "Several women are playing volleyball.", "Woman is looking something.", "A boy plays basketball.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9479", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9483", "candidate_text": "I wonder whether a boy is happy to be so close to the water?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9483:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9251:premise", "snli:test:9484:premise", "snli:test:9244:premise", "snli:test:9483:premise", "style:personachat:test:354:self:6:positive:0", "style:personachat:test:354:self:6:positive:1", "style:personachat:test:354:self:6:positive:2", "style:personachat:test:354:self:6:positive:3", "style:personachat:test:354:self:6:positive:4", "style:personachat:test:354:self:6:positive:5"], "evidence_texts": ["A boy splashing through the ocean.", "two dogs run across the woods.", "Two men outside of a restaurant.", "A boy jumps on the beach.", "i love the mall , shopping is my hobby . do you have any ?", "really ? me too , are you in a relationship ? i am not as of yet .", "i got me some nice blue jean thigh high boots . any siblings ?", "oh no sorry to hear that ! we could go out on a date !", "i bet you are a great catch , i will be a full time nurse soon .", "lol see , where are you located at ?"], "speaker_id": "snli:test:9483", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9483", "candidate_text": "well, a boy is happy to be so close to the water", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9483:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9251:premise", "snli:test:9484:premise", "snli:test:9244:premise", "snli:test:9483:premise", "style:personachat:test:591:self:7:positive:0", "style:personachat:test:591:self:7:positive:1", "style:personachat:test:591:self:7:positive:2", "style:personachat:test:591:self:7:positive:3", "style:personachat:test:591:self:7:positive:4", "style:personachat:test:591:self:7:positive:5", "style:personachat:test:591:self:7:positive:6"], "evidence_texts": ["A boy splashing through the ocean.", "two dogs run across the woods.", "Two men outside of a restaurant.", "A boy jumps on the beach.", "hey i am fine how about you", "oh what major are you going for", "oh i would do history its great learning new things", "i like barbecue and sushi", "whats you favorite type of sushi", "i could go for some now lol", "depends on where you go there is a buffet out here 8 before 330"], "speaker_id": "snli:test:9483", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9484", "candidate_text": "I wonder whether the dogs laid down on the rug?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9484:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9484:premise", "snli:test:9483:premise", "snli:test:9524:premise", "snli:test:9251:premise", "style:personachat:test:355:self:1:positive:0"], "evidence_texts": ["two dogs run across the woods.", "A boy jumps on the beach.", "A dog jumping over a beam", "A boy splashing through the ocean.", "what kind of teacher ? you enjoy it ?"], "speaker_id": "snli:test:9484", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9484", "candidate_text": "Honestly, The dogs laid down on the rug!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9484:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9484:premise", "snli:test:9483:premise", "snli:test:9524:premise", "snli:test:9251:premise", "style:personachat:test:627:self:5:positive:0", "style:personachat:test:627:self:5:positive:1", "style:personachat:test:627:self:5:positive:2", "style:personachat:test:627:self:5:positive:3", "style:personachat:test:627:self:5:positive:4"], "evidence_texts": ["two dogs run across the woods.", "A boy jumps on the beach.", "A dog jumping over a beam", "A boy splashing through the ocean.", "i am 30 years old . what is your favorite sport ?", "i like soccer . i like watching it on tv .", "world cup if i had to choose . are you married ?", "what is your favorite food or restaurant ?", "i am craving cake and pizza ! !"], "speaker_id": "snli:test:9484", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9488", "candidate_text": "I wonder whether a young asian couple are swimming in the pool?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9488:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9488:premise", "snli:test:9394:premise", "snli:test:9323:premise", "snli:test:9493:premise", "style:personachat:test:355:self:2:positive:0", "style:personachat:test:355:self:2:positive:1"], "evidence_texts": ["A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A brown and white dog swims in a backyard swimming pool.", "Boy sitting looking through a glass window at a train station.", "i am a medium and read tarot cards", "what kind of teacher ? you enjoy it ?"], "speaker_id": "snli:test:9488", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9488", "candidate_text": "A young Asian couple are swimming in the pool.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9488:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9488:premise", "snli:test:9394:premise", "snli:test:9323:premise", "snli:test:9493:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A brown and white dog swims in a backyard swimming pool.", "Boy sitting looking through a glass window at a train station.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9488", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9489", "candidate_text": "I wonder whether the young asian couple are walking together?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9489:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9493:premise", "snli:test:9489:premise", "snli:test:9394:premise", "snli:test:9604:premise", "style:personachat:test:355:self:5:positive:0", "style:personachat:test:355:self:5:positive:1", "style:personachat:test:355:self:5:positive:2", "style:personachat:test:355:self:5:positive:3", "style:personachat:test:355:self:5:positive:4"], "evidence_texts": ["Boy sitting looking through a glass window at a train station.", "A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A boy stands in the middle of a crowd of teenagers.", "same . this is my favorite time of year .", "i read tarot cards and have a bunch of tattoos", "they are addicting . you will want more and more lol", "i am a medium and read tarot cards", "what kind of teacher ? you enjoy it ?"], "speaker_id": "snli:test:9489", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9489", "candidate_text": "well, the young asian couple are walking together", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9489:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9493:premise", "snli:test:9489:premise", "snli:test:9394:premise", "snli:test:9604:premise", "style:personachat:test:594:self:5:positive:0", "style:personachat:test:594:self:5:positive:1", "style:personachat:test:594:self:5:positive:2", "style:personachat:test:594:self:5:positive:3", "style:personachat:test:594:self:5:positive:4"], "evidence_texts": ["Boy sitting looking through a glass window at a train station.", "A young japanese couple holding an umbrella walking down a street.", "A public speaker in a plaid shirt speaks on the microphone.", "A boy stands in the middle of a crowd of teenagers.", "i had a choice , go in the army or get a computer science degree .", "so i got a computer science degree and now i sit on my butt all day", "my friend could not get in the military because he is colorblind", "no pets right now but i had plenty in the past . all kid of pets", "i am usually real shy but i feel comfortable talking to you . i like both equal"], "speaker_id": "snli:test:9489", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9493", "candidate_text": "I wonder whether a boy is looking through a window at another boy at the train station?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9493:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9489:premise", "snli:test:9493:premise", "snli:test:9604:premise", "snli:test:9488:premise", "style:personachat:test:355:self:7:positive:0", "style:personachat:test:355:self:7:positive:1", "style:personachat:test:355:self:7:positive:2", "style:personachat:test:355:self:7:positive:3", "style:personachat:test:355:self:7:positive:4", "style:personachat:test:355:self:7:positive:5", "style:personachat:test:355:self:7:positive:6"], "evidence_texts": ["A young japanese couple holding an umbrella walking down a street.", "Boy sitting looking through a glass window at a train station.", "A boy stands in the middle of a crowd of teenagers.", "A young japanese couple holding an umbrella walking down a street.", "not too bad . how about you", "gardening is always fun . i like the guitar", "same . this is my favorite time of year .", "i read tarot cards and have a bunch of tattoos", "they are addicting . you will want more and more lol", "i am a medium and read tarot cards", "what kind of teacher ? you enjoy it ?"], "speaker_id": "snli:test:9493", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9493", "candidate_text": "Honestly, A boy is looking through a window at another boy at the train station!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9493:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9489:premise", "snli:test:9493:premise", "snli:test:9604:premise", "snli:test:9488:premise", "style:personachat:test:636:self:2:positive:0", "style:personachat:test:636:self:2:positive:1"], "evidence_texts": ["A young japanese couple holding an umbrella walking down a street.", "Boy sitting looking through a glass window at a train station.", "A boy stands in the middle of a crowd of teenagers.", "A young japanese couple holding an umbrella walking down a street.", "absolutely ! my mom and dad were always gone growing up so i am used to eating junk .", "oh most definitely ! the pink ones are the best !"], "speaker_id": "snli:test:9493", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9524", "candidate_text": "well, an animal is outdoors", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9524:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9524:premise", "snli:test:9484:premise", "snli:test:9620:premise", "snli:test:9483:premise", "style:personachat:test:356:self:1:positive:0"], "evidence_texts": ["A dog jumping over a beam", "two dogs run across the woods.", "A woman holding a newborn baby.", "A boy jumps on the beach.", "sorry , no . i only date those in my profession ."], "speaker_id": "snli:test:9524", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9524", "candidate_text": "I wonder whether an animal is outdoors?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9524:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9524:premise", "snli:test:9484:premise", "snli:test:9620:premise", "snli:test:9483:premise", "style:personachat:test:364:self:1:positive:0"], "evidence_texts": ["A dog jumping over a beam", "two dogs run across the woods.", "A woman holding a newborn baby.", "A boy jumps on the beach.", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9524", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9530", "candidate_text": "well, the children are eating pb&j sandwiches inside their kitchen", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9530:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9078:premise", "snli:test:9577:premise", "snli:test:9530:premise", "snli:test:9076:premise", "style:personachat:test:356:self:2:positive:0", "style:personachat:test:356:self:2:positive:1"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "i have visited several times , i prefer hawaii", "sorry , no . i only date those in my profession ."], "speaker_id": "snli:test:9530", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9530", "candidate_text": "Honestly, The children are eating PB&J sandwiches inside their kitchen!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9530:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9078:premise", "snli:test:9577:premise", "snli:test:9530:premise", "snli:test:9076:premise", "style:personachat:test:636:self:7:positive:0", "style:personachat:test:636:self:7:positive:1", "style:personachat:test:636:self:7:positive:2", "style:personachat:test:636:self:7:positive:3", "style:personachat:test:636:self:7:positive:4", "style:personachat:test:636:self:7:positive:5", "style:personachat:test:636:self:7:positive:6"], "evidence_texts": ["A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "sure ! can always use more friends . tell me about you", "awesome ! i would love to be a fashion designer ! are you married ? kids ?", "congrats on graduating ! i am married but no kids yet . we want kids in the future !", "i had a friend in high school named lucy but she stole my boyfriend lol", "thanks ; hehe so what do you like to do ?", "absolutely ! my mom and dad were always gone growing up so i am used to eating junk .", "oh most definitely ! the pink ones are the best !"], "speaker_id": "snli:test:9530", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9532", "candidate_text": "well, man is a girl", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9532:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9532:premise", "snli:test:9343:premise", "snli:test:9612:premise", "snli:test:9327:premise", "style:personachat:test:356:self:5:positive:0", "style:personachat:test:356:self:5:positive:1", "style:personachat:test:356:self:5:positive:2", "style:personachat:test:356:self:5:positive:3", "style:personachat:test:356:self:5:positive:4"], "evidence_texts": ["A man in a red headband an shirt is standing in front of a display case.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "A lab dog is standing on a fallen tree trunk with a guy in the background.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "i do photography and paint a little .", "do you have any pets ? i have two pups .", "i am in oklahoma , where are you located ?", "i have visited several times , i prefer hawaii", "sorry , no . i only date those in my profession ."], "speaker_id": "snli:test:9532", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9532", "candidate_text": "man is a girl.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9532:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9532:premise", "snli:test:9343:premise", "snli:test:9612:premise", "snli:test:9327:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A man in a red headband an shirt is standing in front of a display case.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "A lab dog is standing on a fallen tree trunk with a guy in the background.", "A mother shading her baby with an umbrella as she and her partner climb the stairs.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9532", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9541", "candidate_text": "well, a person in a robe sits on a couch", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9541:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9254:premise", "snli:test:9541:premise", "snli:test:9640:premise", "snli:test:9236:premise", "style:personachat:test:356:self:7:positive:0", "style:personachat:test:356:self:7:positive:1", "style:personachat:test:356:self:7:positive:2", "style:personachat:test:356:self:7:positive:3", "style:personachat:test:356:self:7:positive:4", "style:personachat:test:356:self:7:positive:5", "style:personachat:test:356:self:7:positive:6"], "evidence_texts": ["A little white dog runs outside, their ears blown back in the wind.", "A person in a green robe sits on a couch with a blanket.", "A group of young men in a gym take turns scoring in basketball.", "A woman is cutting into a sandwich while sitting at an outdoor table.", "i do , though not the dogs or the cactus i am growing", "i am a criminal defense lawyer .", "i do photography and paint a little .", "do you have any pets ? i have two pups .", "i am in oklahoma , where are you located ?", "i have visited several times , i prefer hawaii", "sorry , no . i only date those in my profession ."], "speaker_id": "snli:test:9541", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9541", "candidate_text": "I wonder whether a person in a robe sits on a couch?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9541:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9254:premise", "snli:test:9541:premise", "snli:test:9640:premise", "snli:test:9236:premise", "style:personachat:test:364:self:7:positive:0", "style:personachat:test:364:self:7:positive:1", "style:personachat:test:364:self:7:positive:2", "style:personachat:test:364:self:7:positive:3", "style:personachat:test:364:self:7:positive:4", "style:personachat:test:364:self:7:positive:5", "style:personachat:test:364:self:7:positive:6"], "evidence_texts": ["A little white dog runs outside, their ears blown back in the wind.", "A person in a green robe sits on a couch with a blanket.", "A group of young men in a gym take turns scoring in basketball.", "A woman is cutting into a sandwich while sitting at an outdoor table.", "swimmingly . how are you ?", "polar bears ? are those like , mythical creatures who cannot swim and live above ?", "oh no ! do you hunt mer people also ? ?", "geology ? is that rocks or something ? i hide behind those sometimes .", "oh no . please tell me where he fishes so i can stay away ! !", "my sister got stuck in a fishing net once !", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9541", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9570", "candidate_text": "well, everyone is in a restaurant", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9570:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9472:premise", "snli:test:9607:premise", "snli:test:9570:premise", "snli:test:9349:premise", "style:personachat:test:357:self:1:positive:0"], "evidence_texts": ["An elderly male is blowing air into an object.", "A man and his camel in the dusty desert.", "A man painting a sign onto his food cart.", "The dog's mouth is open like he is yawning.", "erm , no . but i like weird stickers and i put them on my face ."], "speaker_id": "snli:test:9570", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9570", "candidate_text": "Honestly, Everyone is in a restaurant!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9570:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9472:premise", "snli:test:9607:premise", "snli:test:9570:premise", "snli:test:9349:premise", "style:personachat:test:637:self:5:positive:0", "style:personachat:test:637:self:5:positive:1", "style:personachat:test:637:self:5:positive:2", "style:personachat:test:637:self:5:positive:3", "style:personachat:test:637:self:5:positive:4"], "evidence_texts": ["An elderly male is blowing air into an object.", "A man and his camel in the dusty desert.", "A man painting a sign onto his food cart.", "The dog's mouth is open like he is yawning.", "i also attend school . what are you going to school for ?", "awesome ! music is what i am studying since i like it so much .", "i saw them in concert . could not see much since i am only five feet .", "did you come from a big family ? it is just me , parents and a sibling .", "how late ? my parents were also older . in their 50s !"], "speaker_id": "snli:test:9570", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9577", "candidate_text": "well, a woman rests on the shoulder of a man", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9577:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9530:premise", "snli:test:9577:premise", "snli:test:9750:premise", "snli:test:9078:premise", "style:personachat:test:357:self:2:positive:0", "style:personachat:test:357:self:2:positive:1"], "evidence_texts": ["One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "well yes . yes i have both legs .", "erm , no . but i like weird stickers and i put them on my face ."], "speaker_id": "snli:test:9577", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9577", "candidate_text": "A woman rests on the shoulder of a man.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9577:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9530:premise", "snli:test:9577:premise", "snli:test:9750:premise", "snli:test:9078:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man wearing a brightly green camouflage patterned denim jacket and boots walks by a building labeled Lottery Ticket Centre.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9577", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9580", "candidate_text": "well, a basketball player dunking the ball", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9580:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9580:premise", "snli:test:9189:premise", "snli:test:9601:premise", "snli:test:9125:premise", "style:personachat:test:357:self:5:positive:0", "style:personachat:test:357:self:5:positive:1", "style:personachat:test:357:self:5:positive:2", "style:personachat:test:357:self:5:positive:3", "style:personachat:test:357:self:5:positive:4"], "evidence_texts": ["A baseball player in the act of pick up a baseball with his hand.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A person wearing a blue backpack stands on a field above a scenic view.", "A man throws his young boy playfully into the air standing on a beach.", "why did he do that ?", "oh . gosh . i guess you would not enjoy hiking , then .", "sixty nine is a respectable age .", "well yes . yes i have both legs .", "erm , no . but i like weird stickers and i put them on my face ."], "speaker_id": "snli:test:9580", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9580", "candidate_text": "I wonder whether a basketball player dunking the ball?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9580:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9580:premise", "snli:test:9189:premise", "snli:test:9601:premise", "snli:test:9125:premise", "style:personachat:test:365:self:5:positive:0", "style:personachat:test:365:self:5:positive:1", "style:personachat:test:365:self:5:positive:2", "style:personachat:test:365:self:5:positive:3", "style:personachat:test:365:self:5:positive:4"], "evidence_texts": ["A baseball player in the act of pick up a baseball with his hand.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "A person wearing a blue backpack stands on a field above a scenic view.", "A man throws his young boy playfully into the air standing on a beach.", "going back to asia , just can not decide the destination . any hobbies ? i love travel .", "i love the culture . its very rich and diverse . i spent half of college there .", "i studied anthropology and world religion .", "i work for a group that proves the validity of the jewish faith .", "what kind of job do you have ?"], "speaker_id": "snli:test:9580", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9588", "candidate_text": "well, a dog is warm in it's bed", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9588:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9588:premise", "snli:test:9399:premise", "snli:test:9673:premise", "snli:test:9341:premise", "style:personachat:test:357:self:6:positive:0", "style:personachat:test:357:self:6:positive:1", "style:personachat:test:357:self:6:positive:2", "style:personachat:test:357:self:6:positive:3", "style:personachat:test:357:self:6:positive:4", "style:personachat:test:357:self:6:positive:5"], "evidence_texts": ["A black and white dog jumps up and down in the snow.", "Men playing basketball, the guy with the ball is attempting a layup.", "Two women are laughing and smiling while they eat a meal together.", "a woman wearing a pink tank top holding a mug of liquid", "interesting combo . i am a big robert frost fan .", "why did he do that ?", "oh . gosh . i guess you would not enjoy hiking , then .", "sixty nine is a respectable age .", "well yes . yes i have both legs .", "erm , no . but i like weird stickers and i put them on my face ."], "speaker_id": "snli:test:9588", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9588", "candidate_text": "Honestly, A dog is warm in it's bed!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9588:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9588:premise", "snli:test:9399:premise", "snli:test:9673:premise", "snli:test:9341:premise", "style:personachat:test:646:self:2:positive:0", "style:personachat:test:646:self:2:positive:1"], "evidence_texts": ["A black and white dog jumps up and down in the snow.", "Men playing basketball, the guy with the ball is attempting a layup.", "Two women are laughing and smiling while they eat a meal together.", "a woman wearing a pink tank top holding a mug of liquid", "now i just play a lot of games .", "fantasy videogames ! ! do you like them ?"], "speaker_id": "snli:test:9588", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9601", "candidate_text": "well, the person is outdoors", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9601:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9580:premise", "snli:test:9601:premise", "snli:test:9731:premise", "snli:test:9189:premise", "style:personachat:test:358:self:1:positive:0"], "evidence_texts": ["A baseball player in the act of pick up a baseball with his hand.", "A person wearing a blue backpack stands on a field above a scenic view.", "A group of Asian men are sitting or standing around a small square table.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "animals scare me . i was bitten by a dog"], "speaker_id": "snli:test:9601", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9601", "candidate_text": "The person is outdoors.", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9601:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9580:premise", "snli:test:9601:premise", "snli:test:9731:premise", "snli:test:9189:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A baseball player in the act of pick up a baseball with his hand.", "A person wearing a blue backpack stands on a field above a scenic view.", "A group of Asian men are sitting or standing around a small square table.", "A man in a yellow shirt and helmet mountain biking down a dusty path.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9601", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9604", "candidate_text": "well, a person stands in the middle of a crowd of teenagers", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9604:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9604:premise", "snli:test:9493:premise", "snli:test:9489:premise", "snli:test:9814:premise", "style:personachat:test:358:self:2:positive:0", "style:personachat:test:358:self:2:positive:1"], "evidence_texts": ["A boy stands in the middle of a crowd of teenagers.", "Boy sitting looking through a glass window at a train station.", "A young japanese couple holding an umbrella walking down a street.", "A child being watched while getting something out of a tank.", "i do not really like kale . about as much as i dislike animals", "animals scare me . i was bitten by a dog"], "speaker_id": "snli:test:9604", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9604", "candidate_text": "I wonder whether a person stands in the middle of a crowd of teenagers?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9604:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9604:premise", "snli:test:9493:premise", "snli:test:9489:premise", "snli:test:9814:premise", "style:personachat:test:366:self:2:positive:0", "style:personachat:test:366:self:2:positive:1"], "evidence_texts": ["A boy stands in the middle of a crowd of teenagers.", "Boy sitting looking through a glass window at a train station.", "A young japanese couple holding an umbrella walking down a street.", "A child being watched while getting something out of a tank.", "are you working as a nurse ?", "ops you are selling by phone ?"], "speaker_id": "snli:test:9604", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9606", "candidate_text": "well, a sad boy stands in the middle of a crowd of teenagers", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9606:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9814:premise", "snli:test:9493:premise", "snli:test:9606:premise", "snli:test:9854:premise", "style:personachat:test:358:self:5:positive:0", "style:personachat:test:358:self:5:positive:1", "style:personachat:test:358:self:5:positive:2", "style:personachat:test:358:self:5:positive:3", "style:personachat:test:358:self:5:positive:4"], "evidence_texts": ["A child being watched while getting something out of a tank.", "Boy sitting looking through a glass window at a train station.", "A boy stands in the middle of a crowd of teenagers.", "Kids play in the water in the middle of the street.", "i did not finish high school", "it is hell living here", "in canada , i am a fast runner because of all the crime around me", "i do not really like kale . about as much as i dislike animals", "animals scare me . i was bitten by a dog"], "speaker_id": "snli:test:9606", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9606", "candidate_text": "Honestly, A sad boy stands in the middle of a crowd of teenagers!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9606:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9814:premise", "snli:test:9493:premise", "snli:test:9606:premise", "snli:test:9854:premise", "style:personachat:test:648:self:1:positive:0"], "evidence_texts": ["A child being watched while getting something out of a tank.", "Boy sitting looking through a glass window at a train station.", "A boy stands in the middle of a crowd of teenagers.", "Kids play in the water in the middle of the street.", "maybe i will revisit the idea in the future !"], "speaker_id": "snli:test:9606", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9607", "candidate_text": "well, a man and his camel", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9607:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:9570:premise", "snli:test:9828:premise", "snli:test:9472:premise", "snli:test:9607:premise", "style:personachat:test:358:self:6:positive:0", "style:personachat:test:358:self:6:positive:1", "style:personachat:test:358:self:6:positive:2", "style:personachat:test:358:self:6:positive:3", "style:personachat:test:358:self:6:positive:4", "style:personachat:test:358:self:6:positive:5"], "evidence_texts": ["A man painting a sign onto his food cart.", "A closeup of an older man with glasses speaking.", "An elderly male is blowing air into an object.", "A man and his camel in the dusty desert.", "not really , i work for a fast food company . . .", "i did not finish high school", "it is hell living here", "in canada , i am a fast runner because of all the crime around me", "i do not really like kale . about as much as i dislike animals", "animals scare me . i was bitten by a dog"], "speaker_id": "snli:test:9607", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9607", "candidate_text": "a man and his camel.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9607:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9570:premise", "snli:test:9828:premise", "snli:test:9472:premise", "snli:test:9607:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man painting a sign onto his food cart.", "A closeup of an older man with glasses speaking.", "An elderly male is blowing air into an object.", "A man and his camel in the dusty desert.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9607", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9612", "candidate_text": "well, there is a guy carrying his pet with him near fallen tree", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9612:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9612:premise", "snli:test:9532:premise", "snli:test:9703:premise", "snli:test:9343:premise", "style:personachat:test:359:self:1:positive:0"], "evidence_texts": ["A lab dog is standing on a fallen tree trunk with a guy in the background.", "A man in a red headband an shirt is standing in front of a display case.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "oh okay i am lost but i get it now"], "speaker_id": "snli:test:9612", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9612", "candidate_text": "I wonder whether there is a guy carrying his pet with him near fallen tree?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9612:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9612:premise", "snli:test:9532:premise", "snli:test:9703:premise", "snli:test:9343:premise", "style:personachat:test:370:self:1:positive:0"], "evidence_texts": ["A lab dog is standing on a fallen tree trunk with a guy in the background.", "A man in a red headband an shirt is standing in front of a display case.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "A race car, numbered 104 is rounding a bend as two people watch and take pictures.", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9612", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9617", "candidate_text": "well, a group of officers are standing", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9617:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:8889:premise", "snli:test:9617:premise", "snli:test:9712:premise", "snli:test:8888:premise", "style:personachat:test:359:self:2:positive:0", "style:personachat:test:359:self:2:positive:1"], "evidence_texts": ["A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "this is not any good mr hunter", "oh okay i am lost but i get it now"], "speaker_id": "snli:test:9617", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9617", "candidate_text": "Honestly, A group of officers are standing!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9617:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:8889:premise", "snli:test:9617:premise", "snli:test:9712:premise", "snli:test:8888:premise", "style:personachat:test:648:self:7:positive:0", "style:personachat:test:648:self:7:positive:1", "style:personachat:test:648:self:7:positive:2", "style:personachat:test:648:self:7:positive:3", "style:personachat:test:648:self:7:positive:4", "style:personachat:test:648:self:7:positive:5", "style:personachat:test:648:self:7:positive:6"], "evidence_texts": ["A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "oh hey , i am a 5 foot , 35 year old woman who has a dog , pedro .", "i work in healthcare . it is boring .", "in new york . it is in a great town with film festivals .", "i have a two year old daughter . . she inherited my love of muffins .", "i tried getting a real estate licence but it seemed too tedious .", "i believe it . it seemed like something i would like .", "maybe i will revisit the idea in the future !"], "speaker_id": "snli:test:9617", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9620", "candidate_text": "well, woman holds baby", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9620:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9620:premise", "snli:test:9524:premise", "snli:test:9996:premise", "snli:test:9484:premise", "style:personachat:test:359:self:5:positive:0", "style:personachat:test:359:self:5:positive:1", "style:personachat:test:359:self:5:positive:2", "style:personachat:test:359:self:5:positive:3", "style:personachat:test:359:self:5:positive:4"], "evidence_texts": ["A woman holding a newborn baby.", "A dog jumping over a beam", "Two women are observing something together.", "two dogs run across the woods.", "oh why ? i am schooled at home", "really ? i am to young i am a teen", "where you live me an my bff robot live in cali", "this is not any good mr hunter", "oh okay i am lost but i get it now"], "speaker_id": "snli:test:9620", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9620", "candidate_text": "woman holds baby.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9620:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9620:premise", "snli:test:9524:premise", "snli:test:9996:premise", "snli:test:9484:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A woman holding a newborn baby.", "A dog jumping over a beam", "Two women are observing something together.", "two dogs run across the woods.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9620", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9640", "candidate_text": "well, guys are playing shirts vs skins", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9640:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9640:premise", "snli:test:9541:premise", "snli:test:9910:premise", "snli:test:9254:premise", "style:personachat:test:359:self:7:positive:0", "style:personachat:test:359:self:7:positive:1", "style:personachat:test:359:self:7:positive:2", "style:personachat:test:359:self:7:positive:3", "style:personachat:test:359:self:7:positive:4", "style:personachat:test:359:self:7:positive:5", "style:personachat:test:359:self:7:positive:6"], "evidence_texts": ["A group of young men in a gym take turns scoring in basketball.", "A person in a green robe sits on a couch with a blanket.", "A woman in a black coat eats dinner while her dog looks on.", "A little white dog runs outside, their ears blown back in the wind.", "oh just hungry an you ?", "cool , where at ? my hair blue", "oh why ? i am schooled at home", "really ? i am to young i am a teen", "where you live me an my bff robot live in cali", "this is not any good mr hunter", "oh okay i am lost but i get it now"], "speaker_id": "snli:test:9640", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9640", "candidate_text": "I wonder whether guys are playing shirts vs skins?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9640:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9640:premise", "snli:test:9541:premise", "snli:test:9910:premise", "snli:test:9254:premise", "style:personachat:test:370:self:7:positive:0", "style:personachat:test:370:self:7:positive:1", "style:personachat:test:370:self:7:positive:2", "style:personachat:test:370:self:7:positive:3", "style:personachat:test:370:self:7:positive:4", "style:personachat:test:370:self:7:positive:5", "style:personachat:test:370:self:7:positive:6"], "evidence_texts": ["A group of young men in a gym take turns scoring in basketball.", "A person in a green robe sits on a couch with a blanket.", "A woman in a black coat eats dinner while her dog looks on.", "A little white dog runs outside, their ears blown back in the wind.", "very good , how are you today ?", "yummy ! i love to cook , but that makes me unhappy about my weight", "yes i should , but my three kids like junk food lol", "it is hard to eat together , since i work days and my husband works overnights", "do you have any pets ? we have cats , which we love", "that would be nice , i love dogs too .", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9640", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9643", "candidate_text": "I wonder whether a barber waits for business?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9643:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9330:premise", "snli:test:9649:premise", "snli:test:9329:premise", "snli:test:9643:premise", "style:personachat:test:360:self:1:positive:0"], "evidence_texts": ["A man on a skateboard on steps in front glass doors of a brick building.", "The man in black is flying off the full in front of a red fence.", "A man on a skateboard on steps in front glass doors of a brick building.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "what are your plans for this evening ?"], "speaker_id": "snli:test:9643", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9643", "candidate_text": "well, a barber waits for business", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9643:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9330:premise", "snli:test:9649:premise", "snli:test:9329:premise", "snli:test:9643:premise", "style:personachat:test:599:self:1:positive:0"], "evidence_texts": ["A man on a skateboard on steps in front glass doors of a brick building.", "The man in black is flying off the full in front of a red fence.", "A man on a skateboard on steps in front glass doors of a brick building.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "anything the kiddo likes , she is my world ."], "speaker_id": "snli:test:9643", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9647", "candidate_text": "I wonder whether two women are having tea?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9647:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9409:premise", "snli:test:9685:premise", "snli:test:9357:premise", "snli:test:9647:premise", "style:personachat:test:360:self:2:positive:0", "style:personachat:test:360:self:2:positive:1"], "evidence_texts": ["A skateboard is grinding on a curb with his skateboard.", "Two humans looking out in the distance from a hill.", "A woman in a gray shirt is looking at something.", "Two women stand in a kitchen and wipe down trays", "i hate cats they make me sneeze", "what are your plans for this evening ?"], "speaker_id": "snli:test:9647", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9647", "candidate_text": "Honestly, Two women are having tea!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9647:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9409:premise", "snli:test:9685:premise", "snli:test:9357:premise", "snli:test:9647:premise", "style:personachat:test:654:self:7:positive:0", "style:personachat:test:654:self:7:positive:1", "style:personachat:test:654:self:7:positive:2", "style:personachat:test:654:self:7:positive:3", "style:personachat:test:654:self:7:positive:4", "style:personachat:test:654:self:7:positive:5", "style:personachat:test:654:self:7:positive:6"], "evidence_texts": ["A skateboard is grinding on a curb with his skateboard.", "Two humans looking out in the distance from a hill.", "A woman in a gray shirt is looking at something.", "Two women stand in a kitchen and wipe down trays", "hey . that is a terrible conundrum !", "wish i could . i keep a large garden as well .", "i am a little busy . i teach a college class in biology .", "maybe when i am free . i like walking in my spare time .", "sure . i also love travelling . been to six countries already !", "yes , but ohio is not a country , you know ?", "haha . i did not say that . ohio is a great state !"], "speaker_id": "snli:test:9647", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9649", "candidate_text": "I wonder whether the man is next to the fence?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9649:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9643:premise", "snli:test:9649:premise", "snli:test:9967:premise", "snli:test:9330:premise", "style:personachat:test:360:self:5:positive:0", "style:personachat:test:360:self:5:positive:1", "style:personachat:test:360:self:5:positive:2", "style:personachat:test:360:self:5:positive:3", "style:personachat:test:360:self:5:positive:4"], "evidence_texts": ["A barber in a sports themed barber shop waits for his next customer to arrive.", "The man in black is flying off the full in front of a red fence.", "Man on bike with female standing on rear of back with arms around his neck.", "A man on a skateboard on steps in front glass doors of a brick building.", "i love creating kids pop up books .", "my town population is only 2000 people .", "i am my daughter teacher at homeschooling .", "i hate cats they make me sneeze", "what are your plans for this evening ?"], "speaker_id": "snli:test:9649", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9649", "candidate_text": "The man is next to the fence.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9649:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9643:premise", "snli:test:9649:premise", "snli:test:9967:premise", "snli:test:9330:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A barber in a sports themed barber shop waits for his next customer to arrive.", "The man in black is flying off the full in front of a red fence.", "Man on bike with female standing on rear of back with arms around his neck.", "A man on a skateboard on steps in front glass doors of a brick building.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9649", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9673", "candidate_text": "I wonder whether the boys had sandwiches?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9673:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9588:premise", "snli:test:9673:premise", "snli:test:9399:premise", "snli:test:9766:premise", "style:personachat:test:360:self:7:positive:0", "style:personachat:test:360:self:7:positive:1", "style:personachat:test:360:self:7:positive:2", "style:personachat:test:360:self:7:positive:3", "style:personachat:test:360:self:7:positive:4", "style:personachat:test:360:self:7:positive:5", "style:personachat:test:360:self:7:positive:6"], "evidence_texts": ["A black and white dog jumps up and down in the snow.", "Two women are laughing and smiling while they eat a meal together.", "Men playing basketball, the guy with the ball is attempting a layup.", "A man squatting in a park looking at something in the distance.", "hello how are you today ?", "i love to wake up before the sun shines .", "i love creating kids pop up books .", "my town population is only 2000 people .", "i am my daughter teacher at homeschooling .", "i hate cats they make me sneeze", "what are your plans for this evening ?"], "speaker_id": "snli:test:9673", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9673", "candidate_text": "well, the boys had sandwiches", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9673:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9588:premise", "snli:test:9673:premise", "snli:test:9399:premise", "snli:test:9766:premise", "style:personachat:test:599:self:7:positive:0", "style:personachat:test:599:self:7:positive:1", "style:personachat:test:599:self:7:positive:2", "style:personachat:test:599:self:7:positive:3", "style:personachat:test:599:self:7:positive:4", "style:personachat:test:599:self:7:positive:5", "style:personachat:test:599:self:7:positive:6"], "evidence_texts": ["A black and white dog jumps up and down in the snow.", "Two women are laughing and smiling while they eat a meal together.", "Men playing basketball, the guy with the ball is attempting a layup.", "A man squatting in a park looking at something in the distance.", "hello , no . i am in the gym as today is leg day .", "that is horrid . did he like tim mcgraw ? my kid sister and i do .", "that is why i am a body builder . i have a blog about rabbits i have rescued .", "mine have long hair , and look cute in bows , especially teddy .", "that would be hilarious on teddy . he is named after the president .", "he would need a monocle and a suit .", "anything the kiddo likes , she is my world ."], "speaker_id": "snli:test:9673", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9674", "candidate_text": "well, lilly and sally were laughing while eating lunch", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9674:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9766:premise", "snli:test:9588:premise", "snli:test:9674:premise", "snli:test:9768:premise", "style:personachat:test:361:self:1:positive:0"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A black and white dog jumps up and down in the snow.", "Two women are laughing and smiling while they eat a meal together.", "A man squatting in a park looking at something in the distance.", "i love the fall also ."], "speaker_id": "snli:test:9674", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9674", "candidate_text": "Lilly and Sally were laughing while eating lunch.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9674:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9766:premise", "snli:test:9588:premise", "snli:test:9674:premise", "snli:test:9768:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A black and white dog jumps up and down in the snow.", "Two women are laughing and smiling while they eat a meal together.", "A man squatting in a park looking at something in the distance.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9674", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9681", "candidate_text": "well, the strollers are moving", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9681:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9475:premise", "snli:test:9681:premise", "snli:test:40:premise", "snli:test:9347:premise", "style:personachat:test:361:self:2:positive:0", "style:personachat:test:361:self:2:positive:1"], "evidence_texts": ["A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "it is really nice . do you like michigan ? i guess it gets cold there ?", "i love the fall also ."], "speaker_id": "snli:test:9681", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9681", "candidate_text": "I wonder whether the strollers are moving?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9681:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9475:premise", "snli:test:9681:premise", "snli:test:40:premise", "snli:test:9347:premise", "style:personachat:test:383:self:2:positive:0", "style:personachat:test:383:self:2:positive:1"], "evidence_texts": ["A man wearing a green apron, working at a street side vendor called NY Dosas, miles at the camera.", "A city street with market vendors and children being pushed in strollers with a large cathedral in the background.", "A person wearing a straw hat, standing outside working a steel apparatus with a pile of coconuts on the ground.", "Two girls, one wearing a pink shirt and the other wearing a blue shirt, are standing against a wall.", "the office i think if ok ; i am not sure about the latter .", "oh yes i have . it was a lot of fun . will you be going anytime soon ?"], "speaker_id": "snli:test:9681", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9685", "candidate_text": "well, two people are outdoors", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9685:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9647:premise", "snli:test:9685:premise", "snli:test:9409:premise", "snli:test:9727:premise", "style:personachat:test:361:self:5:positive:0", "style:personachat:test:361:self:5:positive:1", "style:personachat:test:361:self:5:positive:2", "style:personachat:test:361:self:5:positive:3", "style:personachat:test:361:self:5:positive:4"], "evidence_texts": ["Two women stand in a kitchen and wipe down trays", "Two humans looking out in the distance from a hill.", "A skateboard is grinding on a curb with his skateboard.", "A large African American man sitting on a camping chair.", "so you want to be a nurse ? i would like to be a marine biologist .", "oh ok . are you from the usa ?", "i am also , i live in california .", "it is really nice . do you like michigan ? i guess it gets cold there ?", "i love the fall also ."], "speaker_id": "snli:test:9685", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9685", "candidate_text": "Honestly, Two people are outdoors!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9685:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9647:premise", "snli:test:9685:premise", "snli:test:9409:premise", "snli:test:9727:premise", "style:personachat:test:677:self:1:positive:0"], "evidence_texts": ["Two women stand in a kitchen and wipe down trays", "Two humans looking out in the distance from a hill.", "A skateboard is grinding on a curb with his skateboard.", "A large African American man sitting on a camping chair.", "never ! have you started getting ready for the holidays yet ?"], "speaker_id": "snli:test:9685", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9687", "candidate_text": "well, two people pause to overlook the beautiful veiw before resuming their hike", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9687:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9727:premise", "snli:test:9647:premise", "snli:test:9805:premise", "snli:test:9687:premise", "style:personachat:test:361:self:7:positive:0", "style:personachat:test:361:self:7:positive:1", "style:personachat:test:361:self:7:positive:2", "style:personachat:test:361:self:7:positive:3", "style:personachat:test:361:self:7:positive:4", "style:personachat:test:361:self:7:positive:5", "style:personachat:test:361:self:7:positive:6"], "evidence_texts": ["A large African American man sitting on a camping chair.", "Two women stand in a kitchen and wipe down trays", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "not really . i like to study , usually biology at the university .", "so you understand ! i love to study ! i am not much of a drinker .", "so you want to be a nurse ? i would like to be a marine biologist .", "oh ok . are you from the usa ?", "i am also , i live in california .", "it is really nice . do you like michigan ? i guess it gets cold there ?", "i love the fall also ."], "speaker_id": "snli:test:9687", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9687", "candidate_text": "Two people pause to overlook the beautiful veiw before resuming their hike.", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9687:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9727:premise", "snli:test:9647:premise", "snli:test:9805:premise", "snli:test:9687:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A large African American man sitting on a camping chair.", "Two women stand in a kitchen and wipe down trays", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9687", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9688", "candidate_text": "well, the young girl is wearing a black leotard", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9688:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9688:premise", "snli:test:5427:premise", "snli:test:941:premise", "snli:test:5367:premise", "style:personachat:test:362:self:1:positive:0"], "evidence_texts": ["A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "i love prince . especially purple rain . i love prince and purple so it works out"], "speaker_id": "snli:test:9688", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9688", "candidate_text": "I wonder whether the young girl is wearing a black leotard?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9688:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9688:premise", "snli:test:5427:premise", "snli:test:941:premise", "snli:test:5367:premise", "style:personachat:test:393:self:1:positive:0"], "evidence_texts": ["A young girl in a black leotard swinging on the lower wrong of a pair of parallel bars in a gym while a young man spots her.", "A young man wearing a green shirt and a hat attempts a trick on a skateboard while riding it in a skate park next to a beach.", "A man in a black shirt and brown pants is jumping in the air performing a karate kick towards a man in a red shirt and black hat.", "Three men in hats and a woman in a red shirt examine the produce presented to them by the man in the blue shirt and white hat.", "i like to grow organic and fix meals for my family . do you cook ?"], "speaker_id": "snli:test:9688", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9703", "candidate_text": "well, the children are yelling loudly", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9703:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9612:premise", "snli:test:9782:premise", "snli:test:9532:premise", "snli:test:9703:premise", "style:personachat:test:362:self:2:positive:0", "style:personachat:test:362:self:2:positive:1"], "evidence_texts": ["A lab dog is standing on a fallen tree trunk with a guy in the background.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A man in a red headband an shirt is standing in front of a display case.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "14 10 . my boys the ohio state buckeyes are up by 4 ! what music ?", "i love prince . especially purple rain . i love prince and purple so it works out"], "speaker_id": "snli:test:9703", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9703", "candidate_text": "Honestly, The children are yelling loudly!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9703:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9612:premise", "snli:test:9782:premise", "snli:test:9532:premise", "snli:test:9703:premise", "style:personachat:test:677:self:7:positive:0", "style:personachat:test:677:self:7:positive:1", "style:personachat:test:677:self:7:positive:2", "style:personachat:test:677:self:7:positive:3", "style:personachat:test:677:self:7:positive:4", "style:personachat:test:677:self:7:positive:5", "style:personachat:test:677:self:7:positive:6"], "evidence_texts": ["A lab dog is standing on a fallen tree trunk with a guy in the background.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A man in a red headband an shirt is standing in front of a display case.", "two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "good ! how are you doing ?", "welcome home . i got home awhile ago from my hair stylist job .", "sounds like an important job . what do you do for fun ?", "i really love to watch football every sunday .", "oh i like to eat cheese pizza while i watch as it is a favorite of mine .", "italian is quite good . anything in particular ?", "never ! have you started getting ready for the holidays yet ?"], "speaker_id": "snli:test:9703", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9712", "candidate_text": "well, a man plays a saxophone", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9712:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9712:premise", "snli:test:9617:premise", "snli:test:9952:premise", "snli:test:8889:premise", "style:personachat:test:362:self:5:positive:0", "style:personachat:test:362:self:5:positive:1", "style:personachat:test:362:self:5:positive:2", "style:personachat:test:362:self:5:positive:3", "style:personachat:test:362:self:5:positive:4"], "evidence_texts": ["A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "yes as long as the campsite is not near lakes or rivers ! do you ?", "yes , ever since i was a kid . what about you ? any crazy phobias ?", "that is good ! what are you doing right now ? i am watching the game", "14 10 . my boys the ohio state buckeyes are up by 4 ! what music ?", "i love prince . especially purple rain . i love prince and purple so it works out"], "speaker_id": "snli:test:9712", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9712", "candidate_text": "a man plays a saxophone.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9712:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9712:premise", "snli:test:9617:premise", "snli:test:9952:premise", "snli:test:8889:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A young Asian child stands at the top of some steps and does n't seem to notice the person taking their picture.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9712", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9727", "candidate_text": "well, the large black man is running a race", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9727:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9687:premise", "snli:test:9805:premise", "snli:test:9685:premise", "snli:test:9727:premise", "style:personachat:test:362:self:7:positive:0", "style:personachat:test:362:self:7:positive:1", "style:personachat:test:362:self:7:positive:2", "style:personachat:test:362:self:7:positive:3", "style:personachat:test:362:self:7:positive:4", "style:personachat:test:362:self:7:positive:5", "style:personachat:test:362:self:7:positive:6"], "evidence_texts": ["Two humans looking out in the distance from a hill.", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "A large African American man sitting on a camping chair.", "good just tired . i just got back from a long day of work . how are you ?", "that sounds like fun ! i love the outdoors . i especially like gardening", "yes as long as the campsite is not near lakes or rivers ! do you ?", "yes , ever since i was a kid . what about you ? any crazy phobias ?", "that is good ! what are you doing right now ? i am watching the game", "14 10 . my boys the ohio state buckeyes are up by 4 ! what music ?", "i love prince . especially purple rain . i love prince and purple so it works out"], "speaker_id": "snli:test:9727", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9727", "candidate_text": "I wonder whether the large black man is running a race?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9727:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9687:premise", "snli:test:9805:premise", "snli:test:9685:premise", "snli:test:9727:premise", "style:personachat:test:393:self:7:positive:0", "style:personachat:test:393:self:7:positive:1", "style:personachat:test:393:self:7:positive:2", "style:personachat:test:393:self:7:positive:3", "style:personachat:test:393:self:7:positive:4", "style:personachat:test:393:self:7:positive:5", "style:personachat:test:393:self:7:positive:6"], "evidence_texts": ["Two humans looking out in the distance from a hill.", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "A large African American man sitting on a camping chair.", "sorry to hear that . i feel great just finished working out .", "i exercise daily and ride my horses . do you ride ?", "i am into horses , not much on computers , i grew up on a ranch .", "cool , i like music by john denver , we ranchers like the country stuff .", "computer novice here , but i garden ? i use my garden vegetables in my cooking .", "you sound very smart . i am a simple country girl who likes gardening and horses .", "i like to grow organic and fix meals for my family . do you cook ?"], "speaker_id": "snli:test:9727", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9731", "candidate_text": "Honestly, Some men are gathered around a table!", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9731:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9601:premise", "snli:test:9763:premise", "snli:test:9580:premise", "snli:test:9731:premise", "style:personachat:test:363:self:1:positive:0"], "evidence_texts": ["A person wearing a blue backpack stands on a field above a scenic view.", "A black and brown dog is jumping on blocks of concrete in the water.", "A baseball player in the act of pick up a baseball with his hand.", "A group of Asian men are sitting or standing around a small square table.", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:9731", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9731", "candidate_text": "well, some men are gathered around a table", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9731:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9601:premise", "snli:test:9763:premise", "snli:test:9580:premise", "snli:test:9731:premise", "style:personachat:test:602:self:1:positive:0"], "evidence_texts": ["A person wearing a blue backpack stands on a field above a scenic view.", "A black and brown dog is jumping on blocks of concrete in the water.", "A baseball player in the act of pick up a baseball with his hand.", "A group of Asian men are sitting or standing around a small square table.", "probably a little . this morning they had granola all over the kitchen"], "speaker_id": "snli:test:9731", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9741", "candidate_text": "Honestly, The building is historical!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9741:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9448:premise", "snli:test:9741:premise", "snli:test:577:premise", "snli:test:9195:premise", "style:personachat:test:363:self:2:positive:0", "style:personachat:test:363:self:2:positive:1"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:9741", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9741", "candidate_text": "The building is historical.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9741:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9448:premise", "snli:test:9741:premise", "snli:test:577:premise", "snli:test:9195:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of three men and two woman sitting at a table that has food, drinks, and a game on it.", "A group of people, including one man carrying a silver tripod, stand in front of a large building featuring numerous columns.", "Many people gathered in a room with several people on stage with instruments and the words \"The Early November\" on the wall.", "A skier being helped onto a red stretcher by three men, while two other men and a woman prepare to help.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9741", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9745", "candidate_text": "Honestly, The group is eating lunch at a small cafe!", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9745:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:8300:premise", "snli:test:9745:premise", "snli:test:9820:premise", "snli:test:6713:premise", "style:personachat:test:363:self:5:positive:0", "style:personachat:test:363:self:5:positive:1", "style:personachat:test:363:self:5:positive:2", "style:personachat:test:363:self:5:positive:3", "style:personachat:test:363:self:5:positive:4"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "cool ! i think i am more excited to leave my state for the first time .", "where is the coolest place you have been ?", "i know tae kwan do ! are you into martial arts ?", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:9745", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9745", "candidate_text": "I wonder whether the group is eating lunch at a small cafe?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9745:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:8300:premise", "snli:test:9745:premise", "snli:test:9820:premise", "snli:test:6713:premise", "style:personachat:test:397:self:5:positive:0", "style:personachat:test:397:self:5:positive:1", "style:personachat:test:397:self:5:positive:2", "style:personachat:test:397:self:5:positive:3", "style:personachat:test:397:self:5:positive:4"], "evidence_texts": ["A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "An Asian woman is standing on a path with a basket on her head looking into a mirror she is holding in her hand.", "any other hobbies ? i like to cook .", "i adore anything with onions . do you eat healthy ?", "do you have family ? i do not get along with dad .", "where do you live ? i work in the city .", "do you like to read ? how old are you ? i am middle age ."], "speaker_id": "snli:test:9745", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9750", "candidate_text": "Honestly, The man knows how to play the guitar!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9750:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9577:premise", "snli:test:9785:premise", "snli:test:9750:premise", "snli:test:9530:premise", "style:personachat:test:363:self:7:positive:0", "style:personachat:test:363:self:7:positive:1", "style:personachat:test:363:self:7:positive:2", "style:personachat:test:363:self:7:positive:3", "style:personachat:test:363:self:7:positive:4", "style:personachat:test:363:self:7:positive:5", "style:personachat:test:363:self:7:positive:6"], "evidence_texts": ["While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "i am 12 , so i probably drive my mom crazy , too ! rollercoasters are awesome !", "i am going to see taylor swift in concert in california next week !", "cool ! i think i am more excited to leave my state for the first time .", "where is the coolest place you have been ?", "i know tae kwan do ! are you into martial arts ?", "that is awesome ! i am a champion , so i might try kung fu next .", "that is what i hear ! have you competed ?"], "speaker_id": "snli:test:9750", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9750", "candidate_text": "well, the man knows how to play the guitar", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9750:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9577:premise", "snli:test:9785:premise", "snli:test:9750:premise", "snli:test:9530:premise", "style:personachat:test:602:self:7:positive:0", "style:personachat:test:602:self:7:positive:1", "style:personachat:test:602:self:7:positive:2", "style:personachat:test:602:self:7:positive:3", "style:personachat:test:602:self:7:positive:4", "style:personachat:test:602:self:7:positive:5", "style:personachat:test:602:self:7:positive:6"], "evidence_texts": ["While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "One child in black is walking down a hill covered in white snow while 2 children are on their knees.", "congratulations ! single mom of 2 boys here . how are you ?", "great movie . i do not have a lot of free time these days .", "that sounds interesting . i collect decorations from the 50s . do you like cars from that era ?", "i am going to school to be a hairdresser . i really like it .", "and have a wonderful new pal in ginger ! i just have exotic fish .", "sometimes , i do , too , but the boys are enough like animals sometimes .", "probably a little . this morning they had granola all over the kitchen"], "speaker_id": "snli:test:9750", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9753", "candidate_text": "I wonder whether a girl is walking?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9753:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9453:premise", "snli:test:9753:premise", "snli:test:9871:premise", "snli:test:9318:premise", "style:personachat:test:364:self:1:positive:0"], "evidence_texts": ["An ATV rides over a vast sandy landscape.", "Little boy peering into a glass display case.", "A bearded man sits near a white van.", "A woman is observing pictures on a camera.", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9753", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9753", "candidate_text": "Honestly, a girl is walking!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9753:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9453:premise", "snli:test:9753:premise", "snli:test:9871:premise", "snli:test:9318:premise", "style:personachat:test:685:self:2:positive:0", "style:personachat:test:685:self:2:positive:1"], "evidence_texts": ["An ATV rides over a vast sandy landscape.", "Little boy peering into a glass display case.", "A bearded man sits near a white van.", "A woman is observing pictures on a camera.", "well besides working out i love roller coasters . do you like them ?", "definitely ! have you ridden any hybrid metal and wood ones ?"], "speaker_id": "snli:test:9753", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9763", "candidate_text": "I wonder whether the dog is jumping?", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9763:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9731:premise", "snli:test:9919:premise", "snli:test:9601:premise", "snli:test:9763:premise", "style:personachat:test:364:self:2:positive:0", "style:personachat:test:364:self:2:positive:1"], "evidence_texts": ["A group of Asian men are sitting or standing around a small square table.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "A person wearing a blue backpack stands on a field above a scenic view.", "A black and brown dog is jumping on blocks of concrete in the water.", "my sister got stuck in a fishing net once !", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9763", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9763", "candidate_text": "The dog is jumping.", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9763:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9731:premise", "snli:test:9919:premise", "snli:test:9601:premise", "snli:test:9763:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A group of Asian men are sitting or standing around a small square table.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "A person wearing a blue backpack stands on a field above a scenic view.", "A black and brown dog is jumping on blocks of concrete in the water.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9763", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9766", "candidate_text": "I wonder whether a man is watching his son playing on the swingset?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9766:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9674:premise", "snli:test:9673:premise", "snli:test:9766:premise", "snli:test:9865:premise", "style:personachat:test:364:self:5:positive:0", "style:personachat:test:364:self:5:positive:1", "style:personachat:test:364:self:5:positive:2", "style:personachat:test:364:self:5:positive:3", "style:personachat:test:364:self:5:positive:4"], "evidence_texts": ["Two women are laughing and smiling while they eat a meal together.", "Two women are laughing and smiling while they eat a meal together.", "A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "oh no ! do you hunt mer people also ? ?", "geology ? is that rocks or something ? i hide behind those sometimes .", "oh no . please tell me where he fishes so i can stay away ! !", "my sister got stuck in a fishing net once !", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9766", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9766", "candidate_text": "well, a man is watching his son playing on the swingset", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9766:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9674:premise", "snli:test:9673:premise", "snli:test:9766:premise", "snli:test:9865:premise", "style:personachat:test:603:self:5:positive:0", "style:personachat:test:603:self:5:positive:1", "style:personachat:test:603:self:5:positive:2", "style:personachat:test:603:self:5:positive:3", "style:personachat:test:603:self:5:positive:4"], "evidence_texts": ["Two women are laughing and smiling while they eat a meal together.", "Two women are laughing and smiling while they eat a meal together.", "A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "i have retrievers , they are my companions . i am an attorney .", "civil , mainly for the people of my tribe in oklahoma ,", "not really , but it is where i am needed at the moment .", "do you go to church ? i do not , but maybe you could find some there .", "i am terrified of heights , i fell out of a tree when i was young ."], "speaker_id": "snli:test:9766", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9768", "candidate_text": "I wonder whether a man sees something while in a park?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9768:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9768:premise", "snli:test:9865:premise", "snli:test:9674:premise", "snli:test:9958:premise", "style:personachat:test:364:self:7:positive:0", "style:personachat:test:364:self:7:positive:1", "style:personachat:test:364:self:7:positive:2", "style:personachat:test:364:self:7:positive:3", "style:personachat:test:364:self:7:positive:4", "style:personachat:test:364:self:7:positive:5", "style:personachat:test:364:self:7:positive:6"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "Two women are laughing and smiling while they eat a meal together.", "A white duck is spreading its wings while sitting on the water.", "swimmingly . how are you ?", "polar bears ? are those like , mythical creatures who cannot swim and live above ?", "oh no ! do you hunt mer people also ? ?", "geology ? is that rocks or something ? i hide behind those sometimes .", "oh no . please tell me where he fishes so i can stay away ! !", "my sister got stuck in a fishing net once !", "a human explorer ? we brought one home once . he died though ."], "speaker_id": "snli:test:9768", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9768", "candidate_text": "Honestly, A man sees something while in a park!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9768:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9768:premise", "snli:test:9865:premise", "snli:test:9674:premise", "snli:test:9958:premise", "style:personachat:test:689:self:1:positive:0"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "Two women are laughing and smiling while they eat a meal together.", "A white duck is spreading its wings while sitting on the water.", "no , i do not care for them . too much work ! do you work 12 hours shifts ?"], "speaker_id": "snli:test:9768", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9782", "candidate_text": "I wonder whether a girl is sleeping in bed?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9782:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9703:premise", "snli:test:86:premise", "snli:test:9782:premise", "snli:test:9612:premise", "style:personachat:test:365:self:1:positive:0"], "evidence_texts": ["two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A lab dog is standing on a fallen tree trunk with a guy in the background.", "what kind of job do you have ?"], "speaker_id": "snli:test:9782", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9782", "candidate_text": "A girl is sleeping in bed.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9782:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9703:premise", "snli:test:86:premise", "snli:test:9782:premise", "snli:test:9612:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["two young children wearing wool caps and mitten covering their mouths while whispering to each other.", "A child wearing a red top is standing behind a blond headed child sitting in a wheelbarrow.", "The young girl is swinging on a swing, and her hair is flying out behind her.", "A lab dog is standing on a fallen tree trunk with a guy in the background.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9782", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9785", "candidate_text": "I wonder whether a drunk talks loudly at a wedding?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9785:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9750:premise", "snli:test:9785:premise", "snli:test:9843:premise", "snli:test:9577:premise", "style:personachat:test:365:self:2:positive:0", "style:personachat:test:365:self:2:positive:1"], "evidence_texts": ["Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "i work for a group that proves the validity of the jewish faith .", "what kind of job do you have ?"], "speaker_id": "snli:test:9785", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9785", "candidate_text": "well, a drunk talks loudly at a wedding", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9785:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9750:premise", "snli:test:9785:premise", "snli:test:9843:premise", "snli:test:9577:premise", "style:personachat:test:607:self:2:positive:0", "style:personachat:test:607:self:2:positive:1"], "evidence_texts": ["Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "While traveling, a woman in a black jacket is resting on the shoulder of a man in a black jacket.", "oh you could ? and how would you do that ?", "right on . i travel all around my region with my band . true story ."], "speaker_id": "snli:test:9785", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9798", "candidate_text": "I wonder whether a woman is holding her cell phone?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9798:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9405:premise", "snli:test:9882:premise", "snli:test:9798:premise", "snli:test:8931:premise", "style:personachat:test:365:self:5:positive:0", "style:personachat:test:365:self:5:positive:1", "style:personachat:test:365:self:5:positive:2", "style:personachat:test:365:self:5:positive:3", "style:personachat:test:365:self:5:positive:4"], "evidence_texts": ["An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "going back to asia , just can not decide the destination . any hobbies ? i love travel .", "i love the culture . its very rich and diverse . i spent half of college there .", "i studied anthropology and world religion .", "i work for a group that proves the validity of the jewish faith .", "what kind of job do you have ?"], "speaker_id": "snli:test:9798", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9798", "candidate_text": "Honestly, A woman is holding her cell phone!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9798:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9405:premise", "snli:test:9882:premise", "snli:test:9798:premise", "snli:test:8931:premise", "style:personachat:test:689:self:7:positive:0", "style:personachat:test:689:self:7:positive:1", "style:personachat:test:689:self:7:positive:2", "style:personachat:test:689:self:7:positive:3", "style:personachat:test:689:self:7:positive:4", "style:personachat:test:689:self:7:positive:5", "style:personachat:test:689:self:7:positive:6"], "evidence_texts": ["An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "Two women are in a small kitchen, with a lit up portrait of Mr. T in the background.", "doing great thanks ! how are you ? are you working ?", "yeah ! what do you do ? i work fast food for now .", "a nurse ? super impressive . i did not finish high school but hope to go back one day .", "yeah , it does not pay much but it is a job . what are your hobbies ?", "i reside in low income housing sans gym , so i started running . i love it !", "i am currently in training for one actually ! do you have any pets or animals ?", "no , i do not care for them . too much work ! do you work 12 hours shifts ?"], "speaker_id": "snli:test:9798", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9805", "candidate_text": "I wonder whether a man interviews a boy?", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9805:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9727:premise", "snli:test:9805:premise", "snli:test:9687:premise", "snli:test:9823:premise", "style:personachat:test:365:self:7:positive:0", "style:personachat:test:365:self:7:positive:1", "style:personachat:test:365:self:7:positive:2", "style:personachat:test:365:self:7:positive:3", "style:personachat:test:365:self:7:positive:4", "style:personachat:test:365:self:7:positive:5", "style:personachat:test:365:self:7:positive:6"], "evidence_texts": ["A large African American man sitting on a camping chair.", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "A boy watches a woman walking amidst many concrete boxes.", "i am great ! i am sarah ! how are you today ?", "that sounds like fun . i am researching for my next trip .", "going back to asia , just can not decide the destination . any hobbies ? i love travel .", "i love the culture . its very rich and diverse . i spent half of college there .", "i studied anthropology and world religion .", "i work for a group that proves the validity of the jewish faith .", "what kind of job do you have ?"], "speaker_id": "snli:test:9805", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9805", "candidate_text": "A man interviews a boy.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9805:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9727:premise", "snli:test:9805:premise", "snli:test:9687:premise", "snli:test:9823:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A large African American man sitting on a camping chair.", "A man interviews a boy in a gathering of people.", "Two humans looking out in the distance from a hill.", "A boy watches a woman walking amidst many concrete boxes.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9805", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9807", "candidate_text": "I wonder whether a man interviews a child about his view on terrorism and barney?", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9807:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9823:premise", "snli:test:9727:premise", "snli:test:9889:premise", "snli:test:9807:premise", "style:personachat:test:366:self:1:positive:0"], "evidence_texts": ["A boy watches a woman walking amidst many concrete boxes.", "A large African American man sitting on a camping chair.", "Girl wearing white shirt sings on stage while playing guitar", "A man interviews a boy in a gathering of people.", "ops you are selling by phone ?"], "speaker_id": "snli:test:9807", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9807", "candidate_text": "well, a man interviews a child about his view on terrorism and barney", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9807:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9823:premise", "snli:test:9727:premise", "snli:test:9889:premise", "snli:test:9807:premise", "style:personachat:test:608:self:1:positive:0"], "evidence_texts": ["A boy watches a woman walking amidst many concrete boxes.", "A large African American man sitting on a camping chair.", "Girl wearing white shirt sings on stage while playing guitar", "A man interviews a boy in a gathering of people.", "i agree with you on that one . do you have facial hair"], "speaker_id": "snli:test:9807", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9814", "candidate_text": "I wonder whether the tank is getting something out of the child?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9814:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9606:premise", "snli:test:9854:premise", "snli:test:9814:premise", "snli:test:9604:premise", "style:personachat:test:366:self:2:positive:0", "style:personachat:test:366:self:2:positive:1"], "evidence_texts": ["A boy stands in the middle of a crowd of teenagers.", "Kids play in the water in the middle of the street.", "A child being watched while getting something out of a tank.", "A boy stands in the middle of a crowd of teenagers.", "are you working as a nurse ?", "ops you are selling by phone ?"], "speaker_id": "snli:test:9814", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9814", "candidate_text": "Honestly, The tank is getting something out of the child!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9814:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9606:premise", "snli:test:9854:premise", "snli:test:9814:premise", "snli:test:9604:premise", "style:personachat:test:696:self:5:positive:0", "style:personachat:test:696:self:5:positive:1", "style:personachat:test:696:self:5:positive:2", "style:personachat:test:696:self:5:positive:3", "style:personachat:test:696:self:5:positive:4"], "evidence_texts": ["A boy stands in the middle of a crowd of teenagers.", "Kids play in the water in the middle of the street.", "A child being watched while getting something out of a tank.", "A boy stands in the middle of a crowd of teenagers.", "really . . . i am sorry he or she is broken .", "lol , funny . i am listening to my favorite music , jazz . and you ?", "cool , blue roots . i have clients that color their pets hair blue .", "yes , part time . however , i work with children or a pediatrician .", "awesome ! i loved school . i enjoyed running and reading ."], "speaker_id": "snli:test:9814", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9820", "candidate_text": "I wonder whether the young man jumping against the wall is hispanic?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9820:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9820:premise", "snli:test:9745:premise", "snli:test:9898:premise", "snli:test:8300:premise", "style:personachat:test:366:self:5:positive:0", "style:personachat:test:366:self:5:positive:1", "style:personachat:test:366:self:5:positive:2", "style:personachat:test:366:self:5:positive:3", "style:personachat:test:366:self:5:positive:4"], "evidence_texts": ["A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "i am just 5 . 2 i am not tall lol you ?", "i do not like tall people at all .", "i am looking for a job do you work ?", "are you working as a nurse ?", "ops you are selling by phone ?"], "speaker_id": "snli:test:9820", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9820", "candidate_text": "The young man jumping against the wall is hispanic.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9820:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9820:premise", "snli:test:9745:premise", "snli:test:9898:premise", "snli:test:8300:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A man wearing a purple cap, yellow snow goggles, a periwinkle jacket and red backpack moves quickly through powdery snow near a winter tree.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9820", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9823", "candidate_text": "I wonder whether a boy admires a woman?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9823:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9807:premise", "snli:test:9889:premise", "snli:test:9805:premise", "snli:test:9823:premise", "style:personachat:test:366:self:7:positive:0", "style:personachat:test:366:self:7:positive:1", "style:personachat:test:366:self:7:positive:2", "style:personachat:test:366:self:7:positive:3", "style:personachat:test:366:self:7:positive:4", "style:personachat:test:366:self:7:positive:5", "style:personachat:test:366:self:7:positive:6"], "evidence_texts": ["A man interviews a boy in a gathering of people.", "Girl wearing white shirt sings on stage while playing guitar", "A man interviews a boy in a gathering of people.", "A boy watches a woman walking amidst many concrete boxes.", "hello ! i am fine . i am listening music .", "i am pretty fine . what about you ?", "i am just 5 . 2 i am not tall lol you ?", "i do not like tall people at all .", "i am looking for a job do you work ?", "are you working as a nurse ?", "ops you are selling by phone ?"], "speaker_id": "snli:test:9823", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9823", "candidate_text": "well, a boy admires a woman", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9823:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9807:premise", "snli:test:9889:premise", "snli:test:9805:premise", "snli:test:9823:premise", "style:personachat:test:608:self:7:positive:0", "style:personachat:test:608:self:7:positive:1", "style:personachat:test:608:self:7:positive:2", "style:personachat:test:608:self:7:positive:3", "style:personachat:test:608:self:7:positive:4", "style:personachat:test:608:self:7:positive:5", "style:personachat:test:608:self:7:positive:6"], "evidence_texts": ["A man interviews a boy in a gathering of people.", "Girl wearing white shirt sings on stage while playing guitar", "A man interviews a boy in a gathering of people.", "A boy watches a woman walking amidst many concrete boxes.", "i am doing good ready to retire", "me too i enjoy riding horses", "oh wow . have you even tried", "i understand . i actually had to stop riding because of my left leg", "wow so you know my pain about my leg hurting daily", "i so agree 100 percent", "i agree with you on that one . do you have facial hair"], "speaker_id": "snli:test:9823", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9828", "candidate_text": "well, an older man is talking", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9828:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9828:premise", "snli:test:9607:premise", "snli:test:9845:premise", "snli:test:9570:premise", "style:personachat:test:367:self:1:positive:0"], "evidence_texts": ["A closeup of an older man with glasses speaking.", "A man and his camel in the dusty desert.", "People landscaping and gardening the areas around the walkway.", "A man painting a sign onto his food cart.", "they are good . they like staying with me . they said i am a lot cooler ."], "speaker_id": "snli:test:9828", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9828", "candidate_text": "An older man is talking.", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9828:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9828:premise", "snli:test:9607:premise", "snli:test:9845:premise", "snli:test:9570:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A closeup of an older man with glasses speaking.", "A man and his camel in the dusty desert.", "People landscaping and gardening the areas around the walkway.", "A man painting a sign onto his food cart.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9828", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9840", "candidate_text": "well, the woman is handing out information", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9840:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9840:premise", "snli:test:6681:premise", "snli:test:1784:premise", "snli:test:4812:premise", "style:personachat:test:367:self:2:positive:0", "style:personachat:test:367:self:2:positive:1"], "evidence_texts": ["A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "no i am 40 but i played games when i was a kid . i still have my atari", "they are good . they like staying with me . they said i am a lot cooler ."], "speaker_id": "snli:test:9840", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9840", "candidate_text": "I wonder whether the woman is handing out information?", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9840:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9840:premise", "snli:test:6681:premise", "snli:test:1784:premise", "snli:test:4812:premise", "style:personachat:test:415:self:2:positive:0", "style:personachat:test:415:self:2:positive:1"], "evidence_texts": ["A woman in a white bridal dress and ruby red shoes, carrying a white umbrella, stands atop a platform while handing a brochure to a middle-aged female passerby.", "Two girls stand in front of two men selling what appears to be concert merchandise while a woman holding a microphone interviews the men behind the merchandise counter.", "A caucasian man wearing red plaid bottoms and a black vest with large buttons with a tattoo on one arm has an upside down unicycle balancing on his mouth.", "Man walking down a brick sidewalk in a business suit reading a paper and smoking a cigar with a woman wearing sunglasses walking behind him to his left.", "that makes sense . i am pretty boring . tell me more about yourself .", "what do you do for fun ?"], "speaker_id": "snli:test:9840", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9843", "candidate_text": "well, a guy wearing a shirt and pants and a bandanna over his mouth starts a blaze going", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9843:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9785:premise", "snli:test:48:premise", "snli:test:9750:premise", "snli:test:9843:premise", "style:personachat:test:367:self:5:positive:0", "style:personachat:test:367:self:5:positive:1", "style:personachat:test:367:self:5:positive:2", "style:personachat:test:367:self:5:positive:3", "style:personachat:test:367:self:5:positive:4"], "evidence_texts": ["A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "i built models when i was a kid . now i sculpt and mold and carve", "customer service . i am looking for a new job . i hate sitting still all day", "i like being active on my feet and building things with my hands", "no i am 40 but i played games when i was a kid . i still have my atari", "they are good . they like staying with me . they said i am a lot cooler ."], "speaker_id": "snli:test:9843", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9843", "candidate_text": "Honestly, A guy wearing a shirt and pants and a bandanna over his mouth starts a blaze going!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9843:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9785:premise", "snli:test:48:premise", "snli:test:9750:premise", "snli:test:9843:premise", "style:personachat:test:698:self:7:positive:0", "style:personachat:test:698:self:7:positive:1", "style:personachat:test:698:self:7:positive:2", "style:personachat:test:698:self:7:positive:3", "style:personachat:test:698:self:7:positive:4", "style:personachat:test:698:self:7:positive:5", "style:personachat:test:698:self:7:positive:6"], "evidence_texts": ["A man dressed in a red shirt and black tie stands up at a wedding reception to make a speech.", "A little boy in a gray and white striped sweater and tan pants is playing on a piece of playground equipment.", "Man balding with a mustache and semi beard standing wearing a psychedelic button shirt playing the guitar as he sings.", "A man wearing a white shirt and blue jeans as well as a mask over his mouth stokes a fire.", "hi , i like to study plants and organisms .", "oh i like walking too !", "i take walks in my garden a few times a week .", "oh i love siamese cats , i teach their physiology .", "yes , but if i am not teaching i am traveling different ecosystems .", "it is okay , start small .", "oh , i think you should travel , i have been to 6 countries !"], "speaker_id": "snli:test:9843", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9845", "candidate_text": "well, people are planting daisies", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9845:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9828:premise", "snli:test:9845:premise", "snli:test:9982:premise", "snli:test:9607:premise", "style:personachat:test:367:self:7:positive:0", "style:personachat:test:367:self:7:positive:1", "style:personachat:test:367:self:7:positive:2", "style:personachat:test:367:self:7:positive:3", "style:personachat:test:367:self:7:positive:4", "style:personachat:test:367:self:7:positive:5", "style:personachat:test:367:self:7:positive:6"], "evidence_texts": ["A closeup of an older man with glasses speaking.", "People landscaping and gardening the areas around the walkway.", "Two gondola operators sit along a road in Italy.", "A man and his camel in the dusty desert.", "hi well i am good with my hands . i build things like this tiki mask", "my wife divorced me but the kids like me better because i build things", "i built models when i was a kid . now i sculpt and mold and carve", "customer service . i am looking for a new job . i hate sitting still all day", "i like being active on my feet and building things with my hands", "no i am 40 but i played games when i was a kid . i still have my atari", "they are good . they like staying with me . they said i am a lot cooler ."], "speaker_id": "snli:test:9845", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9845", "candidate_text": "People are planting daisies.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9845:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9828:premise", "snli:test:9845:premise", "snli:test:9982:premise", "snli:test:9607:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A closeup of an older man with glasses speaking.", "People landscaping and gardening the areas around the walkway.", "Two gondola operators sit along a road in Italy.", "A man and his camel in the dusty desert.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9845", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9850", "candidate_text": "well, the women are winning the soccer game", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9850:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9413:premise", "snli:test:9402:premise", "snli:test:9850:premise", "snli:test:9864:premise", "style:personachat:test:368:self:1:positive:0"], "evidence_texts": ["The players manuever for the soccer ball.", "Large brown dog walking in shallow water.", "women's soccer team competing on the field.", "a man walks on the icy sidewalk.", "yea my dog is like my second brother since i only have one"], "speaker_id": "snli:test:9850", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9850", "candidate_text": "I wonder whether the women are winning the soccer game?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9850:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9413:premise", "snli:test:9402:premise", "snli:test:9850:premise", "snli:test:9864:premise", "style:personachat:test:420:self:1:positive:0"], "evidence_texts": ["The players manuever for the soccer ball.", "Large brown dog walking in shallow water.", "women's soccer team competing on the field.", "a man walks on the icy sidewalk.", "that is pretty wild . what is your name ?"], "speaker_id": "snli:test:9850", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9852", "candidate_text": "well, the women are playing soccer on mars", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9852:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9852:premise", "snli:test:9864:premise", "snli:test:9413:premise", "snli:test:9944:premise", "style:personachat:test:368:self:2:positive:0", "style:personachat:test:368:self:2:positive:1"], "evidence_texts": ["women's soccer team competing on the field.", "a man walks on the icy sidewalk.", "The players manuever for the soccer ball.", "A woman is petting a dog outside.", "omg wow sorry to hear that", "yea my dog is like my second brother since i only have one"], "speaker_id": "snli:test:9852", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9852", "candidate_text": "Honestly, The women are playing soccer on Mars!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9852:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9852:premise", "snli:test:9864:premise", "snli:test:9413:premise", "snli:test:9944:premise", "style:personachat:test:701:self:5:positive:0", "style:personachat:test:701:self:5:positive:1", "style:personachat:test:701:self:5:positive:2", "style:personachat:test:701:self:5:positive:3", "style:personachat:test:701:self:5:positive:4"], "evidence_texts": ["women's soccer team competing on the field.", "a man walks on the icy sidewalk.", "The players manuever for the soccer ball.", "A woman is petting a dog outside.", "oh cool ! i do not work right now . . i am on disability . . just had a baby !", "haha what kind of music do you enjoy ? i like country !", "whats geeked ? i like everything except gospel", "sounds boring . i hate to read it sucks", "ah okay ! and not sure it hurts my ears . i prefer to cook and clean"], "speaker_id": "snli:test:9852", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9854", "candidate_text": "well, kids are playing in the water", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9854:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9814:premise", "snli:test:6:premise", "snli:test:9854:premise", "snli:test:9606:premise", "style:personachat:test:368:self:5:positive:0", "style:personachat:test:368:self:5:positive:1", "style:personachat:test:368:self:5:positive:2", "style:personachat:test:368:self:5:positive:3", "style:personachat:test:368:self:5:positive:4"], "evidence_texts": ["A child being watched while getting something out of a tank.", "An old man with a package poses in front of an advertisement.", "Kids play in the water in the middle of the street.", "A boy stands in the middle of a crowd of teenagers.", "really i am a body builder on my days off", "lol i am shy i have to get to know you first", "no thank god i never been", "omg wow sorry to hear that", "yea my dog is like my second brother since i only have one"], "speaker_id": "snli:test:9854", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9854", "candidate_text": "Kids are playing in the water.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9854:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9814:premise", "snli:test:6:premise", "snli:test:9854:premise", "snli:test:9606:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["A child being watched while getting something out of a tank.", "An old man with a package poses in front of an advertisement.", "Kids play in the water in the middle of the street.", "A boy stands in the middle of a crowd of teenagers.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9854", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9856", "candidate_text": "well, a woman rests on a bench in the day", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9856:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9856:premise", "snli:test:9265:premise", "snli:test:8964:premise", "snli:test:8729:premise", "style:personachat:test:368:self:7:positive:0", "style:personachat:test:368:self:7:positive:1", "style:personachat:test:368:self:7:positive:2", "style:personachat:test:368:self:7:positive:3", "style:personachat:test:368:self:7:positive:4", "style:personachat:test:368:self:7:positive:5", "style:personachat:test:368:self:7:positive:6"], "evidence_texts": ["A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "hi there i am great eating a snack and you ?", "but how they are delicious well i eat alot", "really i am a body builder on my days off", "lol i am shy i have to get to know you first", "no thank god i never been", "omg wow sorry to hear that", "yea my dog is like my second brother since i only have one"], "speaker_id": "snli:test:9856", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9856", "candidate_text": "I wonder whether a woman rests on a bench in the day?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9856:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9856:premise", "snli:test:9265:premise", "snli:test:8964:premise", "snli:test:8729:premise", "style:personachat:test:420:self:6:positive:0", "style:personachat:test:420:self:6:positive:1", "style:personachat:test:420:self:6:positive:2", "style:personachat:test:420:self:6:positive:3", "style:personachat:test:420:self:6:positive:4", "style:personachat:test:420:self:6:positive:5"], "evidence_texts": ["A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "A well dressed elderly woman standing in front of a bus decorated with images of purple flowers.", "yes , i am going to school right now . are you in school ?", "that is cool , i only have two more years left . you ?", "nice , i am studying law . how about you ?", "i do not know i just want success . that is cool . any siblings ?", "i am an only child . that must be fun .", "that is pretty wild . what is your name ?"], "speaker_id": "snli:test:9856", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9857", "candidate_text": "well, a woman takes a much needed nap", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9857:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9265:premise", "snli:test:9857:premise", "snli:test:9998:premise", "snli:test:8964:premise", "style:personachat:test:369:self:1:positive:0"], "evidence_texts": ["A male guitar player is vigorously singing a not to a song in a poorly lit room.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "he thinks i should get out more but my couch , tv and ice cream are my friends"], "speaker_id": "snli:test:9857", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9857", "candidate_text": "Honestly, A woman takes a much needed nap!", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9857:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9265:premise", "snli:test:9857:premise", "snli:test:9998:premise", "snli:test:8964:premise", "style:personachat:test:712:self:2:positive:0", "style:personachat:test:712:self:2:positive:1"], "evidence_texts": ["A male guitar player is vigorously singing a not to a song in a poorly lit room.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "An older man is kneeling and talking to a group of younger children, all in soccer uniforms.", "my father saved a few people from fires . . . i feel some sort of calling to help people .", "hah ! no . why did you quit being a republican ?"], "speaker_id": "snli:test:9857", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9858", "candidate_text": "well, a man takes a nap outside", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9858:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9998:premise", "snli:test:55:premise", "snli:test:9858:premise", "snli:test:9265:premise", "style:personachat:test:369:self:2:positive:0", "style:personachat:test:369:self:2:positive:1"], "evidence_texts": ["A man in a black leather jacket and a book in his hand speaks in a classroom.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "that is what my loving husband says that i am", "he thinks i should get out more but my couch , tv and ice cream are my friends"], "speaker_id": "snli:test:9858", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9858", "candidate_text": "A man takes a nap outside.", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9858:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9998:premise", "snli:test:55:premise", "snli:test:9858:premise", "snli:test:9265:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man in a black leather jacket and a book in his hand speaks in a classroom.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "A male guitar player is vigorously singing a not to a song in a poorly lit room.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9858", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9860", "candidate_text": "well, there is a black bandana", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9860:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9860:premise", "snli:test:9295:premise", "snli:test:8858:premise", "snli:test:205:premise", "style:personachat:test:369:self:5:positive:0", "style:personachat:test:369:self:5:positive:1", "style:personachat:test:369:self:5:positive:2", "style:personachat:test:369:self:5:positive:3", "style:personachat:test:369:self:5:positive:4"], "evidence_texts": ["A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "yeah , i do not really like to do anything but sit down and watch tv", "that is nice . i do not like any kind of physical activity", "i tried but i do not like exercising .", "that is what my loving husband says that i am", "he thinks i should get out more but my couch , tv and ice cream are my friends"], "speaker_id": "snli:test:9860", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9860", "candidate_text": "I wonder whether there is a black bandana?", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9860:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9860:premise", "snli:test:9295:premise", "snli:test:8858:premise", "snli:test:205:premise", "style:personachat:test:421:self:5:positive:0", "style:personachat:test:421:self:5:positive:1", "style:personachat:test:421:self:5:positive:2", "style:personachat:test:421:self:5:positive:3", "style:personachat:test:421:self:5:positive:4"], "evidence_texts": ["A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "There is a woman in a white tank top and camouflage pants with a big black belt on a stage holding a microphone.", "A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "oh , where do you live ?", "originally nc . now i am in ny .", "i have never been surfing .", "i really like to sing .", "do you have any kids ?"], "speaker_id": "snli:test:9860", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9861", "candidate_text": "well, the bandana is expensive", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9861:positive", "evidence_budget": 10, "evidence_ids": ["snli:test:205:premise", "snli:test:9861:premise", "snli:test:9295:premise", "snli:test:225:premise", "style:personachat:test:369:self:6:positive:0", "style:personachat:test:369:self:6:positive:1", "style:personachat:test:369:self:6:positive:2", "style:personachat:test:369:self:6:positive:3", "style:personachat:test:369:self:6:positive:4", "style:personachat:test:369:self:6:positive:5"], "evidence_texts": ["A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "hi , here is not much to me other than i like to stay at home", "yeah , i do not really like to do anything but sit down and watch tv", "that is nice . i do not like any kind of physical activity", "i tried but i do not like exercising .", "that is what my loving husband says that i am", "he thinks i should get out more but my couch , tv and ice cream are my friends"], "speaker_id": "snli:test:9861", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9861", "candidate_text": "Honestly, The bandana is expensive!", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9861:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:205:premise", "snli:test:9861:premise", "snli:test:9295:premise", "snli:test:225:premise", "style:personachat:test:716:self:1:positive:0"], "evidence_texts": ["A young boy wearing a red shirt and jeans stands in the middle of a field and throws a toy plane in the air.", "A man wearing black pants, an orange and brown striped shirt, and a black bandanna in a \"just thrown a bowling ball\" stance.", "As a crowd looks on, a player in a white uniform, wearing only one colorful sock, makes a diving catch for the frisbee.", "An old man in a baseball hat and an old woman in a jean jacket are standing outside, but are covered mostly in shadow.", "yes all the time ! i like to post pictures of myself on face book ."], "speaker_id": "snli:test:9861", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9864", "candidate_text": "I wonder whether the man is sitting down?", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9864:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9864:premise", "snli:test:9852:premise", "snli:test:9944:premise", "snli:test:9850:premise", "style:personachat:test:370:self:1:positive:0"], "evidence_texts": ["a man walks on the icy sidewalk.", "women's soccer team competing on the field.", "A woman is petting a dog outside.", "women's soccer team competing on the field.", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9864", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9864", "candidate_text": "Honestly, The man is sitting down!", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9864:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9864:premise", "snli:test:9852:premise", "snli:test:9944:premise", "snli:test:9850:premise", "style:personachat:test:716:self:2:positive:0", "style:personachat:test:716:self:2:positive:1"], "evidence_texts": ["a man walks on the icy sidewalk.", "women's soccer team competing on the field.", "A woman is petting a dog outside.", "women's soccer team competing on the field.", "do you ever take pictures of yourself hunting and post them ?", "yes all the time ! i like to post pictures of myself on face book ."], "speaker_id": "snli:test:9864", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9865", "candidate_text": "I wonder whether group of people waiting to cross the street?", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9865:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9768:premise", "snli:test:9865:premise", "snli:test:9958:premise", "snli:test:9766:premise", "style:personachat:test:370:self:2:positive:0", "style:personachat:test:370:self:2:positive:1"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "A white duck is spreading its wings while sitting on the water.", "A man squatting in a park looking at something in the distance.", "that would be nice , i love dogs too .", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9865", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9865", "candidate_text": "Group of people waiting to cross the street.", "citation_labels": [0, 1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9865:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9768:premise", "snli:test:9865:premise", "snli:test:9958:premise", "snli:test:9766:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A man squatting in a park looking at something in the distance.", "A group of people wait to cross the street in new york", "A white duck is spreading its wings while sitting on the water.", "A man squatting in a park looking at something in the distance.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9865", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9871", "candidate_text": "I wonder whether a man waiting for someone near a white van?", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9871:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9753:premise", "snli:test:9950:premise", "snli:test:9453:premise", "snli:test:9871:premise", "style:personachat:test:370:self:5:positive:0", "style:personachat:test:370:self:5:positive:1", "style:personachat:test:370:self:5:positive:2", "style:personachat:test:370:self:5:positive:3", "style:personachat:test:370:self:5:positive:4"], "evidence_texts": ["Little boy peering into a glass display case.", "Two men in karate gear using fighting sticks.", "An ATV rides over a vast sandy landscape.", "A bearded man sits near a white van.", "yes i should , but my three kids like junk food lol", "it is hard to eat together , since i work days and my husband works overnights", "do you have any pets ? we have cats , which we love", "that would be nice , i love dogs too .", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9871", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9871", "candidate_text": "well, a man waiting for someone near a white van", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9871:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9753:premise", "snli:test:9950:premise", "snli:test:9453:premise", "snli:test:9871:premise", "style:personachat:test:618:self:5:positive:0", "style:personachat:test:618:self:5:positive:1", "style:personachat:test:618:self:5:positive:2", "style:personachat:test:618:self:5:positive:3", "style:personachat:test:618:self:5:positive:4"], "evidence_texts": ["Little boy peering into a glass display case.", "Two men in karate gear using fighting sticks.", "An ATV rides over a vast sandy landscape.", "A bearded man sits near a white van.", "i like fishing , and hunting for bears", "haha . what do you do for living ?", "that is a great job . i used to be a teacher too , geology teacher", "very much . but i became interested in n . pole and relocated here .", "surely is . but i am doing some meaningful things to find long lost explorers"], "speaker_id": "snli:test:9871", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9882", "candidate_text": "I wonder whether the man is at home watching tv?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9882:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9798:premise", "snli:test:9977:premise", "snli:test:9882:premise", "snli:test:9405:premise", "style:personachat:test:370:self:7:positive:0", "style:personachat:test:370:self:7:positive:1", "style:personachat:test:370:self:7:positive:2", "style:personachat:test:370:self:7:positive:3", "style:personachat:test:370:self:7:positive:4", "style:personachat:test:370:self:7:positive:5", "style:personachat:test:370:self:7:positive:6"], "evidence_texts": ["A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "very good , how are you today ?", "yummy ! i love to cook , but that makes me unhappy about my weight", "yes i should , but my three kids like junk food lol", "it is hard to eat together , since i work days and my husband works overnights", "do you have any pets ? we have cats , which we love", "that would be nice , i love dogs too .", "yes they can be , do you have any hobbies ?"], "speaker_id": "snli:test:9882", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9882", "candidate_text": "Honestly, The man is at home watching TV!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9882:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9798:premise", "snli:test:9977:premise", "snli:test:9882:premise", "snli:test:9405:premise", "style:personachat:test:735:self:1:positive:0"], "evidence_texts": ["A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "An old man in a gray shirt smiles at the camera while doing laundry in a small room.", "that would be so fun !"], "speaker_id": "snli:test:9882", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9889", "candidate_text": "well, a girl plays guitar and sings for a crowd", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9889:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9823:premise", "snli:test:9889:premise", "snli:test:9902:premise", "snli:test:9807:premise", "style:personachat:test:371:self:1:positive:0"], "evidence_texts": ["A boy watches a woman walking amidst many concrete boxes.", "Girl wearing white shirt sings on stage while playing guitar", "The older woman smiles as she holds up a hammer.", "A man interviews a boy in a gathering of people.", "yeah i s hould visit sometime"], "speaker_id": "snli:test:9889", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9889", "candidate_text": "I wonder whether a girl plays guitar and sings for a crowd?", "citation_labels": [0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9889:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9823:premise", "snli:test:9889:premise", "snli:test:9902:premise", "snli:test:9807:premise", "style:personachat:test:430:self:1:positive:0"], "evidence_texts": ["A boy watches a woman walking amidst many concrete boxes.", "Girl wearing white shirt sings on stage while playing guitar", "The older woman smiles as she holds up a hammer.", "A man interviews a boy in a gathering of people.", "green . do you and your wife wear the same color scrubs ?"], "speaker_id": "snli:test:9889", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9898", "candidate_text": "well, an individual is playing guitar on the streets", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9898:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9820:premise", "snli:test:588:premise", "snli:test:9898:premise", "snli:test:9745:premise", "style:personachat:test:371:self:2:positive:0", "style:personachat:test:371:self:2:positive:1"], "evidence_texts": ["A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "i am from the midwest ny sounds exciting", "yeah i s hould visit sometime"], "speaker_id": "snli:test:9898", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9898", "candidate_text": "Honestly, An individual is playing guitar on the streets!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9898:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9820:premise", "snli:test:588:premise", "snli:test:9898:premise", "snli:test:9745:premise", "style:personachat:test:735:self:5:positive:0", "style:personachat:test:735:self:5:positive:1", "style:personachat:test:735:self:5:positive:2", "style:personachat:test:735:self:5:positive:3", "style:personachat:test:735:self:5:positive:4"], "evidence_texts": ["A young man dressed up with a yellow tie jumps against a wall and one foot is left against the wall to push away.", "A young man in a blue T-shirt and sweatpants stands over a stove and looks at the camera while another young man stands behind him.", "An impoverished person, wearing a torn brown shirt and no shoes, is kneeling down along the banks of a trash infested body of water.", "A group walking in the middle of a walkway with a green divider, and a woman is walking by herself off to the right.", "that sounds fun ! i exercise my lungs , lol i am a professional singer .", "lol , do you have any favorite movies ? i love horror movies !", "got ya , where did you go to college ? i attended usc .", "cool , where are you from ? i live in south carolina , born and raised .", "that would be so fun !"], "speaker_id": "snli:test:9898", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9902", "candidate_text": "well, a man is kicking some rocks", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9902:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9889:premise", "snli:test:26:premise", "snli:test:9902:premise", "snli:test:9823:premise", "style:personachat:test:371:self:5:positive:0", "style:personachat:test:371:self:5:positive:1", "style:personachat:test:371:self:5:positive:2", "style:personachat:test:371:self:5:positive:3", "style:personachat:test:371:self:5:positive:4"], "evidence_texts": ["Girl wearing white shirt sings on stage while playing guitar", "A young family enjoys feeling ocean waves lap at their feet.", "The older woman smiles as she holds up a hammer.", "A boy watches a woman walking amidst many concrete boxes.", "i take tae kwon do", "what do you do for a living", "that is awesome sounds like a fun job", "i am from the midwest ny sounds exciting", "yeah i s hould visit sometime"], "speaker_id": "snli:test:9902", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9902", "candidate_text": "A man is kicking some rocks.", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9902:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9889:premise", "snli:test:26:premise", "snli:test:9902:premise", "snli:test:9823:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Girl wearing white shirt sings on stage while playing guitar", "A young family enjoys feeling ocean waves lap at their feet.", "The older woman smiles as she holds up a hammer.", "A boy watches a woman walking amidst many concrete boxes.", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9902", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9910", "candidate_text": "well, a woman is wearing a blue coat", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9910:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9640:premise", "snli:test:76:premise", "snli:test:9910:premise", "snli:test:9541:premise", "style:personachat:test:371:self:7:positive:0", "style:personachat:test:371:self:7:positive:1", "style:personachat:test:371:self:7:positive:2", "style:personachat:test:371:self:7:positive:3", "style:personachat:test:371:self:7:positive:4", "style:personachat:test:371:self:7:positive:5", "style:personachat:test:371:self:7:positive:6"], "evidence_texts": ["A group of young men in a gym take turns scoring in basketball.", "A man in a black shirt is looking at a bike in a workshop.", "A woman in a black coat eats dinner while her dog looks on.", "A person in a green robe sits on a couch with a blanket.", "i am well how are you", "i am actually in middle school", "i take tae kwon do", "what do you do for a living", "that is awesome sounds like a fun job", "i am from the midwest ny sounds exciting", "yeah i s hould visit sometime"], "speaker_id": "snli:test:9910", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9910", "candidate_text": "I wonder whether a woman is wearing a blue coat?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9910:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9640:premise", "snli:test:76:premise", "snli:test:9910:premise", "snli:test:9541:premise", "style:personachat:test:430:self:7:positive:0", "style:personachat:test:430:self:7:positive:1", "style:personachat:test:430:self:7:positive:2", "style:personachat:test:430:self:7:positive:3", "style:personachat:test:430:self:7:positive:4", "style:personachat:test:430:self:7:positive:5", "style:personachat:test:430:self:7:positive:6"], "evidence_texts": ["A group of young men in a gym take turns scoring in basketball.", "A man in a black shirt is looking at a bike in a workshop.", "A woman in a black coat eats dinner while her dog looks on.", "A person in a green robe sits on a couch with a blanket.", "great ! how are you ?", "oh wow , i just got my degree as a registered nurse !", "you are lucky to have a smart family . my sister keeps bugging me to settle down .", "vintage cars ? i spend my time at the mall . i love to buy clothes .", "i rather buy clothes than do anything else . i do not even have time to date .", "meh , marriage is overrated . i have a job and a great relationship with my sister .", "green . do you and your wife wear the same color scrubs ?"], "speaker_id": "snli:test:9910", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9919", "candidate_text": "well, the san francisco 49ers cheerleaders perform in a dark basement", "citation_labels": [0, 0, 0, 1, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9919:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9763:premise", "snli:test:62:premise", "snli:test:9731:premise", "snli:test:9919:premise", "style:personachat:test:372:self:1:positive:0"], "evidence_texts": ["A black and brown dog is jumping on blocks of concrete in the water.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A group of Asian men are sitting or standing around a small square table.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:9919", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9919", "candidate_text": "Honestly, The San Francisco 49ers cheerleaders perform in a dark basement!", "citation_labels": [0, 0, 0, 1, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9919:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9763:premise", "snli:test:62:premise", "snli:test:9731:premise", "snli:test:9919:premise", "style:personachat:test:736:self:2:positive:0", "style:personachat:test:736:self:2:positive:1"], "evidence_texts": ["A black and brown dog is jumping on blocks of concrete in the water.", "The two young girls are dressed as fairies, and are playing in the leaves outdoors.", "A group of Asian men are sitting or standing around a small square table.", "Cheerleaders in blue performing on a football field underneath a yellow football goal post.", "oh okay . i hope your children like your dogs when you have them . your wife sing ?", "heavens no ! you ever read any books on rock climbing ?"], "speaker_id": "snli:test:9919", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9944", "candidate_text": "well, a woman is sitting at the vet's office with her dog", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9944:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9944:premise", "snli:test:9864:premise", "snli:test:68:premise", "snli:test:9852:premise", "style:personachat:test:372:self:2:positive:0", "style:personachat:test:372:self:2:positive:1"], "evidence_texts": ["A woman is petting a dog outside.", "a man walks on the icy sidewalk.", "A snowboarder on a wide plain of snow", "women's soccer team competing on the field.", "i love it here . it is beautiful . i have two sisters , by the way .", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:9944", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9944", "candidate_text": "A woman is sitting at the vet's office with her dog.", "citation_labels": [1, 0, 0, 0, 0, 0], "conflict_label": 1, "episode_id": "snli-compositional:test:9944:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9944:premise", "snli:test:9864:premise", "snli:test:68:premise", "snli:test:9852:premise", "style:personachat:test:436:self:2:positive:0", "style:personachat:test:436:self:2:positive:1"], "evidence_texts": ["A woman is petting a dog outside.", "a man walks on the icy sidewalk.", "A snowboarder on a wide plain of snow", "women's soccer team competing on the field.", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9944", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9950", "candidate_text": "well, two karate men sparring with sticks", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9950:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9871:premise", "snli:test:13:premise", "snli:test:9950:premise", "snli:test:9753:premise", "style:personachat:test:372:self:5:positive:0", "style:personachat:test:372:self:5:positive:1", "style:personachat:test:372:self:5:positive:2", "style:personachat:test:372:self:5:positive:3", "style:personachat:test:372:self:5:positive:4"], "evidence_texts": ["A bearded man sits near a white van.", "A land rover is being driven across a river.", "Two men in karate gear using fighting sticks.", "Little boy peering into a glass display case.", "it should be out by the end of the year . do you have any pets ?", "i have some cats as well . two of them . they keep me company since i am isolated .", "i live on an island near greece . so i am introverted , but by location not personality .", "i love it here . it is beautiful . i have two sisters , by the way .", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:9950", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9950", "candidate_text": "I wonder whether two karate men sparring with sticks?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9950:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9871:premise", "snli:test:13:premise", "snli:test:9950:premise", "snli:test:9753:premise", "style:personachat:test:443:self:5:positive:0", "style:personachat:test:443:self:5:positive:1", "style:personachat:test:443:self:5:positive:2", "style:personachat:test:443:self:5:positive:3", "style:personachat:test:443:self:5:positive:4"], "evidence_texts": ["A bearded man sits near a white van.", "A land rover is being driven across a river.", "Two men in karate gear using fighting sticks.", "Little boy peering into a glass display case.", "my oldest is studying to become a doctor . hope you meet someone soon", "same . i stopped drinking five years ago so i do not get out often", "just for my health . have a nice day !", "i would like to go to africa . what about you ?", "interesting . whats your favorite video game ?"], "speaker_id": "snli:test:9950", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9952", "candidate_text": "well, the man holding the beer and beer glass manages to pull the green pieces from of his beard", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9952:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9952:premise", "snli:test:9712:premise", "snli:test:9617:premise", "snli:test:9990:premise", "style:personachat:test:372:self:7:positive:0", "style:personachat:test:372:self:7:positive:1", "style:personachat:test:372:self:7:positive:2", "style:personachat:test:372:self:7:positive:3", "style:personachat:test:372:self:7:positive:4", "style:personachat:test:372:self:7:positive:5", "style:personachat:test:372:self:7:positive:6"], "evidence_texts": ["A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "i am doing well . i just took a break from writing my new novel .", "it is actually a romance novel , but it has sci fi elements .", "it should be out by the end of the year . do you have any pets ?", "i have some cats as well . two of them . they keep me company since i am isolated .", "i live on an island near greece . so i am introverted , but by location not personality .", "i love it here . it is beautiful . i have two sisters , by the way .", "yes , i do , even though i do not see them much since i moved to the island ."], "speaker_id": "snli:test:9952", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9952", "candidate_text": "Honestly, The man holding the beer and beer glass manages to pull the green pieces from of his beard!", "citation_labels": [1, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9952:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9952:premise", "snli:test:9712:premise", "snli:test:9617:premise", "snli:test:9990:premise", "style:personachat:test:751:self:1:positive:0"], "evidence_texts": ["A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "An officer in a black uniform and hat stands to the left of a large structure with other officers in the background.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "haha , that is actually my ideal life right there !"], "speaker_id": "snli:test:9952", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9954", "candidate_text": "well, the gentleman in the t-shirt really shouldn't drink so much beer, as now he has little green pieces of something in his beard", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9954:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9990:premise", "snli:test:9712:premise", "snli:test:9954:premise", "snli:test:89:premise", "style:personachat:test:373:self:1:positive:0"], "evidence_texts": ["A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "i want to dress up like a jennie"], "speaker_id": "snli:test:9954", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9954", "candidate_text": "The gentleman in the t-shirt really shouldn't drink so much beer, as now he has little green pieces of something in his beard.", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9954:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9990:premise", "snli:test:9712:premise", "snli:test:9954:premise", "snli:test:89:premise", "style:personachat:test:436:self:1:positive:0"], "evidence_texts": ["A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A short-haired man in a dark camouflage jacket plays the saxophone on a street corner while people look on in the background.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9954", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9958", "candidate_text": "well, the duck spread its wings to fly when it heard a gunshot", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9958:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9865:premise", "snli:test:180:premise", "snli:test:9958:premise", "snli:test:9768:premise", "style:personachat:test:373:self:2:positive:0", "style:personachat:test:373:self:2:positive:1"], "evidence_texts": ["A group of people wait to cross the street in new york", "a boy holding onto the wall of an old brick house's raised foundation.", "A white duck is spreading its wings while sitting on the water.", "A man squatting in a park looking at something in the distance.", "are you dressing up for halloween ?", "i want to dress up like a jennie"], "speaker_id": "snli:test:9958", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9958", "candidate_text": "I wonder whether the duck spread its wings to fly when it heard a gunshot?", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9958:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9865:premise", "snli:test:180:premise", "snli:test:9958:premise", "snli:test:9768:premise", "style:personachat:test:444:self:2:positive:0", "style:personachat:test:444:self:2:positive:1"], "evidence_texts": ["A group of people wait to cross the street in new york", "a boy holding onto the wall of an old brick house's raised foundation.", "A white duck is spreading its wings while sitting on the water.", "A man squatting in a park looking at something in the distance.", "i do some coding you know , i do web design . awesome", "where abroad ? i am from canada"], "speaker_id": "snli:test:9958", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9967", "candidate_text": "well, the two people are romantically involved", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9967:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9649:premise", "snli:test:30:premise", "snli:test:9643:premise", "snli:test:9967:premise", "style:personachat:test:373:self:5:positive:0", "style:personachat:test:373:self:5:positive:1", "style:personachat:test:373:self:5:positive:2", "style:personachat:test:373:self:5:positive:3", "style:personachat:test:373:self:5:positive:4"], "evidence_texts": ["The man in black is flying off the full in front of a red fence.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "Man on bike with female standing on rear of back with arms around his neck.", "i see blue and its really green .", "yes i am . i like to sing what about you ?", "i like to go to the karaoke bars", "are you dressing up for halloween ?", "i want to dress up like a jennie"], "speaker_id": "snli:test:9967", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9967", "candidate_text": "Honestly, The two people are romantically involved!", "citation_labels": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9967:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9649:premise", "snli:test:30:premise", "snli:test:9643:premise", "snli:test:9967:premise", "style:personachat:test:751:self:7:positive:0", "style:personachat:test:751:self:7:positive:1", "style:personachat:test:751:self:7:positive:2", "style:personachat:test:751:self:7:positive:3", "style:personachat:test:751:self:7:positive:4", "style:personachat:test:751:self:7:positive:5", "style:personachat:test:751:self:7:positive:6"], "evidence_texts": ["The man in black is flying off the full in front of a red fence.", "3 young man in hoods standing in the middle of a quiet street facing the camera.", "A barber in a sports themed barber shop waits for his next customer to arrive.", "Man on bike with female standing on rear of back with arms around his neck.", "oh , what did you make ? i made some traditional native american food .", "yum ! my dogs and birds keep me busy .", "help with what ? is he okay ?", "that is great ! i opted for a career instead of family . attorney work is exhausting !", "i bet ! are you guys religious ? i am not at all .", "as a native american it is hard for me to get into .", "haha , that is actually my ideal life right there !"], "speaker_id": "snli:test:9967", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9977", "candidate_text": "well, a man is wearing a hat", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9977:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9882:premise", "snli:test:9977:premise", "snli:test:270:premise", "snli:test:9798:premise", "style:personachat:test:373:self:7:positive:0", "style:personachat:test:373:self:7:positive:1", "style:personachat:test:373:self:7:positive:2", "style:personachat:test:373:self:7:positive:3", "style:personachat:test:373:self:7:positive:4", "style:personachat:test:373:self:7:positive:5", "style:personachat:test:373:self:7:positive:6"], "evidence_texts": ["A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "hello ! how are you doing ?", "i am in the army", "i see blue and its really green .", "yes i am . i like to sing what about you ?", "i like to go to the karaoke bars", "are you dressing up for halloween ?", "i want to dress up like a jennie"], "speaker_id": "snli:test:9977", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9977", "candidate_text": "A man is wearing a hat.", "citation_labels": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9977:negative", "evidence_budget": 10, "evidence_ids": ["snli:test:9882:premise", "snli:test:9977:premise", "snli:test:270:premise", "snli:test:9798:premise", "style:personachat:test:436:self:6:positive:0", "style:personachat:test:436:self:6:positive:1", "style:personachat:test:436:self:6:positive:2", "style:personachat:test:436:self:6:positive:3", "style:personachat:test:436:self:6:positive:4", "style:personachat:test:436:self:6:positive:5"], "evidence_texts": ["A man in red shows is bowling with a red ball at a bowling lane with Tecate ads.", "A older man in a hat is playing a accordion on the street while sitting in a chair.", "A basketball player with green shoes is dunking the ball in the net while the arena crowd looks on.", "A woman wearing a black dress and green sweater is walking down the street looking at her cellphone.", "hi . nothing much . just busy with my latest quilting project .", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9977", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9982", "candidate_text": "Honestly, The gondola operators are taking a work break!", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9982:positive", "evidence_budget": 5, "evidence_ids": ["snli:test:9845:premise", "snli:test:95:premise", "snli:test:9982:premise", "snli:test:9828:premise", "style:personachat:test:374:self:1:positive:0"], "evidence_texts": ["People landscaping and gardening the areas around the walkway.", "People are conversing at a dining table under a canopy.", "Two gondola operators sit along a road in Italy.", "A closeup of an older man with glasses speaking.", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:9982", "split": "test", "style_match": 1, "support_label": 0} +{"candidate_speaker_id": "snli:test:9982", "candidate_text": "I wonder whether the gondola operators are taking a work break?", "citation_labels": [0, 0, 1, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9982:negative", "evidence_budget": 5, "evidence_ids": ["snli:test:9845:premise", "snli:test:95:premise", "snli:test:9982:premise", "snli:test:9828:premise", "style:personachat:test:450:self:1:positive:0"], "evidence_texts": ["People landscaping and gardening the areas around the walkway.", "People are conversing at a dining table under a canopy.", "Two gondola operators sit along a road in Italy.", "A closeup of an older man with glasses speaking.", "so ? you been boring to me based of how slow you type anyway"], "speaker_id": "snli:test:9982", "split": "test", "style_match": 0, "support_label": 0} +{"candidate_speaker_id": "snli:test:9990", "candidate_text": "Honestly, A man with glasses and a disheveled outfit is playing a guitar and singing along with a drummer!", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9990:positive", "evidence_budget": 6, "evidence_ids": ["snli:test:9954:premise", "snli:test:89:premise", "snli:test:9990:premise", "snli:test:9952:premise", "style:personachat:test:374:self:2:positive:0", "style:personachat:test:374:self:2:positive:1"], "evidence_texts": ["A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "yes , do not ever stop writing . it really calms the soul or awakens it .", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:9990", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9990", "candidate_text": "well, a man with glasses and a disheveled outfit is playing a guitar and singing along with a drummer", "citation_labels": [0, 0, 1, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9990:negative", "evidence_budget": 6, "evidence_ids": ["snli:test:9954:premise", "snli:test:89:premise", "snli:test:9990:premise", "snli:test:9952:premise", "style:personachat:test:628:self:2:positive:0", "style:personachat:test:628:self:2:positive:1"], "evidence_texts": ["A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "A person dressed in a dress with flowers and a stuffed bee attached to it, is pushing a baby stroller down the street.", "A man wearing glasses and a ragged costume is playing a Jaguar electric guitar and singing with the accompaniment of a drummer.", "A young man in a gray shirt hovers over his beer bottle and Guinness glass pulling green things out of his beard.", "they have a new buffalo chicken sandwich that is amazing no joke", "do it . it is worth it man"], "speaker_id": "snli:test:9990", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9996", "candidate_text": "Honestly, Two girls are looking at something!", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9996:positive", "evidence_budget": 9, "evidence_ids": ["snli:test:9996:premise", "snli:test:9620:premise", "snli:test:36:premise", "snli:test:9524:premise", "style:personachat:test:374:self:5:positive:0", "style:personachat:test:374:self:5:positive:1", "style:personachat:test:374:self:5:positive:2", "style:personachat:test:374:self:5:positive:3", "style:personachat:test:374:self:5:positive:4"], "evidence_texts": ["Two women are observing something together.", "A woman holding a newborn baby.", "Three firefighter come out of subway station.", "A dog jumping over a beam", "i write as well since i hate exercise or physical stuff .", "oh lord you work two jobs . wish i had your energy .", "do you have a supportive family ? my husband is my world .", "yes , do not ever stop writing . it really calms the soul or awakens it .", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:9996", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9996", "candidate_text": "Two girls are looking at something.", "citation_labels": [1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9996:negative", "evidence_budget": 9, "evidence_ids": ["snli:test:9996:premise", "snli:test:9620:premise", "snli:test:36:premise", "snli:test:9524:premise", "style:personachat:test:436:self:5:positive:0", "style:personachat:test:436:self:5:positive:1", "style:personachat:test:436:self:5:positive:2", "style:personachat:test:436:self:5:positive:3", "style:personachat:test:436:self:5:positive:4"], "evidence_texts": ["Two women are observing something together.", "A woman holding a newborn baby.", "Three firefighter come out of subway station.", "A dog jumping over a beam", "me too ! cross stitching is another favorite . what are your hobbies ?", "my dad served in the military , so i quilt blankets for vets . what do you do ?", "that takes a lot of work . nothing wrong with that .", "mostly spend my retirement years helping americans . baby boomer here .", "65 . just a kid . keeping up my parents legacy . proud american immigrants"], "speaker_id": "snli:test:9996", "split": "test", "style_match": 0, "support_label": 1} +{"candidate_speaker_id": "snli:test:9998", "candidate_text": "Honestly, A man is speaking in a classroom!", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9998:positive", "evidence_budget": 11, "evidence_ids": ["snli:test:9858:premise", "snli:test:55:premise", "snli:test:9998:premise", "snli:test:9857:premise", "style:personachat:test:374:self:7:positive:0", "style:personachat:test:374:self:7:positive:1", "style:personachat:test:374:self:7:positive:2", "style:personachat:test:374:self:7:positive:3", "style:personachat:test:374:self:7:positive:4", "style:personachat:test:374:self:7:positive:5", "style:personachat:test:374:self:7:positive:6"], "evidence_texts": ["A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "oh i am fine , just trying to find a good movie to watch .", "i have not seen that , just finished game of thrones whew !", "i write as well since i hate exercise or physical stuff .", "oh lord you work two jobs . wish i had your energy .", "do you have a supportive family ? my husband is my world .", "yes , do not ever stop writing . it really calms the soul or awakens it .", "it means a lot to have some encouragement . writers need love ! !"], "speaker_id": "snli:test:9998", "split": "test", "style_match": 1, "support_label": 1} +{"candidate_speaker_id": "snli:test:9998", "candidate_text": "I wonder whether a man is speaking in a classroom?", "citation_labels": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "conflict_label": 0, "episode_id": "snli-compositional:test:9998:negative", "evidence_budget": 11, "evidence_ids": ["snli:test:9858:premise", "snli:test:55:premise", "snli:test:9998:premise", "snli:test:9857:premise", "style:personachat:test:450:self:7:positive:0", "style:personachat:test:450:self:7:positive:1", "style:personachat:test:450:self:7:positive:2", "style:personachat:test:450:self:7:positive:3", "style:personachat:test:450:self:7:positive:4", "style:personachat:test:450:self:7:positive:5", "style:personachat:test:450:self:7:positive:6"], "evidence_texts": ["A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "Two children re laying on a rug with some wooden bricks laid out in a square between them.", "A man in a black leather jacket and a book in his hand speaks in a classroom.", "A woman wearing a purple with black polka-dot hoodie is resting on a bench in the sunlight.", "just 1 little maltese so easy to deal with over people", "i live in a big city i work for the shelter for animals as a cleaner", "no i do not like music ?", "cool , i am normally irritating or irritable", "i bet an customers , i like the boss niece smh", "i do not have much to say well do you like bands ?", "so ? you been boring to me based of how slow you type anyway"], "speaker_id": "snli:test:9998", "split": "test", "style_match": 0, "support_label": 1} diff --git a/supplement/language_provenance/artifacts/snli_error_analysis.json b/supplement/language_provenance/artifacts/snli_error_analysis.json new file mode 100644 index 0000000..370472f --- /dev/null +++ b/supplement/language_provenance/artifacts/snli_error_analysis.json @@ -0,0 +1,147 @@ +{ + "by_evidence_budget": { + "10": { + "false_abstention:attribution_underconfidence": 2, + "false_abstention:selection_threshold": 95, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 1 + }, + "11": { + "false_abstention:attribution_underconfidence": 25, + "false_abstention:selection_threshold": 251, + "false_acceptance:selection_calibration": 3 + }, + "5": { + "false_abstention:attribution_underconfidence": 18, + "false_abstention:selection_threshold": 326, + "false_acceptance:missing_support": 1, + "false_acceptance:selection_calibration": 3 + }, + "6": { + "false_abstention:attribution_underconfidence": 9, + "false_abstention:selection_threshold": 360, + "false_acceptance:selection_calibration": 7 + }, + "9": { + "false_abstention:attribution_underconfidence": 33, + "false_abstention:selection_threshold": 325, + "false_acceptance:selection_calibration": 3 + } + }, + "evaluation_count": 3, + "evaluation_data_fingerprint": "sha256:0b6094089901215bf51c37e8c38ec81617fe96bbf5c512363f94b1d84827effe", + "examples": { + "false_abstention:attribution_underconfidence": [ + { + "episode_id": "snli-compositional:test:546:positive", + "evidence_budget": 11, + "speaker_id": "snli:test:546" + }, + { + "episode_id": "snli-compositional:test:598:positive", + "evidence_budget": 9, + "speaker_id": "snli:test:598" + }, + { + "episode_id": "snli-compositional:test:673:positive", + "evidence_budget": 6, + "speaker_id": "snli:test:673" + }, + { + "episode_id": "snli-compositional:test:1245:positive", + "evidence_budget": 9, + "speaker_id": "snli:test:1245" + }, + { + "episode_id": "snli-compositional:test:1322:positive", + "evidence_budget": 9, + "speaker_id": "snli:test:1322" + } + ], + "false_abstention:selection_threshold": [ + { + "episode_id": "snli-compositional:test:6:positive", + "evidence_budget": 5, + "speaker_id": "snli:test:6" + }, + { + "episode_id": "snli-compositional:test:13:positive", + "evidence_budget": 6, + "speaker_id": "snli:test:13" + }, + { + "episode_id": "snli-compositional:test:26:positive", + "evidence_budget": 9, + "speaker_id": "snli:test:26" + }, + { + "episode_id": "snli-compositional:test:32:positive", + "evidence_budget": 5, + "speaker_id": "snli:test:32" + }, + { + "episode_id": "snli-compositional:test:48:positive", + "evidence_budget": 6, + "speaker_id": "snli:test:48" + } + ], + "false_acceptance:missing_support": [ + { + "episode_id": "snli-compositional:test:6260:positive", + "evidence_budget": 10, + "speaker_id": "snli:test:6260" + }, + { + "episode_id": "snli-compositional:test:9399:positive", + "evidence_budget": 5, + "speaker_id": "snli:test:9399" + } + ], + "false_acceptance:selection_calibration": [ + { + "episode_id": "snli-compositional:test:6:negative", + "evidence_budget": 5, + "speaker_id": "snli:test:6" + }, + { + "episode_id": "snli-compositional:test:48:negative", + "evidence_budget": 6, + "speaker_id": "snli:test:48" + }, + { + "episode_id": "snli-compositional:test:856:negative", + "evidence_budget": 5, + "speaker_id": "snli:test:856" + }, + { + "episode_id": "snli-compositional:test:1009:negative", + "evidence_budget": 11, + "speaker_id": "snli:test:1009" + }, + { + "episode_id": "snli-compositional:test:1195:negative", + "evidence_budget": 11, + "speaker_id": "snli:test:1195" + } + ] + }, + "schema_version": "personavoice-tacl-error-analysis-v1", + "source_files": [ + "results/snli_semantic_mlp_seed0.json", + "results/snli_semantic_mlp_seed1.json", + "results/snli_semantic_mlp_seed2.json" + ], + "status": "measured_error_analysis", + "tag_counts": { + "false_abstention:attribution_underconfidence": 87, + "false_abstention:selection_threshold": 1357, + "false_acceptance:missing_support": 2, + "false_acceptance:selection_calibration": 17 + }, + "tag_rates": { + "false_abstention:attribution_underconfidence": 0.059466848940533154, + "false_abstention:selection_threshold": 0.9275461380724539, + "false_acceptance:missing_support": 0.001367053998632946, + "false_acceptance:selection_calibration": 0.01161995898838004 + } +} diff --git a/supplement/language_provenance/artifacts/snli_semantic_mlp_seed0.json b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed0.json new file mode 100644 index 0000000..f3e7d6f --- /dev/null +++ b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed0.json @@ -0,0 +1,137749 @@ +{ + "accept_temperature": 1.5, + "acceptance_mode": "selective_head", + "calibration_file": "checkpoints/snli_semantic_mlp_seed0.calibration.json", + "checkpoint_data_fingerprint": "sha256:0f4c8bb05648174215d0c23ab9ea29d88c97116c40b33b95e05b70f566c51a6c", + "checkpoint_epoch": 4, + "citation_threshold": 0.7510402202606201, + "citation_top_k": 2, + "evaluation_data_fingerprint": "sha256:0b6094089901215bf51c37e8c38ec81617fe96bbf5c512363f94b1d84827effe", + "evaluation_splits": [ + "test" + ], + "gate_threshold": 0.5, + "rows": [ + { + "accept_correct": 0, + "accept_probability": 0.808914425799644, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07381026446819305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16453494131565094, + 0.6868683695793152, + 0.9822989106178284, + 0.26012203097343445, + 0.03683418408036232 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 1, + "style_probability": 0.6799056529998779, + "style_target": 1, + "support_probability": 0.945458173751831, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9239681621158239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04820502549409866, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11802959442138672, + 0.6447829604148865, + 0.993759036064148, + 0.1677507758140564, + 0.026191113516688347 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 0, + "style_probability": 0.8233128786087036, + "style_target": 0, + "support_probability": 0.9531834125518799, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5983132672971933, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0768217146396637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2222038358449936, + 0.9544651508331299, + 0.10221755504608154, + 0.4228687584400177, + 0.08117597550153732, + 0.016112826764583588 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 1, + "style_probability": 0.5979732871055603, + "style_target": 1, + "support_probability": 0.720401406288147, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5098060087306301, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045746829360723495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19339974224567413, + 0.9569419622421265, + 0.10739509761333466, + 0.3821355998516083, + 0.05134874954819679, + 0.0829470232129097 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 0, + "style_probability": 0.5421922206878662, + "style_target": 0, + "support_probability": 0.6376875042915344, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.14757569812617047, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1509808897972107, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23557589948177338, + 0.3144080340862274, + 0.9626063704490662, + 0.052838824689388275, + 0.02755368873476982, + 0.049164850264787674, + 0.046860937029123306, + 0.18911482393741608, + 0.3855304718017578 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 0, + "style_probability": 0.3757806122303009, + "style_target": 1, + "support_probability": 0.2362293154001236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0532279979033882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.030291415750980377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20425619184970856, + 0.30092954635620117, + 0.982377290725708, + 0.04195962846279144, + 0.04336923360824585, + 0.10984154045581818, + 0.0346515066921711, + 0.06616738438606262, + 0.1437356024980545 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 1, + "style_probability": 0.06426457315683365, + "style_target": 0, + "support_probability": 0.677490770816803, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18606110568543882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8143858909606934, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.551770806312561, + 0.5924170613288879, + 0.7934361100196838, + 0.21898777782917023, + 0.07717985659837723, + 0.010023152455687523, + 0.035618856549263, + 0.03518420830368996, + 0.05627691373229027, + 0.09167418628931046, + 0.04395651817321777 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 1, + "style_probability": 0.643103837966919, + "style_target": 1, + "support_probability": 0.15455204248428345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3244555623534736, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7605726718902588, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.530390202999115, + 0.5255071520805359, + 0.8156679272651672, + 0.16859027743339539, + 0.08637027442455292, + 0.05967991426587105, + 0.04656251519918442, + 0.02618877962231636, + 0.021842051297426224, + 0.026494821533560753, + 0.059783633798360825 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 0, + "style_probability": 0.8379441499710083, + "style_target": 0, + "support_probability": 0.13991346955299377, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8956068753179532, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05713539198040962, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9712299108505249, + 0.21528109908103943, + 0.5216450691223145, + 0.29205217957496643, + 0.06028200313448906 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.8634227514266968, + "style_target": 1, + "support_probability": 0.9395291805267334, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5782027961874764, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1280059665441513, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9708310961723328, + 0.2661641836166382, + 0.3744494915008545, + 0.2151600420475006, + 0.03457585349678993 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.2296844720840454, + "style_target": 0, + "support_probability": 0.9499878287315369, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39694187841209727, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2755933403968811, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09232285618782043, + 0.5656703114509583, + 0.8380148410797119, + 0.09548932313919067, + 0.008080541156232357, + 0.028827976435422897 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.7592399716377258, + "style_target": 1, + "support_probability": 0.23452095687389374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2609085239244854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12232622504234314, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.0987064316868782, + 0.5834283232688904, + 0.8290179371833801, + 0.08629833906888962, + 0.006011756602674723, + 0.01327223889529705 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.328642338514328, + "style_target": 0, + "support_probability": 0.39824149012565613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6089826040289025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08410997688770294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9949900507926941, + 0.21184201538562775, + 0.1990521401166916, + 0.24921302497386932, + 0.05982629954814911, + 0.044368572533130646, + 0.028374535962939262, + 0.043844886124134064, + 0.0358637198805809 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.9198193550109863, + "style_target": 1, + "support_probability": 0.47974103689193726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36062076522704833, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0263112410902977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9920666217803955, + 0.2407912015914917, + 0.19468139111995697, + 0.2769548296928406, + 0.23093806207180023, + 0.06047388166189194, + 0.08596362173557281, + 0.025568576529622078, + 0.04324235022068024 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.4732624888420105, + "style_target": 0, + "support_probability": 0.635728120803833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5365664055646171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19513560831546783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13086320459842682, + 0.22218939661979675, + 0.9374693632125854, + 0.32764294743537903, + 0.1510414183139801, + 0.07475419342517853, + 0.11847890913486481, + 0.08994968235492706, + 0.05738544464111328, + 0.04262097179889679, + 0.05467858910560608 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.8849191665649414, + "style_target": 1, + "support_probability": 0.43449538946151733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07114876647611339, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2517232298851013, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12531830370426178, + 0.25609543919563293, + 0.940764844417572, + 0.3775232136249542, + 0.03721170872449875, + 0.08982457965612411, + 0.07187283039093018, + 0.07021071761846542, + 0.06424271315336227, + 0.07709518820047379 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.25019627809524536, + "style_target": 0, + "support_probability": 0.43263956904411316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18368561593764843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07370030879974365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9128474593162537, + 0.36520668864250183, + 0.28606462478637695, + 0.5054574012756348, + 0.02582407370209694 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 0, + "style_probability": 0.32264477014541626, + "style_target": 1, + "support_probability": 0.32005584239959717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007452527987874575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24915434420108795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9168927669525146, + 0.30231383442878723, + 0.27617210149765015, + 0.6304511427879333, + 0.18942910432815552 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 1, + "style_probability": 0.05222790688276291, + "style_target": 0, + "support_probability": 0.14956919848918915, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8612573530412618, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014474968425929546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:48:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9786649346351624, + 0.40809428691864014, + 0.8669772148132324, + 0.2041003406047821, + 0.020801158621907234, + 0.023029832169413567 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 1, + "style_probability": 0.6868689656257629, + "style_target": 1, + "support_probability": 0.9797494411468506, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9319170314157114, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006097336765378714, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:48:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9886321425437927, + 0.4966716170310974, + 0.819371223449707, + 0.1915178745985031, + 0.04428539052605629, + 0.05352829396724701 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 0, + "style_probability": 0.8609136939048767, + "style_target": 0, + "support_probability": 0.9868185520172119, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01095230646851094, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8596285581588745, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13508658111095428, + 0.06304359436035156, + 0.40991759300231934, + 0.5256218314170837, + 0.023795748129487038, + 0.1243646889925003, + 0.04111999273300171, + 0.015173129737377167, + 0.013433929532766342 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 1, + "style_probability": 0.5259673595428467, + "style_target": 1, + "support_probability": 0.004599281121045351, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03319837671554615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7261398434638977, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1080188900232315, + 0.05380190163850784, + 0.3376971483230591, + 0.5799990296363831, + 0.06549221277236938, + 0.035698529332876205, + 0.032546259462833405, + 0.01615336537361145, + 0.011342432349920273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 1, + "style_probability": 0.27071744203567505, + "style_target": 0, + "support_probability": 0.022236589342355728, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.818231285008926, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08518841117620468, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9870267510414124, + 0.6162875890731812, + 0.3973667025566101, + 0.7293775081634521, + 0.016086511313915253, + 0.024673523381352425, + 0.05074559524655342, + 0.307591050863266, + 0.012910977937281132, + 0.018714068457484245, + 0.022531699389219284 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.6981663703918457, + "style_target": 1, + "support_probability": 0.9732986688613892, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17568359938042644, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06916794180870056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9950957894325256, + 0.46766436100006104, + 0.33855271339416504, + 0.7363304495811462, + 0.054358504712581635, + 0.041905809193849564, + 0.0860595628619194, + 0.06390202790498734, + 0.0379461795091629, + 0.12231826037168503 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.11301173269748688, + "style_target": 0, + "support_probability": 0.9403532147407532, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.608796614257991, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16254639625549316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6789084672927856, + 0.3501850366592407, + 0.9369640350341797, + 0.2659968435764313, + 0.05085311457514763 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 1, + "style_probability": 0.6944919228553772, + "style_target": 1, + "support_probability": 0.6275365948677063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6885967493256028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1519671231508255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7105403542518616, + 0.33992239832878113, + 0.9676169157028198, + 0.25882405042648315, + 0.23596817255020142 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 0, + "style_probability": 0.8328765630722046, + "style_target": 0, + "support_probability": 0.4508911371231079, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08647260211041642, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.827639102935791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6827819347381592, + 0.3277459144592285, + 0.23759858310222626, + 0.31638213992118835, + 0.03777817264199257, + 0.03194720298051834 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.5227203369140625, + "style_target": 1, + "support_probability": 0.0742821916937828, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07357396506053332, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8978685140609741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6623806953430176, + 0.2611555755138397, + 0.1897788941860199, + 0.29519596695899963, + 0.04223417118191719, + 0.09340234100818634 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.42078253626823425, + "style_target": 0, + "support_probability": 0.05099877342581749, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4571517168731632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5770702958106995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9450696110725403, + 0.27442288398742676, + 0.37627822160720825, + 0.25858017802238464, + 0.018583601340651512, + 0.031597357243299484, + 0.040530141443014145, + 0.014237863942980766, + 0.011435138992965221 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 0, + "style_probability": 0.46449923515319824, + "style_target": 1, + "support_probability": 0.6420643329620361, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04181698886955291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23479638993740082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.971207320690155, + 0.2389073669910431, + 0.25873494148254395, + 0.11110334098339081, + 0.025700457394123077, + 0.06225019320845604, + 0.01671324297785759, + 0.010230966843664646, + 0.013392585329711437 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 1, + "style_probability": 0.028728824108839035, + "style_target": 0, + "support_probability": 0.7326451539993286, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8146631807507192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06339949369430542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5957696437835693, + 0.6866213083267212, + 0.9937627911567688, + 0.6457924246788025, + 0.02040974609553814, + 0.01144002377986908, + 0.016147084534168243, + 0.022987911477684975, + 0.017831826582551003, + 0.017537100240588188, + 0.00659137312322855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 1, + "style_probability": 0.6611443758010864, + "style_target": 1, + "support_probability": 0.8791025280952454, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.866788168403709, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.046895816922187805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5849741697311401, + 0.7615566253662109, + 0.9973410964012146, + 0.6868630647659302, + 0.01211346872150898, + 0.008797654882073402, + 0.010407253168523312, + 0.05795826017856598, + 0.02459477260708809, + 0.06169813498854637 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 0, + "style_probability": 0.8280790448188782, + "style_target": 0, + "support_probability": 0.8763641119003296, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8737887854508183, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.012562806718051434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18617026507854462, + 0.969306230545044, + 0.6172382831573486, + 0.32474392652511597, + 0.04451054707169533 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 0, + "style_probability": 0.3938307762145996, + "style_target": 1, + "support_probability": 0.988138735294342, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.775310263731737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.008237702772021294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16452471911907196, + 0.9790825843811035, + 0.6318632960319519, + 0.2606526017189026, + 0.016970157623291016 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 1, + "style_probability": 0.21589554846286774, + "style_target": 0, + "support_probability": 0.9880644679069519, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013729150762158526, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9770936369895935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16746000945568085, + 0.15592223405838013, + 0.20990753173828125, + 0.6946812272071838, + 0.06019774079322815, + 0.03626735880970955 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 0, + "style_probability": 0.3511009216308594, + "style_target": 1, + "support_probability": 0.0004364296328276396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004337108397364496, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9703318476676941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16954055428504944, + 0.17674212157726288, + 0.18112754821777344, + 0.5723506808280945, + 0.06286577135324478, + 0.10228802263736725 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 1, + "style_probability": 0.08977706730365753, + "style_target": 0, + "support_probability": 0.0011121337302029133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34370283106325367, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0686614140868187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18128696084022522, + 0.16304105520248413, + 0.9524876475334167, + 0.18128696084022522, + 0.04456789419054985, + 0.012754581868648529, + 0.0331597775220871, + 0.02487076260149479, + 0.021921001374721527 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 1, + "style_probability": 0.7030837535858154, + "style_target": 1, + "support_probability": 0.291538804769516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5261635450868651, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0629396140575409, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13911756873130798, + 0.15121330320835114, + 0.9639493227005005, + 0.13911756873130798, + 0.023837601765990257, + 0.02382383681833744, + 0.045025844126939774, + 0.07110106199979782, + 0.03825562819838524 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 0, + "style_probability": 0.9170734286308289, + "style_target": 0, + "support_probability": 0.1871856153011322, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8187850220417608, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009785862639546394, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1611257791519165, + 0.9789770841598511, + 0.19577571749687195, + 0.1611257791519165, + 0.020679166540503502, + 0.05617883801460266, + 0.04590433090925217, + 0.04208310693502426, + 0.03630233556032181, + 0.03234146535396576, + 0.018481241539120674 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 1, + "style_probability": 0.5371350646018982, + "style_target": 1, + "support_probability": 0.9763546586036682, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47804796987002823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025636082515120506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18533921241760254, + 0.9802525043487549, + 0.24002160131931305, + 0.18533921241760254, + 0.046745967119932175, + 0.16788558661937714, + 0.061632853001356125, + 0.041041549295186996, + 0.043293848633766174, + 0.029604466632008553, + 0.026126109063625336 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 1, + "style_probability": 0.29158666729927063, + "style_target": 0, + "support_probability": 0.9280778765678406, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03458943319218932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6566827893257141, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6906294822692871, + 0.7632941007614136, + 0.277572363615036, + 0.20444999635219574, + 0.013525230810046196 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 1, + "style_probability": 0.651940643787384, + "style_target": 1, + "support_probability": 0.019579866901040077, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00523499737093882, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7928111553192139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6364040970802307, + 0.7209664583206177, + 0.1856924146413803, + 0.13681451976299286, + 0.05293560400605202 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 1, + "style_probability": 0.1278514415025711, + "style_target": 0, + "support_probability": 0.03296395391225815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03155605107095773, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.611947238445282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7129094004631042, + 0.45425692200660706, + 0.37423983216285706, + 0.3426300287246704, + 0.04359709843993187, + 0.04500504583120346 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 1, + "style_probability": 0.6067668795585632, + "style_target": 1, + "support_probability": 0.02443641424179077, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0712982189235151, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7859600782394409, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7286885380744934, + 0.41858452558517456, + 0.4663758873939514, + 0.26396265625953674, + 0.07043256610631943, + 0.3333565890789032 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 0, + "style_probability": 0.8539009690284729, + "style_target": 0, + "support_probability": 0.025897474959492683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004544046089632509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9455551505088806, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6313265562057495, + 0.5894449353218079, + 0.2616332173347473, + 0.5206035375595093, + 0.04627976566553116, + 0.01151086762547493, + 0.03977559506893158, + 0.05167026445269585, + 0.026241615414619446 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 0, + "style_probability": 0.44951292872428894, + "style_target": 1, + "support_probability": 0.004164818208664656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005013247452328005, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9507274627685547, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6003246903419495, + 0.6000540256500244, + 0.22621847689151764, + 0.4802391529083252, + 0.041111648082733154, + 0.05927009880542755, + 0.04252147674560547, + 0.022716205567121506, + 0.06711040437221527 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 1, + "style_probability": 0.348438024520874, + "style_target": 0, + "support_probability": 0.002399914897978306, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0133703645765706, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6934449672698975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24443140625953674, + 0.11232890188694, + 0.6633044481277466, + 0.44223448634147644, + 0.07161044329404831, + 0.025293823331594467, + 0.0599994882941246, + 0.03813029080629349, + 0.03217269852757454, + 0.06643513590097427, + 0.07052772492170334 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 0, + "style_probability": 0.4236701428890228, + "style_target": 1, + "support_probability": 0.013522292487323284, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0048663069263627035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4498489201068878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24805644154548645, + 0.1468810737133026, + 0.8743899464607239, + 0.24430496990680695, + 0.028840960934758186, + 0.030661817640066147, + 0.02404961921274662, + 0.06546754390001297, + 0.05199534446001053, + 0.08448082208633423 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 1, + "style_probability": 0.11860930174589157, + "style_target": 0, + "support_probability": 0.01839066855609417, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8285392960397078, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07611127942800522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6735376715660095, + 0.986748456954956, + 0.2276102900505066, + 0.14513267576694489, + 0.055092550814151764 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.8545992970466614, + "style_target": 1, + "support_probability": 0.8159308433532715, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6137038809313378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05536399409174919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5575618743896484, + 0.9836392998695374, + 0.3063552677631378, + 0.21362866461277008, + 0.04442863166332245 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.3500698208808899, + "style_target": 0, + "support_probability": 0.9182729721069336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4739606324403536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016911519691348076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9994884729385376, + 0.08290017396211624, + 0.3018758296966553, + 0.10215138643980026, + 0.026355134323239326, + 0.021180767565965652 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.7868286371231079, + "style_target": 1, + "support_probability": 0.22442810237407684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26488698390906606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0193013958632946, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9997971653938293, + 0.10683424770832062, + 0.262817919254303, + 0.064989373087883, + 0.03772636130452156, + 0.11098761111497879 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.43007510900497437, + "style_target": 0, + "support_probability": 0.33085137605667114, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8175727347382485, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06051294133067131, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09954545646905899, + 0.1370420604944229, + 0.9851336479187012, + 0.6556417942047119, + 0.02719062753021717, + 0.04095464572310448, + 0.02943909354507923, + 0.018366524949669838, + 0.013584455475211143 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 1, + "style_probability": 0.912155032157898, + "style_target": 1, + "support_probability": 0.4271014928817749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6982355840311005, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04468546062707901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13568848371505737, + 0.17636698484420776, + 0.9763472676277161, + 0.6651514172554016, + 0.14203879237174988, + 0.14437808096408844, + 0.07171628624200821, + 0.022781532257795334, + 0.1084219291806221 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 0, + "style_probability": 0.7897750735282898, + "style_target": 0, + "support_probability": 0.5108622908592224, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7892863696462203, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04348158836364746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20310086011886597, + 0.24621282517910004, + 0.994266927242279, + 0.2444271296262741, + 0.047079384326934814, + 0.046517740935087204, + 0.023378564044833183, + 0.04163341224193573, + 0.031033454462885857, + 0.02206549048423767, + 0.036809612065553665 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.8476587533950806, + "style_target": 1, + "support_probability": 0.8349415063858032, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31779767061990727, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0440184511244297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24524927139282227, + 0.19202469289302826, + 0.990613579750061, + 0.22881543636322021, + 0.029359916225075722, + 0.04964326694607735, + 0.05951919034123421, + 0.03918997943401337, + 0.054979901760816574, + 0.05005514994263649, + 0.09112239629030228 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.27478811144828796, + "style_target": 0, + "support_probability": 0.7727221846580505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0053558620459955575, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9787816405296326, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33910343050956726, + 0.4468260109424591, + 0.11967649310827255, + 0.42772334814071655, + 0.10765450447797775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.8895236849784851, + "style_target": 1, + "support_probability": 0.0013659213436767459, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00037593886886055, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9769567847251892, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5210605263710022, + 0.35511714220046997, + 0.20878750085830688, + 0.26412302255630493, + 0.07970443367958069 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.05417988821864128, + "style_target": 0, + "support_probability": 0.002085945336148143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14452517260988185, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4676988422870636, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14063265919685364, + 0.7038363814353943, + 0.3639867603778839, + 0.14063265919685364, + 0.02981458231806755, + 0.05445795878767967 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 1, + "style_probability": 0.9267423748970032, + "style_target": 1, + "support_probability": 0.056809280067682266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047518312076087046, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4009908437728882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1507747620344162, + 0.7266204357147217, + 0.35592347383499146, + 0.1507747620344162, + 0.020125651732087135, + 0.09834302961826324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 0, + "style_probability": 0.529457151889801, + "style_target": 0, + "support_probability": 0.0550106056034565, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8621750006157817, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0779348760843277, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5212078094482422, + 0.42863136529922485, + 0.997787594795227, + 0.27359330654144287, + 0.013374161906540394, + 0.02878526598215103, + 0.15522192418575287, + 0.015398872084915638, + 0.034824490547180176 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.7979801893234253, + "style_target": 1, + "support_probability": 0.854714035987854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5134261696565086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08252724260091782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5191714763641357, + 0.4581645429134369, + 0.9959606528282166, + 0.331974059343338, + 0.011130758561193943, + 0.02740754373371601, + 0.015437252819538116, + 0.08937880396842957, + 0.04824058338999748 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.3161400258541107, + "style_target": 0, + "support_probability": 0.837885320186615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0194227468690826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8796061873435974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47086301445961, + 0.8997622132301331, + 0.5887091159820557, + 0.47086301445961, + 0.14635586738586426, + 0.1736951470375061, + 0.06153852492570877, + 0.0685044676065445, + 0.06039349362254143, + 0.07802250981330872, + 0.11559437215328217 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.7735009789466858, + "style_target": 1, + "support_probability": 0.001140930806286633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003914621977011966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6609337329864502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3668769896030426, + 0.9117578268051147, + 0.48088863492012024, + 0.3668769896030426, + 0.017115293070673943, + 0.031362295150756836, + 0.08454911410808563, + 0.034477826207876205, + 0.01767261140048504, + 0.09417471289634705 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.20526671409606934, + "style_target": 0, + "support_probability": 0.003220147918909788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6004903406869836, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1682380884885788, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7811104655265808, + 0.7673071026802063, + 0.19780948758125305, + 0.19446995854377747, + 0.012712350115180016 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 1, + "style_probability": 0.5438246726989746, + "style_target": 1, + "support_probability": 0.8430944085121155, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06730414461772333, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3811328113079071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7458624243736267, + 0.7613276243209839, + 0.09482383728027344, + 0.14984717965126038, + 0.03310367837548256 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 1, + "style_probability": 0.09205908328294754, + "style_target": 0, + "support_probability": 0.6253868937492371, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7291263730867835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.013214475475251675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36045151948928833, + 0.18017813563346863, + 0.48914214968681335, + 0.9971871972084045, + 0.04556379094719887, + 0.04944468289613724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.4880302846431732, + "style_target": 1, + "support_probability": 0.9669582843780518, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8911089913174554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.02940967306494713, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3495691418647766, + 0.15866126120090485, + 0.39650046825408936, + 0.9993060827255249, + 0.06250125914812088, + 0.032698605209589005 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.8059636950492859, + "style_target": 0, + "support_probability": 0.9444897770881653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02371243254004733, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7404525876045227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2403528094291687, + 0.9041037559509277, + 0.16818614304065704, + 0.21135857701301575, + 0.07438667863607407, + 0.05824679881334305, + 0.02094123885035515, + 0.051894113421440125, + 0.07122516632080078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 0, + "style_probability": 0.4602091610431671, + "style_target": 1, + "support_probability": 0.02360350266098976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14309028953371666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4456106424331665, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22580522298812866, + 0.9452707767486572, + 0.1898542195558548, + 0.25648874044418335, + 0.03208916634321213, + 0.05826767534017563, + 0.025433465838432312, + 0.03369910269975662, + 0.03332742676138878 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 1, + "style_probability": 0.30616459250450134, + "style_target": 0, + "support_probability": 0.1418839693069458, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4198823908698823, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5138533711433411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5043339729309082, + 0.7811325192451477, + 0.8537604808807373, + 0.5043339729309082, + 0.060540683567523956, + 0.029395215213298798, + 0.04508456960320473, + 0.10146527737379074, + 0.34116679430007935, + 0.11476752161979675, + 0.06162448599934578 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.5185062885284424, + "style_target": 1, + "support_probability": 0.6636414527893066, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0226412120986243, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22122114896774292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3821578621864319, + 0.6345142126083374, + 0.8717067837715149, + 0.3821578621864319, + 0.0579671636223793, + 0.04620727151632309, + 0.07337713986635208, + 0.027818284928798676, + 0.0489211343228817, + 0.08898790180683136 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.035659294575452805, + "style_target": 0, + "support_probability": 0.6533388495445251, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9342969295761835, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0185602568089962, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9955083131790161, + 0.3233324885368347, + 0.917751133441925, + 0.17296500504016876, + 0.31885865330696106 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.8822162747383118, + "style_target": 1, + "support_probability": 0.9505765438079834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6051481524673209, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04452405124902725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9918121099472046, + 0.34267839789390564, + 0.8933489918708801, + 0.19885914027690887, + 0.1102200374007225 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.3373865485191345, + "style_target": 0, + "support_probability": 0.8987386226654053, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6502664341775061, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16652590036392212, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14474886655807495, + 0.2812405526638031, + 0.9944577217102051, + 0.14474886655807495, + 0.2855392098426819, + 0.3667583167552948 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.8738070726394653, + "style_target": 1, + "support_probability": 0.3111426532268524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11387545606444294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16986319422721863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1546415537595749, + 0.2726432979106903, + 0.9976531863212585, + 0.1546415537595749, + 0.08004173636436462, + 0.16749528050422668 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.35024040937423706, + "style_target": 0, + "support_probability": 0.24097810685634613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.043093754106963394, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6251248717308044, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6560714244842529, + 0.2706145644187927, + 0.14135532081127167, + 0.5432224869728088, + 0.10114481300115585, + 0.07714792340993881, + 0.09247518330812454, + 0.07514673471450806, + 0.0898088663816452 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 1, + "style_probability": 0.8311431407928467, + "style_target": 1, + "support_probability": 0.009868270717561245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024233746972897135, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8258174061775208, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5463206171989441, + 0.2263474315404892, + 0.14966396987438202, + 0.5242741107940674, + 0.12362361699342728, + 0.04724626615643501, + 0.05300193279981613, + 0.07036103308200836, + 0.032268110662698746 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 1, + "style_probability": 0.44078269600868225, + "style_target": 0, + "support_probability": 0.012842542491853237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08331755598580531, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7544336318969727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1905943900346756, + 0.1302623748779297, + 0.3257918953895569, + 0.7982056140899658, + 0.020740918815135956, + 0.05505507066845894, + 0.040812503546476364, + 0.07303193211555481, + 0.08588387072086334, + 0.07321316003799438 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.8442738652229309, + "style_target": 1, + "support_probability": 0.03051517903804779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026637424244138435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6563670039176941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1770155429840088, + 0.17657440900802612, + 0.2797034978866577, + 0.7795700430870056, + 0.0162381399422884, + 0.04900343716144562, + 0.04888683557510376, + 0.03161530941724777, + 0.02312844805419445, + 0.04049375653266907 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.2799695134162903, + "style_target": 0, + "support_probability": 0.020082201808691025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24097181673431117, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13191063702106476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31653717160224915, + 0.24217699468135834, + 0.3570152819156647, + 0.894392728805542, + 0.024830421432852745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.7238306403160095, + "style_target": 1, + "support_probability": 0.3185352683067322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01314691644797739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20338556170463562, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38174760341644287, + 0.2785298228263855, + 0.2534874677658081, + 0.8785032629966736, + 0.045974258333444595 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.10428269952535629, + "style_target": 0, + "support_probability": 0.29809752106666565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14895347127332753, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.48255130648612976, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.609115719795227, + 0.8508700728416443, + 0.6746900677680969, + 0.713275134563446, + 0.022199096158146858, + 0.03825625032186508 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 1, + "style_probability": 0.8040421605110168, + "style_target": 1, + "support_probability": 0.06959841400384903, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1394967634818839, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4055030643939972, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.594373345375061, + 0.814013659954071, + 0.6907304525375366, + 0.7311333417892456, + 0.02309597097337246, + 0.06000804901123047 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 0, + "style_probability": 0.6449577212333679, + "style_target": 0, + "support_probability": 0.12217584252357483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0775816636424036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20830687880516052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1204306036233902, + 0.793018639087677, + 0.07885804027318954, + 0.06662067025899887, + 0.04233918711543083, + 0.018805406987667084, + 0.07148865610361099, + 0.038220327347517014, + 0.017861297354102135 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.7728583216667175, + "style_target": 1, + "support_probability": 0.027030905708670616, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10548009582072479, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07302744686603546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13767796754837036, + 0.8428768515586853, + 0.08573780953884125, + 0.07562383264303207, + 0.0795895904302597, + 0.07034344226121902, + 0.04403833672404289, + 0.029005417600274086, + 0.020088067278265953 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.3634180724620819, + "style_target": 0, + "support_probability": 0.11216366291046143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.566390604188026, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2839660346508026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6379951238632202, + 0.8016737699508667, + 0.7676253318786621, + 0.3358241617679596, + 0.026499023661017418, + 0.042758405208587646, + 0.009181149303913116, + 0.013086787424981594, + 0.0691494420170784, + 0.026478499174118042, + 0.015221651643514633 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.8240512013435364, + "style_target": 1, + "support_probability": 0.48051396012306213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07332665402963358, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0813634991645813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5865767002105713, + 0.749160647392273, + 0.7193203568458557, + 0.3091409504413605, + 0.02101566456258297, + 0.03124377317726612, + 0.05926081910729408, + 0.015404729172587395, + 0.010236777365207672, + 0.038059938699007034 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.06958784908056259, + "style_target": 0, + "support_probability": 0.752194344997406, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7821257810138112, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027614109218120575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9946809411048889, + 0.29511719942092896, + 0.4502541720867157, + 0.22349977493286133, + 0.018743149936199188 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 1, + "style_probability": 0.5234779119491577, + "style_target": 1, + "support_probability": 0.9293365478515625, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7742513843491634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025737358257174492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9903590083122253, + 0.35092583298683167, + 0.49702009558677673, + 0.2636236250400543, + 0.008696289733052254 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 1, + "style_probability": 0.49666768312454224, + "style_target": 0, + "support_probability": 0.9492748379707336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3153661756517982, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6586012244224548, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2887551784515381, + 0.6872439384460449, + 0.12202110141515732, + 0.2721961736679077, + 0.025821374729275703, + 0.01854053884744644 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7725372314453125, + "style_target": 1, + "support_probability": 0.27050942182540894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.247989668040185, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5004452466964722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3302932381629944, + 0.7426692247390747, + 0.2065480351448059, + 0.21643038094043732, + 0.02111106552183628, + 0.05610685050487518 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 1, + "style_probability": 0.355894535779953, + "style_target": 0, + "support_probability": 0.3754056692123413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18432653513472144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6193707585334778, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5675750374794006, + 0.446238249540329, + 0.42261236906051636, + 0.7542107701301575, + 0.019977346062660217, + 0.06079055368900299, + 0.009863249026238918, + 0.05266623944044113, + 0.05251337215304375 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.7890900373458862, + "style_target": 1, + "support_probability": 0.12665590643882751, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002192368751529628, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7568562030792236, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41541147232055664, + 0.351266086101532, + 0.4512665867805481, + 0.630409836769104, + 0.058308664709329605, + 0.06513438373804092, + 0.02795802801847458, + 0.03211917728185654, + 0.08661146461963654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.06709573417901993, + "style_target": 0, + "support_probability": 0.039346061646938324, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8643403326248076, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0927528589963913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21721597015857697, + 0.3565707504749298, + 0.21510513126850128, + 0.9688991904258728, + 0.031035970896482468, + 0.011674112640321255, + 0.0290889460593462, + 0.03138300031423569, + 0.014750328846275806, + 0.019265683367848396, + 0.07467802613973618 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 1, + "style_probability": 0.846162736415863, + "style_target": 1, + "support_probability": 0.9274676442146301, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6321218287376387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25861164927482605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.176951065659523, + 0.36673054099082947, + 0.2861471176147461, + 0.9306625723838806, + 0.025643130764365196, + 0.10174715518951416, + 0.039841603487730026, + 0.09137748926877975, + 0.022459492087364197, + 0.021433159708976746, + 0.032477010041475296 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 0, + "style_probability": 0.5081210732460022, + "style_target": 0, + "support_probability": 0.7801255583763123, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6178026152251881, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.040240850299596786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9988963603973389, + 0.4042770564556122, + 0.5294154286384583, + 0.21262100338935852, + 0.07295280694961548 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 1, + "style_probability": 0.696054220199585, + "style_target": 1, + "support_probability": 0.6570178270339966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7063388423195728, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03816952183842659, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.999875545501709, + 0.2956617772579193, + 0.49964678287506104, + 0.22454765439033508, + 0.02992928959429264 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 0, + "style_probability": 0.8024643659591675, + "style_target": 0, + "support_probability": 0.7153445482254028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3955450374105864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20607520639896393, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1763213872909546, + 0.9965142607688904, + 0.4085133671760559, + 0.12910130620002747, + 0.096644826233387, + 0.08087513595819473 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 1, + "style_probability": 0.7215181589126587, + "style_target": 1, + "support_probability": 0.3052537143230438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39559705923726424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22620104253292084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21839433908462524, + 0.9985584616661072, + 0.37208491563796997, + 0.11699958145618439, + 0.06613782793283463, + 0.08666327595710754 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 0, + "style_probability": 0.5846271514892578, + "style_target": 0, + "support_probability": 0.28950589895248413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4768766706301461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1544993370771408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.04510088637471199, + 0.8980891704559326, + 0.04510088637471199, + 0.1777404397726059, + 0.10177584737539291, + 0.046109892427921295, + 0.059164900332689285, + 0.03731256723403931, + 0.09868384897708893 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 1, + "style_probability": 0.7139998078346252, + "style_target": 1, + "support_probability": 0.3572884798049927, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13863917523317607, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3633784055709839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.0403507836163044, + 0.936342716217041, + 0.0403507836163044, + 0.1782870590686798, + 0.028842369094491005, + 0.0501631498336792, + 0.03085837885737419, + 0.021665751934051514, + 0.0601741261780262 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 1, + "style_probability": 0.4564947485923767, + "style_target": 0, + "support_probability": 0.11544744670391083, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3670661177064502, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3074672818183899, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.048532772809267044, + 0.09287044405937195, + 0.92434161901474, + 0.048532772809267044, + 0.01818249747157097, + 0.1062803640961647, + 0.06578521430492401, + 0.04611210525035858, + 0.030887091532349586, + 0.07398159801959991 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 0, + "style_probability": 0.47810474038124084, + "style_target": 1, + "support_probability": 0.44710513949394226, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.274562521507803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.577923059463501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.041440777480602264, + 0.09653221815824509, + 0.9489820003509521, + 0.041440777480602264, + 0.04911758378148079, + 0.05757896974682808, + 0.02453056536614895, + 0.015547310002148151, + 0.0888981819152832, + 0.030262647196650505 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 0, + "style_probability": 0.6418275833129883, + "style_target": 0, + "support_probability": 0.2841743528842926, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.062263459196750304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9445247054100037, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12140091508626938, + 0.07593906670808792, + 0.06982193887233734, + 0.8218879699707031, + 0.19689233601093292 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.6453636288642883, + "style_target": 1, + "support_probability": 0.03022012673318386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016450577073257675, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8925697803497314, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14676091074943542, + 0.10173389315605164, + 0.07039882242679596, + 0.8813511729240417, + 0.038871362805366516 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.20091672241687775, + "style_target": 0, + "support_probability": 0.05002380162477493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6592159091670549, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22587089240550995, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7231230735778809, + 0.23414060473442078, + 0.8077074289321899, + 0.13375329971313477, + 0.008551710285246372, + 0.09947873651981354 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 1, + "style_probability": 0.9397350549697876, + "style_target": 1, + "support_probability": 0.7907079458236694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3165188361086023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3913249373435974, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.612585723400116, + 0.19114099442958832, + 0.7771515250205994, + 0.11202538758516312, + 0.05788678303360939, + 0.04265216365456581 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 0, + "style_probability": 0.5404056906700134, + "style_target": 0, + "support_probability": 0.568828284740448, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8138303841902712, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07278681546449661, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25418853759765625, + 0.31348782777786255, + 0.4312196373939514, + 0.9656945466995239, + 0.033022310584783554, + 0.03967607766389847, + 0.12143681198358536, + 0.021355589851737022, + 0.12340860068798065 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 1, + "style_probability": 0.9051157236099243, + "style_target": 1, + "support_probability": 0.8443499207496643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5880239423037925, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04803181067109108, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27191662788391113, + 0.2623673379421234, + 0.3648582696914673, + 0.9710217118263245, + 0.04832259193062782, + 0.019229711964726448, + 0.03893282637000084, + 0.09101729840040207, + 0.04776306822896004 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 0, + "style_probability": 0.5384284257888794, + "style_target": 0, + "support_probability": 0.932676374912262, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8864170610259139, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02439570054411888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6219640374183655, + 0.9986883997917175, + 0.8711728453636169, + 0.1219194158911705, + 0.012239528819918633, + 0.016313867643475533, + 0.027096904814243317, + 0.013892995193600655, + 0.008509159088134766, + 0.02928544208407402 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 1, + "style_probability": 0.8043826222419739, + "style_target": 1, + "support_probability": 0.9789880514144897, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3036959643933126, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04186899960041046, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6507140398025513, + 0.9982410669326782, + 0.8644105195999146, + 0.10939135402441025, + 0.035930950194597244, + 0.01904892548918724, + 0.035609856247901917, + 0.050367314368486404, + 0.027212494984269142, + 0.04983092099428177 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 1, + "style_probability": 0.17300638556480408, + "style_target": 0, + "support_probability": 0.9611030220985413, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5553850108365151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7812890410423279, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3180570602416992, + 0.4226851463317871, + 0.8608099818229675, + 0.24475184082984924, + 0.04773795232176781 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.7175632119178772, + "style_target": 1, + "support_probability": 0.4146183431148529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4883299520903134, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5385458469390869, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3106063902378082, + 0.4275798499584198, + 0.8560919761657715, + 0.34783995151519775, + 0.03428197279572487 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.37871885299682617, + "style_target": 0, + "support_probability": 0.7648730874061584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6699548293199784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03113923966884613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9159445762634277, + 0.6506223678588867, + 0.24706526100635529, + 0.6755391955375671, + 0.03217439725995064, + 0.12063755840063095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.7760521173477173, + "style_target": 1, + "support_probability": 0.5607715845108032, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45460764313078356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02170935459434986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.924142599105835, + 0.6885933876037598, + 0.2641156017780304, + 0.7409126162528992, + 0.07570834457874298, + 0.04756646975874901 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.3299868404865265, + "style_target": 0, + "support_probability": 0.5841510891914368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08401422903218404, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7209157943725586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17962832748889923, + 0.5804732441902161, + 0.32687610387802124, + 0.30816102027893066, + 0.04767600819468498, + 0.03443818911910057, + 0.057781320065259933, + 0.10887972265481949, + 0.03776201978325844 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.8936923146247864, + "style_target": 1, + "support_probability": 0.06548255681991577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018242801058588574, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7287925481796265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19203662872314453, + 0.6204344034194946, + 0.3137419521808624, + 0.15403486788272858, + 0.02736339531838894, + 0.025454582646489143, + 0.028188226744532585, + 0.02482701651751995, + 0.04827291518449783 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.17985044419765472, + "style_target": 0, + "support_probability": 0.17918676137924194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006999337213578478, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9512547850608826, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.29514092206954956, + 0.2993730306625366, + 0.35159268975257874, + 0.29514092206954956, + 0.05676991492509842, + 0.10541954636573792, + 0.04743530973792076, + 0.0765271782875061, + 0.11250298470258713, + 0.08901607245206833 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 1, + "style_probability": 0.7675918340682983, + "style_target": 1, + "support_probability": 0.00024176490842364728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000760160026150846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9228576421737671, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2425641268491745, + 0.2848237156867981, + 0.33530014753341675, + 0.2425641268491745, + 0.11592333018779755, + 0.048173826187849045, + 0.1404729187488556, + 0.1366063505411148, + 0.056670013815164566, + 0.044793501496315, + 0.05860366299748421 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 0, + "style_probability": 0.5491324067115784, + "style_target": 0, + "support_probability": 0.0002738448674790561, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21542240790760228, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09727556258440018, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13557825982570648, + 0.34724825620651245, + 0.9153843522071838, + 0.3406186103820801, + 0.059745606034994125 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 0, + "style_probability": 0.4913540780544281, + "style_target": 1, + "support_probability": 0.15182778239250183, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2903130282730566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15794037282466888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1291012316942215, + 0.3509204387664795, + 0.9109432697296143, + 0.3736295998096466, + 0.08551901578903198 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 0, + "style_probability": 0.8476824760437012, + "style_target": 0, + "support_probability": 0.05491084232926369, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1051539447475871, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21882231533527374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7064774036407471, + 0.14126721024513245, + 0.24175716936588287, + 0.26349663734436035, + 0.02864811196923256, + 0.03303049877285957 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 0, + "style_probability": 0.3740224242210388, + "style_target": 1, + "support_probability": 0.1664549708366394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03585243753675002, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3401879072189331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6133047938346863, + 0.2307892143726349, + 0.21060039103031158, + 0.2723187208175659, + 0.05865095555782318, + 0.09150341153144836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 1, + "style_probability": 0.24260535836219788, + "style_target": 0, + "support_probability": 0.04391776770353317, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.035382880778351875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8192018270492554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6182699799537659, + 0.24801044166088104, + 0.8691315054893494, + 0.7102115154266357, + 0.05287458002567291, + 0.10737085342407227, + 0.6640397310256958, + 0.021607795730233192, + 0.04479426145553589 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 0, + "style_probability": 0.4242117702960968, + "style_target": 1, + "support_probability": 0.03479398041963577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00732771173499994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15347230434417725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4958936274051666, + 0.23998147249221802, + 0.8727291226387024, + 0.6123954653739929, + 0.030987899750471115, + 0.04562559351325035, + 0.15098673105239868, + 0.05375788360834122, + 0.0673072412610054 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 1, + "style_probability": 0.034004420042037964, + "style_target": 0, + "support_probability": 0.3183212876319885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0819148136314922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7172028422355652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08891509473323822, + 0.1502634733915329, + 0.5579496026039124, + 0.9367890954017639, + 0.04255089536309242, + 0.04404837638139725, + 0.04225064814090729, + 0.04209809750318527, + 0.24267101287841797, + 0.07555633038282394, + 0.12511084973812103 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 1, + "style_probability": 0.5246069431304932, + "style_target": 1, + "support_probability": 0.06372439861297607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7028145302348149, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.438660591840744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05485853925347328, + 0.1871260404586792, + 0.5794596672058105, + 0.9628220200538635, + 0.04665716737508774, + 0.054678674787282944, + 0.0501369833946228, + 0.043640684336423874, + 0.047052353620529175, + 0.1080700159072876, + 0.02149438112974167 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 0, + "style_probability": 0.775122344493866, + "style_target": 0, + "support_probability": 0.671425998210907, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.537687014842447, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07827446609735489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2710273861885071, + 0.2527824640274048, + 0.8649731874465942, + 0.5325688123703003, + 0.013397090137004852 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 1, + "style_probability": 0.5140444040298462, + "style_target": 1, + "support_probability": 0.7405012249946594, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40516635277627144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11117814481258392, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32381772994995117, + 0.2295779138803482, + 0.8747475147247314, + 0.507132887840271, + 0.062314290553331375 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 1, + "style_probability": 0.3091976046562195, + "style_target": 0, + "support_probability": 0.6460940837860107, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012485362329566003, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8591263890266418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28158196806907654, + 0.2864610254764557, + 0.14435023069381714, + 0.643257737159729, + 0.015579529106616974, + 0.04207489639520645 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 0, + "style_probability": 0.40801486372947693, + "style_target": 1, + "support_probability": 0.009006262756884098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012209496314548282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.702268660068512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3236386775970459, + 0.3123711049556732, + 0.22955012321472168, + 0.7829136252403259, + 0.030229682102799416, + 0.07008261978626251 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 1, + "style_probability": 0.305372416973114, + "style_target": 0, + "support_probability": 0.0433669351041317, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7604767895435496, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031589098274707794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:386:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29669106006622314, + 0.39998719096183777, + 0.9935989379882812, + 0.6157981157302856, + 0.08148729056119919, + 0.035967420786619186, + 0.024303436279296875, + 0.0391862727701664, + 0.03674882650375366 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 1, + "style_probability": 0.5662265419960022, + "style_target": 1, + "support_probability": 0.9393001794815063, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8604972645163184, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027404585853219032, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:386:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2921270430088043, + 0.3762526512145996, + 0.9991012811660767, + 0.6040228009223938, + 0.07401183992624283, + 0.035033512860536575, + 0.06041314825415611, + 0.0558520145714283, + 0.02418380044400692 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 0, + "style_probability": 0.8439685702323914, + "style_target": 0, + "support_probability": 0.9267725944519043, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06856121630207426, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6239228844642639, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16252544522285461, + 0.13336636126041412, + 0.2685723602771759, + 0.6457601189613342, + 0.0133057264611125, + 0.020416101440787315, + 0.01868797466158867, + 0.034557074308395386, + 0.004627196583896875, + 0.01934053562581539, + 0.06971552968025208 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 0, + "style_probability": 0.4992941915988922, + "style_target": 1, + "support_probability": 0.05746612697839737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09425224756663945, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31109511852264404, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17365236580371857, + 0.13765080273151398, + 0.21384868025779724, + 0.6847214698791504, + 0.023002175614237785, + 0.08429592102766037, + 0.015876198187470436, + 0.034436482936143875, + 0.014676226302981377, + 0.01292237639427185, + 0.02930988371372223 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 1, + "style_probability": 0.2809159755706787, + "style_target": 0, + "support_probability": 0.1343623697757721, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18325542678664122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12313736975193024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06087709218263626, + 0.9887527227401733, + 0.3142109215259552, + 0.06087709218263626, + 0.04139287397265434 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 1, + "style_probability": 0.6323317885398865, + "style_target": 1, + "support_probability": 0.1084185391664505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3258559631804209, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08590782433748245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07259280979633331, + 0.9812429547309875, + 0.31865307688713074, + 0.07259280979633331, + 0.019631385803222656 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 0, + "style_probability": 0.6577061414718628, + "style_target": 0, + "support_probability": 0.28435370326042175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09957427541400454, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8558590412139893, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19251187145709991, + 0.4456601142883301, + 0.35818517208099365, + 0.20128297805786133, + 0.17659708857536316, + 0.13264043629169464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.9173645377159119, + "style_target": 1, + "support_probability": 0.18570174276828766, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013110181343956766, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8691886067390442, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22039353847503662, + 0.40194961428642273, + 0.38328319787979126, + 0.1750396341085434, + 0.16077545285224915, + 0.15422554314136505 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.3117072284221649, + "style_target": 0, + "support_probability": 0.029732976108789444, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4346507416550666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041664037853479385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9206996560096741, + 0.2797311246395111, + 0.11090590059757233, + 0.1811150163412094, + 0.11590676009654999, + 0.03305675461888313, + 0.05056340992450714, + 0.1201070249080658, + 0.06137273460626602 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.8234445452690125, + "style_target": 1, + "support_probability": 0.19836966693401337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04051172965813328, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07193353027105331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9333120584487915, + 0.2581815719604492, + 0.12114305794239044, + 0.20328915119171143, + 0.10161490738391876, + 0.047257743775844574, + 0.05431225523352623, + 0.07512889057397842, + 0.07241048663854599 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.14423881471157074, + "style_target": 0, + "support_probability": 0.22899384796619415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6619125288328037, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17078226804733276, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.831380307674408, + 0.6830432415008545, + 0.29838910698890686, + 0.7918005585670471, + 0.048146944493055344, + 0.0179926548153162, + 0.012774278409779072, + 0.008686560206115246, + 0.0574040561914444, + 0.03371470421552658 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 1, + "style_probability": 0.8231298923492432, + "style_target": 1, + "support_probability": 0.8530183434486389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4534346986831058, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4810863733291626, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8371272683143616, + 0.6821767687797546, + 0.3804883062839508, + 0.5751278400421143, + 0.020184913650155067, + 0.022595785558223724, + 0.01359530445188284, + 0.01806379109621048, + 0.06250686943531036, + 0.04354208707809448, + 0.0204020906239748 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 1, + "style_probability": 0.4323153793811798, + "style_target": 0, + "support_probability": 0.7143185138702393, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23534909990446579, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39949485659599304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8764632940292358, + 0.20201362669467926, + 0.6388468146324158, + 0.45466551184654236, + 0.02498852089047432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.7367613315582275, + "style_target": 1, + "support_probability": 0.2484940141439438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08869506556913877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47449174523353577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8505621552467346, + 0.214921772480011, + 0.6291520595550537, + 0.4649692475795746, + 0.05870981141924858 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.306036114692688, + "style_target": 0, + "support_probability": 0.14404363930225372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7726191120125351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15019410848617554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9810815453529358, + 0.09440450370311737, + 0.14393430948257446, + 0.23593994975090027, + 0.009666724130511284, + 0.014850451610982418 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.8281408548355103, + "style_target": 1, + "support_probability": 0.45426133275032043, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15470983901690555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17366249859333038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.987518846988678, + 0.07395473122596741, + 0.13366541266441345, + 0.19630210101604462, + 0.019014496356248856, + 0.0133229224011302 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.15078291296958923, + "style_target": 0, + "support_probability": 0.4256167411804199, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8476149338916044, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10301762074232101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1264140009880066, + 0.9602231383323669, + 0.7246891260147095, + 0.07852020859718323, + 0.06413877755403519, + 0.02913823537528515, + 0.00969032570719719, + 0.0037102235946804285, + 0.0092400424182415 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 1, + "style_probability": 0.8107765316963196, + "style_target": 1, + "support_probability": 0.894219696521759, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7763804135216771, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15580911934375763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16098380088806152, + 0.946878969669342, + 0.5460394024848938, + 0.09816743433475494, + 0.00787760503590107, + 0.010208822786808014, + 0.06807760149240494, + 0.007177942432463169, + 0.01355593092739582 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 1, + "style_probability": 0.44259628653526306, + "style_target": 0, + "support_probability": 0.9482789039611816, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005858788383663375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9961839318275452, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3736460506916046, + 0.2870762050151825, + 0.7625786662101746, + 0.14596204459667206, + 0.07499658316373825, + 0.10213426500558853, + 0.1603884994983673, + 0.11615093052387238, + 0.05062644183635712, + 0.022241266444325447, + 0.017575252801179886 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.7458460927009583, + "style_target": 1, + "support_probability": 0.001502571627497673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003888490781445094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.996672511100769, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38272392749786377, + 0.3433701992034912, + 0.7822681069374084, + 0.14392375946044922, + 0.042333848774433136, + 0.05292718857526779, + 0.035365600138902664, + 0.038553040474653244, + 0.035683006048202515, + 0.03433769568800926, + 0.07150095701217651 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.3946298062801361, + "style_target": 0, + "support_probability": 0.001062353840097785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07718355014268483, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8175647854804993, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8569578528404236, + 0.13641291856765747, + 0.23252476751804352, + 0.35003405809402466, + 0.036490242928266525 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.7874210476875305, + "style_target": 1, + "support_probability": 0.024445783346891403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010086417119039641, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8859449625015259, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8256461024284363, + 0.08773753046989441, + 0.2091912180185318, + 0.33490195870399475, + 0.10408003628253937 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.15383468568325043, + "style_target": 0, + "support_probability": 0.026175150647759438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015724462735497914, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9022417664527893, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12863600254058838, + 0.6121176481246948, + 0.19296209514141083, + 0.12863600254058838, + 0.020916832610964775, + 0.03657327964901924 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 1, + "style_probability": 0.8029567003250122, + "style_target": 1, + "support_probability": 0.011480642482638359, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011154654651322763, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.92572021484375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15802980959415436, + 0.5207934379577637, + 0.19357642531394958, + 0.15802980959415436, + 0.03451406955718994, + 0.06485648453235626 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 1, + "style_probability": 0.44175297021865845, + "style_target": 0, + "support_probability": 0.010094397701323032, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6539380151941964, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1423005759716034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13567720353603363, + 0.14445732533931732, + 0.10181046277284622, + 0.9805151224136353, + 0.04155104234814644, + 0.027730673551559448, + 0.018588637933135033, + 0.01179320365190506, + 0.0270735714584589 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.8484758734703064, + "style_target": 1, + "support_probability": 0.6221656203269958, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1952593829401635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12594231963157654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.139298677444458, + 0.12947596609592438, + 0.10442797094583511, + 0.9704015254974365, + 0.014654714614152908, + 0.02569797821342945, + 0.04350704327225685, + 0.03782077506184578, + 0.06361056119203568 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.29118338227272034, + "style_target": 0, + "support_probability": 0.5499983429908752, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5837104009113224, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0869981124997139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3371874690055847, + 0.1547722965478897, + 0.9809440970420837, + 0.3464644253253937, + 0.06480777263641357, + 0.01857193559408188, + 0.023470234125852585, + 0.028639312833547592, + 0.028910892084240913, + 0.03138505294919014 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.833503782749176, + "style_target": 1, + "support_probability": 0.5342696905136108, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010357519283551539, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3504432439804077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2176806628704071, + 0.16915489733219147, + 0.9778491258621216, + 0.23131436109542847, + 0.052380312234163284, + 0.06503903865814209, + 0.06708719581365585, + 0.05347124859690666, + 0.07316622883081436, + 0.05820641666650772 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.04620683938264847, + "style_target": 0, + "support_probability": 0.1612185388803482, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05539962801763544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7674807906150818, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6079575419425964, + 0.4193617105484009, + 0.19359077513217926, + 0.25191667675971985, + 0.026610827073454857 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 1, + "style_probability": 0.8093419075012207, + "style_target": 1, + "support_probability": 0.03491746634244919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001568303301583963, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9446442127227783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5555365681648254, + 0.48090964555740356, + 0.26405346393585205, + 0.2612365484237671, + 0.05676387995481491 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 1, + "style_probability": 0.34759536385536194, + "style_target": 0, + "support_probability": 0.00294104334898293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009199028934969536, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9037116765975952, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38851746916770935, + 0.12907840311527252, + 0.4971446394920349, + 0.22048714756965637, + 0.011966182850301266, + 0.02562631666660309 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.7671375274658203, + "style_target": 1, + "support_probability": 0.005806491710245609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034928281403754794, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9397196769714355, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3787580132484436, + 0.1532030552625656, + 0.4415401816368103, + 0.2972387671470642, + 0.03412037715315819, + 0.0328994058072567 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.2710752487182617, + "style_target": 0, + "support_probability": 0.003577030962333083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12944156936206377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6525046229362488, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26745226979255676, + 0.37062540650367737, + 0.06842014193534851, + 0.6565966606140137, + 0.09242673218250275, + 0.033271439373493195, + 0.03392352536320686, + 0.02247789315879345, + 0.010856674052774906 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.9403676390647888, + "style_target": 1, + "support_probability": 0.020606106147170067, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038626368835548645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5155940055847168, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21654276549816132, + 0.45022502541542053, + 0.027632275596261024, + 0.7231581807136536, + 0.042054712772369385, + 0.17040961980819702, + 0.021308979019522667, + 0.05596990883350372, + 0.07155423611402512 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.3381105661392212, + "style_target": 0, + "support_probability": 0.08642483502626419, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6429372983575233, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044225577265024185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.528011679649353, + 0.5282576084136963, + 0.7172194123268127, + 0.5153486728668213, + 0.028464028611779213, + 0.01938880793750286, + 0.09146369248628616, + 0.039995286613702774, + 0.012510725297033787, + 0.009135158732533455, + 0.007966892793774605 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.8657968044281006, + "style_target": 1, + "support_probability": 0.9587765336036682, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10195325512563076, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.609309196472168, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36510756611824036, + 0.4619910418987274, + 0.5363498330116272, + 0.39446577429771423, + 0.02363462746143341, + 0.020197516307234764, + 0.015405511483550072, + 0.016563676297664642, + 0.015963083133101463, + 0.022470712661743164, + 0.04559623450040817 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.3216674029827118, + "style_target": 0, + "support_probability": 0.3223644495010376, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13540464934202304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31573352217674255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48919612169265747, + 0.43799498677253723, + 0.40129512548446655, + 0.43799498677253723, + 0.20645259320735931 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.30695492029190063, + "style_target": 1, + "support_probability": 0.3507850468158722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5518405268607564, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10971769690513611, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6229279041290283, + 0.5078586935997009, + 0.47731125354766846, + 0.5078586935997009, + 0.06640054285526276 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.8810180425643921, + "style_target": 0, + "support_probability": 0.7931584715843201, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010381709945008265, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8721172213554382, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4512103199958801, + 0.0853004977107048, + 0.15098130702972412, + 0.35574695467948914, + 0.057274967432022095, + 0.06169957295060158 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 0, + "style_probability": 0.20943044126033783, + "style_target": 1, + "support_probability": 0.026682069525122643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018142001117767805, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9574359655380249, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4657239019870758, + 0.09051328152418137, + 0.147420272231102, + 0.3698749542236328, + 0.04953092709183693, + 0.04348268359899521 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 1, + "style_probability": 0.08944530040025711, + "style_target": 0, + "support_probability": 0.021757634356617928, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7505729343643577, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007918532937765121, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9876412749290466, + 0.26292845606803894, + 0.2693935036659241, + 0.23296506702899933, + 0.012412370182573795, + 0.0624220073223114, + 0.023676615208387375, + 0.04956125468015671, + 0.0591600276529789 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.3560572862625122, + "style_target": 1, + "support_probability": 0.9736790060997009, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8565001672686833, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.00956769660115242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9855565428733826, + 0.31294527649879456, + 0.25685131549835205, + 0.23613257706165314, + 0.024248819798231125, + 0.0337531678378582, + 0.06605451554059982, + 0.024530775845050812, + 0.02117331512272358 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.6906790733337402, + "style_target": 0, + "support_probability": 0.9790661931037903, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26198687147223054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14125660061836243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2344912439584732, + 0.2344912439584732, + 0.12471508234739304, + 0.9487574696540833, + 0.01433954481035471, + 0.043183986097574234, + 0.01253117062151432, + 0.016315927729010582, + 0.019117536023259163, + 0.04127221554517746, + 0.17077882587909698 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 1, + "style_probability": 0.5466756224632263, + "style_target": 1, + "support_probability": 0.09775664657354355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48471334065231747, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07333194464445114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2524699866771698, + 0.2524699866771698, + 0.09864142537117004, + 0.9687841534614563, + 0.1000063493847847, + 0.031592387706041336, + 0.00801089871674776, + 0.014010416343808174, + 0.01900736428797245, + 0.09349468350410461, + 0.047820720821619034 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 0, + "style_probability": 0.8364368081092834, + "style_target": 0, + "support_probability": 0.11876419931650162, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7146017327179601, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.006802820134907961, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10188273340463638, + 0.9751921892166138, + 0.326158344745636, + 0.170403391122818, + 0.04845830425620079 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 0, + "style_probability": 0.4336818754673004, + "style_target": 1, + "support_probability": 0.8676663637161255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5661757390264391, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.012157279066741467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1327609121799469, + 0.9858830571174622, + 0.2918578088283539, + 0.16801393032073975, + 0.01822337694466114 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 1, + "style_probability": 0.43685823678970337, + "style_target": 0, + "support_probability": 0.8355728387832642, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7765977725029071, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1046319380402565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3480457365512848, + 0.07087855786085129, + 0.3480457365512848, + 0.9528536200523376, + 0.04581607133150101, + 0.06488350033760071 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 1, + "style_probability": 0.5149281620979309, + "style_target": 1, + "support_probability": 0.9762985706329346, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5343942940623428, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06243821978569031, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37431028485298157, + 0.0437404066324234, + 0.37431028485298157, + 0.9873649477958679, + 0.014774630777537823, + 0.06391982734203339 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 1, + "style_probability": 0.2727130055427551, + "style_target": 0, + "support_probability": 0.9808452129364014, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.061733796461669516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.735332190990448, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37878885865211487, + 0.6622723340988159, + 0.4895867109298706, + 0.31517836451530457, + 0.12266308069229126, + 0.03542766720056534, + 0.02196704037487507, + 0.03499714657664299, + 0.03995935991406441 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6093059182167053, + "style_target": 1, + "support_probability": 0.022220484912395477, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1118453292984843, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5886207818984985, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5057477355003357, + 0.7049098610877991, + 0.5376850366592407, + 0.37049198150634766, + 0.033808790147304535, + 0.0794980376958847, + 0.08699517697095871, + 0.056667499244213104, + 0.17649522423744202 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 0, + "style_probability": 0.7760353088378906, + "style_target": 0, + "support_probability": 0.03221122920513153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45812202134109836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1321718543767929, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43891483545303345, + 0.8994983434677124, + 0.07630235701799393, + 0.17740578949451447, + 0.013936033472418785, + 0.203854501247406, + 0.5057482719421387, + 0.028913138434290886, + 0.03596150875091553, + 0.031025003641843796, + 0.02859785035252571 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 0, + "style_probability": 0.40040335059165955, + "style_target": 1, + "support_probability": 0.6634227633476257, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12986608478016057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24492457509040833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3878503739833832, + 0.8592523336410522, + 0.12429700791835785, + 0.11887460201978683, + 0.07783187180757523, + 0.12561564147472382, + 0.14749737083911896, + 0.014677868224680424, + 0.043263643980026245, + 0.027784505859017372, + 0.04277443140745163 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 1, + "style_probability": 0.27299582958221436, + "style_target": 0, + "support_probability": 0.1793593317270279, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10390104265338965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17962519824504852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.826810359954834, + 0.10875716060400009, + 0.13106755912303925, + 0.17340748012065887, + 0.02120552584528923 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 1, + "style_probability": 0.6518390774726868, + "style_target": 1, + "support_probability": 0.07029799371957779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05782567989575249, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34743186831474304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7794263362884521, + 0.11397082358598709, + 0.15573664009571075, + 0.20797759294509888, + 0.06551235169172287 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 0, + "style_probability": 0.5781214237213135, + "style_target": 0, + "support_probability": 0.026097727939486504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30271120605101354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5102893710136414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3395041823387146, + 0.3073360323905945, + 0.3395041823387146, + 0.805095911026001, + 0.14314258098602295, + 0.030333170667290688 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.645828366279602, + "style_target": 1, + "support_probability": 0.31563133001327515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2581308478719949, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.31157606840133667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4075888693332672, + 0.3441344201564789, + 0.4075888693332672, + 0.7968089580535889, + 0.06761696189641953, + 0.04632338136434555 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.1920071691274643, + "style_target": 0, + "support_probability": 0.32294216752052307, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4406049818906707, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2000843584537506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14818283915519714, + 0.2919794023036957, + 0.46285080909729004, + 0.041255462914705276, + 0.026616398245096207, + 0.005170256365090609, + 0.004091858863830566, + 0.008851173333823681, + 0.004226328805088997 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.6439298391342163, + "style_target": 1, + "support_probability": 0.873979926109314, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05211273814587473, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24183019995689392, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21756286919116974, + 0.2867119312286377, + 0.6516695022583008, + 0.053140655159950256, + 0.03263268619775772, + 0.02510889619588852, + 0.011564552783966064, + 0.011047730222344398, + 0.03005920723080635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.03614155575633049, + "style_target": 0, + "support_probability": 0.8743882179260254, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006498770389050794, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5699430108070374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22894257307052612, + 0.7085689902305603, + 0.07115055620670319, + 0.15126408636569977, + 0.020233768969774246, + 0.040173448622226715, + 0.02654673531651497, + 0.022262072190642357, + 0.01979573629796505, + 0.02322644554078579 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 1, + "style_probability": 0.8013906478881836, + "style_target": 1, + "support_probability": 0.0014332304708659649, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009588734643443718, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6504629254341125, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2699538767337799, + 0.7288696765899658, + 0.09765037894248962, + 0.18813841044902802, + 0.07521022856235504, + 0.023862237110733986, + 0.022765547037124634, + 0.009810803458094597, + 0.05095411092042923, + 0.028235794976353645 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 0, + "style_probability": 0.7152588963508606, + "style_target": 0, + "support_probability": 0.0016229848843067884, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7823763570474392, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.02962901070713997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21609579026699066, + 0.03262033686041832, + 0.912373960018158, + 0.11985205113887787, + 0.06881830096244812 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.8807604908943176, + "style_target": 1, + "support_probability": 0.8286068439483643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4711137992891964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.01789066568017006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26696962118148804, + 0.032061174511909485, + 0.8816941976547241, + 0.10523197054862976, + 0.18738919496536255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.22107936441898346, + "style_target": 0, + "support_probability": 0.8095635771751404, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7338000192939249, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10021864622831345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9163289070129395, + 0.7320201396942139, + 0.6308382153511047, + 0.31741172075271606, + 0.012884248048067093, + 0.04464392364025116 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 1, + "style_probability": 0.8131385445594788, + "style_target": 1, + "support_probability": 0.6118422150611877, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32039289244379265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17785990238189697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9193508625030518, + 0.7501809597015381, + 0.5253236293792725, + 0.29318374395370483, + 0.017195837572216988, + 0.0527382530272007 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 1, + "style_probability": 0.3978096544742584, + "style_target": 0, + "support_probability": 0.46256616711616516, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018071683972586607, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7380503416061401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8227536082267761, + 0.11797726154327393, + 0.2205565720796585, + 0.0895233154296875, + 0.018688971176743507, + 0.0579802542924881, + 0.06493563205003738, + 0.04645857959985733, + 0.09432169795036316 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 1, + "style_probability": 0.7820866107940674, + "style_target": 1, + "support_probability": 0.006176626775413752, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006498206599080509, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8288520574569702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.769111692905426, + 0.09881913661956787, + 0.28539198637008667, + 0.11343187838792801, + 0.11506149172782898, + 0.06634342670440674, + 0.04875154793262482, + 0.030978986993432045, + 0.030918173491954803 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 0, + "style_probability": 0.5724874138832092, + "style_target": 0, + "support_probability": 0.0024165003560483456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2049441072659855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5197796821594238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1444171816110611, + 0.05022081360220909, + 0.3453611433506012, + 0.2496165633201599, + 0.012329642660915852, + 0.013867468573153019, + 0.025651006028056145, + 0.020305605605244637, + 0.025162912905216217, + 0.03441443666815758 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 1, + "style_probability": 0.8572139143943787, + "style_target": 1, + "support_probability": 0.11612376570701599, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044531323226797814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7766422629356384, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15271355211734772, + 0.06552959233522415, + 0.2768547534942627, + 0.26114779710769653, + 0.03016304410994053, + 0.04654122516512871, + 0.03468208760023117, + 0.04016025736927986, + 0.0381360799074173, + 0.06131604686379433, + 0.14040279388427734 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 1, + "style_probability": 0.39261171221733093, + "style_target": 0, + "support_probability": 0.03154296427965164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024172994503964304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7164251804351807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39195704460144043, + 0.4987218976020813, + 0.6504631042480469, + 0.39195704460144043, + 0.021269487217068672 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 1, + "style_probability": 0.7490217089653015, + "style_target": 1, + "support_probability": 0.0041830879636108875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007297880246133776, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7629483342170715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39971980452537537, + 0.5113167762756348, + 0.6038057804107666, + 0.39971980452537537, + 0.027477366849780083 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 1, + "style_probability": 0.35794785618782043, + "style_target": 0, + "support_probability": 0.003691233927384019, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2805826463023063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27667802572250366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4236205220222473, + 0.24592076241970062, + 0.470509797334671, + 0.9739032983779907, + 0.031020810827612877, + 0.02279568836092949 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 1, + "style_probability": 0.8467410802841187, + "style_target": 1, + "support_probability": 0.06587324291467667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2643637144011228, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29765433073043823, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3798631727695465, + 0.28479865193367004, + 0.44253069162368774, + 0.9739534854888916, + 0.07323161512613297, + 0.06117123365402222 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 0, + "style_probability": 0.7154897451400757, + "style_target": 0, + "support_probability": 0.13294623792171478, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6966839627267115, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.036116063594818115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9625853896141052, + 0.1586121916770935, + 0.16834811866283417, + 0.1586121916770935, + 0.03440525010228157, + 0.046511270105838776, + 0.025865303352475166, + 0.035344623029232025, + 0.03359023109078407 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.8801835775375366, + "style_target": 1, + "support_probability": 0.7945923209190369, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3867241105390075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05216072499752045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9442896246910095, + 0.11449356377124786, + 0.19870024919509888, + 0.11449356377124786, + 0.03398776799440384, + 0.07342047989368439, + 0.01803918182849884, + 0.018136443570256233, + 0.05260396748781204 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.33078745007514954, + "style_target": 0, + "support_probability": 0.7454159259796143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1398117492843097, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7331069707870483, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5216909050941467, + 0.3405378460884094, + 0.5216909050941467, + 0.7469097971916199, + 0.03857724368572235, + 0.04151294752955437, + 0.05457933992147446, + 0.03500426560640335, + 0.05172529071569443, + 0.03047124482691288, + 0.02657688781619072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.8445454239845276, + "style_target": 1, + "support_probability": 0.08330105990171432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004284143492855139, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8736239671707153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3976261019706726, + 0.352839857339859, + 0.3976261019706726, + 0.669711709022522, + 0.05477569252252579, + 0.06894288957118988, + 0.049499452114105225, + 0.03836081176996231, + 0.03866090252995491, + 0.0802551656961441 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.02526765689253807, + "style_target": 0, + "support_probability": 0.12840744853019714, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04848409973800358, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7574804425239563, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.571858823299408, + 0.1767270714044571, + 0.2965983748435974, + 0.6127839684486389, + 0.0701947957277298 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.6941908001899719, + "style_target": 1, + "support_probability": 0.027495360001921654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005293062537031715, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.92926424741745, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5249268412590027, + 0.1176799088716507, + 0.24633941054344177, + 0.4972095191478729, + 0.055898070335388184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.19569019973278046, + "style_target": 0, + "support_probability": 0.011880450882017612, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.40902107596325826, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4630756974220276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5185564160346985, + 0.4216238558292389, + 0.2752443850040436, + 0.06157652288675308, + 0.016305698081851006, + 0.02001791074872017 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.4633370637893677, + "style_target": 1, + "support_probability": 0.8490554094314575, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5771999369626146, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20989567041397095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5630386471748352, + 0.6243629455566406, + 0.20651991665363312, + 0.02577272057533264, + 0.018224969506263733, + 0.013582922518253326 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.960369348526001, + "style_target": 0, + "support_probability": 0.902187705039978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4281980007520233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12403009831905365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1906886249780655, + 0.9952804446220398, + 0.16130003333091736, + 0.25052326917648315, + 0.035247553139925, + 0.10025952756404877, + 0.04586097225546837, + 0.05010470375418663, + 0.0397087000310421 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 1, + "style_probability": 0.5885393023490906, + "style_target": 1, + "support_probability": 0.3642587959766388, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28260156605769865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12927302718162537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23700226843357086, + 0.9963093400001526, + 0.14624284207820892, + 0.2024131864309311, + 0.08492525666952133, + 0.0351666584610939, + 0.017495224252343178, + 0.07796420156955719, + 0.06424097716808319 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 1, + "style_probability": 0.45747947692871094, + "style_target": 0, + "support_probability": 0.277363121509552, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6132454323220523, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1794588714838028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6678018569946289, + 0.9084851741790771, + 0.17553363740444183, + 0.13825221359729767, + 0.041435759514570236, + 0.01625688187777996, + 0.14725767076015472, + 0.12113004177808762, + 0.2182319164276123, + 0.057839639484882355, + 0.04692160338163376 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 0, + "style_probability": 0.4334765672683716, + "style_target": 1, + "support_probability": 0.7635195851325989, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05542934147538046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23747627437114716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.49824532866477966, + 0.9767787456512451, + 0.16753366589546204, + 0.20181915163993835, + 0.037700455635786057, + 0.15287509560585022, + 0.03862784802913666, + 0.0783226415514946, + 0.03273748606443405, + 0.06730124354362488 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 1, + "style_probability": 0.04303915798664093, + "style_target": 0, + "support_probability": 0.7712822556495667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7900667956079064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.041789744049310684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21498936414718628, + 0.16411715745925903, + 0.11087775230407715, + 0.9811862707138062, + 0.047804273664951324 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 1, + "style_probability": 0.8361242413520813, + "style_target": 1, + "support_probability": 0.8560540676116943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.579060532004099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.06546930968761444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2806970477104187, + 0.20378002524375916, + 0.16595764458179474, + 0.9768170714378357, + 0.047152869403362274 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 1, + "style_probability": 0.47277000546455383, + "style_target": 0, + "support_probability": 0.8492549061775208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5373858380263161, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10639632493257523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2349894642829895, + 0.38763904571533203, + 0.9954453706741333, + 0.4080885946750641, + 0.11755556613206863, + 0.040856990963220596 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.8232907056808472, + "style_target": 1, + "support_probability": 0.5225077271461487, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14363967430716476, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14037221670150757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17689549922943115, + 0.35649558901786804, + 0.9978158473968506, + 0.35807913541793823, + 0.02084699086844921, + 0.03745657950639725 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.24651245772838593, + "style_target": 0, + "support_probability": 0.6192392110824585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040855622338511685, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7501400709152222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28405970335006714, + 0.7443616986274719, + 0.08712223172187805, + 0.4056747257709503, + 0.060682084411382675, + 0.1820526421070099, + 0.15779276192188263, + 0.14574435353279114, + 0.07538972795009613 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 1, + "style_probability": 0.7186683416366577, + "style_target": 1, + "support_probability": 0.014452002011239529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008141185915011817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8281733989715576, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3237169086933136, + 0.6385661363601685, + 0.1367957890033722, + 0.3828561305999756, + 0.1452665627002716, + 0.1322144716978073, + 0.02285328321158886, + 0.11496727168560028, + 0.1300773173570633 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 1, + "style_probability": 0.4550611972808838, + "style_target": 0, + "support_probability": 0.005994193255901337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022830966671792297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41218283772468567, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47906962037086487, + 0.13434536755084991, + 0.5118895173072815, + 0.7753210663795471, + 0.012469390407204628, + 0.029646756127476692, + 0.13728740811347961, + 0.07670257985591888, + 0.08777783811092377, + 0.06242170184850693, + 0.054372869431972504 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.9220051765441895, + "style_target": 1, + "support_probability": 0.0029086442664265633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015233086563845193, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24200327694416046, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36108168959617615, + 0.11383365839719772, + 0.5523520112037659, + 0.7461461424827576, + 0.03601035103201866, + 0.038692522794008255, + 0.12551775574684143, + 0.06201945245265961, + 0.03125733509659767, + 0.04409632086753845, + 0.031028904020786285 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.2526494264602661, + "style_target": 0, + "support_probability": 0.005269545596092939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04527399632758152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30240678787231445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16056370735168457, + 0.29474112391471863, + 0.8811943531036377, + 0.16056370735168457, + 0.0291704460978508 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 1, + "style_probability": 0.5260316729545593, + "style_target": 1, + "support_probability": 0.039130691438913345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014805101251192178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4147043526172638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12796510756015778, + 0.29031264781951904, + 0.8526705503463745, + 0.12796510756015778, + 0.055203910917043686 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 1, + "style_probability": 0.26382559537887573, + "style_target": 0, + "support_probability": 0.01144530437886715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0035812716211436656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9845060706138611, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5651865005493164, + 0.17587251961231232, + 0.403604120016098, + 0.2080669403076172, + 0.04362277686595917, + 0.018765166401863098 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 0, + "style_probability": 0.25478145480155945, + "style_target": 1, + "support_probability": 0.003393713152036071, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008955000769669429, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9780248403549194, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3686612546443939, + 0.11075104027986526, + 0.22873800992965698, + 0.13742132484912872, + 0.07680024206638336, + 0.19253087043762207 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 1, + "style_probability": 0.06567631661891937, + "style_target": 0, + "support_probability": 0.021089205518364906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33235085917541307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12089128792285919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4487786293029785, + 0.7758006453514099, + 0.11416133493185043, + 0.38520705699920654, + 0.02914341539144516, + 0.06294956803321838, + 0.04908525571227074, + 0.06420687586069107, + 0.018043890595436096 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 1, + "style_probability": 0.571092426776886, + "style_target": 1, + "support_probability": 0.5149880051612854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.562815808357, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08292105048894882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.485927551984787, + 0.7115681171417236, + 0.13246452808380127, + 0.2451394647359848, + 0.042365897446870804, + 0.013280559331178665, + 0.10355746001005173, + 0.04024206101894379, + 0.027665570378303528 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 0, + "style_probability": 0.926958441734314, + "style_target": 0, + "support_probability": 0.6391115188598633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5967439408895612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02935357764363289, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.964523434638977, + 0.36659306287765503, + 0.16230246424674988, + 0.23951733112335205, + 0.015683650970458984, + 0.032207831740379333, + 0.012017661705613136, + 0.09598350524902344, + 0.00985797494649887, + 0.013543891720473766, + 0.013047344982624054 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 1, + "style_probability": 0.6129915118217468, + "style_target": 1, + "support_probability": 0.6314006447792053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5627870911358708, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026075562462210655, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9719657897949219, + 0.36432501673698425, + 0.16753868758678436, + 0.26151129603385925, + 0.05199392884969711, + 0.017190471291542053, + 0.01041531004011631, + 0.05379727855324745, + 0.025239063426852226, + 0.021614011377096176, + 0.07034634053707123 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 1, + "style_probability": 0.4698733687400818, + "style_target": 0, + "support_probability": 0.7130764722824097, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7175156857918048, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09050191938877106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.981020450592041, + 0.13124847412109375, + 0.46912825107574463, + 0.3605204224586487, + 0.015436114743351936 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 0, + "style_probability": 0.4709049165248871, + "style_target": 1, + "support_probability": 0.9803652763366699, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7668720942892164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06941040605306625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9701557755470276, + 0.14534112811088562, + 0.5104313492774963, + 0.2959538996219635, + 0.03475243225693703 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 1, + "style_probability": 0.4666410982608795, + "style_target": 0, + "support_probability": 0.9772759079933167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1458820447943593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.45695704221725464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17127537727355957, + 0.11781686544418335, + 0.38610681891441345, + 0.9228885173797607, + 0.016649216413497925, + 0.028531938791275024 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 1, + "style_probability": 0.5655041933059692, + "style_target": 1, + "support_probability": 0.08851058781147003, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11137557930354706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7841062545776367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17728793621063232, + 0.10633495450019836, + 0.3217264413833618, + 0.9313808083534241, + 0.0495004840195179, + 0.06226816773414612 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 0, + "style_probability": 0.887153148651123, + "style_target": 0, + "support_probability": 0.0268959179520607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4369206351979059, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20930956304073334, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8137043118476868, + 0.2422727793455124, + 0.80571448802948, + 0.8137043118476868, + 0.13373994827270508, + 0.027117883786559105, + 0.08105536550283432, + 0.012291144579648972, + 0.038403384387493134 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 0, + "style_probability": 0.3847569525241852, + "style_target": 1, + "support_probability": 0.7626447677612305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11068832310043002, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.314272940158844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.768036425113678, + 0.1863187998533249, + 0.7371078133583069, + 0.768036425113678, + 0.07565343379974365, + 0.07101873308420181, + 0.06031924486160278, + 0.021911241114139557, + 0.04659944027662277 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 1, + "style_probability": 0.09896121174097061, + "style_target": 0, + "support_probability": 0.6828521490097046, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03685049088661744, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8090037703514099, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3917745351791382, + 0.3049924373626709, + 0.47479328513145447, + 0.18979153037071228, + 0.05696765333414078, + 0.10789959132671356, + 0.071375273168087, + 0.02802661433815956, + 0.06495543569326401, + 0.013332615606486797, + 0.044858239591121674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 0, + "style_probability": 0.40747570991516113, + "style_target": 1, + "support_probability": 0.031045343726873398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012432499169030213, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9516854882240295, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35279226303100586, + 0.3701803386211395, + 0.4805164337158203, + 0.2722322940826416, + 0.01767759583890438, + 0.08331522345542908, + 0.056088268756866455, + 0.04456965997815132, + 0.0432610884308815, + 0.022646259516477585, + 0.019064560532569885 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 0, + "style_probability": 0.5630351901054382, + "style_target": 0, + "support_probability": 0.010952885262668133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5307117556067054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06445077806711197, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37948402762413025, + 0.9965813755989075, + 0.2911693751811981, + 0.12455599009990692, + 0.051300305873155594 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 1, + "style_probability": 0.7789562344551086, + "style_target": 1, + "support_probability": 0.37448152899742126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39104288247649904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03617282211780548, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.375272274017334, + 0.994758665561676, + 0.3450818657875061, + 0.1742948591709137, + 0.05487598851323128 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 1, + "style_probability": 0.3819330036640167, + "style_target": 0, + "support_probability": 0.5207455158233643, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7271561471588831, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038662880659103394, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22503994405269623, + 0.828575611114502, + 0.4363090693950653, + 0.22235843539237976, + 0.025348646566271782, + 0.01827920973300934 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.928643524646759, + "style_target": 1, + "support_probability": 0.9403768181800842, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03977363104300234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08096195012331009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2122143656015396, + 0.8500038385391235, + 0.2725689113140106, + 0.17331993579864502, + 0.03307543694972992, + 0.05748039111495018 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.04139407351613045, + "style_target": 0, + "support_probability": 0.8781200647354126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005611989103886774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8513680696487427, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3096778690814972, + 0.22429461777210236, + 0.25221556425094604, + 0.2916325032711029, + 0.06842644512653351, + 0.033390581607818604, + 0.031632017344236374, + 0.13026754558086395, + 0.10484306514263153 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 1, + "style_probability": 0.8001407384872437, + "style_target": 1, + "support_probability": 0.0017618797719478607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011398751847559166, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9354157447814941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3058348298072815, + 0.24408859014511108, + 0.202656552195549, + 0.25957190990448, + 0.037253569811582565, + 0.03126806393265724, + 0.03883016109466553, + 0.031834911555051804, + 0.04472906142473221 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 1, + "style_probability": 0.33529382944107056, + "style_target": 0, + "support_probability": 0.000685181119479239, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011668316309989244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21938131749629974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3064892888069153, + 0.1347743719816208, + 0.9092009663581848, + 0.26269835233688354, + 0.033680375665426254, + 0.049301303923130035, + 0.09743601828813553, + 0.044935908168554306, + 0.04309181123971939, + 0.0746532678604126, + 0.02879968099296093 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.6810340881347656, + "style_target": 1, + "support_probability": 0.0030224735382944345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008475665724737425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5050740838050842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3954821228981018, + 0.16037589311599731, + 0.9064400792121887, + 0.27486586570739746, + 0.07451052963733673, + 0.09509075433015823, + 0.06459175050258636, + 0.06198921054601669, + 0.023914122954010963, + 0.09473219513893127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.3662855327129364, + "style_target": 0, + "support_probability": 0.001065600081346929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2018802402095806, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1579875946044922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35424235463142395, + 0.6827203631401062, + 0.6086714863777161, + 0.35424235463142395, + 0.015138682909309864 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.8264946937561035, + "style_target": 1, + "support_probability": 0.04821096733212471, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00811239130043026, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3184624910354614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3316856622695923, + 0.5557417273521423, + 0.543346643447876, + 0.3316856622695923, + 0.04314936697483063 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.1701970398426056, + "style_target": 0, + "support_probability": 0.01865244284272194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0792410923055513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30738377571105957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19562004506587982, + 0.19492584466934204, + 0.19218088686466217, + 0.9687419533729553, + 0.02731049805879593, + 0.02909715473651886 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 0, + "style_probability": 0.48795783519744873, + "style_target": 1, + "support_probability": 0.042168840765953064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32356818532293447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16153405606746674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12946633994579315, + 0.217217817902565, + 0.18306083977222443, + 0.972252368927002, + 0.08313991874456406, + 0.038423649966716766 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 0, + "style_probability": 0.6653086543083191, + "style_target": 0, + "support_probability": 0.18844202160835266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8324361381902157, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08445779234170914, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20136067271232605, + 0.2920723557472229, + 0.977438747882843, + 0.14884231984615326, + 0.02557199075818062, + 0.09689954668283463, + 0.1254005879163742, + 0.037891555577516556, + 0.020312663167715073 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.8692310452461243, + "style_target": 1, + "support_probability": 0.8451296091079712, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29357607642710937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15616516768932343, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25377604365348816, + 0.277797669172287, + 0.9722267985343933, + 0.13767492771148682, + 0.03589925542473793, + 0.032267309725284576, + 0.205158993601799, + 0.23539645969867706, + 0.05135186016559601 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.2931484878063202, + "style_target": 0, + "support_probability": 0.6670064926147461, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27660459944341953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3414382040500641, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25665363669395447, + 0.8591426014900208, + 0.21861203014850616, + 0.14055395126342773, + 0.0605628527700901, + 0.26491039991378784, + 0.03507797420024872, + 0.06653851270675659, + 0.0873357281088829, + 0.03633278235793114, + 0.04580342024564743 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.8988832831382751, + "style_target": 1, + "support_probability": 0.2036280483007431, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03453330515458641, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24547743797302246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2085612714290619, + 0.8330408334732056, + 0.12941637635231018, + 0.24731820821762085, + 0.04614373296499252, + 0.04068949073553085, + 0.03636613488197327, + 0.02621946483850479, + 0.01809208281338215, + 0.04092521220445633 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.046685703098773956, + "style_target": 0, + "support_probability": 0.5694715976715088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021219021627407422, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.680627703666687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7063462138175964, + 0.39459386467933655, + 0.41099512577056885, + 0.5188941955566406, + 0.03099171444773674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 0, + "style_probability": 0.27085232734680176, + "style_target": 1, + "support_probability": 0.009386842139065266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016011238711238412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7779648303985596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6621886491775513, + 0.39347314834594727, + 0.4384794235229492, + 0.5070929527282715, + 0.13514739274978638 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 1, + "style_probability": 0.4853994846343994, + "style_target": 0, + "support_probability": 0.014385120943188667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.745547649675832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026028765365481377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.969694972038269, + 0.2504896819591522, + 0.3176262080669403, + 0.2504896819591522, + 0.0070908465422689915, + 0.01910248212516308 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.342086523771286, + "style_target": 1, + "support_probability": 0.9879032969474792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.880283961615177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03222346305847168, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9746935963630676, + 0.20220385491847992, + 0.289522647857666, + 0.20220385491847992, + 0.00970278400927782, + 0.03336136415600777 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.5892877578735352, + "style_target": 0, + "support_probability": 0.9845204949378967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25731169857304353, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28445863723754883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.04215005040168762, + 0.10929154604673386, + 0.27580225467681885, + 0.9771300554275513, + 0.02993302047252655, + 0.02714894339442253, + 0.011751387268304825, + 0.012983095832169056, + 0.01959974132478237 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 0, + "style_probability": 0.3334258794784546, + "style_target": 1, + "support_probability": 0.15327970683574677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13516086619239065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23276260495185852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.03765375167131424, + 0.0862128734588623, + 0.24727043509483337, + 0.9893904328346252, + 0.01440960168838501, + 0.0410456620156765, + 0.03383258357644081, + 0.01035465206950903, + 0.0187153872102499 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 1, + "style_probability": 0.3380439579486847, + "style_target": 0, + "support_probability": 0.09109088033437729, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22737494582459566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1769116222858429, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21080374717712402, + 0.8169418573379517, + 0.22716565430164337, + 0.4821881055831909, + 0.04574568569660187, + 0.06940984725952148, + 0.03567337617278099, + 0.04726782441139221, + 0.053904566913843155, + 0.018560128286480904, + 0.030079923570156097 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 0, + "style_probability": 0.1117347851395607, + "style_target": 1, + "support_probability": 0.48966076970100403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40010996096497725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13086575269699097, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.306110143661499, + 0.8666101694107056, + 0.3042207360267639, + 0.48952728509902954, + 0.02906912751495838, + 0.019531458616256714, + 0.03459502011537552, + 0.011471208184957504, + 0.05699003115296364, + 0.014154675416648388, + 0.18214672803878784 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 1, + "style_probability": 0.21572273969650269, + "style_target": 0, + "support_probability": 0.7767000198364258, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8248865043573647, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1071380004286766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:856:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9887674450874329, + 0.2381635457277298, + 0.2563917934894562, + 0.20289449393749237, + 0.09269320964813232 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 1, + "style_probability": 0.7319155931472778, + "style_target": 1, + "support_probability": 0.9470374584197998, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.912429530880306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10991060733795166, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:856:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9946268200874329, + 0.2661483585834503, + 0.21023717522621155, + 0.17725200951099396, + 0.01297830045223236 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 0, + "style_probability": 0.8617051839828491, + "style_target": 0, + "support_probability": 0.9306949973106384, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7707313797432791, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009828630834817886, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9821532368659973, + 0.25650086998939514, + 0.2596084475517273, + 0.14201156795024872, + 0.09204501658678055, + 0.07270314544439316 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 1, + "style_probability": 0.5751940608024597, + "style_target": 1, + "support_probability": 0.8900532126426697, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5096882543473045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014907022006809711, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9778008460998535, + 0.2272620052099228, + 0.18605269491672516, + 0.1470707654953003, + 0.03731221333146095, + 0.10025984048843384 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 1, + "style_probability": 0.2822360694408417, + "style_target": 0, + "support_probability": 0.8011894822120667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36882207366014463, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2516144812107086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5007842183113098, + 0.9219362139701843, + 0.19841566681861877, + 0.319087952375412, + 0.006996973883360624, + 0.012022638693451881, + 0.020813653245568275, + 0.043061286211013794, + 0.016741979867219925 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 1, + "style_probability": 0.5747687220573425, + "style_target": 1, + "support_probability": 0.3446948826313019, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04196243739643514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2609308063983917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5955719947814941, + 0.9694499969482422, + 0.12094278633594513, + 0.25924214720726013, + 0.027114279568195343, + 0.06385961174964905, + 0.06399844586849213, + 0.02887522056698799, + 0.08156221359968185 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 1, + "style_probability": 0.07644302397966385, + "style_target": 0, + "support_probability": 0.3288237154483795, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5448278887877978, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16778849065303802, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8697337508201599, + 0.30627548694610596, + 0.5854927897453308, + 0.3428078889846802, + 0.027545683085918427, + 0.06633016467094421, + 0.03207290172576904, + 0.0176194217056036, + 0.025103939697146416, + 0.07264388352632523, + 0.050898097455501556 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.31976714730262756, + "style_target": 1, + "support_probability": 0.7034898400306702, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7482678370530186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0869438499212265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8837583661079407, + 0.3662736713886261, + 0.638344407081604, + 0.4030735194683075, + 0.025525936856865883, + 0.010850746184587479, + 0.017314117401838303, + 0.005543249659240246, + 0.02341294288635254, + 0.029456308111548424, + 0.02282395027577877 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.8218203783035278, + "style_target": 0, + "support_probability": 0.7621610164642334, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12399974808202247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8298948407173157, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8485339879989624, + 0.16016535460948944, + 0.10370476543903351, + 0.3968467116355896, + 0.07439558207988739 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.8253759741783142, + "style_target": 1, + "support_probability": 0.1534074991941452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0024840813726643343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8640549778938293, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8376272916793823, + 0.13937199115753174, + 0.06068503484129906, + 0.3564571142196655, + 0.0313233956694603 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.055977482348680496, + "style_target": 0, + "support_probability": 0.07874106615781784, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6602414226383914, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13732293248176575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09627711772918701, + 0.19762009382247925, + 0.039430879056453705, + 0.9587981104850769, + 0.013286027126014233, + 0.0853235274553299 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 1, + "style_probability": 0.7621297240257263, + "style_target": 1, + "support_probability": 0.6060948371887207, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5594381778591597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.18357832729816437, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11087486892938614, + 0.21395546197891235, + 0.04038846865296364, + 0.9506069421768188, + 0.31817135214805603, + 0.030616773292422295 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 1, + "style_probability": 0.45516306161880493, + "style_target": 0, + "support_probability": 0.721275806427002, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5921856631591842, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5517450571060181, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2846558392047882, + 0.491066038608551, + 0.8664259910583496, + 0.6670809388160706, + 0.00940045528113842, + 0.024077195674180984, + 0.028460422530770302, + 0.00773240951821208, + 0.08944623172283173 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.8708398938179016, + "style_target": 1, + "support_probability": 0.5827209949493408, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20846093081162861, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6577706336975098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3618999719619751, + 0.5022093057632446, + 0.8446060419082642, + 0.6758471727371216, + 0.028537482023239136, + 0.15519729256629944, + 0.02445942722260952, + 0.02652842551469803, + 0.050735555589199066 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.46384772658348083, + "style_target": 0, + "support_probability": 0.34426945447921753, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6613999455552684, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23896025121212006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8702759146690369, + 0.6046289205551147, + 0.2861412763595581, + 0.14226485788822174, + 0.008771046064794064, + 0.03503691405057907, + 0.011422716081142426, + 0.020261472091078758, + 0.007282858714461327, + 0.009335396811366081, + 0.03531211242079735 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.7681668400764465, + "style_target": 1, + "support_probability": 0.7191542983055115, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04411280223754827, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3726835250854492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8424192667007446, + 0.5331170558929443, + 0.2657754123210907, + 0.10847820341587067, + 0.015755638480186462, + 0.023468540981411934, + 0.055220428854227066, + 0.03809134662151337, + 0.013561454601585865, + 0.02627301961183548 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.07434871792793274, + "style_target": 0, + "support_probability": 0.555075466632843, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8373373342451247, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014711185358464718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9977726340293884, + 0.5859773755073547, + 0.43793728947639465, + 0.294939249753952, + 0.035277895629405975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 1, + "style_probability": 0.5835575461387634, + "style_target": 1, + "support_probability": 0.9203842878341675, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7503195894896735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021181806921958923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9940965175628662, + 0.5111185908317566, + 0.5865665078163147, + 0.3020898997783661, + 0.2019967883825302 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 0, + "style_probability": 0.5322616100311279, + "style_target": 0, + "support_probability": 0.8750454187393188, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1628082013661803, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13953082263469696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3363322615623474, + 0.7275922894477844, + 0.3671547770500183, + 0.2658642530441284, + 0.09298746287822723, + 0.06827914714813232 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.8803123235702515, + "style_target": 1, + "support_probability": 0.08878225088119507, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01956447251822671, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3223290741443634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3074124753475189, + 0.6686418652534485, + 0.28139808773994446, + 0.2070854902267456, + 0.0933561846613884, + 0.0583096407353878 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.30530452728271484, + "style_target": 0, + "support_probability": 0.03199697658419609, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8581459373356062, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02448461949825287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.707074761390686, + 0.9482212662696838, + 0.4296127259731293, + 0.3557182550430298, + 0.02309185080230236, + 0.01094114501029253, + 0.007337587419897318, + 0.019060110673308372, + 0.0276828333735466 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.6953559517860413, + "style_target": 1, + "support_probability": 0.9843965172767639, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34004459273482307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0329725481569767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.64421147108078, + 0.9487398862838745, + 0.3304492235183716, + 0.32541248202323914, + 0.0276156198233366, + 0.036649055778980255, + 0.02855307050049305, + 0.00794703047722578, + 0.02190445549786091 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.0738716572523117, + "style_target": 0, + "support_probability": 0.9727941155433655, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8997519047061358, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009632864966988564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24486567080020905, + 0.27013227343559265, + 0.989521861076355, + 0.32793718576431274, + 0.010857181623578072, + 0.07981785386800766, + 0.014976798556745052, + 0.017358560115098953, + 0.007100208662450314, + 0.009766189381480217, + 0.0740620344877243 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 1, + "style_probability": 0.8034537434577942, + "style_target": 1, + "support_probability": 0.9744570851325989, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.696518441162548, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014740590006113052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2241828888654709, + 0.3085505962371826, + 0.9786655306816101, + 0.35575592517852783, + 0.023868674412369728, + 0.008375750854611397, + 0.017543936148285866, + 0.03310306742787361, + 0.0646260604262352, + 0.08391021192073822 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 1, + "style_probability": 0.4875638484954834, + "style_target": 0, + "support_probability": 0.9406673312187195, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34491692064122137, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2718801498413086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7299070358276367, + 0.6810858845710754, + 0.4258844256401062, + 0.4943264424800873, + 0.030479004606604576 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 1, + "style_probability": 0.543667197227478, + "style_target": 1, + "support_probability": 0.49506136775016785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6270570399838981, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25013890862464905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7277354598045349, + 0.6494032740592957, + 0.5877235531806946, + 0.32498404383659363, + 0.02018004283308983 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 0, + "style_probability": 0.9538447856903076, + "style_target": 0, + "support_probability": 0.7163287401199341, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10909896931050087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1522812694311142, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11602511256933212, + 0.30791527032852173, + 0.15677067637443542, + 0.9276592135429382, + 0.09337442368268967, + 0.04992855712771416 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 1, + "style_probability": 0.5201303362846375, + "style_target": 1, + "support_probability": 0.06692233681678772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06762145381981342, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1350875049829483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07116696238517761, + 0.26561954617500305, + 0.14251363277435303, + 0.967104434967041, + 0.029197344556450844, + 0.12086986005306244 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 1, + "style_probability": 0.29277291893959045, + "style_target": 0, + "support_probability": 0.17657218873500824, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3715034259966564, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1795908808708191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45144739747047424, + 0.34731325507164, + 0.9398976564407349, + 0.4406118094921112, + 0.04862523823976517, + 0.026111546903848648, + 0.04610374942421913, + 0.04211225360631943, + 0.15512414276599884 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 0, + "style_probability": 0.422238290309906, + "style_target": 1, + "support_probability": 0.623336672782898, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5787668422081635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19600728154182434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4231131672859192, + 0.3071807622909546, + 0.932483971118927, + 0.44736379384994507, + 0.017061814665794373, + 0.019264141097664833, + 0.1234985888004303, + 0.07666561007499695, + 0.0188459400087595 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 0, + "style_probability": 0.539680004119873, + "style_target": 0, + "support_probability": 0.7605413198471069, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7656202940141595, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01669665239751339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24088291823863983, + 0.9919912219047546, + 0.14089500904083252, + 0.15785659849643707, + 0.07960837334394455, + 0.42437806725502014, + 0.02771119214594364, + 0.03913239389657974, + 0.03353428468108177, + 0.03694294020533562, + 0.0447404645383358 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.39040932059288025, + "style_target": 1, + "support_probability": 0.9778002500534058, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9462730523439724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01627560518682003, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1009:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19874727725982666, + 0.9914613962173462, + 0.12199720740318298, + 0.21019503474235535, + 0.0197497196495533, + 0.04330294579267502, + 0.01974140852689743, + 0.04262905940413475, + 0.015409357845783234, + 0.01995387114584446, + 0.15308809280395508 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.9524486660957336, + "style_target": 0, + "support_probability": 0.9862804412841797, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20527379818501174, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3556441366672516, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06651856750249863, + 0.2152044177055359, + 0.9106090664863586, + 0.19784221053123474, + 0.035060856491327286 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 0, + "style_probability": 0.39302003383636475, + "style_target": 1, + "support_probability": 0.48923128843307495, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1521245917240903, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.513739287853241, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.056599851697683334, + 0.2196756899356842, + 0.8886590600013733, + 0.16000531613826752, + 0.024315916001796722 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 1, + "style_probability": 0.26670604944229126, + "style_target": 0, + "support_probability": 0.28328683972358704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10808710952624292, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33140236139297485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11889387667179108, + 0.18538375198841095, + 0.11889387667179108, + 0.9278830885887146, + 0.03547557815909386, + 0.37506866455078125 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 1, + "style_probability": 0.6524317860603333, + "style_target": 1, + "support_probability": 0.06230789050459862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002055794282522837, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5892989635467529, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12992607057094574, + 0.11719737946987152, + 0.12992607057094574, + 0.921731173992157, + 0.05116749182343483, + 0.07831290364265442 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 1, + "style_probability": 0.04894110932946205, + "style_target": 0, + "support_probability": 0.03435961529612541, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.744705544108379, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.018428850919008255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8305698037147522, + 0.1567438244819641, + 0.8129783272743225, + 0.9755075573921204, + 0.016434289515018463, + 0.053234126418828964, + 0.04867618903517723, + 0.04044361412525177, + 0.13592490553855896 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 0, + "style_probability": 0.41262397170066833, + "style_target": 1, + "support_probability": 0.9478491544723511, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8437976940572413, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019533155485987663, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.812652587890625, + 0.13540413975715637, + 0.825854480266571, + 0.9784709811210632, + 0.027308866381645203, + 0.022102851420640945, + 0.052289485931396484, + 0.05151420831680298, + 0.037899408489465714 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 0, + "style_probability": 0.7759711146354675, + "style_target": 0, + "support_probability": 0.9130832552909851, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5474693444323518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02070409059524536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30238476395606995, + 0.17271572351455688, + 0.94691401720047, + 0.37925785779953003, + 0.022329386323690414, + 0.02498650550842285, + 0.009352894499897957, + 0.015050615184009075, + 0.02342822775244713, + 0.05519641563296318, + 0.04872060939669609 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 1, + "style_probability": 0.5070943832397461, + "style_target": 1, + "support_probability": 0.841912031173706, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.437476782979886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01777545176446438, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32835835218429565, + 0.19273462891578674, + 0.9596017003059387, + 0.37750452756881714, + 0.03579635173082352, + 0.03954547271132469, + 0.09571126848459244, + 0.03238480165600777, + 0.03662226349115372, + 0.08111942559480667, + 0.027244113385677338 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 1, + "style_probability": 0.3610447645187378, + "style_target": 0, + "support_probability": 0.7771344184875488, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7900621706651283, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021553080528974533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9915918111801147, + 0.23999173939228058, + 0.1538865864276886, + 0.6519614458084106, + 0.012437406927347183 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 1, + "style_probability": 0.642062783241272, + "style_target": 1, + "support_probability": 0.9249413013458252, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7771016093582835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02779613994061947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9856469035148621, + 0.31973960995674133, + 0.18403945863246918, + 0.691356897354126, + 0.02958606742322445 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 0, + "style_probability": 0.572658121585846, + "style_target": 0, + "support_probability": 0.9456499218940735, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09046885636178431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6913250088691711, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6951020956039429, + 0.9339415431022644, + 0.17040583491325378, + 0.28865766525268555, + 0.049737002700567245, + 0.04746190086007118 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.8673372864723206, + "style_target": 1, + "support_probability": 0.05728471651673317, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011918709989304005, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8491513133049011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6586020588874817, + 0.9032343029975891, + 0.14754101634025574, + 0.2554418444633484, + 0.02128368616104126, + 0.06999377906322479 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.39770543575286865, + "style_target": 0, + "support_probability": 0.009970848448574543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9142887096158769, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007000234443694353, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1050:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33419063687324524, + 0.34870246052742004, + 0.999001681804657, + 0.2627277970314026, + 0.038696080446243286, + 0.040917277336120605, + 0.027762647718191147, + 0.15841476619243622, + 0.08905000239610672 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.6387562155723572, + "style_target": 1, + "support_probability": 0.9930347204208374, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3395552473561565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008194565773010254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1050:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28168460726737976, + 0.3753179907798767, + 0.9996200799942017, + 0.3215237259864807, + 0.06446925550699234, + 0.0239702221006155, + 0.08649209141731262, + 0.023232286795973778, + 0.036471132189035416 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.05669836327433586, + "style_target": 0, + "support_probability": 0.986890971660614, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.772941945729427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14170213043689728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22174431383609772, + 0.7374498248100281, + 0.9509376287460327, + 0.49895474314689636, + 0.018239909783005714, + 0.028065310791134834, + 0.03179215267300606, + 0.04131007939577103, + 0.07707394659519196, + 0.086239755153656 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 1, + "style_probability": 0.9382225871086121, + "style_target": 1, + "support_probability": 0.6706505417823792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47905407582637843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18563655018806458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15007753670215607, + 0.6144605278968811, + 0.9073106050491333, + 0.4114513695240021, + 0.032359566539525986, + 0.03476964682340622, + 0.025303401052951813, + 0.023350117728114128, + 0.02265804074704647, + 0.029418256133794785 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 0, + "style_probability": 0.5167067646980286, + "style_target": 0, + "support_probability": 0.6778298020362854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08552336393399837, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8176475167274475, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36074939370155334, + 0.7756500244140625, + 0.39514973759651184, + 0.4834337830543518, + 0.08432076871395111 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 1, + "style_probability": 0.639045000076294, + "style_target": 1, + "support_probability": 0.048279326409101486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019461650726342785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.840464174747467, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.367075651884079, + 0.7672315835952759, + 0.4046238362789154, + 0.4926603138446808, + 0.08182820677757263 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 0, + "style_probability": 0.8159592151641846, + "style_target": 0, + "support_probability": 0.006094831507652998, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053969385556004196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09773144125938416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23938970267772675, + 0.08667714893817902, + 0.14678305387496948, + 0.938075065612793, + 0.016336657106876373, + 0.07123789191246033 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.510162353515625, + "style_target": 1, + "support_probability": 0.061053186655044556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03566305931826325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14795824885368347, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2446756809949875, + 0.1276734620332718, + 0.18823029100894928, + 0.9529110193252563, + 0.03369290009140968, + 0.051064975559711456 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.3320166766643524, + "style_target": 0, + "support_probability": 0.035465143620967865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6993716705466886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08582126349210739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9143631458282471, + 0.4880855977535248, + 0.11394303292036057, + 0.314410924911499, + 0.01984495110809803, + 0.009927870705723763, + 0.01482879277318716, + 0.040465760976076126, + 0.06890066713094711 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 1, + "style_probability": 0.5456422567367554, + "style_target": 1, + "support_probability": 0.962293267250061, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06865236684178115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09165972471237183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8340928554534912, + 0.3705885708332062, + 0.07913566380739212, + 0.2898593842983246, + 0.028136566281318665, + 0.03508538380265236, + 0.0188356414437294, + 0.018645770847797394, + 0.026691358536481857 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 1, + "style_probability": 0.06563448160886765, + "style_target": 0, + "support_probability": 0.8499879837036133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46950992991384854, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35730254650115967, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.604996383190155, + 0.5811721682548523, + 0.8089614510536194, + 0.7710496783256531, + 0.0141298184171319, + 0.024315569549798965, + 0.027669629082083702, + 0.014233673922717571, + 0.014709417708218098, + 0.030514422804117203, + 0.06268612295389175 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 1, + "style_probability": 0.6054914593696594, + "style_target": 1, + "support_probability": 0.7714999318122864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08738620705469297, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5423592329025269, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7114973068237305, + 0.7141527533531189, + 0.8373444676399231, + 0.7724448442459106, + 0.06996634602546692, + 0.023936927318572998, + 0.012370389886200428, + 0.026560138911008835, + 0.0653613731265068, + 0.06226630136370659 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 0, + "style_probability": 0.8403636813163757, + "style_target": 0, + "support_probability": 0.10098063200712204, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5252061265386044, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23905330896377563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34979161620140076, + 0.8625839948654175, + 0.28215453028678894, + 0.18588238954544067, + 0.02946617268025875 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 1, + "style_probability": 0.6207624077796936, + "style_target": 1, + "support_probability": 0.8227449655532837, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43946817993863546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23810535669326782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3703421354293823, + 0.8692793846130371, + 0.27767282724380493, + 0.2207793891429901, + 0.04110733047127724 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 0, + "style_probability": 0.5066731572151184, + "style_target": 0, + "support_probability": 0.738196849822998, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5192709717956768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07574984431266785, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17494727671146393, + 0.33162862062454224, + 0.9961750507354736, + 0.15447761118412018, + 0.09003076702356339, + 0.032897450029850006 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.7705426216125488, + "style_target": 1, + "support_probability": 0.4254070222377777, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06679520442959133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16278834640979767, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09943293035030365, + 0.30357688665390015, + 0.9990904331207275, + 0.09573426097631454, + 0.04465572163462639, + 0.1153910905122757 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.16766194999217987, + "style_target": 0, + "support_probability": 0.2092854082584381, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6274335401839917, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07131636887788773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11573231965303421, + 0.23005375266075134, + 0.34002235531806946, + 0.9919706583023071, + 0.061085592955350876, + 0.09858345985412598, + 0.039075132459402084, + 0.11358526349067688, + 0.03572978079319 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 1, + "style_probability": 0.6868882775306702, + "style_target": 1, + "support_probability": 0.7068510055541992, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5460920605612494, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13374006748199463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11340673267841339, + 0.20086082816123962, + 0.3885143995285034, + 0.9965497255325317, + 0.0514710508286953, + 0.04449765011668205, + 0.05657762661576271, + 0.05383937060832977, + 0.12662935256958008 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 0, + "style_probability": 0.7248626351356506, + "style_target": 0, + "support_probability": 0.4975389838218689, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07645911914969404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34885740280151367, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8446308374404907, + 0.26565778255462646, + 0.05685978755354881, + 0.21501365303993225, + 0.015264325775206089, + 0.014610877260565758, + 0.06392969936132431, + 0.02585497498512268, + 0.043799612671136856, + 0.13204899430274963, + 0.07335465401411057 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 0, + "style_probability": 0.37365975975990295, + "style_target": 1, + "support_probability": 0.08630967885255814, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053927364627579376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4634566605091095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.902370274066925, + 0.289095938205719, + 0.05013314634561539, + 0.2132681906223297, + 0.03166547045111656, + 0.04582179710268974, + 0.37771734595298767, + 0.06203620508313179, + 0.05159616097807884, + 0.03446552902460098, + 0.08093772083520889 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 1, + "style_probability": 0.16786402463912964, + "style_target": 0, + "support_probability": 0.07692468166351318, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5876477201569836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5520027279853821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22401057183742523, + 0.3665178418159485, + 0.262044221162796, + 0.8683562278747559, + 0.07377936691045761 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 1, + "style_probability": 0.8881937265396118, + "style_target": 1, + "support_probability": 0.44206473231315613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15556204892007275, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7116916179656982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2791038453578949, + 0.363955020904541, + 0.31640833616256714, + 0.8584323525428772, + 0.05381615459918976 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 0, + "style_probability": 0.5926361083984375, + "style_target": 0, + "support_probability": 0.13739094138145447, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13244646270160226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6429968476295471, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7747777104377747, + 0.05469167232513428, + 0.36628103256225586, + 0.21146993339061737, + 0.06431803107261658, + 0.021428490057587624 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.7611930966377258, + "style_target": 1, + "support_probability": 0.0746079534292221, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07052402850704227, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5429138541221619, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7297999262809753, + 0.06099861115217209, + 0.4254797101020813, + 0.19889400899410248, + 0.06370984762907028, + 0.14131569862365723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.3560411036014557, + "style_target": 0, + "support_probability": 0.08022058010101318, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.849902394154927, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16658569872379303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28763237595558167, + 0.971782922744751, + 0.12417803704738617, + 0.043914545327425, + 0.019965142011642456, + 0.03285050392150879, + 0.07420597970485687, + 0.052258916199207306, + 0.043348997831344604 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.7629554271697998, + "style_target": 1, + "support_probability": 0.8860183358192444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1359951272102343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.138874813914299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18871170282363892, + 0.9757393598556519, + 0.12885496020317078, + 0.03005392476916313, + 0.04651246592402458, + 0.11447861045598984, + 0.04019841551780701, + 0.03534483537077904, + 0.06320102512836456 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.07217705249786377, + "style_target": 0, + "support_probability": 0.888500452041626, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8265820099543408, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02693478949368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9211194515228271, + 0.10677092522382736, + 0.4380611479282379, + 0.12898209691047668, + 0.0199409332126379, + 0.013107008300721645, + 0.009537690319120884, + 0.010424023494124413, + 0.013269429095089436, + 0.03394182398915291, + 0.026193199679255486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.9076520204544067, + "style_target": 1, + "support_probability": 0.9833499193191528, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7592239868226188, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02136397734284401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9456988573074341, + 0.1028473749756813, + 0.46170350909233093, + 0.08496365696191788, + 0.01921594887971878, + 0.04546896368265152, + 0.01397721003741026, + 0.01616654172539711, + 0.015007337555289268, + 0.05068986490368843, + 0.02255123108625412 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.43421220779418945, + "style_target": 0, + "support_probability": 0.9813982844352722, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05562685871335498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9304065704345703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08046912401914597, + 0.8154193162918091, + 0.1512681394815445, + 0.3914162218570709, + 0.09608589857816696 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.38463106751441956, + "style_target": 1, + "support_probability": 0.1310553401708603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012069685900788767, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9848111867904663, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13001829385757446, + 0.7728004455566406, + 0.15944989025592804, + 0.4476388692855835, + 0.07787062227725983 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.6884595155715942, + "style_target": 0, + "support_probability": 0.011092442087829113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6627117893712737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08907520771026611, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24103078246116638, + 0.18333064019680023, + 0.9355841279029846, + 0.22160182893276215, + 0.05148441717028618, + 0.05771699920296669 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.5492671728134155, + "style_target": 1, + "support_probability": 0.8051497936248779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48471726781524943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05221656709909439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21211543679237366, + 0.19526910781860352, + 0.9449173808097839, + 0.2834179103374481, + 0.12871937453746796, + 0.05058084428310394 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.40575361251831055, + "style_target": 0, + "support_probability": 0.7914764285087585, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6907204405428058, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3387800455093384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19282887876033783, + 0.964293360710144, + 0.3738749921321869, + 0.19282887876033783, + 0.02779184840619564, + 0.04355176165699959, + 0.049185965210199356, + 0.07676037400960922, + 0.11950105428695679 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.6284970045089722, + "style_target": 1, + "support_probability": 0.8633823990821838, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.271479260107863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06062202528119087, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09090831875801086, + 0.9906813502311707, + 0.21618390083312988, + 0.09090831875801086, + 0.018190862610936165, + 0.0340283140540123, + 0.05244196951389313, + 0.048561472445726395, + 0.10221049934625626 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.17598861455917358, + "style_target": 0, + "support_probability": 0.9348633289337158, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07993640817724285, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5947075486183167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10832522809505463, + 0.6930055618286133, + 0.3385132849216461, + 0.3508084714412689, + 0.012263082899153233, + 0.02598240226507187, + 0.02405346930027008, + 0.018688395619392395, + 0.0854794904589653, + 0.1321164220571518, + 0.054832376539707184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 0, + "style_probability": 0.47993603348731995, + "style_target": 1, + "support_probability": 0.1695493459701538, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1227644021459242, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6425530910491943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12853121757507324, + 0.7422418594360352, + 0.4644382894039154, + 0.40250861644744873, + 0.05392247810959816, + 0.017218835651874542, + 0.03672325238585472, + 0.01898394152522087, + 0.020715203136205673, + 0.03946803882718086, + 0.020448783412575722 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 0, + "style_probability": 0.8664832711219788, + "style_target": 0, + "support_probability": 0.060884833335876465, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7913979362946997, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05028547719120979, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9927980899810791, + 0.4157351851463318, + 0.25054407119750977, + 0.03063405118882656, + 0.012824781239032745 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 0, + "style_probability": 0.46570855379104614, + "style_target": 1, + "support_probability": 0.9322612881660461, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5289146331537942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.056594837456941605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9934864044189453, + 0.5473342537879944, + 0.37060439586639404, + 0.03112821653485298, + 0.19213716685771942 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 1, + "style_probability": 0.3285519480705261, + "style_target": 0, + "support_probability": 0.8254718780517578, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01344414851268798, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7770810723304749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.706974446773529, + 0.12032821029424667, + 0.31350916624069214, + 0.14202037453651428, + 0.032414138317108154, + 0.02622631937265396 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 0, + "style_probability": 0.4582059383392334, + "style_target": 1, + "support_probability": 0.008524411357939243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001405710657819093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8144146203994751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7588529586791992, + 0.08938370645046234, + 0.1557900756597519, + 0.11258537322282791, + 0.04518340528011322, + 0.08079761266708374 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 1, + "style_probability": 0.1422608494758606, + "style_target": 0, + "support_probability": 0.006605570204555988, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032843260110497166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5392535924911499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26036909222602844, + 0.828687310218811, + 0.2533273994922638, + 0.3972437083721161, + 0.03781234100461006, + 0.038364190608263016, + 0.2565594017505646, + 0.03535176068544388, + 0.033532436937093735 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 0, + "style_probability": 0.46659108996391296, + "style_target": 1, + "support_probability": 0.05789885297417641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030139754771174025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7863550186157227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20400890707969666, + 0.8150426745414734, + 0.30719566345214844, + 0.3354111611843109, + 0.031237056478857994, + 0.04314476251602173, + 0.029416369274258614, + 0.04732682555913925, + 0.07785633951425552 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 0, + "style_probability": 0.8818396329879761, + "style_target": 0, + "support_probability": 0.020727505907416344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012402768781232712, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9348779320716858, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8883761167526245, + 0.16887669265270233, + 0.28792548179626465, + 0.16887669265270233, + 0.044062186032533646, + 0.02704164944589138, + 0.01816204935312271, + 0.05811268463730812, + 0.041354529559612274, + 0.037522900849580765, + 0.04078919440507889 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 1, + "style_probability": 0.5299130082130432, + "style_target": 1, + "support_probability": 0.007574046961963177, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027830131239318736, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7618516683578491, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8876502513885498, + 0.17199696600437164, + 0.247990682721138, + 0.17199696600437164, + 0.04085136577486992, + 0.06682553142309189, + 0.04677106812596321, + 0.1703367382287979, + 0.37852174043655396, + 0.06635608524084091, + 0.38689708709716797 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 1, + "style_probability": 0.3063211143016815, + "style_target": 0, + "support_probability": 0.01584261655807495, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20094996369822632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5379399061203003, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2270238697528839, + 0.964458703994751, + 0.3341878056526184, + 0.16570758819580078, + 0.18559233844280243 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 1, + "style_probability": 0.8701803088188171, + "style_target": 1, + "support_probability": 0.07744553685188293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05714151394488489, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.44896289706230164, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22585263848304749, + 0.961305558681488, + 0.30840766429901123, + 0.18168140947818756, + 0.08224334567785263 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 0, + "style_probability": 0.5110539197921753, + "style_target": 0, + "support_probability": 0.05039793625473976, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7965457415556668, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.018567971885204315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08457055687904358, + 0.07195205986499786, + 0.4874417781829834, + 0.999446451663971, + 0.014721350744366646, + 0.06350525468587875 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 0, + "style_probability": 0.4543919563293457, + "style_target": 1, + "support_probability": 0.9491569399833679, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5021863033848447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.02700071781873703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10877997428178787, + 0.11158446967601776, + 0.5612388849258423, + 0.998380184173584, + 0.023840876296162605, + 0.05602894723415375 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 1, + "style_probability": 0.20817607641220093, + "style_target": 0, + "support_probability": 0.9268472194671631, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6435617069487056, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22268083691596985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20619292557239532, + 0.8577713966369629, + 0.035598453134298325, + 0.22283875942230225, + 0.0529753640294075, + 0.06327338516712189, + 0.05394340306520462, + 0.01160380244255066, + 0.058236926794052124 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.8692637085914612, + "style_target": 1, + "support_probability": 0.5546630024909973, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03345619985927274, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2787114381790161, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19637396931648254, + 0.8552170991897583, + 0.057366326451301575, + 0.18477009236812592, + 0.06525371968746185, + 0.02334941178560257, + 0.027710357680916786, + 0.031955085694789886, + 0.05777188390493393 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.08480101823806763, + "style_target": 0, + "support_probability": 0.5327686071395874, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4541815189601192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044812168926000595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3545713424682617, + 0.4594176113605499, + 0.19826054573059082, + 0.9676677584648132, + 0.020230310037732124, + 0.011557729914784431, + 0.05430030822753906, + 0.0401499941945076, + 0.028457295149564743, + 0.01898276060819626, + 0.015241878107190132 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 1, + "style_probability": 0.8340662717819214, + "style_target": 1, + "support_probability": 0.30325767397880554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22493369205693867, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06609759479761124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35090792179107666, + 0.5039831399917603, + 0.22776134312152863, + 0.9408514499664307, + 0.059201765805482864, + 0.021066507324576378, + 0.014135804958641529, + 0.06595207005739212, + 0.039405565708875656, + 0.032531727105379105 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 1, + "style_probability": 0.4041818380355835, + "style_target": 0, + "support_probability": 0.3051050305366516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06845780593819695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10692889988422394, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8896947503089905, + 0.2197234332561493, + 0.3063086271286011, + 0.27108317613601685, + 0.0737922415137291 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.26512283086776733, + "style_target": 1, + "support_probability": 0.08166967332363129, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18813930790320846, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07384826242923737, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9005410075187683, + 0.28284403681755066, + 0.2534885108470917, + 0.22970610857009888, + 0.016351858153939247 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.7415616512298584, + "style_target": 0, + "support_probability": 0.12209499627351761, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4346498585925791, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021596690639853477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20501390099525452, + 0.9814314842224121, + 0.18784669041633606, + 0.273392915725708, + 0.06223240867257118, + 0.0838092714548111 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 0, + "style_probability": 0.2795613706111908, + "style_target": 1, + "support_probability": 0.8391550779342651, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12196400642958306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045370835810899734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2025047391653061, + 0.9914650321006775, + 0.19588008522987366, + 0.27992501854896545, + 0.06351584941148758, + 0.1158309206366539 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 1, + "style_probability": 0.055292196571826935, + "style_target": 0, + "support_probability": 0.8827418088912964, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001450162609424576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9955928921699524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6034513115882874, + 0.2101214975118637, + 0.8476946353912354, + 0.15595190227031708, + 0.05284033715724945, + 0.08620335906744003, + 0.0867057666182518, + 0.06354431807994843, + 0.076717808842659 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.41879451274871826, + "style_target": 1, + "support_probability": 0.00020659141591750085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010517287606082233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9680288434028625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5966019630432129, + 0.25582873821258545, + 0.8836915493011475, + 0.18070238828659058, + 0.04462410509586334, + 0.12718023359775543, + 0.5634417533874512, + 0.08092605322599411, + 0.08326016366481781 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.5572900772094727, + "style_target": 0, + "support_probability": 0.002861353103071451, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6565516629490338, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02117510512471199, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09034819900989532, + 0.6153428554534912, + 0.28863731026649475, + 0.9897295236587524, + 0.013177474029362202, + 0.029050204902887344, + 0.021816860884428024, + 0.021794719621539116, + 0.03866683319211006, + 0.025372043251991272, + 0.02507803961634636 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.4414561986923218, + "style_target": 1, + "support_probability": 0.9193791747093201, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9149169981817589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016964638605713844, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1195:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09993604570627213, + 0.631043016910553, + 0.22720059752464294, + 0.9924087524414062, + 0.0574062243103981, + 0.0435570552945137, + 0.011289085261523724, + 0.0223613940179348, + 0.02658199891448021, + 0.04283956065773964, + 0.017732320353388786 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.9511095285415649, + "style_target": 0, + "support_probability": 0.9151676893234253, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17428712193927895, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5223878026008606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0300307497382164, + 0.1442922204732895, + 0.18445293605327606, + 0.6224509477615356, + 0.025254912674427032 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.7837721705436707, + "style_target": 1, + "support_probability": 0.12407776713371277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001369676499773991, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7956139445304871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.016461163759231567, + 0.1030578762292862, + 0.14456014335155487, + 0.589150607585907, + 0.03679429739713669 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.028582334518432617, + "style_target": 0, + "support_probability": 0.03154761716723442, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5616358677981738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32231104373931885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30561116337776184, + 0.9296535849571228, + 0.035756468772888184, + 0.24661751091480255, + 0.02718917839229107, + 0.023675601929426193 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 1, + "style_probability": 0.5764366984367371, + "style_target": 1, + "support_probability": 0.6575543880462646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6021583228272078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28226572275161743, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3495534360408783, + 0.9128593802452087, + 0.06484296917915344, + 0.2733025550842285, + 0.030961591750383377, + 0.015881681814789772 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 1, + "style_probability": 0.3801572024822235, + "style_target": 0, + "support_probability": 0.8496756553649902, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38553019328030735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39491724967956543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4921019971370697, + 0.14253303408622742, + 0.4921019971370697, + 0.8405119776725769, + 0.012374830432236195, + 0.011642975732684135, + 0.023928357288241386, + 0.01877576671540737, + 0.04643506556749344 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.8287379145622253, + "style_target": 1, + "support_probability": 0.2123325765132904, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08255385832356887, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6584222912788391, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43406984210014343, + 0.140035942196846, + 0.43406984210014343, + 0.8311396241188049, + 0.023842213675379753, + 0.06059728190302849, + 0.06627027690410614, + 0.03144415467977524, + 0.03016812913119793 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.48308074474334717, + "style_target": 0, + "support_probability": 0.049268919974565506, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008985707959738009, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9968056678771973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7213507294654846, + 0.4414016008377075, + 0.14501014351844788, + 0.4414016008377075, + 0.02476666495203972, + 0.11175499856472015, + 0.035585056990385056, + 0.04301684349775314, + 0.05901872366666794, + 0.023841748014092445, + 0.03252851590514183 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.7862270474433899, + "style_target": 1, + "support_probability": 0.00010695893433876336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005563796123576412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.995100200176239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6790936589241028, + 0.41644927859306335, + 0.08568105101585388, + 0.41644927859306335, + 0.024460503831505775, + 0.03704081475734711, + 0.12328492105007172, + 0.03745168820023537, + 0.03432510420680046, + 0.10272520035505295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.25130555033683777, + "style_target": 0, + "support_probability": 0.0003765510336961597, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.40472848564013003, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17678311467170715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.957447350025177, + 0.15778332948684692, + 0.07270356267690659, + 0.15778332948684692, + 0.03681018203496933 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.6357055902481079, + "style_target": 1, + "support_probability": 0.48975613713264465, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05105333909351644, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14063794910907745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9840824604034424, + 0.12819533050060272, + 0.05356225743889809, + 0.12819533050060272, + 0.04775218665599823 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.1543108969926834, + "style_target": 0, + "support_probability": 0.5617372393608093, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020748125529580514, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8318600058555603, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2954404652118683, + 0.7178991436958313, + 0.10930398851633072, + 0.6360272765159607, + 0.03228301927447319, + 0.04889453202486038 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 1, + "style_probability": 0.577386736869812, + "style_target": 1, + "support_probability": 0.01686633564531803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020910549721360232, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.956803560256958, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2164497822523117, + 0.6963596940040588, + 0.11795587837696075, + 0.5256751179695129, + 0.036012932658195496, + 0.10780829936265945 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 0, + "style_probability": 0.920995831489563, + "style_target": 0, + "support_probability": 0.0030963493045419455, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5794091965501592, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03347184136509895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1956721991300583, + 0.7434796094894409, + 0.3393939435482025, + 0.34281259775161743, + 0.01603219285607338, + 0.017858220264315605, + 0.06534389406442642, + 0.005972511600703001, + 0.030833056196570396 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 1, + "style_probability": 0.5242388248443604, + "style_target": 1, + "support_probability": 0.9709242582321167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5301746361467772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021745407953858376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16415168344974518, + 0.77242511510849, + 0.27331477403640747, + 0.36866986751556396, + 0.02764143981039524, + 0.01941034384071827, + 0.015145828947424889, + 0.01533361617475748, + 0.03191688284277916 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 1, + "style_probability": 0.423153817653656, + "style_target": 0, + "support_probability": 0.9700709581375122, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11214871424616155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5483049750328064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8202993869781494, + 0.15752087533473969, + 0.13953790068626404, + 0.3868200182914734, + 0.016397910192608833, + 0.0252127293497324, + 0.04127543047070503, + 0.03336465731263161, + 0.03651658445596695, + 0.022377237677574158, + 0.021673772484064102 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 0, + "style_probability": 0.42065122723579407, + "style_target": 1, + "support_probability": 0.22403770685195923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022823110206958114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5142565369606018, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7951802611351013, + 0.15944145619869232, + 0.09514468908309937, + 0.30992162227630615, + 0.11038143187761307, + 0.05067821592092514, + 0.600800096988678, + 0.023599211126565933, + 0.029529307037591934, + 0.07310954481363297 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 1, + "style_probability": 0.13816680014133453, + "style_target": 0, + "support_probability": 0.16218946874141693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27391596891355213, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2215450406074524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6343324184417725, + 0.2060844600200653, + 0.07829972356557846, + 0.2060844600200653, + 0.02542462758719921 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 1, + "style_probability": 0.6114521622657776, + "style_target": 1, + "support_probability": 0.406833291053772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29256921482772186, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5532703399658203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6823004484176636, + 0.23510630428791046, + 0.06916827708482742, + 0.23510630428791046, + 0.031537014991045 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 0, + "style_probability": 0.8354697823524475, + "style_target": 0, + "support_probability": 0.13270539045333862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03883680957815968, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4730866551399231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5240519642829895, + 0.1607096791267395, + 0.6305253505706787, + 0.26474201679229736, + 0.04697214439511299, + 0.09556230157613754 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 0, + "style_probability": 0.4211280643939972, + "style_target": 1, + "support_probability": 0.030290324240922928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05669320013293434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21872557699680328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6800410747528076, + 0.11616921424865723, + 0.6436181664466858, + 0.18017512559890747, + 0.07686351984739304, + 0.22982537746429443 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 1, + "style_probability": 0.198238343000412, + "style_target": 0, + "support_probability": 0.05317654833197594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5483032281859062, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4771275222301483, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2612328827381134, + 0.8395699262619019, + 0.0648338720202446, + 0.20319592952728271, + 0.007780450861901045, + 0.03797169029712677, + 0.02006421983242035, + 0.03173081949353218, + 0.04015421122312546 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.6248939037322998, + "style_target": 1, + "support_probability": 0.7807907462120056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026640548554322595, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7227376103401184, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15646284818649292, + 0.7626718878746033, + 0.03225936368107796, + 0.24394066631793976, + 0.019583038985729218, + 0.014052866958081722, + 0.02437828667461872, + 0.028608502820134163, + 0.044169772416353226 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.09265129268169403, + "style_target": 0, + "support_probability": 0.39694347977638245, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8324515149255629, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007055265363305807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11698240786790848, + 0.11698240786790848, + 0.3086862564086914, + 0.9996732473373413, + 0.0037991395220160484, + 0.011882875114679337, + 0.024465395137667656, + 0.017933672294020653, + 0.019654884934425354, + 0.014319117181003094, + 0.0109260780736804 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 1, + "style_probability": 0.7226660251617432, + "style_target": 1, + "support_probability": 0.9683895707130432, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7083577646444438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004758717026561499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11994350701570511, + 0.11994350701570511, + 0.2566600441932678, + 0.9999500513076782, + 0.011135152541100979, + 0.012298197485506535, + 0.02010517753660679, + 0.012592446990311146, + 0.009400366805493832, + 0.08148807287216187, + 0.018487123772501945 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 0, + "style_probability": 0.6971220374107361, + "style_target": 0, + "support_probability": 0.9327914118766785, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8132739985421087, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13459022343158722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5244495868682861, + 0.9503083825111389, + 0.4073663353919983, + 0.8872883915901184, + 0.10698578506708145 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.940597653388977, + "style_target": 1, + "support_probability": 0.725638747215271, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025661863070736774, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23208566009998322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5369718074798584, + 0.9344534277915955, + 0.3125452697277069, + 0.8139544725418091, + 0.04425456374883652 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.09616415947675705, + "style_target": 0, + "support_probability": 0.4110187292098999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6507726127725177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1145949736237526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.931026041507721, + 0.1004052385687828, + 0.17885200679302216, + 0.4022529423236847, + 0.015700988471508026, + 0.09746421873569489 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 1, + "style_probability": 0.9126578569412231, + "style_target": 1, + "support_probability": 0.3101280927658081, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41544827105910487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1438688337802887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9096419215202332, + 0.10350720584392548, + 0.2250945121049881, + 0.32633525133132935, + 0.011533948592841625, + 0.016692010685801506 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 0, + "style_probability": 0.5128354430198669, + "style_target": 0, + "support_probability": 0.4369753897190094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01744490023719787, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.869733989238739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34547513723373413, + 0.3049190640449524, + 0.256626158952713, + 0.7693957090377808, + 0.10115791112184525, + 0.062107570469379425, + 0.019263911992311478, + 0.02867555059492588, + 0.04977850243449211 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 1, + "style_probability": 0.887576699256897, + "style_target": 1, + "support_probability": 0.0035449767019599676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010679368185880604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.801068902015686, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3366197347640991, + 0.25671425461769104, + 0.2635776698589325, + 0.7554574608802795, + 0.09229519963264465, + 0.0343928299844265, + 0.04421918839216232, + 0.0967903807759285, + 0.032226260751485825 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 0, + "style_probability": 0.5357526540756226, + "style_target": 0, + "support_probability": 0.0027118900325149298, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5094348914187412, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13793021440505981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35412976145744324, + 0.2410728484392166, + 0.34267809987068176, + 0.9651968479156494, + 0.04501479119062424, + 0.0685376450419426, + 0.042619865387678146, + 0.021049128845334053, + 0.023302117362618446, + 0.02628283016383648, + 0.03912360966205597 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.8728581070899963, + "style_target": 1, + "support_probability": 0.16817039251327515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17090460478519337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0946781113743782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2578859329223633, + 0.22284936904907227, + 0.2885873019695282, + 0.9779613018035889, + 0.030376426875591278, + 0.051370665431022644, + 0.06651109457015991, + 0.0252462737262249, + 0.04594389349222183, + 0.05028944090008736 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.40720149874687195, + "style_target": 0, + "support_probability": 0.37103018164634705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3432297293938163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.244324192404747, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8370577692985535, + 0.3931000232696533, + 0.6017276644706726, + 0.21536727249622345, + 0.10134322941303253 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 1, + "style_probability": 0.5454925894737244, + "style_target": 1, + "support_probability": 0.27173957228660583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4917288700147052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3260037899017334, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8234370946884155, + 0.5299882292747498, + 0.5801410675048828, + 0.16116459667682648, + 0.09565835446119308 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 0, + "style_probability": 0.8243792653083801, + "style_target": 0, + "support_probability": 0.2734953463077545, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42012680337766856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04591863974928856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14176233112812042, + 0.19354721903800964, + 0.7758356332778931, + 0.16634781658649445, + 0.03460484370589256, + 0.2070816606283188 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.3372327387332916, + "style_target": 1, + "support_probability": 0.8225440979003906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6921122415025196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03855966776609421, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1480509489774704, + 0.19804728031158447, + 0.8098443746566772, + 0.1905619204044342, + 0.018939364701509476, + 0.052240654826164246 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.8835500478744507, + "style_target": 0, + "support_probability": 0.88663649559021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13258117265232464, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21608690917491913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2211996167898178, + 0.8765118718147278, + 0.554679274559021, + 0.2211996167898178, + 0.017539935186505318, + 0.061790499836206436, + 0.03268086537718773, + 0.009703453630208969, + 0.03452848270535469 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 0, + "style_probability": 0.3202090859413147, + "style_target": 1, + "support_probability": 0.1766473352909088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06252980258551401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28813618421554565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12863433361053467, + 0.8912738561630249, + 0.5624691247940063, + 0.12863433361053467, + 0.0147350849583745, + 0.02540285512804985, + 0.01862149126827717, + 0.01611640304327011, + 0.01784488558769226 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 1, + "style_probability": 0.22957554459571838, + "style_target": 0, + "support_probability": 0.22422121465206146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004802931072191962, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9816617369651794, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32045555114746094, + 0.48649004101753235, + 0.29415440559387207, + 0.5904005169868469, + 0.058096252381801605, + 0.12582190334796906, + 0.0532282292842865, + 0.13735300302505493, + 0.09959626942873001, + 0.04432235658168793, + 0.13405419886112213 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 0, + "style_probability": 0.2739948332309723, + "style_target": 1, + "support_probability": 0.0025573470629751682, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007321437631630878, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9757485389709473, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2971794009208679, + 0.3659780025482178, + 0.2609451413154602, + 0.6155251264572144, + 0.06121504306793213, + 0.05161435529589653, + 0.04641079902648926, + 0.08282803744077682, + 0.12472760677337646, + 0.2861383855342865 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 1, + "style_probability": 0.38627076148986816, + "style_target": 0, + "support_probability": 0.004802791867405176, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8478121465971588, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4128640592098236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26625463366508484, + 0.19287237524986267, + 0.24020549654960632, + 0.9733095169067383, + 0.05407160893082619 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.9696229696273804, + "style_target": 1, + "support_probability": 0.818562924861908, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5464093313460832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2439858466386795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1924152374267578, + 0.1631578803062439, + 0.2173689603805542, + 0.9791204929351807, + 0.07164347916841507 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.49955159425735474, + "style_target": 0, + "support_probability": 0.9507890939712524, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8457870761291614, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.026856305077672005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13375718891620636, + 0.10033966600894928, + 0.9903520941734314, + 0.2383478879928589, + 0.10580097138881683, + 0.061045847833156586 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.8847917318344116, + "style_target": 1, + "support_probability": 0.883625328540802, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21832153331484766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.017440441995859146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10581445693969727, + 0.07046565413475037, + 0.9961829781532288, + 0.24788333475589752, + 0.020005272701382637, + 0.08325057476758957 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.13645590841770172, + "style_target": 0, + "support_probability": 0.8792394995689392, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4873923778572243, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2573513686656952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6216455101966858, + 0.34650498628616333, + 0.6225723028182983, + 0.02501351200044155, + 0.013068975880742073, + 0.03303898870944977, + 0.01911766082048416, + 0.0071461377665400505, + 0.0837743952870369 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 1, + "style_probability": 0.809954047203064, + "style_target": 1, + "support_probability": 0.6183254718780518, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4961809660004591, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18572653830051422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7282976508140564, + 0.3760122060775757, + 0.6191394925117493, + 0.040098488330841064, + 0.02726348489522934, + 0.011201023124158382, + 0.029391847550868988, + 0.02288810722529888, + 0.013815352693200111 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 0, + "style_probability": 0.5036994814872742, + "style_target": 0, + "support_probability": 0.8156463503837585, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17183737110284678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23068979382514954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.08854261040687561, + 0.06636942923069, + 0.462576687335968, + 0.862195611000061, + 0.012413556687533855, + 0.007619068026542664, + 0.040390510112047195, + 0.01986592635512352, + 0.00914742425084114, + 0.02000591531395912 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.7564474940299988, + "style_target": 1, + "support_probability": 0.20826637744903564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17429659846233359, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21701323986053467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10833960026502609, + 0.06973732262849808, + 0.44192999601364136, + 0.8331189155578613, + 0.02209736779332161, + 0.05460501089692116, + 0.014969062060117722, + 0.0986047089099884, + 0.03420914337038994, + 0.024067850783467293, + 0.01179609913378954 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.47962650656700134, + "style_target": 0, + "support_probability": 0.2254945933818817, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.082238894131406, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8694985508918762, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1823570430278778, + 0.34148773550987244, + 0.5593371391296387, + 0.42822495102882385, + 0.028831426054239273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.8771108388900757, + "style_target": 1, + "support_probability": 0.02866464853286743, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003648110227990455, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8929007649421692, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15262962877750397, + 0.31916218996047974, + 0.4647083282470703, + 0.35470178723335266, + 0.06541066616773605 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.0870715007185936, + "style_target": 0, + "support_probability": 0.03946518152952194, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7496956731408856, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.049348507076501846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6271228790283203, + 0.13612553477287292, + 0.9232348799705505, + 0.07092289626598358, + 0.016783136874437332, + 0.015627408400177956 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 1, + "style_probability": 0.8644494414329529, + "style_target": 1, + "support_probability": 0.7599091529846191, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3919236625068708, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09934304654598236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.569512128829956, + 0.16517922282218933, + 0.8743520379066467, + 0.10145967453718185, + 0.037002116441726685, + 0.020405137911438942 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 0, + "style_probability": 0.6411860585212708, + "style_target": 0, + "support_probability": 0.5851921439170837, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0420146748417766, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6634249687194824, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17653323709964752, + 0.4509885907173157, + 0.5010052919387817, + 0.22371269762516022, + 0.10143415629863739, + 0.014352603815495968, + 0.02670007385313511, + 0.023050157353281975, + 0.017762860283255577 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.7157855033874512, + "style_target": 1, + "support_probability": 0.02727215550839901, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02539503827936158, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6771450042724609, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20174483954906464, + 0.4729878604412079, + 0.4521131217479706, + 0.2369937002658844, + 0.009632992558181286, + 0.06960242241621017, + 0.020650029182434082, + 0.03137046843767166, + 0.21751773357391357 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.41034331917762756, + "style_target": 0, + "support_probability": 0.018535319715738297, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.004296457791033729, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9389499425888062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29833436012268066, + 0.521367073059082, + 0.36451637744903564, + 0.5032529234886169, + 0.030516522005200386, + 0.018173830583691597, + 0.032361648976802826, + 0.019268015399575233, + 0.06343621015548706, + 0.032354168593883514, + 0.017636926844716072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.7700179815292358, + "style_target": 1, + "support_probability": 0.0006656233454123139, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015601953979998836, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8562065958976746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30868902802467346, + 0.42070943117141724, + 0.2883814573287964, + 0.48027071356773376, + 0.03197627514600754, + 0.021473491564393044, + 0.07481306791305542, + 0.03287879377603531, + 0.015796128660440445, + 0.02925153076648712 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.322611540555954, + "style_target": 0, + "support_probability": 0.0009608365362510085, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.23803113282957258, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09380313754081726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18662874400615692, + 0.8800947070121765, + 0.0843704491853714, + 0.27441272139549255, + 0.0698915645480156 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 0, + "style_probability": 0.24065573513507843, + "style_target": 1, + "support_probability": 0.35801902413368225, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12263204243269293, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20521022379398346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30503126978874207, + 0.9603893160820007, + 0.1841089278459549, + 0.356990247964859, + 0.2947961091995239 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 1, + "style_probability": 0.2341196984052658, + "style_target": 0, + "support_probability": 0.2953888773918152, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07942305012677717, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3479270339012146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4918789565563202, + 0.2739080488681793, + 0.6407412886619568, + 0.4918789565563202, + 0.05672406405210495, + 0.07922831922769547 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.4016000032424927, + "style_target": 1, + "support_probability": 0.14668776094913483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32374646025677395, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37694790959358215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4497874677181244, + 0.38882288336753845, + 0.7039790749549866, + 0.4497874677181244, + 0.06220972537994385, + 0.08821219205856323 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.8648343086242676, + "style_target": 0, + "support_probability": 0.2733188271522522, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32764092249371896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.322801411151886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9154688119888306, + 0.05460508540272713, + 0.14960536360740662, + 0.7016794681549072, + 0.04109774902462959, + 0.02956339716911316, + 0.019948342815041542, + 0.023841053247451782, + 0.05882660672068596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.5029050707817078, + "style_target": 1, + "support_probability": 0.29460957646369934, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1787874271797605, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4173722565174103, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9036850929260254, + 0.0319041982293129, + 0.12946227192878723, + 0.7551366090774536, + 0.048445411026477814, + 0.044885676354169846, + 0.06294527649879456, + 0.06357380747795105, + 0.06534845381975174 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.3864489793777466, + "style_target": 0, + "support_probability": 0.26472970843315125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030745353988147987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8930108547210693, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.07445833086967468, + 0.8740052580833435, + 0.17411616444587708, + 0.07445833086967468, + 0.06510548293590546, + 0.4678223729133606, + 0.032298970967531204, + 0.04097419232130051, + 0.11496371775865555, + 0.09965657442808151, + 0.076906718313694 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 0, + "style_probability": 0.40302735567092896, + "style_target": 1, + "support_probability": 0.03226272761821747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011860513413085288, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9895225167274475, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.06306460499763489, + 0.8622421622276306, + 0.21614259481430054, + 0.06306460499763489, + 0.09715749323368073, + 0.04105899855494499, + 0.11272317171096802, + 0.14001363515853882, + 0.0483243465423584, + 0.10918930172920227 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 1, + "style_probability": 0.15296149253845215, + "style_target": 0, + "support_probability": 0.0020230801310390234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08301655693085963, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6002562642097473, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21632219851016998, + 0.5067132711410522, + 0.49392402172088623, + 0.23159433901309967, + 0.02303808368742466 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.9218301177024841, + "style_target": 1, + "support_probability": 0.07764546573162079, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005056342442202157, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7599567174911499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18233266472816467, + 0.5196581482887268, + 0.44666680693626404, + 0.22550909221172333, + 0.05204392969608307 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.310308575630188, + "style_target": 0, + "support_probability": 0.0038394255097955465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28826760544876273, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6153658628463745, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5174797177314758, + 0.48203518986701965, + 0.1910335272550583, + 0.7928557395935059, + 0.023974984884262085, + 0.05250659957528114 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.7826976776123047, + "style_target": 1, + "support_probability": 0.2524161636829376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003831749006332521, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8959637880325317, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4986332058906555, + 0.4706798493862152, + 0.17007778584957123, + 0.8517199158668518, + 0.0866137444972992, + 0.08227106183767319 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.049573078751564026, + "style_target": 0, + "support_probability": 0.0397706963121891, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.33384003729205264, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1556098610162735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4371739327907562, + 0.9857211112976074, + 0.5152927041053772, + 0.3059999644756317, + 0.08543410152196884, + 0.15002593398094177, + 0.0867200568318367, + 0.06513290107250214, + 0.05523322895169258 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.6861009001731873, + "style_target": 1, + "support_probability": 0.153010755777359, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2940296211642093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11411809921264648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4297930598258972, + 0.9762046933174133, + 0.5293429493904114, + 0.3045499324798584, + 0.03338029980659485, + 0.0549536794424057, + 0.04615015536546707, + 0.10004301369190216, + 0.03989734128117561 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.4479199945926666, + "style_target": 0, + "support_probability": 0.1807030737400055, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8584860336223858, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02767808921635151, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4547575116157532, + 0.2734115421772003, + 0.8495386242866516, + 0.9873331785202026, + 0.03273680806159973, + 0.01304838340729475, + 0.049185317009687424, + 0.09310334920883179, + 0.02240888401865959, + 0.011341962032020092, + 0.025517884641885757 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.6747735738754272, + "style_target": 1, + "support_probability": 0.9017462134361267, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6308431529218487, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02465616725385189, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.4565635323524475, + 0.31824779510498047, + 0.8415491580963135, + 0.9822964072227478, + 0.0159112811088562, + 0.04310525953769684, + 0.018694128841161728, + 0.08485641330480576, + 0.04178149625658989, + 0.08820270001888275, + 0.024645693600177765, + 0.0193896796554327 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.48401427268981934, + "style_target": 0, + "support_probability": 0.8383830785751343, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12063970643268168, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6805614829063416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1244538277387619, + 0.32669690251350403, + 0.2632996439933777, + 0.6696865558624268, + 0.04616037756204605 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 1, + "style_probability": 0.8976679444313049, + "style_target": 1, + "support_probability": 0.03141144663095474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05151868445918581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5593889355659485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10621441155672073, + 0.39586371183395386, + 0.35783320665359497, + 0.800249457359314, + 0.030883023515343666 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 0, + "style_probability": 0.5585050582885742, + "style_target": 0, + "support_probability": 0.04897201433777809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5697801880259907, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7864282727241516, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3871338963508606, + 0.5379263162612915, + 0.7425503730773926, + 0.62607741355896, + 0.024852940812706947, + 0.015488843433558941 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.9111137390136719, + "style_target": 1, + "support_probability": 0.5580524802207947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0314267788813008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9591619372367859, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25991135835647583, + 0.4184266924858093, + 0.6919132471084595, + 0.5303405523300171, + 0.028043050318956375, + 0.04748721048235893 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.3557392656803131, + "style_target": 0, + "support_probability": 0.09515973180532455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6385120170000222, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4200582206249237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9665282368659973, + 0.5348337888717651, + 0.24886547029018402, + 0.06168138235807419, + 0.07175033539533615, + 0.016082746908068657, + 0.045213546603918076, + 0.03704684600234032, + 0.04499830678105354 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.9146761298179626, + "style_target": 1, + "support_probability": 0.2857910394668579, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3197478801083084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43249860405921936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9640278816223145, + 0.436906635761261, + 0.2730504274368286, + 0.06941830366849899, + 0.03749357908964157, + 0.03301691636443138, + 0.12275879085063934, + 0.02278217114508152, + 0.0868103951215744 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.2849612236022949, + "style_target": 0, + "support_probability": 0.2594318687915802, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28613128176230934, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22933250665664673, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09107376635074615, + 0.4192431569099426, + 0.8640201091766357, + 0.8211069107055664, + 0.04445064067840576, + 0.212198406457901, + 0.12903091311454773, + 0.10225507616996765, + 0.059805482625961304, + 0.033193204551935196, + 0.06479597836732864 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.8475804328918457, + "style_target": 1, + "support_probability": 0.20274342596530914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029626558234544006, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2994954586029053, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.08121533691883087, + 0.3829485774040222, + 0.8696035742759705, + 0.6219708323478699, + 0.027172483503818512, + 0.044909536838531494, + 0.03575747087597847, + 0.09848523885011673, + 0.03317690268158913, + 0.08944298326969147 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.09770068526268005, + "style_target": 0, + "support_probability": 0.24980702996253967, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8867557922599271, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.015897085890173912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31887751817703247, + 0.9750432372093201, + 0.36162009835243225, + 0.2275308221578598, + 0.010277372784912586 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 1, + "style_probability": 0.7536277770996094, + "style_target": 1, + "support_probability": 0.9480301737785339, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7322221055744379, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.054437462240457535, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24647097289562225, + 0.9523007869720459, + 0.2836892306804657, + 0.17205578088760376, + 0.08160271495580673 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 0, + "style_probability": 0.5707510113716125, + "style_target": 0, + "support_probability": 0.8730413317680359, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7021458007754828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03554733842611313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18085628747940063, + 0.41408202052116394, + 0.9800179600715637, + 0.497631311416626, + 0.03703930974006653, + 0.06310559064149857 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.7664167284965515, + "style_target": 1, + "support_probability": 0.5328752398490906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35106221479186994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02180544100701809, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17528530955314636, + 0.354254812002182, + 0.9780363440513611, + 0.43760472536087036, + 0.06127022206783295, + 0.11323125660419464 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.21647100150585175, + "style_target": 0, + "support_probability": 0.6209525465965271, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7272605413760024, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03952094539999962, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3736039102077484, + 0.27850136160850525, + 0.9740270376205444, + 0.2226698249578476, + 0.044332366436719894, + 0.06303905695676804, + 0.034906692802906036, + 0.008508607745170593, + 0.02222338132560253 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.756270170211792, + "style_target": 1, + "support_probability": 0.7375754714012146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35596948459223765, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029568705707788467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3594275116920471, + 0.18903253972530365, + 0.9839269518852234, + 0.21095938980579376, + 0.032927848398685455, + 0.05824209004640579, + 0.0787232369184494, + 0.07764234393835068, + 0.23858031630516052 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.17711316049098969, + "style_target": 0, + "support_probability": 0.8839269280433655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36102278787982767, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10530457645654678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19432763755321503, + 0.9467216730117798, + 0.31582000851631165, + 0.2730642855167389, + 0.013597817160189152, + 0.08035001903772354, + 0.04152536764740944, + 0.062245212495326996, + 0.031979721039533615, + 0.020420510321855545, + 0.008316116407513618 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 1, + "style_probability": 0.8429263234138489, + "style_target": 1, + "support_probability": 0.14528566598892212, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25620815559688853, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12352575361728668, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.233064666390419, + 0.9241977334022522, + 0.28609147667884827, + 0.28264567255973816, + 0.02658890001475811, + 0.04025093466043472, + 0.016076061874628067, + 0.08099992573261261, + 0.014065166935324669, + 0.10042930394411087, + 0.026334382593631744 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 0, + "style_probability": 0.5858986973762512, + "style_target": 0, + "support_probability": 0.20613999664783478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6111739596974131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24243873357772827, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.281622976064682, + 0.2596513330936432, + 0.849456250667572, + 0.6474730968475342, + 0.039648279547691345 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.8351085186004639, + "style_target": 1, + "support_probability": 0.5152009129524231, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3313350926600943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2455112785100937, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2463516891002655, + 0.2726207375526428, + 0.8539707660675049, + 0.6473677158355713, + 0.025987936183810234 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.4004545509815216, + "style_target": 0, + "support_probability": 0.4537174701690674, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6119232005244416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07862262427806854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3331122398376465, + 0.19629999995231628, + 0.9679744839668274, + 0.3331122398376465, + 0.012863450683653355, + 0.009351016953587532 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 0, + "style_probability": 0.432416170835495, + "style_target": 1, + "support_probability": 0.8083137273788452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24393956003372283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0847456306219101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3335138261318207, + 0.19776684045791626, + 0.9660065770149231, + 0.3335138261318207, + 0.011869238689541817, + 0.035798508673906326 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 1, + "style_probability": 0.13740186393260956, + "style_target": 0, + "support_probability": 0.7736942172050476, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09570849976972826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9712380170822144, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9209309220314026, + 0.1491020768880844, + 0.1491020768880844, + 0.6962668895721436, + 0.0607677660882473, + 0.07612244784832001, + 0.08240009844303131, + 0.008935492485761642, + 0.023934360593557358 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.8092961311340332, + "style_target": 1, + "support_probability": 0.032605428248643875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12125808744639777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8811070919036865, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9336949586868286, + 0.08865676075220108, + 0.08865676075220108, + 0.7296252250671387, + 0.018468527123332024, + 0.08103833347558975, + 0.07926400750875473, + 0.02567114308476448, + 0.030355708673596382 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.33772405982017517, + "style_target": 0, + "support_probability": 0.141451895236969, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.887704618437303, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04607810825109482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6106693744659424, + 0.9725233316421509, + 0.041003480553627014, + 0.6106693744659424, + 0.016926072537899017, + 0.03703020513057709, + 0.06857838481664658, + 0.03351904824376106, + 0.027902599424123764, + 0.01626940816640854, + 0.02260393463075161 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.8830391764640808, + "style_target": 1, + "support_probability": 0.9210614562034607, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.641580437075492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04210382327437401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6162071824073792, + 0.9635932445526123, + 0.04254573956131935, + 0.6162071824073792, + 0.011734133586287498, + 0.10399732738733292, + 0.047967467457056046, + 0.024515988305211067, + 0.018361229449510574, + 0.015524089336395264, + 0.011179395951330662 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.36076587438583374, + "style_target": 0, + "support_probability": 0.9002258777618408, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007337389253940746, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9893748760223389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2200789898633957, + 0.8118250966072083, + 0.39665475487709045, + 0.2987179756164551, + 0.06987570971250534 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.7749598026275635, + "style_target": 1, + "support_probability": 0.0013259730767458677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002185017603627565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9884263277053833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2203339785337448, + 0.7903957962989807, + 0.3671342730522156, + 0.28090131282806396, + 0.1572216898202896 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.35383957624435425, + "style_target": 0, + "support_probability": 0.0018949827644973993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.347538590633057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06332899630069733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27508533000946045, + 0.1520928293466568, + 0.8357653021812439, + 0.27508533000946045, + 0.1878732442855835, + 0.058001842349767685 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 1, + "style_probability": 0.5959801077842712, + "style_target": 1, + "support_probability": 0.3483695983886719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18373874818151628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0838603600859642, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3024628162384033, + 0.22259967029094696, + 0.7963123321533203, + 0.3024628162384033, + 0.030655724927783012, + 0.07093942165374756 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 0, + "style_probability": 0.5050747990608215, + "style_target": 0, + "support_probability": 0.25986817479133606, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4898218605447648, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38910213112831116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3003953993320465, + 0.3003953993320465, + 0.33665144443511963, + 0.8554850220680237, + 0.03150999918580055, + 0.06285572797060013, + 0.11934547126293182, + 0.02330463007092476, + 0.041466906666755676 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.8339660167694092, + "style_target": 1, + "support_probability": 0.20578238368034363, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11757264273233838, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41882967948913574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27344438433647156, + 0.27344438433647156, + 0.3354407548904419, + 0.8406709432601929, + 0.05008241906762123, + 0.03843973949551582, + 0.03960635885596275, + 0.0626731812953949, + 0.04089580848813057 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.4210955500602722, + "style_target": 0, + "support_probability": 0.07035300135612488, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7904754667149659, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05290127173066139, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9744069576263428, + 0.5348095297813416, + 0.47720396518707275, + 0.19645942747592926, + 0.023710405454039574, + 0.03181244805455208, + 0.02233271859586239, + 0.033597711473703384, + 0.05012607201933861, + 0.012057601474225521, + 0.028576040640473366 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.799382209777832, + "style_target": 1, + "support_probability": 0.5243291854858398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19422773969730214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07341587543487549, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9720908999443054, + 0.6156517863273621, + 0.49129679799079895, + 0.1242949441075325, + 0.04164664074778557, + 0.02885388769209385, + 0.238906130194664, + 0.013101877644658089, + 0.013332502916455269, + 0.02000933699309826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.19827455282211304, + "style_target": 0, + "support_probability": 0.3989461064338684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9068250425076, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.032838016748428345, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1626:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12911595404148102, + 0.18846286833286285, + 0.994419515132904, + 0.08481407910585403, + 0.13355910778045654 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 1, + "style_probability": 0.8145692348480225, + "style_target": 1, + "support_probability": 0.9631736874580383, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7712456881432891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05114733427762985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1626:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19874708354473114, + 0.22766704857349396, + 0.9776846766471863, + 0.0664333924651146, + 0.05729381740093231 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 0, + "style_probability": 0.5479939579963684, + "style_target": 0, + "support_probability": 0.9170449376106262, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.757770760404671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01427090261131525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4433324337005615, + 0.932412326335907, + 0.06687238067388535, + 0.27534499764442444, + 0.018082788214087486, + 0.07390198111534119 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.6207485198974609, + "style_target": 1, + "support_probability": 0.9263678193092346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6774930263007413, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01130818109959364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4439490735530853, + 0.938450038433075, + 0.07929041981697083, + 0.29045361280441284, + 0.04051179438829422, + 0.02326023206114769 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.4103478491306305, + "style_target": 0, + "support_probability": 0.9434637427330017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033514372076039016, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7827659249305725, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43037155270576477, + 0.5536943078041077, + 0.7504114508628845, + 0.40533626079559326, + 0.16401061415672302, + 0.0925178974866867, + 0.13833823800086975, + 0.07388928532600403, + 0.11262659728527069 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.6888670325279236, + "style_target": 1, + "support_probability": 0.011754782870411873, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036776327574011676, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6280924677848816, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33098870515823364, + 0.5418490171432495, + 0.7776653170585632, + 0.44144272804260254, + 0.03509698435664177, + 0.05188896134495735, + 0.025275086984038353, + 0.04452132806181908, + 0.18865323066711426 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.4092501103878021, + "style_target": 0, + "support_probability": 0.0553319975733757, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9262966745996276, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0557536743581295, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1636:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9980106949806213, + 0.29607197642326355, + 0.2428983747959137, + 0.277373731136322, + 0.07516869157552719, + 0.06551718711853027, + 0.0639582946896553, + 0.0333278514444828, + 0.02026493288576603, + 0.018149040639400482, + 0.08192697912454605 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 1, + "style_probability": 0.764823853969574, + "style_target": 1, + "support_probability": 0.9686456322669983, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8140026985441295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.055505361407995224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1636:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9942208528518677, + 0.3877723217010498, + 0.29506129026412964, + 0.3271354138851166, + 0.044511448591947556, + 0.017788011580705643, + 0.013625743798911572, + 0.014310600236058235, + 0.028671380132436752, + 0.1622229814529419, + 0.04725611209869385 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 1, + "style_probability": 0.47884610295295715, + "style_target": 0, + "support_probability": 0.9610118269920349, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4215526912756518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06349364668130875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6011353135108948, + 0.43486255407333374, + 0.9882130026817322, + 0.1631341278553009, + 0.10532406717538834 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 0, + "style_probability": 0.4225865304470062, + "style_target": 1, + "support_probability": 0.6696189641952515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6914032151066377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10561343282461166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5853409171104431, + 0.48981529474258423, + 0.9875636696815491, + 0.13928160071372986, + 0.07731285691261292 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 0, + "style_probability": 0.8617714047431946, + "style_target": 0, + "support_probability": 0.6285507678985596, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12946707843812974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6283497214317322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28070318698883057, + 0.07998615503311157, + 0.5016313195228577, + 0.8467612266540527, + 0.024393940344452858, + 0.04875513166189194 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.5231190323829651, + "style_target": 1, + "support_probability": 0.18525783717632294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.052297610359508466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7304834127426147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23096810281276703, + 0.11983929574489594, + 0.4984932243824005, + 0.8184531331062317, + 0.053372591733932495, + 0.07329614460468292 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.4336182177066803, + "style_target": 0, + "support_probability": 0.0534093901515007, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6444465267804081, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038279224187135696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9634584188461304, + 0.10230155289173126, + 0.29145026206970215, + 0.27778947353363037, + 0.03316057100892067, + 0.11467765271663666, + 0.04336026683449745, + 0.009809289127588272, + 0.02129918709397316 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.6052217483520508, + "style_target": 1, + "support_probability": 0.7531572580337524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20463114970839513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0941837802529335, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9292430877685547, + 0.0766352191567421, + 0.3490949273109436, + 0.19737321138381958, + 0.02022901549935341, + 0.013925977982580662, + 0.014649957418441772, + 0.016204969957470894, + 0.029424048960208893 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.13279449939727783, + "style_target": 0, + "support_probability": 0.5728209614753723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007506482743720387, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9245036840438843, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5173200368881226, + 0.2661510407924652, + 0.4988251030445099, + 0.6398533582687378, + 0.06333345174789429, + 0.06376531720161438, + 0.05389523133635521, + 0.09281791001558304, + 0.032808996737003326, + 0.06938882172107697, + 0.05287909880280495 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 0, + "style_probability": 0.4234171211719513, + "style_target": 1, + "support_probability": 0.008821617811918259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01619292483254267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.855012059211731, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.37641045451164246, + 0.23497074842453003, + 0.4045470356941223, + 0.6861714720726013, + 0.10240709036588669, + 0.08890662342309952, + 0.058917608112096786, + 0.030176425352692604, + 0.021626291796565056, + 0.04323255643248558 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 0, + "style_probability": 0.8484318852424622, + "style_target": 0, + "support_probability": 0.008298435248434544, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6224369932242789, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1970394402742386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7268893122673035, + 0.10326487571001053, + 0.24112357199192047, + 0.9648953080177307, + 0.020386306568980217 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.8387477397918701, + "style_target": 1, + "support_probability": 0.3903108239173889, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.042218585332340824, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24506597220897675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8275797963142395, + 0.09161675721406937, + 0.24348780512809753, + 0.9858623147010803, + 0.10429292917251587 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.13586261868476868, + "style_target": 0, + "support_probability": 0.39033523201942444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44580088985077704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.48059725761413574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7310461401939392, + 0.673991858959198, + 0.13489307463169098, + 0.9517746567726135, + 0.053905609995126724, + 0.04763787239789963 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 1, + "style_probability": 0.695642352104187, + "style_target": 1, + "support_probability": 0.3773820996284485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5210093935645965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.307279497385025, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5605520606040955, + 0.611491858959198, + 0.2025240957736969, + 0.9506803750991821, + 0.05184542015194893, + 0.07222156226634979 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 0, + "style_probability": 0.576568067073822, + "style_target": 0, + "support_probability": 0.5500800609588623, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.31591868914658083, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46763473749160767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5671483874320984, + 0.4453599750995636, + 0.5671483874320984, + 0.7108416557312012, + 0.022730596363544464, + 0.014658413827419281, + 0.048348937183618546, + 0.05363807454705238, + 0.019954146817326546 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.9468840956687927, + "style_target": 1, + "support_probability": 0.1804046481847763, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.023671925764454038, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7264774441719055, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45175155997276306, + 0.42023760080337524, + 0.45175155997276306, + 0.6738405823707581, + 0.043862953782081604, + 0.04826711118221283, + 0.014796164818108082, + 0.0526685006916523, + 0.05351719632744789 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.3812614679336548, + "style_target": 0, + "support_probability": 0.026840848848223686, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8067586502125963, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16360586881637573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23192967474460602, + 0.42522570490837097, + 0.7232178449630737, + 0.9651811122894287, + 0.013848108239471912, + 0.02353791892528534, + 0.01770961843430996, + 0.02157682366669178, + 0.026558561250567436, + 0.0847514420747757, + 0.03928554058074951 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.8823366165161133, + "style_target": 1, + "support_probability": 0.8281322717666626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05889276162252396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3919712007045746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21608909964561462, + 0.2594051659107208, + 0.5815195441246033, + 0.9608024954795837, + 0.030126435682177544, + 0.043176114559173584, + 0.06518514454364777, + 0.058278243988752365, + 0.07696032524108887, + 0.05703752487897873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.08352111279964447, + "style_target": 0, + "support_probability": 0.5890277624130249, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7719126975481869, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1059383973479271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21399615705013275, + 0.8714526295661926, + 0.35240498185157776, + 0.6130536198616028, + 0.05208415910601616 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 1, + "style_probability": 0.920642614364624, + "style_target": 1, + "support_probability": 0.9618757367134094, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6574768475742222, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24670203030109406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3032657206058502, + 0.8264263272285461, + 0.26488780975341797, + 0.5765430331230164, + 0.02207631804049015 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 0, + "style_probability": 0.5770884156227112, + "style_target": 0, + "support_probability": 0.9281464219093323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7729586335104425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010850122198462486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9733864068984985, + 0.2620461583137512, + 0.5629741549491882, + 0.49895480275154114, + 0.029253900051116943, + 0.18497636914253235 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 1, + "style_probability": 0.8086169362068176, + "style_target": 1, + "support_probability": 0.8373361825942993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5554343560894628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0154050188139081, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9694420099258423, + 0.27797427773475647, + 0.5325080752372742, + 0.5403940677642822, + 0.07657371461391449, + 0.028230352327227592 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 0, + "style_probability": 0.5151392817497253, + "style_target": 0, + "support_probability": 0.7745164036750793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6200415518652654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04776310548186302, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2849547266960144, + 0.16812852025032043, + 0.9787619113922119, + 0.6814969182014465, + 0.01846039853990078, + 0.03228573128581047, + 0.024098524823784828, + 0.02103414200246334, + 0.09332495182752609 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 1, + "style_probability": 0.6581974625587463, + "style_target": 1, + "support_probability": 0.6302588582038879, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1336810597496255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07070483267307281, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1744941771030426, + 0.1332947164773941, + 0.9745716452598572, + 0.6564393639564514, + 0.03394636511802673, + 0.09679903090000153, + 0.03329722583293915, + 0.030164044350385666, + 0.06065507233142853 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 1, + "style_probability": 0.18124297261238098, + "style_target": 0, + "support_probability": 0.47392573952674866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11747487329692358, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10595298558473587, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.210419163107872, + 0.8795861601829529, + 0.1740618646144867, + 0.11755669862031937, + 0.03966614976525307, + 0.031018950045108795, + 0.01832999475300312, + 0.05969742685556412, + 0.023704176768660545, + 0.027445796877145767, + 0.19398543238639832 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 1, + "style_probability": 0.715904176235199, + "style_target": 1, + "support_probability": 0.09216021001338959, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1938485093173174, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06261616945266724, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2440195083618164, + 0.8801823854446411, + 0.25998058915138245, + 0.11762101948261261, + 0.022967485710978508, + 0.014008788391947746, + 0.017234286293387413, + 0.035796936601400375, + 0.02182605117559433, + 0.04570925980806351, + 0.011988685466349125 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 0, + "style_probability": 0.6010831594467163, + "style_target": 0, + "support_probability": 0.1701824814081192, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6428995599676497, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08257438987493515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9507442116737366, + 0.40928199887275696, + 0.42924079298973083, + 0.2712833285331726, + 0.01222957018762827 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 0, + "style_probability": 0.3558991849422455, + "style_target": 1, + "support_probability": 0.9342196583747864, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8835405466881369, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021477052941918373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9818981289863586, + 0.4861072599887848, + 0.6548503637313843, + 0.28263983130455017, + 0.024239828810095787 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 0, + "style_probability": 0.6982837319374084, + "style_target": 0, + "support_probability": 0.9778035283088684, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5175979693434266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020257070660591125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08616653084754944, + 0.9746274948120117, + 0.07251442223787308, + 0.1707727164030075, + 0.0537819042801857, + 0.06159932538866997 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 0, + "style_probability": 0.4582163095474243, + "style_target": 1, + "support_probability": 0.740665853023529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18531442821274907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08349483460187912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12446852773427963, + 0.9707217812538147, + 0.1281643509864807, + 0.22680765390396118, + 0.22248272597789764, + 0.1214623674750328 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 1, + "style_probability": 0.28368470072746277, + "style_target": 0, + "support_probability": 0.3158036172389984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42013424781879477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028118837624788284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.094700388610363, + 0.1882028877735138, + 0.9726476073265076, + 0.22168245911598206, + 0.019379708915948868, + 0.070982925593853, + 0.04297592490911484, + 0.0799020454287529, + 0.04087837412953377 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 0, + "style_probability": 0.34329915046691895, + "style_target": 1, + "support_probability": 0.8620670437812805, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0896015238513805, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03412891924381256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08734611421823502, + 0.1397675722837448, + 0.9903954863548279, + 0.21413426101207733, + 0.07270106673240662, + 0.03606068715453148, + 0.06889034062623978, + 0.04856322705745697, + 0.07761526852846146 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 1, + "style_probability": 0.07001790404319763, + "style_target": 0, + "support_probability": 0.8783980011940002, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06332387637299075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6437003016471863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23662324249744415, + 0.08027008920907974, + 0.825495719909668, + 0.1764443963766098, + 0.0240451879799366, + 0.017262350767850876, + 0.025559868663549423, + 0.05883026123046875, + 0.03456239774823189, + 0.02729746140539646, + 0.01404201053082943 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 1, + "style_probability": 0.6400195360183716, + "style_target": 1, + "support_probability": 0.016450004652142525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021047910819297765, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7243907451629639, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1905224472284317, + 0.0550670400261879, + 0.7793177962303162, + 0.19082769751548767, + 0.022081283852458, + 0.027889234945178032, + 0.022744376212358475, + 0.017832454293966293, + 0.04441966116428375, + 0.016635870561003685, + 0.009483168832957745 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 0, + "style_probability": 0.8024184703826904, + "style_target": 0, + "support_probability": 0.005790536757558584, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.887155668808581, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03470978885889053, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.993569016456604, + 0.15746960043907166, + 0.5858640074729919, + 0.6158406138420105, + 0.059616703540086746 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 1, + "style_probability": 0.6722190380096436, + "style_target": 1, + "support_probability": 0.9890437126159668, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7785796800320365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024126654490828514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.996202290058136, + 0.09432235360145569, + 0.6769869327545166, + 0.6099031567573547, + 0.15747293829917908 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 1, + "style_probability": 0.4222269058227539, + "style_target": 0, + "support_probability": 0.9887853860855103, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012991016813755299, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8918601274490356, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3731222450733185, + 0.27818673849105835, + 0.32547762989997864, + 0.3731222450733185, + 0.021701214835047722, + 0.06055513024330139 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 0, + "style_probability": 0.4675469696521759, + "style_target": 1, + "support_probability": 0.0076005589216947556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004226619231973342, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8954251408576965, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3664720356464386, + 0.33339548110961914, + 0.3140832185745239, + 0.3664720356464386, + 0.016230424866080284, + 0.022891510277986526 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 1, + "style_probability": 0.36469918489456177, + "style_target": 0, + "support_probability": 0.004653901793062687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011093446519741616, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9512709975242615, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6824707388877869, + 0.4183041453361511, + 0.3332445025444031, + 0.23969607055187225, + 0.11008935421705246, + 0.20426256954669952, + 0.1441778540611267, + 0.09854041785001755, + 0.2729175388813019 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 0, + "style_probability": 0.3977925479412079, + "style_target": 1, + "support_probability": 0.007464143447577953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015639045808341658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9745210409164429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.56093430519104, + 0.3847349286079407, + 0.3760797083377838, + 0.3060295879840851, + 0.07675665616989136, + 0.20565879344940186, + 0.16273875534534454, + 0.15007098019123077, + 0.08373639732599258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 0, + "style_probability": 0.9003153443336487, + "style_target": 0, + "support_probability": 0.006799853406846523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014793039817521125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9976676106452942, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7923211455345154, + 0.5166531801223755, + 0.07447692006826401, + 0.5166531801223755, + 0.054574791342020035, + 0.05850054696202278, + 0.05555054172873497, + 0.05050158500671387, + 0.03659602999687195, + 0.06888823211193085 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 0, + "style_probability": 0.4230121374130249, + "style_target": 1, + "support_probability": 0.0004968214780092239, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025825617615304742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9955605268478394, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8555001020431519, + 0.45060041546821594, + 0.055490680038928986, + 0.45060041546821594, + 0.10167867690324783, + 0.04000195115804672, + 0.17833757400512695, + 0.08315247297286987, + 0.05697033926844597, + 0.0761057659983635 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 1, + "style_probability": 0.3493320047855377, + "style_target": 0, + "support_probability": 0.0009862530278041959, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25036181758058373, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7499004006385803, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6451225876808167, + 0.34446173906326294, + 0.38852179050445557, + 0.1718631088733673, + 0.039254527539014816 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 1, + "style_probability": 0.771472692489624, + "style_target": 1, + "support_probability": 0.13619503378868103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3459768556870008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5611037015914917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6294448971748352, + 0.40374094247817993, + 0.48454567790031433, + 0.21332113444805145, + 0.024570629000663757 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 0, + "style_probability": 0.5002760291099548, + "style_target": 0, + "support_probability": 0.4392377436161041, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20578728622508488, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.57051020860672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2321997582912445, + 0.1121564731001854, + 0.42862018942832947, + 0.7530398368835449, + 0.003647560253739357, + 0.0094820037484169 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.8273686766624451, + "style_target": 1, + "support_probability": 0.21743585169315338, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05023066823708428, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6871060132980347, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2339300513267517, + 0.1469884067773819, + 0.43301960825920105, + 0.7378122210502625, + 0.04174685478210449, + 0.02246427722275257 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.4068636894226074, + "style_target": 0, + "support_probability": 0.11979927867650986, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7975606998083482, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016556259244680405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9871068596839905, + 0.13543064892292023, + 0.10013805329799652, + 0.38838985562324524, + 0.05518558621406555, + 0.074326291680336, + 0.04581088945269585, + 0.0275412630289793, + 0.01590084657073021 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.8256768584251404, + "style_target": 1, + "support_probability": 0.8150832653045654, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10362590669198046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01773831807076931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9970487952232361, + 0.07808010280132294, + 0.08409810811281204, + 0.3043525815010071, + 0.07889381051063538, + 0.027341265231370926, + 0.041015490889549255, + 0.07330043613910675, + 0.07970651984214783 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.035876404494047165, + "style_target": 0, + "support_probability": 0.9246094226837158, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00434597801772496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9796452522277832, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6989662647247314, + 0.6989662647247314, + 0.8736169338226318, + 0.27302637696266174, + 0.038865722715854645, + 0.19444847106933594, + 0.05482567846775055, + 0.10984256118535995, + 0.09671743214130402, + 0.0366179496049881, + 0.12062793970108032 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 1, + "style_probability": 0.8294000625610352, + "style_target": 1, + "support_probability": 0.0005062191048637033, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021358958283115605, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9361562728881836, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7043765187263489, + 0.7043765187263489, + 0.8543243408203125, + 0.24369500577449799, + 0.04479696974158287, + 0.04642561450600624, + 0.06382658332586288, + 0.10783004015684128, + 0.040349382907152176, + 0.02771824598312378, + 0.021649261936545372 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 0, + "style_probability": 0.653235673904419, + "style_target": 0, + "support_probability": 0.004048814065754414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.052129979697949184, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6511151194572449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8325688242912292, + 0.2502962648868561, + 0.7141238451004028, + 0.16156400740146637, + 0.02479447051882744 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 0, + "style_probability": 0.3830443024635315, + "style_target": 1, + "support_probability": 0.016570448875427246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013809565479047914, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8006350994110107, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8226768374443054, + 0.21832136809825897, + 0.748645544052124, + 0.12167050689458847, + 0.028550803661346436 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 0, + "style_probability": 0.6953378915786743, + "style_target": 0, + "support_probability": 0.002445320365950465, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5908227911156926, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07029182463884354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.883912980556488, + 0.1663433313369751, + 0.21875271201133728, + 0.2818000912666321, + 0.010806698352098465, + 0.011898551136255264 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 1, + "style_probability": 0.6832648515701294, + "style_target": 1, + "support_probability": 0.6989704370498657, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5516667187090979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03520713374018669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8967354893684387, + 0.17603179812431335, + 0.17323103547096252, + 0.2519647479057312, + 0.01621091552078724, + 0.02486078254878521 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 0, + "style_probability": 0.5630886554718018, + "style_target": 0, + "support_probability": 0.7710456848144531, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6658179528493126, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011780117638409138, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20957744121551514, + 0.11780201643705368, + 0.9486926794052124, + 0.28389403223991394, + 0.24601057171821594, + 0.026774976402521133, + 0.013371299020946026, + 0.01113097369670868, + 0.01201050728559494 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 1, + "style_probability": 0.5510751605033875, + "style_target": 1, + "support_probability": 0.9125348329544067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07785759041706378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01784571260213852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08080314099788666, + 0.1639605015516281, + 0.9696921706199646, + 0.13262255489826202, + 0.057921092957258224, + 0.04579496756196022, + 0.042125482112169266, + 0.018791958689689636, + 0.12097592651844025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 1, + "style_probability": 0.057155441492795944, + "style_target": 0, + "support_probability": 0.9106274247169495, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6073196954047735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0257878415286541, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35059645771980286, + 0.27064818143844604, + 0.35059645771980286, + 0.9541823267936707, + 0.023250916972756386, + 0.42285630106925964, + 0.03217986226081848, + 0.02524743601679802, + 0.013281183317303658, + 0.01677221991121769 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 1, + "style_probability": 0.5144561529159546, + "style_target": 1, + "support_probability": 0.8015748262405396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.55169979241499, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.049038179218769073, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2178945243358612, + 0.19063670933246613, + 0.2178945243358612, + 0.9547162055969238, + 0.007494674064218998, + 0.01722760871052742, + 0.023895390331745148, + 0.02247319556772709, + 0.004221426788717508, + 0.007503528613597155, + 0.006959288846701384 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 0, + "style_probability": 0.7116362452507019, + "style_target": 0, + "support_probability": 0.619197428226471, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8762497067623969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008631845936179161, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9935431480407715, + 0.1964108794927597, + 0.3019934594631195, + 0.10204332321882248, + 0.02473902702331543 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 1, + "style_probability": 0.7028314471244812, + "style_target": 1, + "support_probability": 0.9751613736152649, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7787776790503143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005562204867601395, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.995992124080658, + 0.1335013210773468, + 0.3178199231624603, + 0.07733739912509918, + 0.019443374127149582 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 1, + "style_probability": 0.48114266991615295, + "style_target": 0, + "support_probability": 0.9808322191238403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15521965843793703, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39859580993652344, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.061232876032590866, + 0.6641138195991516, + 0.28700172901153564, + 0.5234455466270447, + 0.019947368651628494, + 0.030716117471456528 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.5373271703720093, + "style_target": 1, + "support_probability": 0.16121871769428253, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018043725457513107, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5116509795188904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.029466649517416954, + 0.6659196615219116, + 0.2033754587173462, + 0.258624792098999, + 0.0306896660476923, + 0.22250500321388245 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.05830756202340126, + "style_target": 0, + "support_probability": 0.33286023139953613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18934198441652683, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4199674427509308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.784898579120636, + 0.13053563237190247, + 0.5202163457870483, + 0.22964885830879211, + 0.06496001034975052, + 0.021851222962141037, + 0.028877707198262215, + 0.026697330176830292, + 0.06328227370977402 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 0, + "style_probability": 0.45396801829338074, + "style_target": 1, + "support_probability": 0.25880861282348633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08761556455240334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.591713547706604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7751212120056152, + 0.1749240905046463, + 0.6094292402267456, + 0.20255567133426666, + 0.029233749955892563, + 0.03385044261813164, + 0.0387573204934597, + 0.03865082934498787, + 0.12764717638492584 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 0, + "style_probability": 0.7817983031272888, + "style_target": 0, + "support_probability": 0.06819992512464523, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.26821550296227414, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4509131610393524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39834752678871155, + 0.19634030759334564, + 0.2970353364944458, + 0.8681638240814209, + 0.2443833202123642, + 0.04345197230577469, + 0.04609303176403046, + 0.04783691465854645, + 0.05065802484750748, + 0.0445534773170948, + 0.1035492941737175 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 0, + "style_probability": 0.43360379338264465, + "style_target": 1, + "support_probability": 0.5711084604263306, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15779578558975804, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3954892158508301, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41428399085998535, + 0.21606937050819397, + 0.3132452070713043, + 0.8938108682632446, + 0.10382108390331268, + 0.0942457988858223, + 0.07310419529676437, + 0.03524673730134964, + 0.0736156478524208, + 0.09491807222366333, + 0.09005195647478104 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 1, + "style_probability": 0.28881746530532837, + "style_target": 0, + "support_probability": 0.28726068139076233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7852020038919093, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05722515657544136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45242395997047424, + 0.9480611085891724, + 0.7725141644477844, + 0.25235453248023987, + 0.01334290113300085 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 1, + "style_probability": 0.8963326811790466, + "style_target": 1, + "support_probability": 0.6975072026252747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6192852509931256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06971251964569092, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3841908872127533, + 0.9302823543548584, + 0.6651612520217896, + 0.2590654194355011, + 0.016532672569155693 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 0, + "style_probability": 0.5812963247299194, + "style_target": 0, + "support_probability": 0.7598122358322144, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8306565098634647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06193588301539421, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22268103063106537, + 0.41436660289764404, + 0.9497140049934387, + 0.14997367560863495, + 0.016245149075984955, + 0.014143228530883789 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.809645414352417, + "style_target": 1, + "support_probability": 0.9722447991371155, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5958376100525077, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05194252356886864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23624823987483978, + 0.4033799469470978, + 0.9491701126098633, + 0.1746906042098999, + 0.006723536644130945, + 0.03854033723473549 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.30942127108573914, + "style_target": 0, + "support_probability": 0.9790825843811035, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8654901210651327, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.003003958845511079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6428675055503845, + 0.5468452572822571, + 0.9995880722999573, + 0.37041646242141724, + 0.0044983453117311, + 0.02141433022916317, + 0.014936206862330437, + 0.013612804003059864, + 0.009402205236256123 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 1, + "style_probability": 0.7572572231292725, + "style_target": 1, + "support_probability": 0.9490079879760742, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5484491637728287, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004097750876098871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6258823871612549, + 0.46398523449897766, + 0.9997501969337463, + 0.32158800959587097, + 0.014374841004610062, + 0.008425007574260235, + 0.013937631621956825, + 0.010741976089775562, + 0.026872199028730392 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 1, + "style_probability": 0.3985854983329773, + "style_target": 0, + "support_probability": 0.9319535493850708, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3322799495982411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2488485872745514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13597916066646576, + 0.896163284778595, + 0.1117040291428566, + 0.12923137843608856, + 0.01867646537721157, + 0.01773514412343502, + 0.009266761131584644, + 0.045623261481523514, + 0.01442254800349474, + 0.05491293594241142, + 0.01912103034555912 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 1, + "style_probability": 0.5453535318374634, + "style_target": 1, + "support_probability": 0.2781369388103485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09031467533272755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40967825055122375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21102868020534515, + 0.8000738620758057, + 0.18224553763866425, + 0.304492324590683, + 0.029935065656900406, + 0.02669578045606613, + 0.04363177716732025, + 0.018643822520971298, + 0.039436403661966324, + 0.03144831582903862, + 0.029115872457623482 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 1, + "style_probability": 0.4322488307952881, + "style_target": 0, + "support_probability": 0.09278285503387451, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7269589078065128, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017582137137651443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9934529066085815, + 0.06003228574991226, + 0.5728349089622498, + 0.39724069833755493, + 0.0230878759175539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.4343753457069397, + "style_target": 1, + "support_probability": 0.9428333640098572, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8677732991403999, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05087660998106003, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9945791959762573, + 0.056826334446668625, + 0.6231250762939453, + 0.3988122045993805, + 0.018983155488967896 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.753763735294342, + "style_target": 0, + "support_probability": 0.8268897533416748, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6302318784580125, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008162700571119785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9502553343772888, + 0.12649637460708618, + 0.3475840091705322, + 0.2942996323108673, + 0.02431350015103817, + 0.05214138701558113 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 0, + "style_probability": 0.2875520586967468, + "style_target": 1, + "support_probability": 0.9827994704246521, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1632055410661342, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01931803859770298, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9660972356796265, + 0.0790039524435997, + 0.3726072311401367, + 0.2002134621143341, + 0.015422220341861248, + 0.049920085817575455 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 1, + "style_probability": 0.06091514602303505, + "style_target": 0, + "support_probability": 0.9617186188697815, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2989853636485657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2573125958442688, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21180780231952667, + 0.9961383938789368, + 0.2832363545894623, + 0.21180780231952667, + 0.02387763001024723, + 0.02217552438378334, + 0.023189708590507507, + 0.013820591382682323, + 0.016013287007808685 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 0, + "style_probability": 0.21681417524814606, + "style_target": 1, + "support_probability": 0.4256435036659241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4742882970961408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21151186525821686, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2079872488975525, + 0.9944199919700623, + 0.2918233871459961, + 0.2079872488975525, + 0.03636711463332176, + 0.06186928600072861, + 0.0247135441750288, + 0.05782955884933472, + 0.049474332481622696 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 1, + "style_probability": 0.36355599761009216, + "style_target": 0, + "support_probability": 0.5889919400215149, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07034197739281207, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4945353865623474, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09014011174440384, + 0.4978248178958893, + 0.6975735425949097, + 0.5091212391853333, + 0.008723626844584942, + 0.011856293305754662, + 0.009767747484147549, + 0.011196350678801537, + 0.028605284169316292, + 0.00881181750446558, + 0.010908525437116623 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.2564537227153778, + "style_target": 1, + "support_probability": 0.048612866550683975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20526715227924192, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46879813075065613, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05987562611699104, + 0.4023987054824829, + 0.6826820969581604, + 0.41419264674186707, + 0.015370305627584457, + 0.009222728200256824, + 0.00617713900282979, + 0.0027619870379567146, + 0.006580725312232971, + 0.015573511831462383, + 0.009371734224259853 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.7304366827011108, + "style_target": 0, + "support_probability": 0.0821656733751297, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032510132284231864, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7322958707809448, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1464381217956543, + 0.06319481879472733, + 0.5036124587059021, + 0.5355627536773682, + 0.028659816831350327 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 0, + "style_probability": 0.48829129338264465, + "style_target": 1, + "support_probability": 0.05107521638274193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028361742284451806, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7092506289482117, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11546741425991058, + 0.03558643162250519, + 0.5728310942649841, + 0.53679358959198, + 0.02255084738135338 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 1, + "style_probability": 0.11995787173509598, + "style_target": 0, + "support_probability": 0.05710193142294884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1354958314590068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2383958399295807, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46376657485961914, + 0.9627130627632141, + 0.5311474800109863, + 0.3366282880306244, + 0.08549350500106812, + 0.0756424143910408 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 0, + "style_probability": 0.23636125028133392, + "style_target": 1, + "support_probability": 0.3085860311985016, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17172342856374095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16718406975269318, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45712536573410034, + 0.9433428049087524, + 0.5667018294334412, + 0.384720116853714, + 0.027326563373208046, + 0.04206561669707298 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 1, + "style_probability": 0.4284389317035675, + "style_target": 0, + "support_probability": 0.2584037780761719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28810077024919173, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08714225888252258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6856169104576111, + 0.27413037419319153, + 0.9471402168273926, + 0.26388445496559143, + 0.034652598202228546, + 0.01918451115489006, + 0.056429244577884674, + 0.2415589541196823, + 0.029505299404263496 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.2672327160835266, + "style_target": 1, + "support_probability": 0.512484073638916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45163912530599815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1097259446978569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7184769511222839, + 0.23457492887973785, + 0.9539107084274292, + 0.224938303232193, + 0.01992705650627613, + 0.028868408873677254, + 0.033449217677116394, + 0.022630508989095688, + 0.03344021737575531 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.6117698550224304, + "style_target": 0, + "support_probability": 0.5686913728713989, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010788860557706817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7851161956787109, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30942901968955994, + 0.17989641427993774, + 0.18657010793685913, + 0.7125331163406372, + 0.0957455113530159, + 0.14119169116020203, + 0.06745670735836029, + 0.0645301342010498, + 0.06046011671423912, + 0.06529594957828522, + 0.11133255809545517 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 0, + "style_probability": 0.33709198236465454, + "style_target": 1, + "support_probability": 0.004671563394367695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004842116066498368, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7479063868522644, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21696145832538605, + 0.17096388339996338, + 0.20822803676128387, + 0.6967734098434448, + 0.020189587026834488, + 0.0180229339748621, + 0.013866986148059368, + 0.05515146255493164, + 0.06377492845058441, + 0.11138226091861725 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 1, + "style_probability": 0.34445127844810486, + "style_target": 0, + "support_probability": 0.01603429950773716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7040378748192865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017678838223218918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9511345028877258, + 0.30239957571029663, + 0.12208737432956696, + 0.6137071251869202, + 0.04655257612466812 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 1, + "style_probability": 0.8090208172798157, + "style_target": 1, + "support_probability": 0.7363638877868652, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27456594640444715, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07664181292057037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9180415868759155, + 0.28450843691825867, + 0.08784191310405731, + 0.6088992953300476, + 0.09539204835891724 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 0, + "style_probability": 0.5610659718513489, + "style_target": 0, + "support_probability": 0.4164539575576782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9595500645424129, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.018858499825000763, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1930:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2888949513435364, + 0.2067379206418991, + 0.9961684346199036, + 0.503433346748352, + 0.024400053545832634, + 0.01074468158185482 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 1, + "style_probability": 0.9523951411247253, + "style_target": 1, + "support_probability": 0.9655621647834778, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5836085950801236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.040154751390218735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1930:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2935148775577545, + 0.33168575167655945, + 0.9931620359420776, + 0.4848681092262268, + 0.011694514192640781, + 0.0189397931098938 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 1, + "style_probability": 0.3618015646934509, + "style_target": 0, + "support_probability": 0.893397331237793, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41314240541250224, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20123375952243805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4067583382129669, + 0.5884789228439331, + 0.4067583382129669, + 0.979490339756012, + 0.013732842169702053, + 0.011090453714132309, + 0.031184891238808632, + 0.03253108263015747, + 0.02676219493150711 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.8500639796257019, + "style_target": 1, + "support_probability": 0.18988372385501862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09190418212057824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11656095087528229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38282549381256104, + 0.5980056524276733, + 0.38282549381256104, + 0.9890283346176147, + 0.014828193001449108, + 0.04825960844755173, + 0.010447999462485313, + 0.01037736888974905, + 0.03058096021413803 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.13798722624778748, + "style_target": 0, + "support_probability": 0.5270566344261169, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.829574533226223, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.037124816328287125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5437757968902588, + 0.9389441609382629, + 0.31462931632995605, + 0.19213399291038513, + 0.108095683157444, + 0.03144701570272446, + 0.019310569390654564, + 0.018510261550545692, + 0.06269577890634537, + 0.024492358788847923, + 0.011061383411288261 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 1, + "style_probability": 0.8522956371307373, + "style_target": 1, + "support_probability": 0.9395617842674255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7162272132448667, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09885547310113907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6969467401504517, + 0.9101828336715698, + 0.39699336886405945, + 0.35976600646972656, + 0.011987387202680111, + 0.1081770658493042, + 0.027801519259810448, + 0.020181290805339813, + 0.033227771520614624, + 0.014186417683959007, + 0.02323651872575283 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 0, + "style_probability": 0.5496311783790588, + "style_target": 0, + "support_probability": 0.9672412872314453, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0037013147627030624, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9900807738304138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17904499173164368, + 0.2570854127407074, + 0.22919127345085144, + 0.6356267333030701, + 0.24046136438846588 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.2971676290035248, + "style_target": 1, + "support_probability": 0.0008793832967057824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018561070574980357, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9981813430786133, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18564850091934204, + 0.19543617963790894, + 0.23908911645412445, + 0.6319453120231628, + 0.05412585288286209 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.7947818040847778, + "style_target": 0, + "support_probability": 0.0005169500946067274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3424886364391564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11495789140462875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6824696063995361, + 0.07096255570650101, + 0.774606466293335, + 0.27424389123916626, + 0.05779216066002846, + 0.02328098565340042 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 0, + "style_probability": 0.21187235414981842, + "style_target": 1, + "support_probability": 0.8531686663627625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053844544024431146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1627098023891449, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6787406802177429, + 0.08572378009557724, + 0.8393594026565552, + 0.14885731041431427, + 0.008570472709834576, + 0.013430838473141193 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 1, + "style_probability": 0.06536772102117538, + "style_target": 0, + "support_probability": 0.7394324541091919, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.541794876261919, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10557388514280319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9302347898483276, + 0.6152780055999756, + 0.6152780055999756, + 0.09873858839273453, + 0.053548119962215424, + 0.04251810535788536, + 0.01430263090878725, + 0.05486823618412018, + 0.04830382764339447 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 0, + "style_probability": 0.32424166798591614, + "style_target": 1, + "support_probability": 0.885506272315979, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7341475463725202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10872577875852585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9287540912628174, + 0.6609644293785095, + 0.6609644293785095, + 0.146397665143013, + 0.01564239338040352, + 0.07692370563745499, + 0.020449260249733925, + 0.021095484495162964, + 0.005564230959862471 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 1, + "style_probability": 0.37925469875335693, + "style_target": 0, + "support_probability": 0.9711507558822632, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0036259333866524796, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9282304644584656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.0569581463932991, + 0.5620800256729126, + 0.36723095178604126, + 0.24578642845153809, + 0.02707183174788952, + 0.05695139616727829, + 0.09442268311977386, + 0.11940967291593552, + 0.033477310091257095, + 0.056632500141859055, + 0.1191679835319519 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.29733729362487793, + "style_target": 1, + "support_probability": 0.0006094048731029034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015323262577797377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9322179555892944, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.04990842938423157, + 0.5399320125579834, + 0.35470810532569885, + 0.2424336075782776, + 0.03419199958443642, + 0.018561003729701042, + 0.030959881842136383, + 0.041469164192676544, + 0.017673935741186142, + 0.03148932382464409 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.7998510003089905, + "style_target": 0, + "support_probability": 0.00040886117494665086, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48968694092296416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2041146159172058, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9943755865097046, + 0.12929007411003113, + 0.4049184322357178, + 0.021140865981578827, + 0.11450579017400742 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.7892021536827087, + "style_target": 1, + "support_probability": 0.3495662212371826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06354418511595859, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18823973834514618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9963827133178711, + 0.12850560247898102, + 0.34916797280311584, + 0.020294737070798874, + 0.042535435408353806 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.16097962856292725, + "style_target": 0, + "support_probability": 0.4174228012561798, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8121013624740968, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038748130202293396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.02782900258898735, + 0.9189416170120239, + 0.20187163352966309, + 0.047747902572155, + 0.020100394263863564, + 0.02081187441945076 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 1, + "style_probability": 0.865527868270874, + "style_target": 1, + "support_probability": 0.9739254713058472, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5176726720542589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2293206751346588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.039232105016708374, + 0.7805810570716858, + 0.244230255484581, + 0.06011994555592537, + 0.047440316528081894, + 0.046258095651865005 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 0, + "style_probability": 0.5749553442001343, + "style_target": 0, + "support_probability": 0.8102372288703918, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27560839587119645, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19607171416282654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9809792637825012, + 0.8520943522453308, + 0.4449174106121063, + 0.17158877849578857, + 0.024864796549081802, + 0.030156413093209267, + 0.010258904658257961, + 0.011239837855100632, + 0.02466908097267151 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 1, + "style_probability": 0.9049590826034546, + "style_target": 1, + "support_probability": 0.03742020204663277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21313780067928825, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22871476411819458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9739591479301453, + 0.8263285160064697, + 0.4565972685813904, + 0.16078808903694153, + 0.1495288461446762, + 0.05769062787294388, + 0.06425183266401291, + 0.04891955852508545, + 0.03830640763044357 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 0, + "style_probability": 0.6878873705863953, + "style_target": 0, + "support_probability": 0.05834582448005676, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8590924434343793, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027431704103946686, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9618316292762756, + 0.7666318416595459, + 0.8093045353889465, + 0.44256284832954407, + 0.0069916811771690845, + 0.012784981168806553, + 0.013212664984166622, + 0.03340361639857292, + 0.013554312288761139, + 0.015557371079921722, + 0.010108616203069687 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.8820835947990417, + "style_target": 1, + "support_probability": 0.8839413523674011, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17519039718469984, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044780828058719635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9479060173034668, + 0.7634505033493042, + 0.8286864161491394, + 0.29726192355155945, + 0.03331021964550018, + 0.02085191383957863, + 0.01814536564052105, + 0.01853153295814991, + 0.030074266716837883, + 0.042848821729421616 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.11954472213983536, + "style_target": 0, + "support_probability": 0.8507353067398071, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5745026584693629, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4589959383010864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2071179747581482, + 0.9228135943412781, + 0.12168622016906738, + 0.36785948276519775, + 0.028528718277812004 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 1, + "style_probability": 0.6064752340316772, + "style_target": 1, + "support_probability": 0.8445561528205872, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6772711474880823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42671746015548706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20765410363674164, + 0.899424135684967, + 0.139093816280365, + 0.38007527589797974, + 0.03438758850097656 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 0, + "style_probability": 0.7256981730461121, + "style_target": 0, + "support_probability": 0.8821874856948853, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7126065292452219, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0419282428920269, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2017:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4112790822982788, + 0.9899765849113464, + 0.52275151014328, + 0.25438010692596436, + 0.01886926032602787, + 0.024576915428042412 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 1, + "style_probability": 0.6076780557632446, + "style_target": 1, + "support_probability": 0.8065219521522522, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9101888330826928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04186837002635002, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:2017:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4624474346637726, + 0.9922422170639038, + 0.572986364364624, + 0.21647123992443085, + 0.03363031521439552, + 0.035365115851163864 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 0, + "style_probability": 0.9128079414367676, + "style_target": 0, + "support_probability": 0.7960805296897888, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.016476018034550916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9328617453575134, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3182242214679718, + 0.4146641790866852, + 0.7210410833358765, + 0.20076295733451843, + 0.04458494111895561, + 0.0397302508354187, + 0.041714128106832504, + 0.04651370272040367, + 0.06990136951208115 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 0, + "style_probability": 0.3939048647880554, + "style_target": 1, + "support_probability": 0.02139686606824398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008122282200464525, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8245773911476135, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2677503526210785, + 0.32314395904541016, + 0.6945967674255371, + 0.17717793583869934, + 0.026684731245040894, + 0.05083144083619118, + 0.046355053782463074, + 0.023332197219133377, + 0.09337614476680756 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 1, + "style_probability": 0.0614677332341671, + "style_target": 0, + "support_probability": 0.10201890021562576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004486464984494579, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9317759871482849, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3489784300327301, + 0.44185638427734375, + 0.46876081824302673, + 0.3918238878250122, + 0.03637808561325073, + 0.0387069471180439, + 0.017075303941965103, + 0.033707451075315475, + 0.09060930460691452, + 0.07067859917879105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.3661816418170929, + "style_target": 1, + "support_probability": 0.0013766774209216237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01575826225944962, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8745610117912292, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35597312450408936, + 0.4055556058883667, + 0.47026607394218445, + 0.31286466121673584, + 0.0500781312584877, + 0.10298151522874832, + 0.26536744832992554, + 0.1152728721499443, + 0.06595220416784286, + 0.11046033352613449 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.7130870223045349, + "style_target": 0, + "support_probability": 0.0051681846380233765, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7958036614975738, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.299279123544693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.916403591632843, + 0.3998720943927765, + 0.4369927942752838, + 0.3998720943927765, + 0.06436469405889511 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.8904016017913818, + "style_target": 1, + "support_probability": 0.8596916198730469, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5552630812332078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21676352620124817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9224895238876343, + 0.43628576397895813, + 0.45419424772262573, + 0.43628576397895813, + 0.00796525739133358 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.2695942223072052, + "style_target": 0, + "support_probability": 0.92545485496521, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7098632692513008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19822630286216736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40699100494384766, + 0.3901395797729492, + 0.6123605370521545, + 0.989021360874176, + 0.021505476906895638, + 0.03971400856971741 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.8283567428588867, + "style_target": 1, + "support_probability": 0.5915445685386658, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07522566728026149, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3821009695529938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36275988817214966, + 0.3433486521244049, + 0.6264394521713257, + 0.9906099438667297, + 0.06225961819291115, + 0.12540778517723083 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.17432796955108643, + "style_target": 0, + "support_probability": 0.4641217589378357, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21065260683894171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15490320324897766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4026510417461395, + 0.22869659960269928, + 0.12776480615139008, + 0.908271849155426, + 0.014337361790239811, + 0.018024031072854996, + 0.041353363543748856, + 0.045746829360723495, + 0.02012971043586731 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 1, + "style_probability": 0.5740412473678589, + "style_target": 1, + "support_probability": 0.34179016947746277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05584780888749085, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3697827458381653, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4361193776130676, + 0.19407838582992554, + 0.14346455037593842, + 0.8149897456169128, + 0.06621214002370834, + 0.048365965485572815, + 0.05071574077010155, + 0.17607931792736053, + 0.08805041760206223 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 0, + "style_probability": 0.5710288286209106, + "style_target": 0, + "support_probability": 0.06627324223518372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.034053716149677976, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6054929494857788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3440224528312683, + 0.19482716917991638, + 0.46649882197380066, + 0.3440224528312683, + 0.05068197101354599, + 0.02093740738928318, + 0.011676610447466373, + 0.010300971567630768, + 0.02566569298505783, + 0.03386637941002846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.9186761379241943, + "style_target": 1, + "support_probability": 0.014489715918898582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026827477437104105, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4088543951511383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2837318480014801, + 0.20556412637233734, + 0.4344605505466461, + 0.2837318480014801, + 0.0249998290091753, + 0.018555471673607826, + 0.015920817852020264, + 0.025028852745890617, + 0.02414676919579506, + 0.01879957877099514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.19469769299030304, + "style_target": 0, + "support_probability": 0.05239945277571678, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.059577173966468194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.42421993613243103, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13064582645893097, + 0.5812965631484985, + 0.6755929589271545, + 0.8138763308525085, + 0.023533279076218605 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 0, + "style_probability": 0.476296603679657, + "style_target": 1, + "support_probability": 0.06508389115333557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05665912896760696, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5943169593811035, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10670816153287888, + 0.6403292417526245, + 0.7447819709777832, + 0.794685959815979, + 0.04263204708695412 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 1, + "style_probability": 0.45373865962028503, + "style_target": 0, + "support_probability": 0.03465765714645386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13066488429386358, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1887151300907135, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1310122162103653, + 0.265758752822876, + 0.18394070863723755, + 0.7910856604576111, + 0.17931611835956573, + 0.017190231010317802 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 1, + "style_probability": 0.5042794346809387, + "style_target": 1, + "support_probability": 0.1044173315167427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007206027940011596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.606033444404602, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.058562468737363815, + 0.20159466564655304, + 0.07849109917879105, + 0.7102881073951721, + 0.04824798181653023, + 0.07767511904239655 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 1, + "style_probability": 0.05521290376782417, + "style_target": 0, + "support_probability": 0.06284677237272263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.538575931522575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0070462883450090885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13578753173351288, + 0.340210497379303, + 0.9932700991630554, + 0.20351958274841309, + 0.039786916226148605, + 0.020873166620731354, + 0.08557690680027008, + 0.09124119579792023, + 0.02749820239841938 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 0, + "style_probability": 0.4490887522697449, + "style_target": 1, + "support_probability": 0.5997503995895386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.535255787008879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011862333863973618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1503840833902359, + 0.3408023715019226, + 0.9947428703308105, + 0.17591185867786407, + 0.025075608864426613, + 0.01840747892856598, + 0.07236278057098389, + 0.37952741980552673, + 0.02972056157886982 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 0, + "style_probability": 0.7928469777107239, + "style_target": 0, + "support_probability": 0.2559531033039093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30065737169097984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23686432838439941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.45846042037010193, + 0.9606287479400635, + 0.42715317010879517, + 0.3409915864467621, + 0.10598190128803253, + 0.0205831378698349, + 0.0245375894010067, + 0.048556264489889145, + 0.05839444696903229, + 0.021070048213005066 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 1, + "style_probability": 0.7314900755882263, + "style_target": 1, + "support_probability": 0.21244871616363525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2510331289278214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15602591633796692, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.46985694766044617, + 0.972889244556427, + 0.3520398736000061, + 0.29255780577659607, + 0.06708185374736786, + 0.04125069081783295, + 0.08644994348287582, + 0.06956600397825241, + 0.03413073718547821, + 0.03981570154428482 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 0, + "style_probability": 0.60838383436203, + "style_target": 0, + "support_probability": 0.2380814105272293, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8897720995967054, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3301154375076294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07337704300880432, + 0.9942183494567871, + 0.6023023128509521, + 0.07337704300880432, + 0.07128056138753891 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 1, + "style_probability": 0.8343783617019653, + "style_target": 1, + "support_probability": 0.8085858225822449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8130799552685516, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23467481136322021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10565847903490067, + 0.9836642146110535, + 0.5386533737182617, + 0.10565847903490067, + 0.07568983733654022 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 0, + "style_probability": 0.6027848124504089, + "style_target": 0, + "support_probability": 0.8988366723060608, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8230478614003354, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05017635226249695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7647222280502319, + 0.12699441611766815, + 0.13284794986248016, + 0.9713773727416992, + 0.01517928671091795, + 0.033627595752477646 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.6908488869667053, + "style_target": 1, + "support_probability": 0.8883193135261536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.644319727233022, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0467025563120842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7806187272071838, + 0.19225214421749115, + 0.15876279771327972, + 0.9658248424530029, + 0.013958551920950413, + 0.017590902745723724 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.3016076385974884, + "style_target": 0, + "support_probability": 0.8846877217292786, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7930531703731363, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09656628221273422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5921346545219421, + 0.6160570979118347, + 0.306780070066452, + 0.9051150679588318, + 0.027914635837078094, + 0.01035003550350666, + 0.016730735078454018, + 0.013604898937046528, + 0.045196350663900375 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.8996267318725586, + "style_target": 1, + "support_probability": 0.9296855926513672, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03743246589603131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16306914389133453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42981261014938354, + 0.5641558170318604, + 0.3082403540611267, + 0.9141148328781128, + 0.023908523842692375, + 0.01661018840968609, + 0.022924767807126045, + 0.0124078169465065, + 0.025651710107922554 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.04494870826601982, + "style_target": 0, + "support_probability": 0.8286058306694031, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7324791272881876, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05493110418319702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2990524172782898, + 0.48048460483551025, + 0.9432256817817688, + 0.3414262533187866, + 0.0178375244140625, + 0.018051568418741226, + 0.013172483071684837, + 0.033728986978530884, + 0.01840539090335369, + 0.037084098905324936 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 1, + "style_probability": 0.5564481019973755, + "style_target": 1, + "support_probability": 0.9129812717437744, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8162326047718135, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.040618542581796646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30429786443710327, + 0.5529247522354126, + 0.9470494389533997, + 0.3415391743183136, + 0.02010354772210121, + 0.010175313800573349, + 0.01089651882648468, + 0.023109596222639084, + 0.044686995446681976, + 0.035067640244960785 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 1, + "style_probability": 0.4771517217159271, + "style_target": 0, + "support_probability": 0.9797688126564026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06714813295842295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7148836255073547, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2672620713710785, + 0.2948032319545746, + 0.3093211352825165, + 0.9001553654670715, + 0.2357894629240036 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 1, + "style_probability": 0.5309774279594421, + "style_target": 1, + "support_probability": 0.06550437211990356, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1254700467703553, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6482481956481934, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3066343069076538, + 0.34645602107048035, + 0.26330262422561646, + 0.9211952090263367, + 0.11920970678329468 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 0, + "style_probability": 0.856819212436676, + "style_target": 0, + "support_probability": 0.058194953948259354, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002158243378949143, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8717684745788574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11904232949018478, + 0.36954909563064575, + 0.2514328360557556, + 0.4630070626735687, + 0.19467192888259888, + 0.03638983890414238 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 0, + "style_probability": 0.23248256742954254, + "style_target": 1, + "support_probability": 0.000819976266939193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016152298590979572, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9601549506187439, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14893695712089539, + 0.4124147295951843, + 0.3443371653556824, + 0.5350658297538757, + 0.06974330544471741, + 0.04185715317726135 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 1, + "style_probability": 0.14237159490585327, + "style_target": 0, + "support_probability": 0.00037490588147193193, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.807585261427428, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010575756430625916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10974748432636261, + 0.06883423775434494, + 0.9907099008560181, + 0.32546621561050415, + 0.02489645779132843, + 0.024486448615789413, + 0.03046320751309395, + 0.07533504068851471, + 0.03013361059129238 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 1, + "style_probability": 0.5543299317359924, + "style_target": 1, + "support_probability": 0.9363356828689575, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4675981724047952, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005232466384768486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10400893539190292, + 0.1478574573993683, + 0.9958736300468445, + 0.44695544242858887, + 0.02656540274620056, + 0.08118974417448044, + 0.027677718549966812, + 0.01106457132846117, + 0.019531438127160072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 1, + "style_probability": 0.23317676782608032, + "style_target": 0, + "support_probability": 0.9186960458755493, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07531012383042585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6370458006858826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9311931729316711, + 0.1503002643585205, + 0.1939982920885086, + 0.1503002643585205, + 0.050032082945108414, + 0.0470925010740757, + 0.14007790386676788, + 0.11943739652633667, + 0.06353899836540222, + 0.25892403721809387, + 0.3454967141151428 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 0, + "style_probability": 0.42893123626708984, + "style_target": 1, + "support_probability": 0.15133528411388397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2302362262021857, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5609757900238037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9411277770996094, + 0.12728634476661682, + 0.1746726781129837, + 0.12728634476661682, + 0.06666970252990723, + 0.15215815603733063, + 0.07163666188716888, + 0.10910160839557648, + 0.12706464529037476, + 0.1189335510134697 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 0, + "style_probability": 0.8993232846260071, + "style_target": 0, + "support_probability": 0.13472943007946014, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8953237998892383, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.01123121939599514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5583339929580688, + 0.2897784411907196, + 0.9973662495613098, + 0.278484582901001, + 0.02131471410393715 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.688295304775238, + "style_target": 1, + "support_probability": 0.9532004594802856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5724153162258613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.01609245128929615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.546476423740387, + 0.3483106791973114, + 0.9974573254585266, + 0.2737863063812256, + 0.025465935468673706 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.25936922430992126, + "style_target": 0, + "support_probability": 0.9513718485832214, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7366535457347853, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04746575281023979, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34738895297050476, + 0.48377856612205505, + 0.944128692150116, + 0.1240781620144844, + 0.057728514075279236, + 0.015466870740056038 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 0, + "style_probability": 0.40563955903053284, + "style_target": 1, + "support_probability": 0.9691590070724487, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7042020137638534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04653181880712509, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2979845404624939, + 0.4671167731285095, + 0.9195826053619385, + 0.1128944456577301, + 0.02886210009455681, + 0.12722565233707428 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 1, + "style_probability": 0.31056708097457886, + "style_target": 0, + "support_probability": 0.9441727995872498, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8496824966024196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026243440806865692, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19020220637321472, + 0.28547364473342896, + 0.3353765904903412, + 0.9725048542022705, + 0.03511224687099457, + 0.05180925130844116, + 0.03425006568431854, + 0.03637167438864708, + 0.033001191914081573 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.7951793074607849, + "style_target": 1, + "support_probability": 0.8387761116027832, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2886576388647762, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09677983075380325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18657389283180237, + 0.2655027508735657, + 0.3121728301048279, + 0.9626550674438477, + 0.10927710682153702, + 0.10680831223726273, + 0.0522376112639904, + 0.19001014530658722, + 0.12368229031562805 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.19240812957286835, + "style_target": 0, + "support_probability": 0.346038818359375, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03031008638330847, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8086584806442261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3760719299316406, + 0.6058345437049866, + 0.7086350917816162, + 0.3760719299316406, + 0.1344897747039795, + 0.0255692470818758, + 0.01860606111586094, + 0.02758564054965973, + 0.013769218698143959, + 0.01314020436257124, + 0.029934268444776535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.8424711227416992, + "style_target": 1, + "support_probability": 0.003730504307895899, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004481411454523439, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7784504890441895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31258103251457214, + 0.7185351848602295, + 0.6139783263206482, + 0.31258103251457214, + 0.027529697865247726, + 0.01991460472345352, + 0.01766161434352398, + 0.0732681155204773, + 0.040346045047044754, + 0.055521160364151 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.22886817157268524, + "style_target": 0, + "support_probability": 0.0037730075418949127, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6424470465703095, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14386118948459625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14479860663414001, + 0.08400674164295197, + 0.23772896826267242, + 0.9583314657211304, + 0.027881653979420662 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 1, + "style_probability": 0.8921700119972229, + "style_target": 1, + "support_probability": 0.4607694447040558, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4526608755143365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.31685957312583923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17600487172603607, + 0.09765893220901489, + 0.27749788761138916, + 0.9188823103904724, + 0.021040990948677063 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 0, + "style_probability": 0.7034512162208557, + "style_target": 0, + "support_probability": 0.4431487023830414, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7757807982334435, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1610272228717804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23488181829452515, + 0.9215940237045288, + 0.12489136308431625, + 0.7093831896781921, + 0.013320001773536205, + 0.011623403057456017 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.8129666447639465, + "style_target": 1, + "support_probability": 0.8670482039451599, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5252278625942097, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1384638398885727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3159044682979584, + 0.9376028776168823, + 0.1414407342672348, + 0.7471124529838562, + 0.08520422130823135, + 0.07556293904781342 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.2316804975271225, + "style_target": 0, + "support_probability": 0.9270823001861572, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30311084507047953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5949456095695496, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8869113326072693, + 0.432876855134964, + 0.34283125400543213, + 0.17953021824359894, + 0.02474311739206314, + 0.019574668258428574, + 0.011841916479170322, + 0.03469938412308693, + 0.028572574257850647 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.6850625872612, + "style_target": 1, + "support_probability": 0.15021775662899017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03466711011927961, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5116357803344727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8663881421089172, + 0.3118394613265991, + 0.29350465536117554, + 0.15112097561359406, + 0.033492304384708405, + 0.03931384161114693, + 0.01944955438375473, + 0.028851673007011414, + 0.045801546424627304 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.08925001323223114, + "style_target": 0, + "support_probability": 0.13296163082122803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33240007579078873, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17448534071445465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7755626440048218, + 0.3853858709335327, + 0.6241871118545532, + 0.14975424110889435, + 0.050949983298778534, + 0.02486162632703781, + 0.03832845762372017, + 0.028106266632676125, + 0.018199700862169266, + 0.03966972604393959, + 0.024915821850299835 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.7471773624420166, + "style_target": 1, + "support_probability": 0.294828325510025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05449528678881087, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3202921748161316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.617031991481781, + 0.3352587819099426, + 0.549858570098877, + 0.1252395510673523, + 0.033332645893096924, + 0.0559413805603981, + 0.029148103669285774, + 0.08760424703359604, + 0.0643257349729538, + 0.024089617654681206, + 0.026307180523872375 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.27958664298057556, + "style_target": 0, + "support_probability": 0.09321712702512741, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.377835844023166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07794584333896637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3847145438194275, + 0.21930010616779327, + 0.9364221096038818, + 0.1576177030801773, + 0.08846345543861389 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.8896036148071289, + "style_target": 1, + "support_probability": 0.09498870372772217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1770470866316474, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04626685380935669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38828641176223755, + 0.2574048936367035, + 0.9510875344276428, + 0.1296626180410385, + 0.07949477434158325 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.41814178228378296, + "style_target": 0, + "support_probability": 0.17044714093208313, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8807615770510469, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19296662509441376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.264543741941452, + 0.28500595688819885, + 0.9953501224517822, + 0.39095744490623474, + 0.033713143318891525, + 0.049738798290491104 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.8088963031768799, + "style_target": 1, + "support_probability": 0.9038388729095459, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16554956055338055, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27741336822509766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20915016531944275, + 0.2827988266944885, + 0.9976992011070251, + 0.42017486691474915, + 0.023060370236635208, + 0.07206611335277557 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.12528561055660248, + "style_target": 0, + "support_probability": 0.9107919335365295, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3831864016247042, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22539424896240234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3582911491394043, + 0.08609675616025925, + 0.3361221253871918, + 0.9970011115074158, + 0.01751938834786415, + 0.07620441168546677, + 0.01418166235089302, + 0.04038887470960617, + 0.02751201018691063 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 0, + "style_probability": 0.4827897548675537, + "style_target": 1, + "support_probability": 0.5511212944984436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2452710081182951, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27306491136550903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3497968912124634, + 0.112618088722229, + 0.3085881471633911, + 0.9941741824150085, + 0.03951895609498024, + 0.014191272668540478, + 0.04344785958528519, + 0.07634148746728897, + 0.04898152872920036 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 1, + "style_probability": 0.2227623015642166, + "style_target": 0, + "support_probability": 0.5841238498687744, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7044662730939366, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17176665365695953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18227505683898926, + 0.31437164545059204, + 0.31286147236824036, + 0.9836630821228027, + 0.012023738585412502, + 0.023962698876857758, + 0.019755782559514046, + 0.0763658881187439, + 0.029300782829523087, + 0.045471396297216415, + 0.047571126371622086 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.8315461277961731, + "style_target": 1, + "support_probability": 0.4138023555278778, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37275800905943124, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08894297480583191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16253520548343658, + 0.3068917393684387, + 0.325863778591156, + 0.9841259717941284, + 0.02715756930410862, + 0.08137698471546173, + 0.019090885296463966, + 0.03970835357904434, + 0.024625638499855995, + 0.024056222289800644, + 0.1072695255279541 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.4030977785587311, + "style_target": 0, + "support_probability": 0.5375301837921143, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8923030513036917, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.03346460685133934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2849887013435364, + 0.20306198298931122, + 0.2544602155685425, + 0.9711018800735474, + 0.012146134860813618 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.9413562417030334, + "style_target": 1, + "support_probability": 0.9445776343345642, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0887626930937747, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.04543229565024376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1950666755437851, + 0.1813221573829651, + 0.21076181530952454, + 0.9870162606239319, + 0.041641999036073685 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.05779363960027695, + "style_target": 0, + "support_probability": 0.7886876463890076, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8560632084168572, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022047728300094604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.483279824256897, + 0.961857259273529, + 0.2634691596031189, + 0.483279824256897, + 0.01639719493687153, + 0.012141581624746323 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 1, + "style_probability": 0.8400004506111145, + "style_target": 1, + "support_probability": 0.9384320974349976, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7772692427787826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02195158042013645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47084876894950867, + 0.9329878687858582, + 0.25490084290504456, + 0.47084876894950867, + 0.017540886998176575, + 0.03809204325079918 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 0, + "style_probability": 0.6419705152511597, + "style_target": 0, + "support_probability": 0.9631157517433167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.962119850031616, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0893123522400856, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:2209:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14986151456832886, + 0.9990764856338501, + 0.4704461395740509, + 0.08553039282560349, + 0.04550103098154068, + 0.026661595329642296, + 0.027152784168720245, + 0.01660076528787613, + 0.03514614701271057 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.8939923048019409, + "style_target": 1, + "support_probability": 0.9907942414283752, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6091817579730036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15908117592334747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2209:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16270773112773895, + 0.9976269602775574, + 0.4398665130138397, + 0.11552471667528152, + 0.06541044265031815, + 0.038871318101882935, + 0.024238666519522667, + 0.019092001020908356, + 0.17113076150417328 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.25152587890625, + "style_target": 0, + "support_probability": 0.9785240888595581, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8279945953207898, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024828223511576653, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9923850297927856, + 0.17698527872562408, + 0.45544853806495667, + 0.4846007823944092, + 0.014482690952718258, + 0.030041780322790146, + 0.02155626192688942, + 0.015535871498286724, + 0.020866891369223595, + 0.040218889713287354, + 0.022073114290833473 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.5794656276702881, + "style_target": 1, + "support_probability": 0.9579646587371826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2798498747836021, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012586619704961777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.997312068939209, + 0.21654203534126282, + 0.46756917238235474, + 0.4160762429237366, + 0.0358278714120388, + 0.03930867463350296, + 0.03949211537837982, + 0.03892822191119194, + 0.05309511348605156, + 0.07888074964284897 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.06696800887584686, + "style_target": 0, + "support_probability": 0.9637176394462585, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8863561180691739, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.020586775615811348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2420887053012848, + 0.9952566027641296, + 0.10680883377790451, + 0.04386905953288078, + 0.018674036487936974 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 1, + "style_probability": 0.8089641332626343, + "style_target": 1, + "support_probability": 0.9284875988960266, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6368518752020638, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.036120589822530746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23378390073776245, + 0.9841578006744385, + 0.1201307624578476, + 0.046181995421648026, + 0.03282848373055458 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 1, + "style_probability": 0.4931613802909851, + "style_target": 0, + "support_probability": 0.8969292640686035, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.89145895466845, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012086860835552216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9809526801109314, + 0.29410332441329956, + 0.7102769017219543, + 0.6839510798454285, + 0.018233435228466988, + 0.023420419543981552 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.7659780383110046, + "style_target": 1, + "support_probability": 0.9533252120018005, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6091595905170768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01801172085106373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9708448052406311, + 0.2584221363067627, + 0.7403171062469482, + 0.6661885976791382, + 0.15013544261455536, + 0.08280114829540253 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.3350374698638916, + "style_target": 0, + "support_probability": 0.9407876133918762, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033536215426328924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9530660510063171, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3615565001964569, + 0.8470714688301086, + 0.39274001121520996, + 0.1990240067243576, + 0.03492482379078865, + 0.0576288104057312, + 0.03176460415124893, + 0.0455317422747612, + 0.05839702859520912 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.81137615442276, + "style_target": 1, + "support_probability": 0.0005946349119767547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008514159015959165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9287900328636169, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2650393545627594, + 0.8014113306999207, + 0.36969131231307983, + 0.1980045586824417, + 0.03561972454190254, + 0.16443519294261932, + 0.07233409583568573, + 0.047857336699962616, + 0.16824084520339966 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.16320191323757172, + "style_target": 0, + "support_probability": 0.001516936463303864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8067532867755915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05712791532278061, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10446427017450333, + 0.3301914632320404, + 0.9875271916389465, + 0.21282225847244263, + 0.0274675115942955, + 0.026765890419483185, + 0.01907878927886486, + 0.042158979922533035, + 0.017931807786226273, + 0.016267171129584312, + 0.026406491175293922 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 1, + "style_probability": 0.7571619153022766, + "style_target": 1, + "support_probability": 0.7995479702949524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7816837753298096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041886620223522186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.119899220764637, + 0.39398443698883057, + 0.9795388579368591, + 0.23231573402881622, + 0.032006680965423584, + 0.013953485526144505, + 0.04847010225057602, + 0.039690051227808, + 0.03080007992684841, + 0.02711370773613453, + 0.022878773510456085 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 0, + "style_probability": 0.7230945825576782, + "style_target": 0, + "support_probability": 0.82289719581604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1451565251264426, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5100445747375488, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9079129099845886, + 0.802635908126831, + 0.2005508989095688, + 0.2022743821144104, + 0.026840481907129288 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.4323432743549347, + "style_target": 1, + "support_probability": 0.0966721624135971, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2939608884420397, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6476137042045593, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9059580564498901, + 0.8154777884483337, + 0.1833132952451706, + 0.20426030457019806, + 0.2279800921678543 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.8713016510009766, + "style_target": 0, + "support_probability": 0.09673749655485153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009197837469745751, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9729294180870056, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5475228428840637, + 0.5139684677124023, + 0.5475228428840637, + 0.1702229529619217, + 0.07127504050731659, + 0.056705910712480545 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 0, + "style_probability": 0.3285185694694519, + "style_target": 1, + "support_probability": 0.026529982686042786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00045425868518249996, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.996464729309082, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41699737310409546, + 0.34859880805015564, + 0.41699737310409546, + 0.12738709151744843, + 0.04989999905228615, + 0.08633968979120255 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 1, + "style_probability": 0.0751332938671112, + "style_target": 0, + "support_probability": 0.0034177221823483706, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4964654892819383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04320577159523964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22090117633342743, + 0.9888267517089844, + 0.0655578225851059, + 0.1109970435500145, + 0.03234412521123886, + 0.040787264704704285, + 0.05530751124024391, + 0.06990159302949905, + 0.12830431759357452 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 1, + "style_probability": 0.5640423893928528, + "style_target": 1, + "support_probability": 0.6258467435836792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7491390671700513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029291383922100067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1731027513742447, + 0.9887097477912903, + 0.06248541548848152, + 0.15301351249217987, + 0.034498460590839386, + 0.04441620036959648, + 0.02708553336560726, + 0.034643422812223434, + 0.06331482529640198 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 0, + "style_probability": 0.703516960144043, + "style_target": 0, + "support_probability": 0.8159503936767578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010392475443728214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.65720134973526, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09847679734230042, + 0.2450024038553238, + 0.9089046120643616, + 0.38504359126091003, + 0.015543954446911812, + 0.02987496368587017, + 0.02555939368903637, + 0.04196974262595177, + 0.03316422551870346, + 0.0647047758102417, + 0.06730873882770538 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.24139507114887238, + "style_target": 1, + "support_probability": 0.005414729006588459, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07761415459814772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5634734630584717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08347571641206741, + 0.17615942656993866, + 0.9368972778320312, + 0.4177861213684082, + 0.041619908064603806, + 0.030227575451135635, + 0.03146478533744812, + 0.038566384464502335, + 0.027619965374469757, + 0.03227733075618744, + 0.08183528482913971 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.9038987755775452, + "style_target": 0, + "support_probability": 0.017327751964330673, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.88974789286737, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.003921798896044493, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0914754867553711, + 0.06207722797989845, + 0.1143602728843689, + 0.9973734617233276, + 0.06793065369129181 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.5289324522018433, + "style_target": 1, + "support_probability": 0.9918972253799438, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6614024629898472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.011182326823472977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09912523627281189, + 0.07574789226055145, + 0.11142364144325256, + 0.9970705509185791, + 0.0417148657143116 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.3034447133541107, + "style_target": 0, + "support_probability": 0.969318687915802, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7178347354301827, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07088752090930939, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4149196445941925, + 0.19806735217571259, + 0.9518424868583679, + 0.2567099928855896, + 0.052738968282938004, + 0.13619747757911682 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 0, + "style_probability": 0.4947645962238312, + "style_target": 1, + "support_probability": 0.9466814994812012, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05199020087006199, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07340505719184875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1997450441122055, + 0.12662526965141296, + 0.9827662706375122, + 0.23103132843971252, + 0.07597322016954422, + 0.11195799708366394 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 1, + "style_probability": 0.02955753728747368, + "style_target": 0, + "support_probability": 0.8494094610214233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013254502073148815, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8775995373725891, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17152008414268494, + 0.13935735821723938, + 0.7466135025024414, + 0.17152008414268494, + 0.07378585636615753, + 0.05618986114859581, + 0.04763931408524513, + 0.07509490847587585, + 0.1547023355960846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 1, + "style_probability": 0.5105432271957397, + "style_target": 1, + "support_probability": 0.008655263110995293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04312809076118057, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8463062644004822, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12309873849153519, + 0.13529767096042633, + 0.7136399149894714, + 0.12309873849153519, + 0.021324114874005318, + 0.017710713669657707, + 0.04840477183461189, + 0.03507184237241745, + 0.05287226662039757 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 0, + "style_probability": 0.8430655598640442, + "style_target": 0, + "support_probability": 0.023427415639162064, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6090673862079348, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12114283442497253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19673892855644226, + 0.08227580040693283, + 0.8926211595535278, + 0.09224265813827515, + 0.025330370292067528, + 0.06850256770849228, + 0.03857912868261337, + 0.04285876825451851, + 0.022010307759046555, + 0.04755927622318268, + 0.10117776691913605 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 0, + "style_probability": 0.38853323459625244, + "style_target": 1, + "support_probability": 0.9220566749572754, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4891429053909841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12753845751285553, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20040850341320038, + 0.0780109316110611, + 0.8743895292282104, + 0.12227299064397812, + 0.017564784735441208, + 0.053731828927993774, + 0.01904783584177494, + 0.019801223650574684, + 0.019972052425146103, + 0.018406108021736145, + 0.06253711134195328 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 1, + "style_probability": 0.20760060846805573, + "style_target": 0, + "support_probability": 0.9170968532562256, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13576316902267196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9016376733779907, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8970420360565186, + 0.6381623148918152, + 0.36112672090530396, + 0.5196526050567627, + 0.1618182510137558 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 1, + "style_probability": 0.7381083369255066, + "style_target": 1, + "support_probability": 0.03711751475930214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2978172746403845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7937343716621399, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8568788170814514, + 0.6560801267623901, + 0.43613895773887634, + 0.4547281265258789, + 0.11036834865808487 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 0, + "style_probability": 0.514985203742981, + "style_target": 0, + "support_probability": 0.29015985131263733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5497508521276118, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054691243916749954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9971197843551636, + 0.3502916991710663, + 0.3250648081302643, + 0.3502916991710663, + 0.03538505360484123, + 0.02408227138221264 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.8302450776100159, + "style_target": 1, + "support_probability": 0.4339335858821869, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43800036699397676, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.032678619027137756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9924899935722351, + 0.31147611141204834, + 0.3247493505477905, + 0.31147611141204834, + 0.026658382266759872, + 0.02364962361752987 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.475753515958786, + "style_target": 0, + "support_probability": 0.703243613243103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9365836946525576, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025039536878466606, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:2355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2649591267108917, + 0.26766884326934814, + 0.2649591267108917, + 0.9947773218154907, + 0.036902058869600296, + 0.06165238469839096, + 0.07776179164648056, + 0.1568443328142166, + 0.09627795964479446 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.8938865661621094, + "style_target": 1, + "support_probability": 0.9822781085968018, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17466628814123591, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028080325573682785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24598009884357452, + 0.23864558339118958, + 0.24598009884357452, + 0.996857762336731, + 0.06814774870872498, + 0.3369062542915344, + 0.06686816364526749, + 0.06961388885974884, + 0.175572469830513 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.04778334125876427, + "style_target": 0, + "support_probability": 0.9610856175422668, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16656699877931536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25301310420036316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5127912759780884, + 0.41443511843681335, + 0.6095470190048218, + 0.910383939743042, + 0.04231252148747444, + 0.03491177782416344, + 0.030767980962991714, + 0.04859258979558945, + 0.0461011566221714, + 0.11377550661563873, + 0.03546790033578873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 1, + "style_probability": 0.832395613193512, + "style_target": 1, + "support_probability": 0.08777612447738647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1677642061882679, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2995949387550354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45046940445899963, + 0.37174510955810547, + 0.5728380680084229, + 0.8984856605529785, + 0.03845333680510521, + 0.027830112725496292, + 0.03132163733243942, + 0.05648417770862579, + 0.057907991111278534, + 0.1773531585931778, + 0.14354446530342102 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 0, + "style_probability": 0.6250424981117249, + "style_target": 0, + "support_probability": 0.10824097692966461, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06367287025362163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5846877694129944, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33669066429138184, + 0.3288910686969757, + 0.33669066429138184, + 0.9569260478019714, + 0.14150017499923706 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 1, + "style_probability": 0.5068987011909485, + "style_target": 1, + "support_probability": 0.047537628561258316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6614492951550187, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08502981066703796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3831899166107178, + 0.3268173038959503, + 0.3831899166107178, + 0.9789586067199707, + 0.016942664980888367 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 0, + "style_probability": 0.7126466631889343, + "style_target": 0, + "support_probability": 0.6652737855911255, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5021772821582022, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06341251730918884, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9622370004653931, + 0.5108799934387207, + 0.10581277310848236, + 0.10995037853717804, + 0.04121079668402672, + 0.03809235244989395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 0, + "style_probability": 0.30980947613716125, + "style_target": 1, + "support_probability": 0.8538163900375366, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31551473764889437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10383099317550659, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9532706141471863, + 0.5498867630958557, + 0.09560714662075043, + 0.11066355556249619, + 0.051276568323373795, + 0.04475868493318558 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 1, + "style_probability": 0.17935100197792053, + "style_target": 0, + "support_probability": 0.7978634834289551, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6424137584809556, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04149051755666733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08035989850759506, + 0.6592904925346375, + 0.9865193963050842, + 0.23122991621494293, + 0.018180830404162407, + 0.018288105726242065, + 0.02492745965719223, + 0.015928052365779877, + 0.014504630118608475 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.6788400411605835, + "style_target": 1, + "support_probability": 0.7538449168205261, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08085701178259928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06498024612665176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07746926695108414, + 0.5661596655845642, + 0.9916090369224548, + 0.16617001593112946, + 0.025696130469441414, + 0.03766012564301491, + 0.040210623294115067, + 0.0474117211997509, + 0.05229909345507622 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.11608175188302994, + "style_target": 0, + "support_probability": 0.5011352300643921, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7710101658588393, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005011639557778835, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25277847051620483, + 0.1028725728392601, + 0.25277847051620483, + 0.9966931343078613, + 0.013151017017662525, + 0.029641248285770416, + 0.01759193278849125, + 0.022130468860268593, + 0.06147385388612747, + 0.018716638907790184, + 0.024521172046661377 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 1, + "style_probability": 0.64191734790802, + "style_target": 1, + "support_probability": 0.9368746876716614, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8211813727896784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004820080474019051, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24386382102966309, + 0.0674842894077301, + 0.24386382102966309, + 0.9989527463912964, + 0.0978606641292572, + 0.01863194815814495, + 0.02893032692372799, + 0.02156026102602482, + 0.01621144823729992, + 0.011724808253347874 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 0, + "style_probability": 0.8506935238838196, + "style_target": 0, + "support_probability": 0.874273419380188, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7542042831212227, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014475077390670776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.709645688533783, + 0.04768458008766174, + 0.9441418647766113, + 0.37457507848739624, + 0.04849429801106453 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 0, + "style_probability": 0.3871294856071472, + "style_target": 1, + "support_probability": 0.9926336407661438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18527987821736294, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028098933398723602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8202678561210632, + 0.0395108126103878, + 0.9561856389045715, + 0.23215024173259735, + 0.02443980798125267 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 1, + "style_probability": 0.04110589250922203, + "style_target": 0, + "support_probability": 0.9834631085395813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13135224740100562, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24124856293201447, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.030534856021404266, + 0.21690772473812103, + 0.16804899275302887, + 0.8762727379798889, + 0.05237079784274101, + 0.051392409950494766 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 0, + "style_probability": 0.3377467393875122, + "style_target": 1, + "support_probability": 0.09841594845056534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0874793262866133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2918398678302765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.02366696111857891, + 0.19038048386573792, + 0.14762242138385773, + 0.8139217495918274, + 0.018397262319922447, + 0.06742560863494873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 1, + "style_probability": 0.4502711296081543, + "style_target": 0, + "support_probability": 0.07342728227376938, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13501415112867546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5330401659011841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9177023768424988, + 0.7167066335678101, + 0.7167066335678101, + 0.18701383471488953, + 0.08383919298648834, + 0.08955773711204529, + 0.03239330276846886, + 0.1253267079591751, + 0.15838460624217987 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.27492621541023254, + "style_target": 1, + "support_probability": 0.31568631529808044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13281079302672266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7502813935279846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8819739818572998, + 0.692668080329895, + 0.692668080329895, + 0.14527620375156403, + 0.025700092315673828, + 0.054550252854824066, + 0.07486077398061752, + 0.04686761647462845, + 0.03373606503009796 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.5884720087051392, + "style_target": 0, + "support_probability": 0.21027112007141113, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6123177814395186, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023544156923890114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09118504077196121, + 0.9871633052825928, + 0.5072383284568787, + 0.3119870722293854, + 0.008307144045829773, + 0.011617235839366913, + 0.0592791847884655, + 0.04374517500400543, + 0.012778169475495815, + 0.025883592665195465, + 0.12241511046886444 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 0, + "style_probability": 0.281490296125412, + "style_target": 1, + "support_probability": 0.9168816804885864, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17563660781461948, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05707943066954613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.03901982679963112, + 0.989492654800415, + 0.27750280499458313, + 0.2518533766269684, + 0.031278856098651886, + 0.033635079860687256, + 0.03970032185316086, + 0.03491348773241043, + 0.012863118201494217, + 0.037976425141096115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 1, + "style_probability": 0.10326638072729111, + "style_target": 0, + "support_probability": 0.8792558312416077, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28266394351134877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09599219262599945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.05905521288514137, + 0.9736843109130859, + 0.400448203086853, + 0.05905521288514137, + 0.03555392846465111 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 1, + "style_probability": 0.8531291484832764, + "style_target": 1, + "support_probability": 0.04147280007600784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.235924368172069, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11717694252729416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08622510731220245, + 0.9572033882141113, + 0.5066250562667847, + 0.08622510731220245, + 0.02001633122563362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 0, + "style_probability": 0.6983778476715088, + "style_target": 0, + "support_probability": 0.08207312226295471, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7018353616382864, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06534647196531296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7924627661705017, + 0.13596390187740326, + 0.7662622928619385, + 0.2216925323009491, + 0.016773076727986336, + 0.010555851273238659 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.8378345966339111, + "style_target": 1, + "support_probability": 0.9687405228614807, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5053698377887821, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1292252540588379, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7512096762657166, + 0.13835076987743378, + 0.78803950548172, + 0.2790820598602295, + 0.00661237258464098, + 0.010573974810540676 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.26804640889167786, + "style_target": 0, + "support_probability": 0.942977786064148, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006809006949258474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.996154248714447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25474169850349426, + 0.6068151593208313, + 0.6160925030708313, + 0.25474169850349426, + 0.11431188136339188, + 0.029818812385201454, + 0.045619647949934006, + 0.04195953533053398, + 0.0640328899025917 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.8275153040885925, + "style_target": 1, + "support_probability": 0.002001516055315733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001721533890228994, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9855042099952698, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17714981734752655, + 0.5848222374916077, + 0.4601277709007263, + 0.17714981734752655, + 0.08353027701377869, + 0.21939131617546082, + 0.0443652905523777, + 0.07857372611761093, + 0.10908126085996628 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.08725272119045258, + "style_target": 0, + "support_probability": 0.007960568182170391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7869441676406866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14960159361362457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2912210524082184, + 0.22296331822872162, + 0.20437030494213104, + 0.9978875517845154, + 0.02756783366203308, + 0.021835898980498314, + 0.022095290943980217, + 0.03675438091158867, + 0.015456483699381351, + 0.03742313012480736, + 0.02077103778719902 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 1, + "style_probability": 0.8865759372711182, + "style_target": 1, + "support_probability": 0.6462486982345581, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5886907720161955, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10857140272855759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23768799006938934, + 0.20422084629535675, + 0.26713964343070984, + 0.9962469935417175, + 0.055671848356723785, + 0.02811238169670105, + 0.021230822429060936, + 0.05920196697115898, + 0.03680099919438362, + 0.06318148225545883, + 0.041366543620824814 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 0, + "style_probability": 0.6601776480674744, + "style_target": 0, + "support_probability": 0.7073593139648438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3959392537262362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6087756156921387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15162114799022675, + 0.9408443570137024, + 0.07556763291358948, + 0.48405134677886963, + 0.020726503804326057 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.9210287928581238, + "style_target": 1, + "support_probability": 0.09465622156858444, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1907725315067129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21206459403038025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14985300600528717, + 0.9577535390853882, + 0.07684658467769623, + 0.6193504929542542, + 0.01862826943397522 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.37928059697151184, + "style_target": 0, + "support_probability": 0.15845516324043274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009230995843876619, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9925475716590881, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21195317804813385, + 0.5964723229408264, + 0.32519805431365967, + 0.2886878252029419, + 0.045661333948373795, + 0.02707068994641304 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.5184881091117859, + "style_target": 1, + "support_probability": 0.0006099147140048444, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003131397647017947, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9924284815788269, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15872670710086823, + 0.6853070259094238, + 0.2730197310447693, + 0.32628047466278076, + 0.13578453660011292, + 0.2350672483444214 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.10527964681386948, + "style_target": 0, + "support_probability": 0.0009577432065270841, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024245838490902286, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8232913613319397, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20436853170394897, + 0.20436853170394897, + 0.4565218389034271, + 0.1118699163198471, + 0.04508618265390396, + 0.05517582967877388, + 0.018451618030667305, + 0.06320660561323166, + 0.026975112035870552 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.8443756103515625, + "style_target": 1, + "support_probability": 0.020454270765185356, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003857433027089694, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9573615789413452, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1514856368303299, + 0.1514856368303299, + 0.40201547741889954, + 0.1304219514131546, + 0.08157502114772797, + 0.050179820507764816, + 0.13130207359790802, + 0.02438431978225708, + 0.11831076443195343 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.4113221764564514, + "style_target": 0, + "support_probability": 0.006631927099078894, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8560713868640584, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.051886703819036484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3664485514163971, + 0.08459686487913132, + 0.3664485514163971, + 0.9937981963157654, + 0.014210805296897888, + 0.013522197492420673, + 0.02444135583937168, + 0.016674254089593887, + 0.010440143756568432, + 0.04885844513773918, + 0.04337868094444275 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 1, + "style_probability": 0.7460450530052185, + "style_target": 1, + "support_probability": 0.9065486192703247, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4910277693975839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057808104902505875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2881815731525421, + 0.09101098775863647, + 0.2881815731525421, + 0.9868068099021912, + 0.19301307201385498, + 0.03094884194433689, + 0.057739030569791794, + 0.05113877356052399, + 0.09540805220603943, + 0.025786222890019417 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 1, + "style_probability": 0.43434733152389526, + "style_target": 0, + "support_probability": 0.6721923351287842, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01622724837718433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20888446271419525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19988860189914703, + 0.8115118741989136, + 0.3195573389530182, + 0.20196904242038727, + 0.03778880089521408 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 0, + "style_probability": 0.34553733468055725, + "style_target": 1, + "support_probability": 0.007382506504654884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018103802690995093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16501402854919434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21455395221710205, + 0.8477963209152222, + 0.35376086831092834, + 0.2316879779100418, + 0.10552993416786194 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 1, + "style_probability": 0.269371896982193, + "style_target": 0, + "support_probability": 0.007422953378409147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017580808891924652, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.41534334421157837, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41625404357910156, + 0.386287122964859, + 0.6122788786888123, + 0.12876205146312714, + 0.03531087934970856, + 0.030023984611034393 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 0, + "style_probability": 0.4822051227092743, + "style_target": 1, + "support_probability": 0.016682608053088188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010294244650431673, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4437362253665924, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25733113288879395, + 0.33021748065948486, + 0.7517767548561096, + 0.10523758083581924, + 0.03260832279920578, + 0.0493127703666687 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 1, + "style_probability": 0.22886112332344055, + "style_target": 0, + "support_probability": 0.05482019484043121, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008658342190694819, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.622218906879425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3311951458454132, + 0.21578910946846008, + 0.23036730289459229, + 0.4008089005947113, + 0.10301202535629272, + 0.05331860110163689, + 0.10550756007432938, + 0.047906748950481415, + 0.05720214918255806 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 1, + "style_probability": 0.5027939081192017, + "style_target": 1, + "support_probability": 0.0028053056448698044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012037798761073363, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.735460102558136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2952190637588501, + 0.24974603950977325, + 0.3090764880180359, + 0.4892313778400421, + 0.06277883052825928, + 0.047830987721681595, + 0.05351369455456734, + 0.08264702558517456, + 0.12308255583047867 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 0, + "style_probability": 0.7708820104598999, + "style_target": 0, + "support_probability": 0.004699364770203829, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14460726392204165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5801955461502075, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8744361996650696, + 0.10615137219429016, + 0.6401340961456299, + 0.6294671297073364, + 0.010098501108586788, + 0.01479407213628292, + 0.022640815004706383, + 0.017676295712590218, + 0.014592448249459267, + 0.020936546847224236 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 1, + "style_probability": 0.7169018983840942, + "style_target": 1, + "support_probability": 0.052735600620508194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053336253469318404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7532392740249634, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.888460636138916, + 0.10019751638174057, + 0.610004723072052, + 0.6649308204650879, + 0.020434150472283363, + 0.06130805239081383, + 0.09079746901988983, + 0.057211123406887054, + 0.026130130514502525, + 0.027990443632006645, + 0.09158249944448471 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 0, + "style_probability": 0.513526976108551, + "style_target": 0, + "support_probability": 0.013786699622869492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5533532460677151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4099021553993225, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5159593820571899, + 0.3830290138721466, + 0.3455931842327118, + 0.8642347455024719, + 0.05126558989286423 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 1, + "style_probability": 0.9562548398971558, + "style_target": 1, + "support_probability": 0.3407292366027832, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38377583368451684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.41941195726394653, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4543647766113281, + 0.3262440264225006, + 0.2600744962692261, + 0.8612865209579468, + 0.2143011838197708 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 0, + "style_probability": 0.5689162611961365, + "style_target": 0, + "support_probability": 0.4302983582019806, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8264996968559432, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1308017522096634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3106932044029236, + 0.3106932044029236, + 0.947536289691925, + 0.21465083956718445, + 0.02238312177360058, + 0.022138938307762146 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 1, + "style_probability": 0.8998726010322571, + "style_target": 1, + "support_probability": 0.7743117213249207, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5310754650859469, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10202452540397644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30629435181617737, + 0.30629435181617737, + 0.9296745657920837, + 0.19150424003601074, + 0.04449700191617012, + 0.04652930423617363 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 1, + "style_probability": 0.4906105399131775, + "style_target": 0, + "support_probability": 0.7384685277938843, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6692718381370122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3278249502182007, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2061423659324646, + 0.2686731219291687, + 0.05910279229283333, + 0.8594286441802979, + 0.02051934227347374, + 0.03384900093078613, + 0.009245338849723339, + 0.012166880071163177, + 0.04547852650284767 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.8235391974449158, + "style_target": 1, + "support_probability": 0.7757413387298584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06239555485202587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4313770532608032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25067421793937683, + 0.18462614715099335, + 0.0441800095140934, + 0.8715031147003174, + 0.054834961891174316, + 0.1217222660779953, + 0.031490255147218704, + 0.014327841810882092, + 0.04749536141753197 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.10132236778736115, + "style_target": 0, + "support_probability": 0.6053338050842285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47800605183416484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7031661868095398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9610340595245361, + 0.11756087839603424, + 0.5620219707489014, + 0.2763422727584839, + 0.031302034854888916, + 0.04368656873703003, + 0.02397787570953369, + 0.07365113496780396, + 0.033946841955184937, + 0.04756874963641167, + 0.04975110664963722 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 1, + "style_probability": 0.8048800230026245, + "style_target": 1, + "support_probability": 0.28173020482063293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4595565557469808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.459314227104187, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9493173956871033, + 0.13192977011203766, + 0.5056383609771729, + 0.21130961179733276, + 0.036154016852378845, + 0.03573964536190033, + 0.1600639820098877, + 0.044948115944862366, + 0.036078013479709625, + 0.1393628567457199 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 1, + "style_probability": 0.3934819996356964, + "style_target": 0, + "support_probability": 0.5097951292991638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6428959500366401, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23141033947467804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6585860252380371, + 0.3308740258216858, + 0.7172029614448547, + 0.5946072340011597, + 0.032762885093688965 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.9419037699699402, + "style_target": 1, + "support_probability": 0.9693395495414734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.155545686191842, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7112098336219788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5702968239784241, + 0.17891612648963928, + 0.6327195763587952, + 0.5208707451820374, + 0.09605163335800171 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.2962174415588379, + "style_target": 0, + "support_probability": 0.3483813405036926, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3280030322841614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5581398606300354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2285909652709961, + 0.1385922133922577, + 0.4083831310272217, + 0.9482437372207642, + 0.04493242874741554, + 0.09325694292783737 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.8323684334754944, + "style_target": 1, + "support_probability": 0.19299767911434174, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0159952053918892, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7226276397705078, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18880701065063477, + 0.10469873249530792, + 0.36234548687934875, + 0.94876629114151, + 0.05424680560827255, + 0.12487439066171646 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.1858956515789032, + "style_target": 0, + "support_probability": 0.07698607444763184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.054440394263307094, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8075901865959167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30886951088905334, + 0.36227717995643616, + 0.30886951088905334, + 0.5209747552871704, + 0.047271959483623505, + 0.026511048898100853, + 0.05099315568804741, + 0.0512368269264698, + 0.05968066304922104 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 1, + "style_probability": 0.9046584963798523, + "style_target": 1, + "support_probability": 0.031343840062618256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050476403550651465, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8097051978111267, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34957537055015564, + 0.44074732065200806, + 0.34957537055015564, + 0.7064635753631592, + 0.025440789759159088, + 0.034549400210380554, + 0.008688556030392647, + 0.016128050163388252, + 0.03148166090250015 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 0, + "style_probability": 0.6092600226402283, + "style_target": 0, + "support_probability": 0.04849304258823395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07835766157937828, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7517759203910828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15958864986896515, + 0.4115048944950104, + 0.38610324263572693, + 0.6134541630744934, + 0.03149190917611122, + 0.02707805670797825, + 0.01924343965947628, + 0.01791595108807087, + 0.016655275598168373, + 0.01725693978369236, + 0.09521394222974777 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.9300708174705505, + "style_target": 1, + "support_probability": 0.044573381543159485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007162763242485124, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8825225234031677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13890130817890167, + 0.3399001657962799, + 0.41562941670417786, + 0.5463458299636841, + 0.13315053284168243, + 0.012782092206180096, + 0.04148368164896965, + 0.027898790314793587, + 0.018917204812169075, + 0.045192696154117584, + 0.06907957792282104 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.21719685196876526, + "style_target": 0, + "support_probability": 0.009454340673983097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023776996821677297, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9405916929244995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2909289598464966, + 0.4347083866596222, + 0.22611699998378754, + 0.3415772616863251, + 0.28917211294174194 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.8325594067573547, + "style_target": 1, + "support_probability": 0.010514060035347939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001145659418948021, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.980882465839386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27270060777664185, + 0.43823644518852234, + 0.17773564159870148, + 0.2735970914363861, + 0.17991164326667786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.22936414182186127, + "style_target": 0, + "support_probability": 0.007342235185205936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05851625766778868, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15950514376163483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15758654475212097, + 0.1500149667263031, + 0.48702022433280945, + 0.3477364182472229, + 0.02740336023271084, + 0.024584034457802773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.8048743605613708, + "style_target": 1, + "support_probability": 0.037468161433935165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007096124569046615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4656141996383667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15034742653369904, + 0.16675248742103577, + 0.38048848509788513, + 0.250742107629776, + 0.03681860864162445, + 0.046671878546476364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.367387592792511, + "style_target": 0, + "support_probability": 0.010056578554213047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48368092806601665, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5357089042663574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7211334109306335, + 0.0653192475438118, + 0.42123910784721375, + 0.7211334109306335, + 0.05040941387414932, + 0.06271152943372726, + 0.04223556071519852, + 0.039411820471286774, + 0.04177745059132576 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.9243584871292114, + "style_target": 1, + "support_probability": 0.35652461647987366, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20646477080557493, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4946872591972351, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7692771553993225, + 0.08346332609653473, + 0.39004555344581604, + 0.7692771553993225, + 0.008344939909875393, + 0.011642827652394772, + 0.09406581521034241, + 0.03303432837128639, + 0.04466058313846588 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.44211286306381226, + "style_target": 0, + "support_probability": 0.35365843772888184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003078854403022238, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7296635508537292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3145509958267212, + 0.5034275650978088, + 0.1423543244600296, + 0.04569113999605179, + 0.05613637715578079, + 0.018794039264321327, + 0.02058425359427929, + 0.06453867256641388, + 0.04833313077688217, + 0.01377001591026783, + 0.02267458848655224 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.6755165457725525, + "style_target": 1, + "support_probability": 0.0005646792706102133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001108476359496547, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.828479528427124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2931089401245117, + 0.47353729605674744, + 0.18465182185173035, + 0.041609182953834534, + 0.04432687163352966, + 0.040294844657182693, + 0.1443386971950531, + 0.02365158498287201, + 0.04794369265437126, + 0.0295633003115654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.28296664357185364, + "style_target": 0, + "support_probability": 0.0006578774773515761, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7424268902634423, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3686758875846863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27201950550079346, + 0.45290565490722656, + 0.34095120429992676, + 0.9752101898193359, + 0.0399370938539505 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 1, + "style_probability": 0.8326082229614258, + "style_target": 1, + "support_probability": 0.6157850623130798, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.484579714992167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.34227073192596436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2532931864261627, + 0.4158664643764496, + 0.3075844645500183, + 0.9636645317077637, + 0.11213190108537674 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 0, + "style_probability": 0.5233169794082642, + "style_target": 0, + "support_probability": 0.5520510077476501, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.872560422886696, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012205453589558601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.584165632724762, + 0.9792649745941162, + 0.4002143442630768, + 0.7647978067398071, + 0.015622136183083057, + 0.007585859391838312 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6256800889968872, + "style_target": 1, + "support_probability": 0.9717844128608704, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43734695637866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01636497862637043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4965708553791046, + 0.9724038243293762, + 0.35775819420814514, + 0.7180543541908264, + 0.01729412190616131, + 0.024634653702378273 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.164255753159523, + "style_target": 0, + "support_probability": 0.902535080909729, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28765837595164195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13642646372318268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8215512037277222, + 0.10939786583185196, + 0.15734045207500458, + 0.08119264245033264, + 0.007293015718460083, + 0.019091282039880753, + 0.08288872987031937, + 0.03359740972518921, + 0.013682707212865353 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.6501216292381287, + "style_target": 1, + "support_probability": 0.4445113241672516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020049516408189003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16618108749389648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.824030339717865, + 0.13886798918247223, + 0.12077005952596664, + 0.09957610070705414, + 0.02754177525639534, + 0.08024968951940536, + 0.03427347168326378, + 0.013062164187431335, + 0.0195473525673151 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.05680914968252182, + "style_target": 0, + "support_probability": 0.5569019317626953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7352715219057246, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07334007322788239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7347344756126404, + 0.15224911272525787, + 0.55419921875, + 0.8774356842041016, + 0.02560262195765972, + 0.016850845888257027, + 0.013760913163423538, + 0.06213051825761795, + 0.035696741193532944, + 0.05463024601340294, + 0.021722419187426567 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.7206441760063171, + "style_target": 1, + "support_probability": 0.919400155544281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6246081539318464, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14269839227199554, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7177903652191162, + 0.18305771052837372, + 0.4731505811214447, + 0.8332393765449524, + 0.025836791843175888, + 0.021943842992186546, + 0.015452733263373375, + 0.028986118733882904, + 0.009158975444734097, + 0.017662011086940765, + 0.014668761752545834 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.4497034549713135, + "style_target": 0, + "support_probability": 0.931476891040802, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04551578114445733, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8644748330116272, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17267753183841705, + 0.8968806266784668, + 0.2530979514122009, + 0.34356382489204407, + 0.043607596307992935 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.7324684858322144, + "style_target": 1, + "support_probability": 0.020786793902516365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005526304383845268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9051149487495422, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1804170161485672, + 0.8783056735992432, + 0.24133820831775665, + 0.32692521810531616, + 0.0892714112997055 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.34848910570144653, + "style_target": 0, + "support_probability": 0.002464371034875512, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.709230119420325, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038018591701984406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27037313580513, + 0.9963610768318176, + 0.11285983771085739, + 0.05201372876763344, + 0.016930246725678444, + 0.02630344219505787 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.6694608330726624, + "style_target": 1, + "support_probability": 0.7354512214660645, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0646926311138541, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03059091418981552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2995203733444214, + 0.9979825019836426, + 0.12409121543169022, + 0.0797690600156784, + 0.013840289786458015, + 0.03347766026854515 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.06252545118331909, + "style_target": 0, + "support_probability": 0.6371558308601379, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05167345831072866, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8351930379867554, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13643018901348114, + 0.7639292478561401, + 0.26214101910591125, + 0.4511365592479706, + 0.1292782872915268, + 0.0329437330365181, + 0.05245059356093407, + 0.018300967290997505, + 0.07859291881322861 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 1, + "style_probability": 0.7619603872299194, + "style_target": 1, + "support_probability": 0.05132393538951874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1033373775179084, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6179360747337341, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08411587029695511, + 0.7814520597457886, + 0.2557016909122467, + 0.501571536064148, + 0.015080303885042667, + 0.1033637747168541, + 0.025879979133605957, + 0.1916535347700119, + 0.04731862619519234 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 0, + "style_probability": 0.5132147073745728, + "style_target": 0, + "support_probability": 0.11230763792991638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32157378514030105, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48613131046295166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.572538435459137, + 0.23243935406208038, + 0.26657551527023315, + 0.7032830715179443, + 0.02673991210758686, + 0.008452937006950378, + 0.12177377939224243, + 0.0370558463037014, + 0.013822248205542564, + 0.011991092003881931, + 0.03615006059408188 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.7632996439933777, + "style_target": 1, + "support_probability": 0.4063856601715088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05467541494989313, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8605988621711731, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:negative", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.5874653458595276, + 0.3045441508293152, + 0.32675033807754517, + 0.7019538879394531, + 0.055494341999292374, + 0.026478338986635208, + 0.0337442010641098, + 0.011890402995049953, + 0.026623107492923737, + 0.060095205903053284, + 0.030866166576743126, + 0.05443323776125908, + 0.03245043382048607, + 0.013515220023691654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.30536648631095886, + "style_target": 0, + "support_probability": 0.0573674812912941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11558822194140718, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.699367105960846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8076967000961304, + 0.3948841691017151, + 0.5781652331352234, + 0.10937108099460602, + 0.1049472764134407 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.8388136625289917, + "style_target": 1, + "support_probability": 0.12322188913822174, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016560116293773224, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7901569604873657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8705441951751709, + 0.38749220967292786, + 0.6104007959365845, + 0.14111679792404175, + 0.1296551525592804 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.10005082190036774, + "style_target": 0, + "support_probability": 0.23427554965019226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14073741348667035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2212783694267273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24542471766471863, + 0.5987434387207031, + 0.20490388572216034, + 0.9983627200126648, + 0.03344366326928139, + 0.036291610449552536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 1, + "style_probability": 0.8455634713172913, + "style_target": 1, + "support_probability": 0.06242863088846207, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16775496111735888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08633295446634293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2779904007911682, + 0.6346531510353088, + 0.2597924768924713, + 0.9970813393592834, + 0.07461690902709961, + 0.022199630737304688 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 0, + "style_probability": 0.6643855571746826, + "style_target": 0, + "support_probability": 0.14708231389522552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39790302351527695, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3361069858074188, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15306897461414337, + 0.8957930207252502, + 0.21239526569843292, + 0.8463579416275024, + 0.021363090723752975, + 0.07397828996181488, + 0.020123668015003204, + 0.020864007994532585, + 0.034131523221731186 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 1, + "style_probability": 0.7050203680992126, + "style_target": 1, + "support_probability": 0.3778369724750519, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2601326078037926, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3177129924297333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1643582284450531, + 0.8928982019424438, + 0.20266424119472504, + 0.8207258582115173, + 0.021839391440153122, + 0.0756576731801033 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 1, + "style_probability": 0.250084787607193, + "style_target": 0, + "support_probability": 0.4879004955291748, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026854838932163318, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9729272723197937, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3626255691051483, + 0.0878089964389801, + 0.607554018497467, + 0.09090039879083633, + 0.032592181116342545, + 0.023888345807790756, + 0.022375840693712234, + 0.019729584455490112, + 0.020168105140328407, + 0.034094180911779404, + 0.08701477944850922 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.8792552947998047, + "style_target": 1, + "support_probability": 0.008214376866817474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0010778191532712553, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9940827488899231, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26401790976524353, + 0.10283171385526657, + 0.3308258056640625, + 0.08242737501859665, + 0.13607920706272125, + 0.03244830667972565, + 0.12463562935590744, + 0.05789109691977501, + 0.03408431634306908, + 0.13243936002254486 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.14421266317367554, + "style_target": 0, + "support_probability": 0.005778229795396328, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8273434330796126, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0822753980755806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9430862665176392, + 0.7961764335632324, + 0.42244547605514526, + 0.08917221426963806, + 0.010305236093699932 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.8235977292060852, + "style_target": 1, + "support_probability": 0.9166834950447083, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6592567010242902, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04093042388558388, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9266447424888611, + 0.7386530637741089, + 0.4827600121498108, + 0.09992987662553787, + 0.08869753032922745 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.3594892621040344, + "style_target": 0, + "support_probability": 0.9518349170684814, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7284884859168164, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10507146269083023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1667935848236084, + 0.07617885619401932, + 0.9972485899925232, + 0.20191608369350433, + 0.046786513179540634, + 0.014158062636852264 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.521591067314148, + "style_target": 1, + "support_probability": 0.8694192171096802, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6273410151675557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.035630397498607635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16367030143737793, + 0.09101815521717072, + 0.9961771965026855, + 0.22216632962226868, + 0.03683171048760414 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.307860404253006, + "style_target": 0, + "support_probability": 0.9320313930511475, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2943274218996774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6062191128730774, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1262790560722351, + 0.28485533595085144, + 0.6023210883140564, + 0.5087813138961792, + 0.03238179162144661, + 0.015054319985210896, + 0.024312566965818405, + 0.03339875116944313, + 0.01631460338830948 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.8015123009681702, + "style_target": 1, + "support_probability": 0.2951418459415436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004566277083448449, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5829437375068665, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21870771050453186, + 0.3647553622722626, + 0.571130633354187, + 0.3791781961917877, + 0.04103437066078186, + 0.012940921820700169, + 0.038237035274505615, + 0.044335488229990005, + 0.052458006888628006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.022982824593782425, + "style_target": 0, + "support_probability": 0.30145347118377686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009309856113013886, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9848453998565674, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6307618618011475, + 0.6402066349983215, + 0.3329410254955292, + 0.540549635887146, + 0.03255514055490494, + 0.03298868611454964, + 0.10163827240467072, + 0.032304901629686356, + 0.028509989380836487, + 0.043407801538705826 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.728253960609436, + "style_target": 1, + "support_probability": 0.008034700527787209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019875646340442952, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9615036845207214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7009770274162292, + 0.6407604813575745, + 0.3137950003147125, + 0.5669935941696167, + 0.019607113674283028, + 0.023009413853287697, + 0.021925974637269974, + 0.03878306224942207, + 0.03206591680645943, + 0.041626106947660446, + 0.03640887141227722 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.4473065435886383, + "style_target": 0, + "support_probability": 0.025296716019511223, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.283519866699317, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13331112265586853, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32322725653648376, + 0.7811103463172913, + 0.1518937200307846, + 0.9731668829917908, + 0.014936144463717937 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 1, + "style_probability": 0.7210656404495239, + "style_target": 1, + "support_probability": 0.12544797360897064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26140415523083094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05777302384376526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3587441146373749, + 0.7303310036659241, + 0.16659250855445862, + 0.9614837169647217, + 0.018394680693745613, + 0.012768909335136414, + 0.06205121800303459, + 0.03356795012950897, + 0.023141685873270035, + 0.01835884526371956, + 0.02029622159898281 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 1, + "style_probability": 0.4564723074436188, + "style_target": 0, + "support_probability": 0.24441778659820557, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7410176842025317, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07951367646455765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9510301351547241, + 0.27046921849250793, + 0.23329298198223114, + 0.32786789536476135, + 0.008143131621181965, + 0.008928696624934673 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.7949230670928955, + "style_target": 1, + "support_probability": 0.744154691696167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1644288887707133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0963018611073494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9757513999938965, + 0.24109488725662231, + 0.15897119045257568, + 0.3065239191055298, + 0.017047762870788574, + 0.05646176636219025 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.0865948349237442, + "style_target": 0, + "support_probability": 0.7018601298332214, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3342444414035166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19286207854747772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1739719808101654, + 0.34675684571266174, + 0.2258576601743698, + 0.879189133644104, + 0.016436215490102768, + 0.17514461278915405, + 0.05252458527684212, + 0.026832250878214836, + 0.028741199523210526 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.8830162882804871, + "style_target": 1, + "support_probability": 0.15333788096904755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13746819086029413, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28022629022598267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.152646005153656, + 0.3509869873523712, + 0.18680647015571594, + 0.8520493507385254, + 0.012443517334759235, + 0.03145230934023857, + 0.03181762620806694, + 0.04254528507590294, + 0.015943193808197975 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.3874382972717285, + "style_target": 0, + "support_probability": 0.27580776810646057, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8618229819630717, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008434277959167957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11640910804271698, + 0.19198445975780487, + 0.2846323847770691, + 0.9483177065849304, + 0.010730033740401268, + 0.011182960122823715, + 0.011135105043649673, + 0.030735528096556664, + 0.02440190501511097, + 0.007869570516049862, + 0.009769504889845848 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.8590517640113831, + "style_target": 1, + "support_probability": 0.9939919114112854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.684924137746975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015677940100431442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11033861339092255, + 0.20091785490512848, + 0.2615382969379425, + 0.9395537376403809, + 0.033858250826597214, + 0.01655104197561741, + 0.04084411635994911, + 0.04303400218486786, + 0.016010446473956108 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.29180431365966797, + "style_target": 0, + "support_probability": 0.9829844236373901, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009987044819135558, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8215928673744202, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3337860405445099, + 0.34503641724586487, + 0.15309922397136688, + 0.5746403932571411, + 0.06129029765725136 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.7833964824676514, + "style_target": 1, + "support_probability": 0.01059894822537899, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0055606695844835595, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8437540531158447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26877519488334656, + 0.3848890960216522, + 0.13474787771701813, + 0.6182278394699097, + 0.05254868417978287 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.14237599074840546, + "style_target": 0, + "support_probability": 0.07653611153364182, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8379291800050774, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05274069681763649, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9558486342430115, + 0.6597257256507874, + 0.15395978093147278, + 0.3452606797218323, + 0.04917824640870094, + 0.031858496367931366 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 1, + "style_probability": 0.9536755084991455, + "style_target": 1, + "support_probability": 0.7658339142799377, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7092147043076507, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01933969184756279, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9706737399101257, + 0.764367401599884, + 0.11838126182556152, + 0.3144770860671997, + 0.044807448983192444, + 0.1394207924604416 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 0, + "style_probability": 0.5200891494750977, + "style_target": 0, + "support_probability": 0.9360848069190979, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8046760550674178, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06380020081996918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9913027286529541, + 0.15298384428024292, + 0.0974430963397026, + 0.23545512557029724, + 0.018157368525862694, + 0.028341690078377724, + 0.0439935028553009, + 0.08443845808506012, + 0.016735810786485672 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.8450194001197815, + "style_target": 1, + "support_probability": 0.6340295672416687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.575411991951718, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054919321089982986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9892628788948059, + 0.19256000220775604, + 0.11193395406007767, + 0.23650310933589935, + 0.03985743224620819, + 0.00896492600440979 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.4071013629436493, + "style_target": 0, + "support_probability": 0.7804596424102783, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8623234083458715, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015553881414234638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16333125531673431, + 0.3125417232513428, + 0.95450359582901, + 0.3323940336704254, + 0.04101138189435005, + 0.01652967371046543, + 0.011468878015875816, + 0.0183881726115942, + 0.02752334624528885, + 0.004979677498340607 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.6666048765182495, + "style_target": 1, + "support_probability": 0.986517071723938, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09259925962361533, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.042035624384880066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11376958340406418, + 0.3353220820426941, + 0.958045482635498, + 0.4157360792160034, + 0.010956517420709133, + 0.030052023008465767, + 0.014012946747243404, + 0.012163637205958366, + 0.012696554884314537, + 0.023142075166106224 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.0261186845600605, + "style_target": 0, + "support_probability": 0.9456450343132019, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4388034294177555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.065599225461483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3199344575405121, + 0.08270464837551117, + 0.18654783070087433, + 0.9704356789588928, + 0.041551023721694946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 1, + "style_probability": 0.7995984554290771, + "style_target": 1, + "support_probability": 0.3944467306137085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2774535441945176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10346852242946625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4749208092689514, + 0.07881072163581848, + 0.21916894614696503, + 0.9519615173339844, + 0.03421082720160484 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 0, + "style_probability": 0.5565552115440369, + "style_target": 0, + "support_probability": 0.2702857553958893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.892197478964409, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1149524450302124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6669611930847168, + 0.9817773699760437, + 0.31582769751548767, + 0.10366461426019669, + 0.01476297527551651, + 0.019330283626914024 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.8522038459777832, + "style_target": 1, + "support_probability": 0.9456981420516968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49151847780433167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1232888251543045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.605508029460907, + 0.9742165803909302, + 0.2781001031398773, + 0.0971146821975708, + 0.032172877341508865 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.3527526259422302, + "style_target": 0, + "support_probability": 0.8397052884101868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.046422392402496286, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9318751692771912, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7268995046615601, + 0.3498215079307556, + 0.38586899638175964, + 0.18658217787742615, + 0.037194132804870605, + 0.02423831820487976, + 0.016584621742367744, + 0.03838548809289932, + 0.02464446797966957 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.7935949563980103, + "style_target": 1, + "support_probability": 0.021999528631567955, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013514890360988647, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.809259831905365, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6812596917152405, + 0.3037275969982147, + 0.40971627831459045, + 0.09215091168880463, + 0.08925431221723557, + 0.2625482380390167, + 0.03688417375087738, + 0.042826179414987564, + 0.09924687445163727 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.14290198683738708, + "style_target": 0, + "support_probability": 0.07747628539800644, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8270759039327835, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.040206242352724075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17300564050674438, + 0.9781286120414734, + 0.3410755693912506, + 0.8178243041038513, + 0.019574692472815514, + 0.0991726815700531, + 0.01363406516611576, + 0.008578105829656124, + 0.012501479126513004, + 0.02353745885193348, + 0.009758532978594303 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 1, + "style_probability": 0.7030928730964661, + "style_target": 1, + "support_probability": 0.9198861122131348, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7288077746870958, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08529757708311081, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23165516555309296, + 0.9649826884269714, + 0.3724203407764435, + 0.8039246201515198, + 0.020881667733192444, + 0.021693743765354156, + 0.02872454561293125, + 0.053606826812028885, + 0.11377161741256714, + 0.0328424908220768, + 0.03218482434749603 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 0, + "style_probability": 0.5712037682533264, + "style_target": 0, + "support_probability": 0.8797961473464966, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.003641885004689361, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9706859588623047, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6993510127067566, + 0.40351325273513794, + 0.24675396084785461, + 0.09624558687210083, + 0.048900019377470016 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 1, + "style_probability": 0.5164510011672974, + "style_target": 1, + "support_probability": 0.0010170944733545184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04150196276336917, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9356547594070435, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.672413170337677, + 0.38774004578590393, + 0.25127726793289185, + 0.14286838471889496, + 0.053256262093782425 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 0, + "style_probability": 0.9315705299377441, + "style_target": 0, + "support_probability": 0.006893723737448454, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.504224080870699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10968530923128128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8386386632919312, + 0.5477203130722046, + 0.6607246994972229, + 0.18264229595661163, + 0.05981732904911041, + 0.03976679965853691 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 0, + "style_probability": 0.40756839513778687, + "style_target": 1, + "support_probability": 0.8020315766334534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4017220582567654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1682407110929489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.810065507888794, + 0.5366660952568054, + 0.709153950214386, + 0.2294982671737671, + 0.0397394523024559 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 1, + "style_probability": 0.3608274757862091, + "style_target": 0, + "support_probability": 0.6981273293495178, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1680705265969631, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5713971853256226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25939685106277466, + 0.3436976671218872, + 0.888457715511322, + 0.13458436727523804, + 0.08206291496753693, + 0.03228772059082985, + 0.03333733230829239, + 0.1176125630736351, + 0.0524091012775898 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 0, + "style_probability": 0.4229736626148224, + "style_target": 1, + "support_probability": 0.33059149980545044, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014801467594706387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41399726271629333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1256575733423233, + 0.21933341026306152, + 0.9437387585639954, + 0.10618752241134644, + 0.050590693950653076, + 0.08691779524087906, + 0.06659132987260818, + 0.05243191868066788, + 0.05400915816426277 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 1, + "style_probability": 0.06700508296489716, + "style_target": 0, + "support_probability": 0.3643589913845062, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.017641011197572817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9420666694641113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3040032684803009, + 0.1919413059949875, + 0.24549053609371185, + 0.9202097058296204, + 0.08381692320108414, + 0.1713990420103073, + 0.19458536803722382, + 0.17886915802955627, + 0.13733017444610596, + 0.21329757571220398, + 0.12901145219802856 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.3829405903816223, + "style_target": 1, + "support_probability": 0.044686295092105865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009707558009532673, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9901509284973145, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33933311700820923, + 0.2458220273256302, + 0.22637440264225006, + 0.9109188318252563, + 0.2094174027442932, + 0.05434630066156387, + 0.14577169716358185, + 0.07577034085988998, + 0.12526285648345947, + 0.10448573529720306, + 0.1285758763551712 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.7327149510383606, + "style_target": 0, + "support_probability": 0.01209349650889635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7473503257180053, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11415174603462219, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7329080700874329, + 0.5263223052024841, + 0.8889606595039368, + 0.3421347141265869, + 0.03925047814846039 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.880246639251709, + "style_target": 1, + "support_probability": 0.9423810839653015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049188547377565006, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1835474669933319, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.640139102935791, + 0.6351457238197327, + 0.9501902461051941, + 0.3786141276359558, + 0.12478562444448471 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.11006443202495575, + "style_target": 0, + "support_probability": 0.8664764165878296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2651288667720923, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8432027697563171, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20912866294384003, + 0.7127649784088135, + 0.714947521686554, + 0.37613755464553833, + 0.117645263671875, + 0.09832970052957535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 1, + "style_probability": 0.7895811200141907, + "style_target": 1, + "support_probability": 0.22833746671676636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18077371286572505, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.745437502861023, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23225319385528564, + 0.6169156432151794, + 0.7020756602287292, + 0.40914738178253174, + 0.025468384847044945, + 0.047028549015522 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 0, + "style_probability": 0.5225821137428284, + "style_target": 0, + "support_probability": 0.2567041218280792, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8904933143005249, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03651103004813194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09237734973430634, + 0.7569795250892639, + 0.285715788602829, + 0.9989053010940552, + 0.17764919996261597, + 0.015435910783708096, + 0.05389159172773361, + 0.024262728169560432, + 0.034615978598594666 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.7418859601020813, + "style_target": 1, + "support_probability": 0.9513581395149231, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5625416425852103, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.049943044781684875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1061796098947525, + 0.7562109231948853, + 0.31702396273612976, + 0.9978256821632385, + 0.04357953369617462, + 0.06889882683753967 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.25503793358802795, + "style_target": 0, + "support_probability": 0.9201801419258118, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6704571996317119, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09744182229042053, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44560980796813965, + 0.6873687505722046, + 0.44560980796813965, + 0.9869100451469421, + 0.019633689895272255, + 0.05943591520190239, + 0.03270113840699196, + 0.09142526984214783, + 0.036326248198747635, + 0.08234154433012009 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.6733810305595398, + "style_target": 1, + "support_probability": 0.7916244268417358, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07839121247881821, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18762081861495972, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33638009428977966, + 0.5443450808525085, + 0.33638009428977966, + 0.9804922342300415, + 0.04114283621311188, + 0.04130524396896362, + 0.09107533097267151, + 0.05354562774300575, + 0.041025158017873764, + 0.09913671761751175 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.12295059114694595, + "style_target": 0, + "support_probability": 0.6057637333869934, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16369532565299816, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2863268554210663, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15013234317302704, + 0.2034866362810135, + 0.15013234317302704, + 0.8918057084083557, + 0.11951611936092377 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.8175309300422668, + "style_target": 1, + "support_probability": 0.13079369068145752, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12646583203581604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14880411326885223, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1507149338722229, + 0.2029227316379547, + 0.1507149338722229, + 0.9039737582206726, + 0.23028752207756042 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.36747804284095764, + "style_target": 0, + "support_probability": 0.24699537456035614, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2066032616752961, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2646605968475342, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9073126316070557, + 0.8521872758865356, + 0.19917865097522736, + 0.365657776594162, + 0.033684536814689636, + 0.12904362380504608 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 1, + "style_probability": 0.873120129108429, + "style_target": 1, + "support_probability": 0.12022825330495834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11942326295204236, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21797941625118256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9123496413230896, + 0.8330273032188416, + 0.18303953111171722, + 0.3497770428657532, + 0.19450120627880096 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 1, + "style_probability": 0.3839113414287567, + "style_target": 0, + "support_probability": 0.15567044913768768, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7419364513093214, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3351423740386963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3953304886817932, + 0.9875680208206177, + 0.6957387328147888, + 0.1405690610408783, + 0.05813377723097801, + 0.060711879283189774, + 0.03869803994894028, + 0.03385239467024803, + 0.09202858060598373 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.9344105124473572, + "style_target": 1, + "support_probability": 0.7698797583580017, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029918913736034735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2693064510822296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4431093633174896, + 0.9962027668952942, + 0.5586951971054077, + 0.11947713047266006, + 0.07254649698734283, + 0.074591726064682, + 0.05730677396059036, + 0.021569713950157166, + 0.07326832413673401 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.06946983188390732, + "style_target": 0, + "support_probability": 0.5870415568351746, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8052207829216372, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06430541723966599, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1575874388217926, + 0.26378095149993896, + 0.9096894264221191, + 0.33003607392311096, + 0.027569983154535294, + 0.030794033780694008, + 0.018833782523870468, + 0.013908538036048412, + 0.010099182836711407, + 0.05651989206671715 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 1, + "style_probability": 0.9216061234474182, + "style_target": 1, + "support_probability": 0.9510935544967651, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7259466873215654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31721246242523193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1564992219209671, + 0.30899137258529663, + 0.8805327415466309, + 0.47641703486442566, + 0.02834184654057026, + 0.011771018616855145, + 0.04313349723815918, + 0.02050880342721939, + 0.06468383967876434, + 0.2755502760410309 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 0, + "style_probability": 0.5796338319778442, + "style_target": 0, + "support_probability": 0.9690598249435425, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18344250763109257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11537030339241028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09187749773263931, + 0.8666064143180847, + 0.13593101501464844, + 0.21806195378303528, + 0.03687838837504387 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.22755874693393707, + "style_target": 1, + "support_probability": 0.395346462726593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3389752905975506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1465146392583847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08632161468267441, + 0.8864536285400391, + 0.14104707539081573, + 0.161692813038826, + 0.006708980537950993 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.6230041980743408, + "style_target": 0, + "support_probability": 0.3407094180583954, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09129990985175526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22210054099559784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35253629088401794, + 0.9037914276123047, + 0.43629440665245056, + 0.218452587723732, + 0.09177510440349579, + 0.1514984667301178 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 0, + "style_probability": 0.4041386842727661, + "style_target": 1, + "support_probability": 0.17801235616207123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17522824047514446, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03460715711116791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2512228786945343, + 0.9587534070014954, + 0.22312059998512268, + 0.18288004398345947, + 0.03240174427628517, + 0.06480918824672699 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 1, + "style_probability": 0.15339699387550354, + "style_target": 0, + "support_probability": 0.7203088998794556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15466428440745814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5789443850517273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36585113406181335, + 0.3596121072769165, + 0.949204683303833, + 0.6458576321601868, + 0.04422815889120102, + 0.07126616686582565, + 0.053963351994752884, + 0.16433294117450714, + 0.08565673232078552 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.4013597369194031, + "style_target": 1, + "support_probability": 0.20509549975395203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08356217240765017, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.746896505355835, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40803098678588867, + 0.38097265362739563, + 0.9348545670509338, + 0.7040289044380188, + 0.11217052489519119, + 0.03831326588988304, + 0.22218815982341766, + 0.030957438051700592, + 0.2614842653274536 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.5604149699211121, + "style_target": 0, + "support_probability": 0.08329533040523529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09634374536534857, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21185553073883057, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5804381966590881, + 0.30609020590782166, + 0.5144773125648499, + 0.2693500816822052, + 0.023904908448457718, + 0.05290573090314865, + 0.01696225255727768, + 0.03727872669696808, + 0.017294252291321754, + 0.05777009576559067, + 0.08433888852596283 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.22388726472854614, + "style_target": 1, + "support_probability": 0.3909941017627716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18563383174127207, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17772065103054047, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5218647718429565, + 0.27669963240623474, + 0.4112454950809479, + 0.2363520860671997, + 0.014929606579244137, + 0.014954185113310814, + 0.0271855890750885, + 0.016801409423351288, + 0.031249921768903732, + 0.028430726379156113 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.543328046798706, + "style_target": 0, + "support_probability": 0.6256940364837646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5282171612437845, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22884131968021393, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1453789621591568, + 0.7124624252319336, + 0.4723798930644989, + 0.4887247383594513, + 0.022914456203579903 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.6589530110359192, + "style_target": 1, + "support_probability": 0.479573130607605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008307750776388997, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5546787977218628, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12889930605888367, + 0.6180739402770996, + 0.38562893867492676, + 0.4391493499279022, + 0.06483221054077148 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.035414181649684906, + "style_target": 0, + "support_probability": 0.3219013810157776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6822595529052944, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0662725642323494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34734195470809937, + 0.8575791120529175, + 0.6776404976844788, + 0.765834629535675, + 0.03346319496631622, + 0.04974072054028511 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 1, + "style_probability": 0.857991635799408, + "style_target": 1, + "support_probability": 0.8653225302696228, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39098474287992596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15212786197662354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37270963191986084, + 0.8383783102035522, + 0.6703411936759949, + 0.5927446484565735, + 0.09077121317386627, + 0.05296357721090317 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 1, + "style_probability": 0.43926405906677246, + "style_target": 0, + "support_probability": 0.7650409936904907, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1652949484413944, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9215070009231567, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2857184112071991, + 0.6270342469215393, + 0.6343002319335938, + 0.21074359118938446, + 0.03656730055809021, + 0.05279020592570305, + 0.030990082770586014, + 0.03636560216546059, + 0.043922312557697296 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.8992272615432739, + "style_target": 1, + "support_probability": 0.09640316665172577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014457578235028296, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9546202421188354, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24269618093967438, + 0.6777281761169434, + 0.6818416714668274, + 0.20229028165340424, + 0.0841827541589737, + 0.20045499503612518 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.24146288633346558, + "style_target": 0, + "support_probability": 0.015606287866830826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0029301479755715885, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9934687614440918, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15473245084285736, + 0.2151595950126648, + 0.18906860053539276, + 0.5888569951057434, + 0.06462611258029938, + 0.07293076813220978, + 0.09825785458087921, + 0.044835057109594345, + 0.07666613161563873, + 0.045226696878671646, + 0.0587577223777771 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.6125943064689636, + "style_target": 1, + "support_probability": 0.001806456595659256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009209772733476913, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9927607774734497, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11604149639606476, + 0.10525419563055038, + 0.1886536329984665, + 0.4420256018638611, + 0.07439358532428741, + 0.1063033789396286, + 0.06994601339101791, + 0.10059801489114761, + 0.14652028679847717, + 0.10917090624570847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.09748592972755432, + "style_target": 0, + "support_probability": 0.0071161407977342606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8319989399511366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.019436288625001907, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5135428309440613, + 0.9994733929634094, + 0.21082350611686707, + 0.07732762396335602, + 0.06683458387851715 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 1, + "style_probability": 0.7373064160346985, + "style_target": 1, + "support_probability": 0.9237964749336243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7061078097524699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.022167256101965904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.464526504278183, + 0.997707724571228, + 0.2107335329055786, + 0.14321869611740112, + 0.0556015819311142 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 0, + "style_probability": 0.5268946290016174, + "style_target": 0, + "support_probability": 0.8888945579528809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6688523617281888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0983588695526123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.988944411277771, + 0.4898048937320709, + 0.35086819529533386, + 0.44435256719589233, + 0.22701255977153778, + 0.08412948250770569 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.835065484046936, + "style_target": 1, + "support_probability": 0.3900148570537567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38621012266184374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09297738969326019, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9874706864356995, + 0.5827448964118958, + 0.34925806522369385, + 0.358053594827652, + 0.19863368570804596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.26358526945114136, + "style_target": 0, + "support_probability": 0.5604391098022461, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.738458242502226, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2999853789806366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9177964329719543, + 0.5092934370040894, + 0.09869396686553955, + 0.4621701240539551, + 0.024710193276405334, + 0.13477031886577606, + 0.016653385013341904, + 0.12314670532941818, + 0.07802657783031464 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.9654224514961243, + "style_target": 1, + "support_probability": 0.6313837766647339, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10455359426991943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18052494525909424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9685689210891724, + 0.3869781196117401, + 0.05512815713882446, + 0.38065794110298157, + 0.058502938598394394, + 0.27252086997032166, + 0.04371434450149536, + 0.028765641152858734, + 0.048983849585056305 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.15830565989017487, + "style_target": 0, + "support_probability": 0.7734796404838562, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.038733032863397486, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7203497290611267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3541206419467926, + 0.27526623010635376, + 0.9343488216400146, + 0.7777568101882935, + 0.07937144488096237, + 0.019553376361727715, + 0.013826212845742702, + 0.04107063263654709, + 0.11283697932958603, + 0.3139333128929138, + 0.0939326360821724 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 1, + "style_probability": 0.7165043354034424, + "style_target": 1, + "support_probability": 0.010261615738272667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06117371997248825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49761685729026794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3175511658191681, + 0.20844870805740356, + 0.8996514081954956, + 0.6607555747032166, + 0.12935952842235565, + 0.03401804342865944, + 0.03230113908648491, + 0.048153433948755264, + 0.09029155224561691, + 0.015933016315102577, + 0.06788472831249237 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 1, + "style_probability": 0.4142459034919739, + "style_target": 0, + "support_probability": 0.027433037757873535, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32555145926201656, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2562820315361023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22022157907485962, + 0.11381269991397858, + 0.325546532869339, + 0.9291753172874451, + 0.042771436274051666 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 1, + "style_probability": 0.6200007796287537, + "style_target": 1, + "support_probability": 0.377649188041687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23927028753811647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4072076678276062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15547692775726318, + 0.0593825988471508, + 0.38902387022972107, + 0.9434038996696472, + 0.03508501127362251 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 0, + "style_probability": 0.5739401578903198, + "style_target": 0, + "support_probability": 0.3561117351055145, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.22162802870555495, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16648849844932556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9983382225036621, + 0.3897309899330139, + 0.5366562604904175, + 0.3759445250034332, + 0.05234122276306152, + 0.08922966569662094 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.6906956434249878, + "style_target": 1, + "support_probability": 0.17600610852241516, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2454574089306316, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11308193951845169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9994433522224426, + 0.2769693434238434, + 0.4898381233215332, + 0.3217677175998688, + 0.0829661563038826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.3584875166416168, + "style_target": 0, + "support_probability": 0.3771613836288452, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.614656671705443, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09723089635372162, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49268797039985657, + 0.1789216250181198, + 0.9656257033348083, + 0.47440582513809204, + 0.07165271043777466, + 0.07154324650764465, + 0.03795216977596283, + 0.048825934529304504, + 0.10141238570213318 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.7740731835365295, + "style_target": 1, + "support_probability": 0.7920881509780884, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07260488552847841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2633246183395386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40539324283599854, + 0.22682636976242065, + 0.9780092239379883, + 0.24730734527111053, + 0.03170390799641609, + 0.048387862741947174, + 0.07697679102420807, + 0.06165072321891785, + 0.22622735798358917 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.12247121334075928, + "style_target": 0, + "support_probability": 0.727618396282196, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.17746223335854225, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.280365914106369, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28933385014533997, + 0.38148048520088196, + 0.5849449634552002, + 0.7228417992591858, + 0.011985269375145435, + 0.024454884231090546, + 0.020917227491736412, + 0.017271671444177628, + 0.03157099336385727, + 0.009757693856954575, + 0.026883957907557487 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 1, + "style_probability": 0.6461061239242554, + "style_target": 1, + "support_probability": 0.2244671881198883, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27121212076269086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43372681736946106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3282986581325531, + 0.4457557201385498, + 0.6470500826835632, + 0.7801700234413147, + 0.009321331046521664, + 0.015448620542883873, + 0.013057639822363853, + 0.011929148808121681, + 0.007914450950920582, + 0.030996480956673622, + 0.13850928843021393 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 0, + "style_probability": 0.8615201711654663, + "style_target": 0, + "support_probability": 0.14156009256839752, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09470666046530303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7110781073570251, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3203517198562622, + 0.4452265501022339, + 0.5599900484085083, + 0.2134024053812027, + 0.029944127425551414 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 1, + "style_probability": 0.5914742946624756, + "style_target": 1, + "support_probability": 0.14790788292884827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006119153899269675, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9645748138427734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35409486293792725, + 0.5553746819496155, + 0.5065781474113464, + 0.20504869520664215, + 0.0529971681535244, + 0.13417495787143707, + 0.02580270916223526, + 0.08336144685745239, + 0.11015652865171432, + 0.05453143268823624, + 0.05081820487976074 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 1, + "style_probability": 0.3139660954475403, + "style_target": 0, + "support_probability": 0.007008698768913746, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5602627149866435, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09520595520734787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.396823525428772, + 0.6283023357391357, + 0.3340759873390198, + 0.9437848329544067, + 0.034523628652095795, + 0.012955737300217152 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.5134204030036926, + "style_target": 1, + "support_probability": 0.8727341294288635, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11929256757624486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.025145096704363823, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.49016663432121277, + 0.5704136490821838, + 0.3856755495071411, + 0.9860868453979492, + 0.04750192165374756, + 0.07574015110731125 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.04721954092383385, + "style_target": 0, + "support_probability": 0.9394136667251587, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08863286279678637, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5001531839370728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7270901799201965, + 0.31427350640296936, + 0.22978810966014862, + 0.08082529902458191, + 0.0320611372590065, + 0.033123381435871124, + 0.09214065968990326, + 0.05040742829442024, + 0.03027690388262272 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 1, + "style_probability": 0.6255855560302734, + "style_target": 1, + "support_probability": 0.07356413453817368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4373672619793713, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18640825152397156, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.720077633857727, + 0.3039914071559906, + 0.18531547486782074, + 0.03902708739042282, + 0.010610325261950493, + 0.12462558597326279, + 0.12708507478237152, + 0.015884999185800552, + 0.02643069624900818 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 0, + "style_probability": 0.728573203086853, + "style_target": 0, + "support_probability": 0.510210394859314, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.894425392688926, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019550276920199394, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28356871008872986, + 0.4923795759677887, + 0.972575306892395, + 0.5917587280273438, + 0.03773771598935127, + 0.03356144577264786, + 0.03561550751328468, + 0.016943490132689476, + 0.07566909492015839, + 0.017249412834644318, + 0.053062133491039276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 1, + "style_probability": 0.6609086394309998, + "style_target": 1, + "support_probability": 0.9971946477890015, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8633123369441275, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.018081052228808403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29621124267578125, + 0.5236445069313049, + 0.9818208813667297, + 0.5217554569244385, + 0.019527224823832512, + 0.028295062482357025, + 0.04180155694484711, + 0.05388491973280907, + 0.06273259222507477 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 1, + "style_probability": 0.49562564492225647, + "style_target": 0, + "support_probability": 0.9957143664360046, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5849251823041323, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6443027853965759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28719422221183777, + 0.8108398914337158, + 0.21391339600086212, + 0.28719422221183777, + 0.0388840027153492 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 1, + "style_probability": 0.9357625842094421, + "style_target": 1, + "support_probability": 0.44755464792251587, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31692315682030414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6521875262260437, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33771631121635437, + 0.8133459687232971, + 0.19827336072921753, + 0.33771631121635437, + 0.033193189650774 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 0, + "style_probability": 0.6806047558784485, + "style_target": 0, + "support_probability": 0.45275574922561646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0058399483409718195, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31466880440711975, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4704377055168152, + 0.4518747329711914, + 0.07965897768735886, + 0.1579795628786087, + 0.04338051751255989, + 0.03493576496839523 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 1, + "style_probability": 0.6378095746040344, + "style_target": 1, + "support_probability": 0.006742092780768871, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030018285309353315, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6115686297416687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5027080774307251, + 0.4601854383945465, + 0.09707290679216385, + 0.16518472135066986, + 0.04663882404565811 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 1, + "style_probability": 0.33010268211364746, + "style_target": 0, + "support_probability": 0.0017600967548787594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2727588461158794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12008703500032425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.049057506024837494, + 0.0774553120136261, + 0.9366670846939087, + 0.049057506024837494, + 0.029028035700321198, + 0.02900310605764389, + 0.01926529034972191, + 0.027719536796212196, + 0.019444026052951813 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.7537252902984619, + "style_target": 1, + "support_probability": 0.1067868247628212, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022332998506883313, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13648450374603271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.06940367817878723, + 0.08593816310167313, + 0.9551082849502563, + 0.06940367817878723, + 0.0310895387083292, + 0.012345951981842518, + 0.02285037748515606, + 0.015629228204488754, + 0.043360013514757156 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.17786285281181335, + "style_target": 0, + "support_probability": 0.10813739895820618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004748192250558176, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9560948610305786, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6880981922149658, + 0.3369409143924713, + 0.3964346647262573, + 0.12803064286708832, + 0.041365258395671844, + 0.0757756382226944, + 0.037011902779340744, + 0.056335508823394775, + 0.02782086282968521, + 0.059089526534080505, + 0.0379030741751194 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 1, + "style_probability": 0.5771549940109253, + "style_target": 1, + "support_probability": 0.003281416604295373, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0077702216240964584, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9208326935768127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7162330150604248, + 0.42065301537513733, + 0.4052698612213135, + 0.10584374517202377, + 0.09289173036813736, + 0.0514155775308609, + 0.05877095088362694, + 0.04333106428384781, + 0.07678459584712982, + 0.1387215554714203, + 0.06531296670436859 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 0, + "style_probability": 0.5095765590667725, + "style_target": 0, + "support_probability": 0.005057487636804581, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007831324481554095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9876489043235779, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40654778480529785, + 0.18860945105552673, + 0.18860945105552673, + 0.17759375274181366, + 0.040550410747528076 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.2709362208843231, + "style_target": 1, + "support_probability": 0.0006936563295312226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0021289559298578223, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.986223042011261, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3972180187702179, + 0.1494259536266327, + 0.1494259536266327, + 0.12175574153661728, + 0.024034632369875908 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.8803768157958984, + "style_target": 0, + "support_probability": 0.0015516550047323108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17552436135349175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14137010276317596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2460223138332367, + 0.9435346722602844, + 0.15943020582199097, + 0.18693113327026367, + 0.030540263280272484, + 0.0977415069937706 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 0, + "style_probability": 0.40852952003479004, + "style_target": 1, + "support_probability": 0.19673191010951996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.185009720127808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.168445885181427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23548837006092072, + 0.9355306029319763, + 0.1498212218284607, + 0.1507740467786789, + 0.08465366065502167 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 1, + "style_probability": 0.1941082924604416, + "style_target": 0, + "support_probability": 0.24603596329689026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0044266082525923935, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9463006258010864, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.289893239736557, + 0.19081410765647888, + 0.10871981829404831, + 0.21215586364269257, + 0.06367968767881393, + 0.09802955389022827, + 0.07322527468204498, + 0.16945497691631317, + 0.08677981793880463 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 0, + "style_probability": 0.2512266933917999, + "style_target": 1, + "support_probability": 0.0046655298210680485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006834044169813822, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9720459580421448, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23906196653842926, + 0.11773717403411865, + 0.1096353530883789, + 0.15650825202465057, + 0.07758982479572296, + 0.0589970126748085, + 0.052959464490413666, + 0.08552861958742142, + 0.1411253660917282 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 1, + "style_probability": 0.10466177016496658, + "style_target": 0, + "support_probability": 0.005195136647671461, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17388829645696827, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.235556498169899, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2043956071138382, + 0.6628651022911072, + 0.2967703342437744, + 0.18512707948684692, + 0.016178201884031296, + 0.12554457783699036, + 0.031695641577243805, + 0.04473850503563881, + 0.011390963569283485, + 0.031806040555238724, + 0.047771159559488297 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 0, + "style_probability": 0.45369037985801697, + "style_target": 1, + "support_probability": 0.24703767895698547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5497335963320191, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10788549482822418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21488961577415466, + 0.7487443089485168, + 0.2658275365829468, + 0.12012480199337006, + 0.03390955924987793, + 0.03701041638851166, + 0.0491408035159111, + 0.07950747758150101, + 0.052015092223882675, + 0.022676073014736176, + 0.02156680077314377 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 0, + "style_probability": 0.635395884513855, + "style_target": 0, + "support_probability": 0.7032673358917236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9136590627695077, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.021660421043634415, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3011:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3490176498889923, + 0.911431610584259, + 0.3490176498889923, + 0.982609748840332, + 0.06213458254933357 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8931877613067627, + "style_target": 1, + "support_probability": 0.9882176518440247, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13751220408409773, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08320898562669754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3011:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2375219166278839, + 0.9322822093963623, + 0.2375219166278839, + 0.9884114265441895, + 0.05821526423096657 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.08278283476829529, + "style_target": 0, + "support_probability": 0.9422311782836914, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.631240013839934, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.062149107456207275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5675377249717712, + 0.34466418623924255, + 0.7557747960090637, + 0.9886612892150879, + 0.02734565921127796, + 0.030337054282426834 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 1, + "style_probability": 0.6849438548088074, + "style_target": 1, + "support_probability": 0.7242419719696045, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.518389855519006, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06068005412817001, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5609396696090698, + 0.3657943904399872, + 0.7666389346122742, + 0.9808382391929626, + 0.028200428932905197, + 0.025474680587649345 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 0, + "style_probability": 0.651390790939331, + "style_target": 0, + "support_probability": 0.6983568072319031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8624631180650614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04163456708192825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13154572248458862, + 0.10475312918424606, + 0.17632736265659332, + 0.9945388436317444, + 0.02495979331433773, + 0.10996715724468231, + 0.020751822739839554, + 0.04564982280135155, + 0.0187694039195776 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.7548313140869141, + "style_target": 1, + "support_probability": 0.8666205406188965, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6189080699142865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.024757757782936096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13692493736743927, + 0.11333155632019043, + 0.21360819041728973, + 0.9930592775344849, + 0.029893390834331512, + 0.04652213305234909 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.3227114677429199, + "style_target": 0, + "support_probability": 0.9316319823265076, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0010201430942746517, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9817905426025391, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1680418998003006, + 0.07024835795164108, + 0.29425424337387085, + 0.24606066942214966, + 0.06921809166669846, + 0.05591708794236183, + 0.10128559917211533, + 0.049348123371601105, + 0.03766028955578804, + 0.08648966997861862 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.8323262333869934, + "style_target": 1, + "support_probability": 0.0036610430106520653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00366148412780341, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9310488700866699, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.138767272233963, + 0.06076318025588989, + 0.38450193405151367, + 0.3667631447315216, + 0.06160798296332359, + 0.05882226303219795, + 0.06601516902446747, + 0.04205494374036789, + 0.02948664501309395, + 0.05049984157085419 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.13769590854644775, + "style_target": 0, + "support_probability": 0.07959605753421783, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.40650419984216907, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18820472061634064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27058765292167664, + 0.6463367342948914, + 0.4393148422241211, + 0.33212199807167053, + 0.04805620387196541 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 1, + "style_probability": 0.5167971253395081, + "style_target": 1, + "support_probability": 0.6134734153747559, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.021083552231344185, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2782284617424011, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14969328045845032, + 0.7395318150520325, + 0.3130514621734619, + 0.3715485632419586, + 0.12769119441509247 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 1, + "style_probability": 0.07609989494085312, + "style_target": 0, + "support_probability": 0.3094457983970642, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5378861892325774, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1902017444372177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9688329100608826, + 0.3115786015987396, + 0.4288938045501709, + 0.6315847635269165, + 0.13481144607067108, + 0.07617592066526413 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 1, + "style_probability": 0.5601257085800171, + "style_target": 1, + "support_probability": 0.7226739525794983, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6706789385187321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12422317266464233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.989206075668335, + 0.3323257565498352, + 0.375538170337677, + 0.6066899299621582, + 0.08865013718605042, + 0.2584245502948761 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 0, + "style_probability": 0.7598773241043091, + "style_target": 0, + "support_probability": 0.665317177772522, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7912179674309412, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029376527294516563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21619102358818054, + 0.9579501152038574, + 0.39604660868644714, + 0.15821586549282074, + 0.019351039081811905, + 0.04202435165643692, + 0.05328328162431717, + 0.07373083382844925, + 0.046193793416023254 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 0, + "style_probability": 0.4929351210594177, + "style_target": 1, + "support_probability": 0.9541540741920471, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7722371174041956, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01434614323079586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24926981329917908, + 0.968255341053009, + 0.3656977415084839, + 0.16020451486110687, + 0.01867726258933544, + 0.02835990861058235 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 1, + "style_probability": 0.32677823305130005, + "style_target": 0, + "support_probability": 0.9757283926010132, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008732553083949723, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7811008095741272, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5612280964851379, + 0.19391758739948273, + 0.4214939475059509, + 0.6001861095428467, + 0.033018358051776886, + 0.10100056976079941, + 0.2348460555076599, + 0.06512507796287537, + 0.09329131990671158, + 0.12084166705608368 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 0, + "style_probability": 0.48244234919548035, + "style_target": 1, + "support_probability": 0.007023880258202553, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003733258092971334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6808703541755676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7034711837768555, + 0.13805420696735382, + 0.2589900493621826, + 0.6451595425605774, + 0.10694430768489838, + 0.06099255755543709, + 0.04079044610261917, + 0.06717821955680847, + 0.04009032994508743, + 0.13447707891464233 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 1, + "style_probability": 0.15400229394435883, + "style_target": 0, + "support_probability": 0.02797751873731613, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8295925549571436, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.020778782665729523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8253981471061707, + 0.25137418508529663, + 0.9374516606330872, + 0.1442812979221344, + 0.020566970109939575 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 1, + "style_probability": 0.8999314308166504, + "style_target": 1, + "support_probability": 0.9381495714187622, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5080188957268795, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04996681213378906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8113413453102112, + 0.3266631066799164, + 0.9185831546783447, + 0.18119217455387115, + 0.14738719165325165, + 0.07169068604707718, + 0.10171540081501007, + 0.09435310959815979, + 0.053758151829242706, + 0.017724571749567986, + 0.04405601695179939 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 0, + "style_probability": 0.5134710669517517, + "style_target": 0, + "support_probability": 0.8348166942596436, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07420598124443226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8049474954605103, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9634451270103455, + 0.01660310849547386, + 0.41865959763526917, + 0.2930898666381836, + 0.10434471070766449, + 0.02632913738489151 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.8507218956947327, + "style_target": 1, + "support_probability": 0.018145354464650154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007602737358056042, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.708061933517456, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9584246873855591, + 0.021383186802268028, + 0.5087184906005859, + 0.2604118585586548, + 0.019573954865336418, + 0.029573960229754448 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.2538546025753021, + "style_target": 0, + "support_probability": 0.030686696991324425, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6199221569669167, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06091685593128204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48501965403556824, + 0.1174360066652298, + 0.993496298789978, + 0.27096542716026306, + 0.240920752286911, + 0.011196778155863285, + 0.010251025669276714, + 0.20571066439151764, + 0.023676317185163498 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 1, + "style_probability": 0.6031907200813293, + "style_target": 1, + "support_probability": 0.8042703866958618, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5031095809037944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09869848191738129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5415316224098206, + 0.11260669678449631, + 0.9889425039291382, + 0.26322340965270996, + 0.021692203357815742, + 0.03883020579814911, + 0.09801498055458069, + 0.02189655415713787, + 0.02260633185505867 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 1, + "style_probability": 0.4028818607330322, + "style_target": 0, + "support_probability": 0.7939997315406799, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2726585427689655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3626236617565155, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10768075287342072, + 0.49736708402633667, + 0.9795973896980286, + 0.10768075287342072, + 0.023763855919241905, + 0.015389839187264442, + 0.06488698720932007, + 0.02063470147550106, + 0.004021971952170134, + 0.07072142511606216, + 0.017969390377402306 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6063525676727295, + "style_target": 1, + "support_probability": 0.1323159784078598, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07810666379229181, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4594666361808777, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13153406977653503, + 0.5548247694969177, + 0.9713423848152161, + 0.13153406977653503, + 0.013973425142467022, + 0.010760258883237839, + 0.013861438259482384, + 0.014613554812967777, + 0.016488732770085335 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 1, + "style_probability": 0.43817317485809326, + "style_target": 0, + "support_probability": 0.04692092910408974, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2538630424573998, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15727508068084717, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09159974008798599, + 0.9724785685539246, + 0.09241689741611481, + 0.15778034925460815, + 0.056314002722501755 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 1, + "style_probability": 0.5596511363983154, + "style_target": 1, + "support_probability": 0.14717906713485718, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17624687446807455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08298949897289276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08429908752441406, + 0.9759101867675781, + 0.13620750606060028, + 0.1362038254737854, + 0.016643384471535683, + 0.015821754932403564, + 0.04205101355910301, + 0.03225579857826233, + 0.024788562208414078, + 0.04521865025162697, + 0.016267508268356323 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 1, + "style_probability": 0.36710813641548157, + "style_target": 0, + "support_probability": 0.1424693465232849, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12735540185461072, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7476603388786316, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7194806933403015, + 0.21150167286396027, + 0.7684857845306396, + 0.7194806933403015, + 0.04206182807683945, + 0.08018989861011505 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 1, + "style_probability": 0.6609720587730408, + "style_target": 1, + "support_probability": 0.06157035380601883, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0069515215654783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9479259252548218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4969903230667114, + 0.05679786950349808, + 0.9142995476722717, + 0.4969903230667114, + 0.08779097348451614, + 0.09937022626399994 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 1, + "style_probability": 0.10657576471567154, + "style_target": 0, + "support_probability": 0.03586513549089432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03898081816857227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5729155540466309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4376382529735565, + 0.26050400733947754, + 0.8388227224349976, + 0.47577938437461853, + 0.033163223415613174, + 0.01606404408812523, + 0.1734515279531479, + 0.03101535700261593, + 0.061232976615428925 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 0, + "style_probability": 0.32556426525115967, + "style_target": 1, + "support_probability": 0.03775819018483162, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19993910348585184, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26399701833724976, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4761338531970978, + 0.2963537275791168, + 0.8935120105743408, + 0.5037320852279663, + 0.07181599736213684, + 0.03887662664055824, + 0.016747530549764633, + 0.031153811141848564, + 0.029129305854439735 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 0, + "style_probability": 0.8366662859916687, + "style_target": 0, + "support_probability": 0.1274951845407486, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1812177737733067, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6454905867576599, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8706642389297485, + 0.13163553178310394, + 0.3554515540599823, + 0.403521865606308, + 0.020082572475075722, + 0.04748723283410072, + 0.035940926522016525, + 0.014363454654812813, + 0.0863751471042633, + 0.0330292247235775, + 0.02989676594734192 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 1, + "style_probability": 0.7532262802124023, + "style_target": 1, + "support_probability": 0.08831935375928879, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20543801741824363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3004547655582428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.875377357006073, + 0.15685781836509705, + 0.4062204360961914, + 0.41855019330978394, + 0.012328509241342545, + 0.012781196273863316, + 0.04110260680317879, + 0.01657603681087494, + 0.0432869978249073 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 0, + "style_probability": 0.5867728590965271, + "style_target": 0, + "support_probability": 0.19365337491035461, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03618615064059179, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8435018062591553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31873810291290283, + 0.3901646137237549, + 0.35624536871910095, + 0.7498533725738525, + 0.01704295352101326 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 1, + "style_probability": 0.7984113693237305, + "style_target": 1, + "support_probability": 0.014332441613078117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023897450139639145, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9154932498931885, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3257869482040405, + 0.5172784328460693, + 0.4171936511993408, + 0.7629702687263489, + 0.09269523620605469 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 0, + "style_probability": 0.6939268112182617, + "style_target": 0, + "support_probability": 0.006507450249046087, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8394514041322806, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026131700724363327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.992071807384491, + 0.6637666821479797, + 0.41115710139274597, + 0.4005421996116638, + 0.01169250812381506, + 0.04433804377913475 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.7936291694641113, + "style_target": 1, + "support_probability": 0.9120264649391174, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6914216065306404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03228147327899933, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9876009225845337, + 0.7158254384994507, + 0.4402501881122589, + 0.4014284610748291, + 0.06350642442703247 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.4687417447566986, + "style_target": 0, + "support_probability": 0.9306114315986633, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5578369331898335, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13369636237621307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6366841793060303, + 0.2115594893693924, + 0.984898030757904, + 0.3073648512363434, + 0.01701841503381729, + 0.0475100539624691, + 0.03191236034035683, + 0.029657039791345596, + 0.06936971098184586 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.859041690826416, + "style_target": 1, + "support_probability": 0.3547436594963074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10460454048768347, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13739976286888123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6155554056167603, + 0.21103881299495697, + 0.9931527376174927, + 0.3017720580101013, + 0.04042971134185791, + 0.047201257199048996, + 0.1161452978849411, + 0.09418950229883194, + 0.14668256044387817 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.2372240275144577, + "style_target": 0, + "support_probability": 0.32372650504112244, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8907391364005326, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034671615809202194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6786680817604065, + 0.09978044033050537, + 0.9664523005485535, + 0.024883121252059937, + 0.014963564462959766, + 0.019997259601950645, + 0.027326924726366997, + 0.018693795427680016, + 0.015240625478327274, + 0.02101394720375538, + 0.019595755264163017 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 1, + "style_probability": 0.9235685467720032, + "style_target": 1, + "support_probability": 0.9931122660636902, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7802862321629219, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10281707346439362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6284520626068115, + 0.10112417489290237, + 0.9563589692115784, + 0.04663944989442825, + 0.03825977072119713, + 0.04847196862101555, + 0.014417601749300957, + 0.06871200352907181, + 0.01311683189123869, + 0.017126990482211113, + 0.028655003756284714 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 0, + "style_probability": 0.5062511563301086, + "style_target": 0, + "support_probability": 0.9803848266601562, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5609314169100242, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05876368284225464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26823216676712036, + 0.8457032442092896, + 0.45374181866645813, + 0.11250724643468857, + 0.020330829545855522 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 1, + "style_probability": 0.532280683517456, + "style_target": 1, + "support_probability": 0.7031311988830566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7709389634394114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05379253253340721, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23706525564193726, + 0.8918282389640808, + 0.4616786241531372, + 0.0725535899400711, + 0.03196052461862564 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 0, + "style_probability": 0.8130653500556946, + "style_target": 0, + "support_probability": 0.7741495370864868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026777453798750784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4594131112098694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4568309187889099, + 0.542941153049469, + 0.7577546238899231, + 0.4532870352268219, + 0.1883675456047058, + 0.05943024903535843 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 0, + "style_probability": 0.455131858587265, + "style_target": 1, + "support_probability": 0.03420039266347885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1064661477251818, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5214791893959045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.420486718416214, + 0.5433911681175232, + 0.7162942886352539, + 0.4226062297821045, + 0.14799626171588898 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 1, + "style_probability": 0.4090127944946289, + "style_target": 0, + "support_probability": 0.14576460421085358, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7065516464393646, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07160858809947968, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26040413975715637, + 0.22862529754638672, + 0.4941057860851288, + 0.9585304260253906, + 0.039251845329999924, + 0.1268693208694458, + 0.04651862010359764, + 0.052402693778276443, + 0.08398644626140594 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 0, + "style_probability": 0.45666712522506714, + "style_target": 1, + "support_probability": 0.9156028032302856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041841461427327385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10083312541246414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1931946575641632, + 0.1717008650302887, + 0.3187451660633087, + 0.9506481885910034, + 0.09171810001134872, + 0.05258854851126671, + 0.05127301812171936, + 0.033316947519779205, + 0.06086011603474617 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 1, + "style_probability": 0.04040169343352318, + "style_target": 0, + "support_probability": 0.7325549721717834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2299622904018642, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38083386421203613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2997417151927948, + 0.6307527422904968, + 0.8709926009178162, + 0.2997417151927948, + 0.06576021015644073, + 0.06884545087814331, + 0.06095413863658905, + 0.051746297627687454, + 0.152548685669899, + 0.1332206130027771, + 0.04294987767934799 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 1, + "style_probability": 0.53268963098526, + "style_target": 1, + "support_probability": 0.402715802192688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6116698096116934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06658848375082016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22452230751514435, + 0.6187820434570312, + 0.8598066568374634, + 0.22452230751514435, + 0.07012087106704712, + 0.011577128432691097, + 0.024253472685813904, + 0.029019523411989212, + 0.05655462294816971, + 0.015775177627801895, + 0.019740229472517967 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 0, + "style_probability": 0.6994137167930603, + "style_target": 0, + "support_probability": 0.8233576416969299, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08068081541260505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9588252902030945, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43308719992637634, + 0.3797716796398163, + 0.9475767612457275, + 0.46056047081947327, + 0.05692429468035698 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.7736310362815857, + "style_target": 1, + "support_probability": 0.025912364944815636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0077553842601055635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9793463945388794, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4726908802986145, + 0.45603853464126587, + 0.9516889452934265, + 0.33730581402778625, + 0.15895825624465942 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.23671533167362213, + "style_target": 0, + "support_probability": 0.01376351434737444, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06205201114260193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6695760488510132, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20875145494937897, + 0.8237059116363525, + 0.6359294652938843, + 0.39325976371765137, + 0.009102091193199158, + 0.04767976328730583 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 1, + "style_probability": 0.8014847040176392, + "style_target": 1, + "support_probability": 0.014735182747244835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03463053795805291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8351026773452759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2630394697189331, + 0.8400080800056458, + 0.6195163130760193, + 0.43382179737091064, + 0.01682857610285282, + 0.028132067993283272 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 0, + "style_probability": 0.677152156829834, + "style_target": 0, + "support_probability": 0.00961829163134098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14790663044057667, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41089439392089844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2689274847507477, + 0.41610145568847656, + 0.7024326324462891, + 0.9394295811653137, + 0.009461273439228535, + 0.011557381600141525, + 0.009778001345694065, + 0.022396650165319443, + 0.08418448269367218 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.7089188098907471, + "style_target": 1, + "support_probability": 0.04527679830789566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19182147727941626, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3273369073867798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2526007294654846, + 0.3551533818244934, + 0.7173983454704285, + 0.9436196684837341, + 0.029728135094046593, + 0.02547437511384487 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.39259105920791626, + "style_target": 0, + "support_probability": 0.10629818588495255, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15189305857409888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07819317281246185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5146580338478088, + 0.6772505640983582, + 0.7740461826324463, + 0.31456422805786133, + 0.04955928772687912, + 0.023633897304534912, + 0.013624102808535099, + 0.01590902730822563, + 0.01191898062825203, + 0.021908888593316078, + 0.0905660092830658 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 1, + "style_probability": 0.7694352865219116, + "style_target": 1, + "support_probability": 0.0704125463962555, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021609969839650352, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05535207688808441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5083392858505249, + 0.6206198334693909, + 0.8165435194969177, + 0.3179587423801422, + 0.03154713660478592, + 0.022660722956061363, + 0.06713256239891052, + 0.042032185941934586, + 0.04841332510113716, + 0.05570688843727112 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 1, + "style_probability": 0.17185772955417633, + "style_target": 0, + "support_probability": 0.07539558410644531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018639769635035226, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3342152237892151, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15269790589809418, + 0.6594942212104797, + 0.52923184633255, + 0.26461777091026306, + 0.07810866832733154 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 0, + "style_probability": 0.348324716091156, + "style_target": 1, + "support_probability": 0.006079709157347679, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010461292095887935, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45137885212898254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10706471651792526, + 0.5220088958740234, + 0.45080846548080444, + 0.3174611032009125, + 0.03057556413114071 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 1, + "style_probability": 0.45276376605033875, + "style_target": 0, + "support_probability": 0.005148217547684908, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.263347533552056, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1798505187034607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19756244122982025, + 0.18576276302337646, + 0.5938851237297058, + 0.9884207248687744, + 0.15371054410934448, + 0.11347749829292297 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 1, + "style_probability": 0.5645158290863037, + "style_target": 1, + "support_probability": 0.22412462532520294, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3437911611728141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33641403913497925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2224377542734146, + 0.2022901028394699, + 0.6465499997138977, + 0.9878459572792053, + 0.039240311831235886, + 0.3437805473804474 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 0, + "style_probability": 0.823019802570343, + "style_target": 0, + "support_probability": 0.12151829153299332, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06675937506636098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16390137374401093, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09207680076360703, + 0.1788490116596222, + 0.0809718668460846, + 0.9181724190711975, + 0.021041005849838257, + 0.017790285870432854, + 0.016712622717022896, + 0.14780771732330322, + 0.030327903106808662 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 0, + "style_probability": 0.48542365431785583, + "style_target": 1, + "support_probability": 0.018524037674069405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019175876469398365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33422496914863586, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09356487542390823, + 0.17998424172401428, + 0.11764878779649734, + 0.9165905117988586, + 0.02062668465077877, + 0.015275244601070881, + 0.00988983828574419, + 0.011864385567605495, + 0.026158789172768593 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 1, + "style_probability": 0.30366089940071106, + "style_target": 0, + "support_probability": 0.017895139753818512, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011340039105870607, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8217783570289612, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16752731800079346, + 0.0938807874917984, + 0.6630399823188782, + 0.11130233854055405, + 0.039317984133958817, + 0.04348938167095184, + 0.04162254184484482, + 0.01523127406835556, + 0.03358369320631027, + 0.07630860805511475, + 0.04905178025364876 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 0, + "style_probability": 0.3120661973953247, + "style_target": 1, + "support_probability": 0.006517384666949511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01315764013186295, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9260902404785156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16578620672225952, + 0.14757201075553894, + 0.5932490229606628, + 0.11533761024475098, + 0.0265519842505455, + 0.0616970993578434, + 0.09620248526334763, + 0.02075236104428768, + 0.08124106377363205, + 0.025641730055212975, + 0.040397606790065765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 1, + "style_probability": 0.463958740234375, + "style_target": 0, + "support_probability": 0.009076669812202454, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2340142215774642, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14061346650123596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33722129464149475, + 0.3426899313926697, + 0.9611234664916992, + 0.25568273663520813, + 0.030086878687143326 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.7609299421310425, + "style_target": 1, + "support_probability": 0.14601992070674896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12450309062166096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1409953534603119, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32356685400009155, + 0.36555105447769165, + 0.9508107900619507, + 0.2740229070186615, + 0.028331486508250237, + 0.030945174396038055, + 0.13596828281879425, + 0.2598099410533905, + 0.12040994316339493, + 0.06890459358692169, + 0.0609380267560482 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.32968994975090027, + "style_target": 0, + "support_probability": 0.12895537912845612, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7929853689352545, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4151148796081543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.89984130859375, + 0.3390944302082062, + 0.7190077900886536, + 0.8187777996063232, + 0.03396841883659363, + 0.021455377340316772 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.8631515502929688, + "style_target": 1, + "support_probability": 0.9385226368904114, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20671024399053922, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5756479501724243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8945186734199524, + 0.22998234629631042, + 0.6215054392814636, + 0.8057895302772522, + 0.019351953640580177, + 0.08683813363313675 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.0922980010509491, + "style_target": 0, + "support_probability": 0.8261932730674744, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1879553272669865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4955843687057495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7279710173606873, + 0.37144380807876587, + 0.4482131600379944, + 0.5365674495697021, + 0.05629792436957359, + 0.07310985028743744, + 0.016116946935653687, + 0.02315434440970421, + 0.028934232890605927 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 1, + "style_probability": 0.9201059937477112, + "style_target": 1, + "support_probability": 0.131433367729187, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011357713004620597, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8183624744415283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.662473201751709, + 0.37174543738365173, + 0.4299217462539673, + 0.41686728596687317, + 0.07047700881958008, + 0.06201334670186043, + 0.09127533435821533, + 0.16241052746772766, + 0.1016089916229248 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 1, + "style_probability": 0.4080793559551239, + "style_target": 0, + "support_probability": 0.009021835401654243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6725735321479944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31312432885169983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36158615350723267, + 0.8907773494720459, + 0.16554926335811615, + 0.09113945066928864, + 0.019084736704826355, + 0.0281108058989048, + 0.01876603439450264, + 0.02880489081144333, + 0.05172770470380783, + 0.022745314985513687, + 0.029070649296045303 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.8561567068099976, + "style_target": 1, + "support_probability": 0.4953930974006653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4083768205045699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2750049829483032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4478611648082733, + 0.8754703402519226, + 0.19154177606105804, + 0.10664903372526169, + 0.07406249642372131, + 0.19760490953922272, + 0.1570856124162674, + 0.02551729790866375, + 0.13782504200935364 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.35290855169296265, + "style_target": 0, + "support_probability": 0.5511998534202576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0463340886224556, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8500540852546692, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18296393752098083, + 0.43893858790397644, + 0.4484385550022125, + 0.6877806186676025, + 0.0194492619484663 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.8261355757713318, + "style_target": 1, + "support_probability": 0.021200884133577347, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002094813916206539, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8921440839767456, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13223059475421906, + 0.3543536365032196, + 0.3342849910259247, + 0.6126732230186462, + 0.06086109206080437 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.16086475551128387, + "style_target": 0, + "support_probability": 0.008385911583900452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6520711505183874, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17762328684329987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.606150209903717, + 0.6552532315254211, + 0.9666233658790588, + 0.35094308853149414, + 0.04891848936676979, + 0.041450563818216324 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 1, + "style_probability": 0.5617457628250122, + "style_target": 1, + "support_probability": 0.5879241228103638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5728801177650105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19198855757713318, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6590911746025085, + 0.6913455128669739, + 0.9583510756492615, + 0.3492863178253174, + 0.06610646843910217, + 0.20966462790966034 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 0, + "style_probability": 0.5784422755241394, + "style_target": 0, + "support_probability": 0.4986613094806671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005236017615355507, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.979518711566925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9147965312004089, + 0.12619256973266602, + 0.32125142216682434, + 0.2726142406463623, + 0.04277662932872772, + 0.05666517838835716, + 0.060813989490270615, + 0.07541794329881668, + 0.03544503077864647 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.7203351259231567, + "style_target": 1, + "support_probability": 0.0032553679775446653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026965111497358548, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9838414192199707, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8894513845443726, + 0.13969004154205322, + 0.3033809959888458, + 0.3217950761318207, + 0.10222667455673218, + 0.0667952224612236 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.25679320096969604, + "style_target": 0, + "support_probability": 0.0012922511668875813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12223452175576595, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5019744634628296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8728215098381042, + 0.21420931816101074, + 0.2619742453098297, + 0.3312402367591858, + 0.05000883713364601, + 0.024246906861662865, + 0.01957235299050808, + 0.020076723769307137, + 0.012176265940070152, + 0.050318486988544464, + 0.020271673798561096 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.7473704218864441, + "style_target": 1, + "support_probability": 0.058688364923000336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025040676945011155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.642742931842804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8031334280967712, + 0.14550115168094635, + 0.275723397731781, + 0.2973804175853729, + 0.0452197827398777, + 0.04183612018823624, + 0.04192047566175461, + 0.060760077089071274, + 0.019009774550795555, + 0.07531657069921494 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.03397643193602562, + "style_target": 0, + "support_probability": 0.03387681767344475, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8800919458702177, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.008790789172053337, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1523018330335617, + 0.2321942001581192, + 0.17182840406894684, + 0.9621984958648682, + 0.03679194301366806 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 1, + "style_probability": 0.9181570410728455, + "style_target": 1, + "support_probability": 0.9543870687484741, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7119796561561028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.01988210715353489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11134739220142365, + 0.23803012073040009, + 0.20316150784492493, + 0.9459527730941772, + 0.1705523282289505 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 0, + "style_probability": 0.5307739973068237, + "style_target": 0, + "support_probability": 0.8843181133270264, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8288731159269414, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1719397008419037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9433586001396179, + 0.5877389907836914, + 0.18841779232025146, + 0.8432709574699402, + 0.056258026510477066, + 0.05385909974575043 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.8722055554389954, + "style_target": 1, + "support_probability": 0.841885507106781, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33734206854304616, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4759748876094818, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9051463007926941, + 0.573983907699585, + 0.21254873275756836, + 0.8603189587593079, + 0.12207779288291931 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.28370970487594604, + "style_target": 0, + "support_probability": 0.5689296126365662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5578083992650322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1960248500108719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.635012686252594, + 0.43334856629371643, + 0.9851073622703552, + 0.2641727328300476, + 0.03746899217367172, + 0.19018691778182983, + 0.28086307644844055, + 0.07302659004926682, + 0.06214514374732971 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.7721184492111206, + "style_target": 1, + "support_probability": 0.22662515938282013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05748609402105099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3055492043495178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6768831014633179, + 0.29831400513648987, + 0.9824385046958923, + 0.19357085227966309, + 0.08699203282594681, + 0.1655324399471283, + 0.06150507926940918, + 0.09370622038841248, + 0.13034944236278534 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.08895278722047806, + "style_target": 0, + "support_probability": 0.16261093318462372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7730997573191731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041820622980594635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06916109472513199, + 0.499185711145401, + 0.9939593076705933, + 0.10887833684682846, + 0.055834755301475525, + 0.017975937575101852, + 0.02557126060128212, + 0.07643302530050278, + 0.059730324894189835, + 0.01847131922841072, + 0.011164925061166286 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 1, + "style_probability": 0.6841966509819031, + "style_target": 1, + "support_probability": 0.7384966015815735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7291199995626091, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08589393645524979, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10335264354944229, + 0.5974301695823669, + 0.977377712726593, + 0.15120767056941986, + 0.025464095175266266, + 0.02877901680767536, + 0.017327288165688515, + 0.03619275614619255, + 0.04887496307492256, + 0.01094219833612442, + 0.037046078592538834 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 0, + "style_probability": 0.5981963276863098, + "style_target": 0, + "support_probability": 0.8159334063529968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37322380214171597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30934250354766846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8788442611694336, + 0.1591404527425766, + 0.25148600339889526, + 0.1134478747844696, + 0.03566534444689751 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.8812496066093445, + "style_target": 1, + "support_probability": 0.3446677625179291, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03430864508982165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6805439591407776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8366576433181763, + 0.10757940262556076, + 0.2697456479072571, + 0.0950646847486496, + 0.0446423664689064, + 0.0592707097530365, + 0.06625108420848846, + 0.049136266112327576, + 0.04496423155069351, + 0.03727908805012703, + 0.13531488180160522 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.2408037632703781, + "style_target": 0, + "support_probability": 0.051177818328142166, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0650660524305536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.784193217754364, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3834264874458313, + 0.3234810531139374, + 0.07151607424020767, + 0.8225430846214294, + 0.10740017145872116, + 0.07964249700307846 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.779643177986145, + "style_target": 1, + "support_probability": 0.07560445368289948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00026639007488888787, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9819942116737366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24738430976867676, + 0.17766019701957703, + 0.04296429455280304, + 0.5999391078948975, + 0.1470506638288498, + 0.12872382998466492 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.04114892706274986, + "style_target": 0, + "support_probability": 0.006234711967408657, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21604756262905575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22233916819095612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7821179628372192, + 0.05701267719268799, + 0.2345259040594101, + 0.13670477271080017, + 0.01852995902299881, + 0.01898840069770813, + 0.02574247121810913, + 0.044472143054008484, + 0.04493183642625809 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 1, + "style_probability": 0.8167832493782043, + "style_target": 1, + "support_probability": 0.19147469103336334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23756272386261937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2251165211200714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8634252548217773, + 0.09570528566837311, + 0.252272367477417, + 0.11817225813865662, + 0.038464128971099854, + 0.015330205671489239, + 0.051761213690042496, + 0.08684325218200684, + 0.04744385555386543 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 0, + "style_probability": 0.6156286001205444, + "style_target": 0, + "support_probability": 0.3376992344856262, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16386384453141326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5089589357376099, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2282804250717163, + 0.0821523517370224, + 0.5129016637802124, + 0.5276041030883789, + 0.009538942016661167, + 0.0105248698964715, + 0.016688810661435127, + 0.009174099192023277, + 0.020263269543647766, + 0.012754485011100769, + 0.05387977883219719 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 1, + "style_probability": 0.9267546534538269, + "style_target": 1, + "support_probability": 0.1451673060655594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14517282598187414, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4062206447124481, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21585361659526825, + 0.08326338231563568, + 0.46559104323387146, + 0.43902772665023804, + 0.016555210575461388, + 0.01167339738458395, + 0.028013667091727257, + 0.019700270146131516, + 0.011845973320305347 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 0, + "style_probability": 0.5741730332374573, + "style_target": 0, + "support_probability": 0.16987353563308716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.64192102979114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04519912600517273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2848445177078247, + 0.8981854319572449, + 0.052673615515232086, + 0.7204457521438599, + 0.08948875218629837 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7686166167259216, + "style_target": 1, + "support_probability": 0.5796700119972229, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07363864431070116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0763404592871666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30000293254852295, + 0.8879943490028381, + 0.04321591556072235, + 0.6635066270828247, + 0.05316973105072975 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.10108063369989395, + "style_target": 0, + "support_probability": 0.5225120186805725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7678650658246864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01828416809439659, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9994595646858215, + 0.26754283905029297, + 0.5146452784538269, + 0.20514357089996338, + 0.01165703684091568, + 0.04251285642385483 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 1, + "style_probability": 0.6636540293693542, + "style_target": 1, + "support_probability": 0.8727418780326843, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6984490417347837, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01346519123762846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9982228875160217, + 0.2662276029586792, + 0.49310335516929626, + 0.19737516343593597, + 0.026584159582853317, + 0.02162439562380314 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 0, + "style_probability": 0.5557094216346741, + "style_target": 0, + "support_probability": 0.858579695224762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.381302035336726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6852832436561584, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18955613672733307, + 0.518988847732544, + 0.9350789189338684, + 0.36873000860214233, + 0.03194446861743927, + 0.024456525221467018, + 0.030738916248083115, + 0.032815683633089066, + 0.06694687902927399 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.7377997636795044, + "style_target": 1, + "support_probability": 0.16920140385627747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13887105024462953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6018615365028381, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20277710258960724, + 0.4401526153087616, + 0.9192003607749939, + 0.3376517593860626, + 0.045962266623973846, + 0.09374761581420898 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.24993054568767548, + "style_target": 0, + "support_probability": 0.10439349710941315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06618868363303447, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6389036178588867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28132104873657227, + 0.5605446100234985, + 0.7152776718139648, + 0.191608265042305, + 0.047250207513570786, + 0.048024386167526245, + 0.03595389798283577, + 0.04083302617073059, + 0.03451746329665184, + 0.088129922747612 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.8581609129905701, + "style_target": 1, + "support_probability": 0.05285452678799629, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009857292847274957, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6844996213912964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35594624280929565, + 0.4604569673538208, + 0.7111197113990784, + 0.1713295876979828, + 0.022787833586335182, + 0.01833643950521946, + 0.05155595391988754, + 0.040760502219200134, + 0.02863464690744877, + 0.07842560857534409 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.07617625594139099, + "style_target": 0, + "support_probability": 0.15345372259616852, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.29642768943373154, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11373385787010193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1618015617132187, + 0.3606014549732208, + 0.16187886893749237, + 0.9275858998298645, + 0.033307164907455444 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 0, + "style_probability": 0.41738903522491455, + "style_target": 1, + "support_probability": 0.4567057192325592, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02617641368812919, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2091054469347, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08242755383253098, + 0.32987266778945923, + 0.13413819670677185, + 0.9833879470825195, + 0.05527877062559128 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 1, + "style_probability": 0.06856448203325272, + "style_target": 0, + "support_probability": 0.22583216428756714, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12042669913591376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6997000575065613, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1461024284362793, + 0.9185872077941895, + 0.24475866556167603, + 0.1461024284362793, + 0.06279751658439636, + 0.0894022211432457 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 1, + "style_probability": 0.5855914950370789, + "style_target": 1, + "support_probability": 0.10340549051761627, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033568199315977466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9770499467849731, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17439118027687073, + 0.9444020986557007, + 0.24568723142147064, + 0.17439118027687073, + 0.11229871213436127, + 0.14719298481941223 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 0, + "style_probability": 0.8463519215583801, + "style_target": 0, + "support_probability": 0.00991964340209961, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4135136258812259, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13931697607040405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8568311333656311, + 0.6683430671691895, + 0.5616521239280701, + 0.5372872948646545, + 0.048689521849155426, + 0.03295455500483513, + 0.0784287303686142, + 0.0966506376862526, + 0.09900375455617905 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 0, + "style_probability": 0.49940282106399536, + "style_target": 1, + "support_probability": 0.5313540697097778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26641961280226895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08221954107284546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8326894640922546, + 0.6784189343452454, + 0.5552780032157898, + 0.544582724571228, + 0.03949381411075592, + 0.10090766847133636 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 1, + "style_probability": 0.24610719084739685, + "style_target": 0, + "support_probability": 0.5366272926330566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3812776021787045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3266432285308838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.574550211429596, + 0.7878307700157166, + 0.4608757495880127, + 0.34136635065078735, + 0.0771690309047699, + 0.04237886890769005, + 0.028110317885875702, + 0.027125682681798935, + 0.03364857658743858, + 0.02890380099415779 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 1, + "style_probability": 0.5191940069198608, + "style_target": 1, + "support_probability": 0.7292490005493164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028767627825344075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30405113101005554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5916118621826172, + 0.7789146900177002, + 0.5689188241958618, + 0.30640527606010437, + 0.015868734568357468, + 0.022726990282535553, + 0.010684000328183174, + 0.03928818553686142, + 0.03549843654036522, + 0.03528657928109169 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 1, + "style_probability": 0.05517956614494324, + "style_target": 0, + "support_probability": 0.8044079542160034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12051340793137712, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6799124479293823, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21280750632286072, + 0.31012681126594543, + 0.3730788826942444, + 0.6351551413536072, + 0.015232720412313938 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.9156574606895447, + "style_target": 1, + "support_probability": 0.05803123861551285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02436198673668769, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6381513476371765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18995647132396698, + 0.31617558002471924, + 0.3970344662666321, + 0.7271410822868347, + 0.07090539485216141, + 0.030876854434609413, + 0.04898884892463684, + 0.04912376403808594, + 0.10372153669595718, + 0.03426061198115349, + 0.23304900527000427 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.37375766038894653, + "style_target": 0, + "support_probability": 0.02239200472831726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7466567804822176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37574300169944763, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5185479521751404, + 0.3539424240589142, + 0.8822718262672424, + 0.1608312577009201, + 0.02616991102695465, + 0.007746837567538023 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.9115976095199585, + "style_target": 1, + "support_probability": 0.88288414478302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18380894979775603, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5211164951324463, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43864402174949646, + 0.2805075943470001, + 0.917643129825592, + 0.13109296560287476, + 0.01429363340139389, + 0.07461170852184296 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.15066221356391907, + "style_target": 0, + "support_probability": 0.8235601186752319, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8846556997694722, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01929974928498268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9935917258262634, + 0.32830166816711426, + 0.22047384083271027, + 0.06578334420919418, + 0.023341398686170578, + 0.09208162128925323, + 0.05789877474308014, + 0.05613191798329353, + 0.019689608365297318 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.824891984462738, + "style_target": 1, + "support_probability": 0.9359068274497986, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7580546962262309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03050290048122406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.983299970626831, + 0.24018332362174988, + 0.18727871775627136, + 0.08849837630987167, + 0.03226243332028389, + 0.08642392605543137, + 0.021025702357292175, + 0.04574515298008919, + 0.01310031209141016 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.49158620834350586, + "style_target": 0, + "support_probability": 0.9448767304420471, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4933969637179591, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2898537218570709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3676532506942749, + 0.41685184836387634, + 0.7274658679962158, + 0.29963192343711853, + 0.04621945321559906, + 0.06676656752824783, + 0.013475766405463219, + 0.03664503991603851, + 0.056468408554792404, + 0.06423703581094742, + 0.011179274879395962 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.9128967523574829, + "style_target": 1, + "support_probability": 0.45286741852760315, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3555759765262354, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10056624561548233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3776594400405884, + 0.5214440226554871, + 0.713935911655426, + 0.16907012462615967, + 0.6497489809989929, + 0.1429964303970337, + 0.030251089483499527, + 0.05182555690407753, + 0.08085473626852036 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.28098106384277344, + "style_target": 0, + "support_probability": 0.6613911390304565, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04749422118274793, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.41545018553733826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12529338896274567, + 0.2862083613872528, + 0.1595771461725235, + 0.5963764786720276, + 0.029521845281124115 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 0, + "style_probability": 0.26567161083221436, + "style_target": 1, + "support_probability": 0.13611310720443726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03386608057502031, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.276867151260376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13106922805309296, + 0.3601611852645874, + 0.12197677046060562, + 0.7605429887771606, + 0.025789404287934303 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 1, + "style_probability": 0.16221480071544647, + "style_target": 0, + "support_probability": 0.4164228141307831, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5276701118460783, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2067553997039795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6632611155509949, + 0.9120531678199768, + 0.4173831343650818, + 0.6162452697753906, + 0.03402712568640709, + 0.01601870357990265 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 0, + "style_probability": 0.35971370339393616, + "style_target": 1, + "support_probability": 0.9030047655105591, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6707017568477474, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22727154195308685, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5413565039634705, + 0.8685480952262878, + 0.3860292136669159, + 0.6002457141876221, + 0.013771653175354004, + 0.013315527699887753 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 0, + "style_probability": 0.5496984720230103, + "style_target": 0, + "support_probability": 0.9405187368392944, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1672159901013094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45684948563575745, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7120291590690613, + 0.25594544410705566, + 0.6388914585113525, + 0.9396506547927856, + 0.02797461859881878, + 0.08278495818376541, + 0.06368812918663025, + 0.03394785895943642, + 0.06644774973392487 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 0, + "style_probability": 0.4994572401046753, + "style_target": 1, + "support_probability": 0.21472600102424622, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24307026722691344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5864730477333069, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.640134334564209, + 0.25021079182624817, + 0.5785958170890808, + 0.9390201568603516, + 0.020988326519727707, + 0.017123974859714508, + 0.008706106804311275, + 0.013504424132406712, + 0.058057792484760284 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 0, + "style_probability": 0.8061835169792175, + "style_target": 0, + "support_probability": 0.20454354584217072, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2451980001134034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.059758611023426056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33618414402008057, + 0.5805597901344299, + 0.6165133714675903, + 0.9165411591529846, + 0.03222348913550377, + 0.04759920760989189, + 0.01823946088552475, + 0.10936246812343597, + 0.0632157251238823, + 0.03085296042263508, + 0.04361898452043533 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 1, + "style_probability": 0.5278493762016296, + "style_target": 1, + "support_probability": 0.23023438453674316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03648045716888661, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11226923018693924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3452202379703522, + 0.45075300335884094, + 0.577126145362854, + 0.9134631752967834, + 0.07233012467622757, + 0.03944436088204384, + 0.06857741624116898, + 0.05043068900704384, + 0.028127890080213547, + 0.023011017590761185 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 1, + "style_probability": 0.3260444402694702, + "style_target": 0, + "support_probability": 0.09506716579198837, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5779010067045726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.048371322453022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3265807032585144, + 0.24977225065231323, + 0.3265807032585144, + 0.9680020213127136, + 0.026989145204424858 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 0, + "style_probability": 0.3989402651786804, + "style_target": 1, + "support_probability": 0.7407521605491638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31069345184649466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0533856637775898, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32971569895744324, + 0.22096803784370422, + 0.32971569895744324, + 0.9843820929527283, + 0.07426873594522476 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 1, + "style_probability": 0.22659234702587128, + "style_target": 0, + "support_probability": 0.572390615940094, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4549269077418493, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19392982125282288, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.345806747674942, + 0.7842605710029602, + 0.42803335189819336, + 0.25141507387161255, + 0.03751111030578613, + 0.028302013874053955 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 1, + "style_probability": 0.5618892312049866, + "style_target": 1, + "support_probability": 0.7516363263130188, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7396069530568343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09263518452644348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.374320924282074, + 0.8444440960884094, + 0.4977352023124695, + 0.2797233462333679, + 0.015393710695207119, + 0.020713254809379578 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 0, + "style_probability": 0.9110274910926819, + "style_target": 0, + "support_probability": 0.9000621438026428, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3477682469959572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1382702887058258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45439597964286804, + 0.3496585488319397, + 0.36872604489326477, + 0.9731492400169373, + 0.022852372378110886, + 0.026495646685361862, + 0.02854769490659237, + 0.03714202716946602, + 0.01868634857237339 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 1, + "style_probability": 0.768562376499176, + "style_target": 1, + "support_probability": 0.19543994963169098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06523710037134457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3409945070743561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37111616134643555, + 0.3575895130634308, + 0.3512212634086609, + 0.967553973197937, + 0.04443681612610817, + 0.07528107613325119 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 0, + "style_probability": 0.5916038751602173, + "style_target": 0, + "support_probability": 0.019667213782668114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01803673872801791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8502364754676819, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22620636224746704, + 0.8915948867797852, + 0.3789260685443878, + 0.22620636224746704, + 0.08233810216188431, + 0.08499055355787277, + 0.2223748415708542, + 0.11538819223642349, + 0.04465239495038986, + 0.04053829237818718, + 0.06126890331506729 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 0, + "style_probability": 0.4243026673793793, + "style_target": 1, + "support_probability": 0.01849174126982689, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027514052632756872, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3182925283908844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1592959761619568, + 0.9167640209197998, + 0.3964195251464844, + 0.1592959761619568, + 0.043490953743457794, + 0.022686658427119255, + 0.08758657425642014, + 0.05662769824266434, + 0.022923367097973824, + 0.034720271825790405 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 1, + "style_probability": 0.1475096344947815, + "style_target": 0, + "support_probability": 0.2304178774356842, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22246231727643037, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15566380321979523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21663977205753326, + 0.2547498941421509, + 0.9872860312461853, + 0.17625853419303894, + 0.02520265243947506 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.65386563539505, + "style_target": 1, + "support_probability": 0.22153614461421967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33369475042877966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04745429381728172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2780316174030304, + 0.29185953736305237, + 0.9844552278518677, + 0.21599166095256805, + 0.05622829869389534, + 0.026053406298160553, + 0.04336212947964668, + 0.0394623763859272, + 0.0173714067786932, + 0.01677817292511463, + 0.09448204189538956 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.35861146450042725, + "style_target": 0, + "support_probability": 0.539400041103363, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3872780769596539, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08759874850511551, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6412429213523865, + 0.29330071806907654, + 0.6412429213523865, + 0.8958755731582642, + 0.024212291464209557, + 0.04600385203957558 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.6649454832077026, + "style_target": 1, + "support_probability": 0.3151510953903198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05122044771592699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13448114693164825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7220969796180725, + 0.38473787903785706, + 0.7220969796180725, + 0.9160280227661133, + 0.020920224487781525, + 0.07815723121166229 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.1182619258761406, + "style_target": 0, + "support_probability": 0.29825857281684875, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7374985155661811, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02065117098391056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5551806092262268, + 0.6264707446098328, + 0.49326497316360474, + 0.8287450671195984, + 0.011336343362927437, + 0.011717705056071281, + 0.035532254725694656, + 0.016077525913715363, + 0.01643461175262928 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 1, + "style_probability": 0.9436622262001038, + "style_target": 1, + "support_probability": 0.9845322966575623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.670306831393465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034177422523498535, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.641932487487793, + 0.527113676071167, + 0.576753556728363, + 0.7739564776420593, + 0.009574535302817822, + 0.02048899047076702, + 0.00982305034995079, + 0.015285247005522251, + 0.009343761950731277 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 0, + "style_probability": 0.8001941442489624, + "style_target": 0, + "support_probability": 0.9711986780166626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4753358994046316, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23887979984283447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3490661084651947, + 0.36831140518188477, + 0.3490661084651947, + 0.7399085164070129, + 0.049943387508392334, + 0.0270371213555336, + 0.014632739126682281, + 0.015972577035427094, + 0.06946387887001038, + 0.015980590134859085, + 0.024441611021757126 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.8854745626449585, + "style_target": 1, + "support_probability": 0.4903338551521301, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27415246466098214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23101195693016052, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3154751658439636, + 0.34091874957084656, + 0.3154751658439636, + 0.7811297178268433, + 0.2694449722766876, + 0.026696685701608658, + 0.013180841691792011, + 0.05177221819758415, + 0.08260419219732285 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.2883651852607727, + "style_target": 0, + "support_probability": 0.5436918139457703, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.08136322026482844, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8461626768112183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1790783703327179, + 0.9220447540283203, + 0.40169650316238403, + 0.6024240851402283, + 0.05309976264834404 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 0, + "style_probability": 0.4930039942264557, + "style_target": 1, + "support_probability": 0.01964062638580799, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010419432562746555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9334575533866882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1887882947921753, + 0.9060678482055664, + 0.2131957858800888, + 0.45369067788124084, + 0.08470826596021652 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 1, + "style_probability": 0.19144375622272491, + "style_target": 0, + "support_probability": 0.010385505855083466, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5886483806135467, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11307066679000854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43659064173698425, + 0.28543636202812195, + 0.8991216421127319, + 0.6823358535766602, + 0.030938798561692238, + 0.04775554686784744 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 0, + "style_probability": 0.37939319014549255, + "style_target": 1, + "support_probability": 0.9532375931739807, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7037267237098538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1686350405216217, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.546025812625885, + 0.18778744339942932, + 0.8868231177330017, + 0.6793593168258667, + 0.005291074980050325, + 0.04951447620987892 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 0, + "style_probability": 0.5753664374351501, + "style_target": 0, + "support_probability": 0.9750037789344788, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4766046355420371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12371416389942169, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4890885353088379, + 0.3861510455608368, + 0.22763320803642273, + 0.9741697907447815, + 0.029510172083973885, + 0.04425349459052086, + 0.0681401789188385, + 0.03834371641278267, + 0.10911142826080322 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.2919218838214874, + "style_target": 1, + "support_probability": 0.8433107733726501, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8243133626141752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10910767316818237, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4504840672016144, + 0.41214460134506226, + 0.20447765290737152, + 0.9834824800491333, + 0.055667974054813385, + 0.02330382727086544, + 0.04109106585383415, + 0.05697345361113548, + 0.05505676567554474 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.7861475348472595, + "style_target": 0, + "support_probability": 0.8869267106056213, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.027862364500326055, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9166250824928284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3094237744808197, + 0.4373924136161804, + 0.19865040481090546, + 0.8160050511360168, + 0.14517222344875336, + 0.17651714384555817, + 0.10988490283489227, + 0.43251270055770874, + 0.24083837866783142, + 0.1695488542318344, + 0.23333227634429932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 0, + "style_probability": 0.3254640996456146, + "style_target": 1, + "support_probability": 0.019470090046525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.053151384056692515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8827219009399414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27758246660232544, + 0.3079202473163605, + 0.16878361999988556, + 0.9103571772575378, + 0.11016533523797989, + 0.043420836329460144, + 0.1375996321439743, + 0.05843551456928253, + 0.04635576158761978, + 0.07777206599712372 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 1, + "style_probability": 0.2872178256511688, + "style_target": 0, + "support_probability": 0.21181906759738922, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8582370897641352, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.016659827902913094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4003961980342865, + 0.265927791595459, + 0.9720788598060608, + 0.4003961980342865, + 0.06130954995751381 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 1, + "style_probability": 0.8292986750602722, + "style_target": 1, + "support_probability": 0.9364704489707947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6347940761551013, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0251552052795887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33761003613471985, + 0.17950722575187683, + 0.9324368238449097, + 0.33761003613471985, + 0.04221484810113907 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 1, + "style_probability": 0.49959298968315125, + "style_target": 0, + "support_probability": 0.8262162208557129, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8807074457516316, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04725508391857147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17565254867076874, + 0.13431528210639954, + 0.9996955394744873, + 0.046587713062763214, + 0.1254824846982956, + 0.04897777736186981 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.7958030700683594, + "style_target": 1, + "support_probability": 0.9641797542572021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6482598313673801, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.033164821565151215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24093802273273468, + 0.15806850790977478, + 0.9991458654403687, + 0.05165284499526024, + 0.022006070241332054 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.25893139839172363, + "style_target": 0, + "support_probability": 0.9761272072792053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5934502165014321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029932480305433273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07257499545812607, + 0.9796664118766785, + 0.325832337141037, + 0.043815113604068756, + 0.005170212592929602, + 0.013756908476352692, + 0.02641117572784424, + 0.013790681026875973, + 0.047819122672080994 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.77049720287323, + "style_target": 1, + "support_probability": 0.6444220542907715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06430348341891139, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05373607948422432, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07451613992452621, + 0.9814189672470093, + 0.3166811764240265, + 0.03586474806070328, + 0.019877851009368896, + 0.023626107722520828, + 0.01781821995973587, + 0.028538068756461143, + 0.0293404720723629 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.1404881775379181, + "style_target": 0, + "support_probability": 0.5125120282173157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02290092638779691, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7574934363365173, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3056557774543762, + 0.781856894493103, + 0.08426857739686966, + 0.48713013529777527, + 0.02278571017086506, + 0.030239738523960114, + 0.06461693346500397, + 0.04032675921916962, + 0.016492892056703568, + 0.28792497515678406, + 0.09474796056747437 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 1, + "style_probability": 0.7327789068222046, + "style_target": 1, + "support_probability": 0.003644804237410426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005152676679455214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9562566876411438, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38901135325431824, + 0.779994010925293, + 0.09480810165405273, + 0.5111846923828125, + 0.02104056254029274, + 0.026221899315714836, + 0.04533543810248375, + 0.06017467752099037, + 0.02230655774474144, + 0.032863181084394455, + 0.0986664816737175 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 1, + "style_probability": 0.4376589357852936, + "style_target": 0, + "support_probability": 0.0027105386834591627, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049371385136218415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5394492149353027, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4927915930747986, + 0.8123180866241455, + 0.6418856382369995, + 0.5136365294456482, + 0.024835364893078804 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.8329479694366455, + "style_target": 1, + "support_probability": 0.017591966316103935, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008186197610399484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7232003808021545, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.508240818977356, + 0.8221011757850647, + 0.616448700428009, + 0.5385523438453674, + 0.043216343969106674, + 0.04195023700594902, + 0.039677202701568604, + 0.04681804031133652, + 0.033326223492622375, + 0.03517604246735573 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.3111996352672577, + "style_target": 0, + "support_probability": 0.005157105159014463, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8463531899144451, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07862070202827454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3127623498439789, + 0.21431681513786316, + 0.987011730670929, + 0.3259485960006714, + 0.06273556500673294, + 0.01527175959199667 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.608608067035675, + "style_target": 1, + "support_probability": 0.9271213412284851, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25394207575946254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08746310323476791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30982109904289246, + 0.20189577341079712, + 0.9888514876365662, + 0.3165361285209656, + 0.01625833660364151, + 0.028469355776906013 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.09866024553775787, + "style_target": 0, + "support_probability": 0.8825186491012573, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6895383124581027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2198905497789383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7053107619285583, + 0.9680233597755432, + 0.1840566247701645, + 0.3000117242336273, + 0.023674437776207924, + 0.01934419572353363, + 0.03252003714442253, + 0.05535281077027321, + 0.009381106123328209 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 1, + "style_probability": 0.789950966835022, + "style_target": 1, + "support_probability": 0.7364723086357117, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6281327266899427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2595711350440979, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6530859470367432, + 0.9600896835327148, + 0.23631201684474945, + 0.33277490735054016, + 0.06660036742687225, + 0.026080982759594917, + 0.025500500574707985, + 0.11323732137680054, + 0.08578488230705261 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 0, + "style_probability": 0.6866438984870911, + "style_target": 0, + "support_probability": 0.7911314964294434, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.036229960637202165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8618019819259644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:positive", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.4012186825275421, + 0.6007133722305298, + 0.7412117123603821, + 0.22293132543563843, + 0.019848860800266266, + 0.0400533452630043, + 0.012983713299036026, + 0.011829608120024204, + 0.03870275989174843, + 0.02921494096517563, + 0.1982036530971527, + 0.020554689690470695 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.8313577771186829, + "style_target": 1, + "support_probability": 0.02792755700647831, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009978677879905652, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8978400826454163, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3462130129337311, + 0.5558650493621826, + 0.7385258674621582, + 0.24601997435092926, + 0.10903360694646835, + 0.017582787200808525, + 0.055951908230781555, + 0.07433508336544037, + 0.03343717381358147 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.26234036684036255, + "style_target": 0, + "support_probability": 0.0172658059746027, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6338752190347525, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07491309940814972, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.503136932849884, + 0.8018636703491211, + 0.43106165528297424, + 0.5038362145423889, + 0.018446844071149826 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 0, + "style_probability": 0.45704737305641174, + "style_target": 1, + "support_probability": 0.9400430917739868, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08204796028105668, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17451690137386322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5086182951927185, + 0.76385498046875, + 0.3753538727760315, + 0.30645596981048584, + 0.018969209864735603 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 1, + "style_probability": 0.03927690163254738, + "style_target": 0, + "support_probability": 0.8690078258514404, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5896199888968885, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04357132688164711, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6041751503944397, + 0.9844862818717957, + 0.3420366644859314, + 0.18632103502750397, + 0.0486280582845211, + 0.027383949607610703 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 1, + "style_probability": 0.5415080785751343, + "style_target": 1, + "support_probability": 0.7990890145301819, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.723786426943784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0712745189666748, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5648770332336426, + 0.9814224243164062, + 0.29492273926734924, + 0.20616783201694489, + 0.07428852468729019, + 0.017037490382790565 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 0, + "style_probability": 0.7449954152107239, + "style_target": 0, + "support_probability": 0.8092338442802429, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.024433760049497245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9517227411270142, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11728356033563614, + 0.3029800057411194, + 0.817297637462616, + 0.17008176445960999, + 0.0395209901034832, + 0.05972064286470413, + 0.03924393281340599, + 0.04670436307787895, + 0.10620752722024918 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 0, + "style_probability": 0.3853079378604889, + "style_target": 1, + "support_probability": 0.009868531487882137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03074618831851909, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9717365503311157, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13435757160186768, + 0.2215660810470581, + 0.8106175065040588, + 0.14766760170459747, + 0.03490884229540825, + 0.039745036512613297, + 0.024326154962182045, + 0.0581771582365036, + 0.013782856054604053 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 0, + "style_probability": 0.7646805047988892, + "style_target": 0, + "support_probability": 0.012221128679811954, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3037104672505202, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4176727533340454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11664017289876938, + 0.10947681963443756, + 0.2920631766319275, + 0.9790175557136536, + 0.0530424527823925, + 0.09227912127971649, + 0.019293369725346565, + 0.047004856169223785, + 0.012106241658329964, + 0.03618807718157768, + 0.09618092328310013 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 0, + "style_probability": 0.38862526416778564, + "style_target": 1, + "support_probability": 0.35874998569488525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04154034617149745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22257103025913239, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.10625988990068436, + 0.10492508858442307, + 0.24674466252326965, + 0.9932587146759033, + 0.016688060015439987, + 0.049312569200992584, + 0.03591802716255188, + 0.015473880805075169, + 0.018056180328130722, + 0.03793865814805031 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 1, + "style_probability": 0.04785919561982155, + "style_target": 0, + "support_probability": 0.4756064713001251, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9184064922912694, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0068401615135371685, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28031232953071594, + 0.2673514485359192, + 0.9972967505455017, + 0.3218016028404236, + 0.05220687761902809 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 1, + "style_probability": 0.9172992706298828, + "style_target": 1, + "support_probability": 0.9509139060974121, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7725472704438089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0017368275439366698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29802536964416504, + 0.2189183533191681, + 0.9972623586654663, + 0.25272879004478455, + 0.11021043360233307 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 1, + "style_probability": 0.48055586218833923, + "style_target": 0, + "support_probability": 0.9846410155296326, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02735848848478754, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.925201952457428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06029047444462776, + 0.06029047444462776, + 0.5937488079071045, + 0.3404354751110077, + 0.05878134071826935, + 0.07755690068006516 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.8031348586082458, + "style_target": 1, + "support_probability": 0.021050749346613884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05240658328738427, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6898074150085449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06144361570477486, + 0.06144361570477486, + 0.5397186875343323, + 0.28593710064888, + 0.014576652087271214 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.3511408865451813, + "style_target": 0, + "support_probability": 0.07687274366617203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2705552208516285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47605571150779724, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6444678902626038, + 0.7635638117790222, + 0.06516855210065842, + 0.5099734663963318, + 0.030213797464966774, + 0.031025661155581474, + 0.018699420616030693, + 0.042055997997522354, + 0.055476825684309006 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.6796704530715942, + "style_target": 1, + "support_probability": 0.20861811935901642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013449764963407364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4920090138912201, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.625130295753479, + 0.7717727422714233, + 0.07342609018087387, + 0.5018318891525269, + 0.03474997729063034, + 0.14394964277744293, + 0.0490349680185318, + 0.02331189252436161, + 0.04515989124774933 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.07499289512634277, + "style_target": 0, + "support_probability": 0.15200559794902802, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.871662345994065, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012229817919433117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5561632513999939, + 0.8441164493560791, + 0.9534971117973328, + 0.259451299905777, + 0.08367644250392914, + 0.024766329675912857, + 0.021610558032989502, + 0.011256251484155655, + 0.05868890509009361, + 0.06433909386396408 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.896520733833313, + "style_target": 1, + "support_probability": 0.9835247993469238, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8270155695607126, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006024204194545746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6003661751747131, + 0.7564225196838379, + 0.9303832650184631, + 0.36951127648353577, + 0.012069120071828365, + 0.011522847227752209, + 0.012493469752371311, + 0.012688356451690197, + 0.02153608202934265, + 0.008358785882592201, + 0.026251837611198425 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.4687131643295288, + "style_target": 0, + "support_probability": 0.9958937168121338, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014163894745809574, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9831505417823792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36059239506721497, + 0.5408232808113098, + 0.5679997205734253, + 0.36059239506721497, + 0.017399029806256294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 1, + "style_probability": 0.6873875856399536, + "style_target": 1, + "support_probability": 0.008097222074866295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014037889407635276, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9876219630241394, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3762555420398712, + 0.5013517141342163, + 0.5180482864379883, + 0.3762555420398712, + 0.031902674585580826 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 0, + "style_probability": 0.9046149253845215, + "style_target": 0, + "support_probability": 0.0031482905615121126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5703461952241954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11896456032991409, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40843164920806885, + 0.5386866927146912, + 0.7648552656173706, + 0.40843164920806885, + 0.030503781512379646, + 0.010973090305924416 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 1, + "style_probability": 0.7232068181037903, + "style_target": 1, + "support_probability": 0.735562801361084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6486173425573251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.052459608763456345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47999876737594604, + 0.5304443836212158, + 0.8558931350708008, + 0.47999876737594604, + 0.010228495113551617 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 0, + "style_probability": 0.5392052531242371, + "style_target": 0, + "support_probability": 0.8242899775505066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42053141624798057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33700335025787354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9006399512290955, + 0.3301781713962555, + 0.43267983198165894, + 0.3301781713962555, + 0.10378377139568329, + 0.013330747373402119, + 0.022274624556303024, + 0.0640963613986969, + 0.02816094644367695 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.6293557286262512, + "style_target": 1, + "support_probability": 0.5992470979690552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03261565367116983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2007773220539093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9337703585624695, + 0.23246373236179352, + 0.2615569233894348, + 0.23246373236179352, + 0.011828995309770107, + 0.02221851795911789, + 0.02086952142417431, + 0.02374517358839512, + 0.034804750233888626 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.11773154139518738, + "style_target": 0, + "support_probability": 0.34622833132743835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8202919564878228, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06793384999036789, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3566:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5631974935531616, + 0.3186853229999542, + 0.9797090888023376, + 0.5631974935531616, + 0.010092220269143581, + 0.0070993490517139435, + 0.016141116619110107, + 0.0028886839281767607, + 0.01962490752339363, + 0.00704013230279088, + 0.007658594287931919 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.4741997718811035, + "style_target": 1, + "support_probability": 0.9775838851928711, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9421533215282208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05743185058236122, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3566:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5421041250228882, + 0.24019360542297363, + 0.9887039065361023, + 0.5421041250228882, + 0.014461628161370754, + 0.008736579678952694, + 0.02203010767698288, + 0.022329293191432953, + 0.022680332884192467, + 0.018318546935915947 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.935211718082428, + "style_target": 0, + "support_probability": 0.974025547504425, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46039827804151034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1035471111536026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38677355647087097, + 0.46336090564727783, + 0.8130142688751221, + 0.27039048075675964, + 0.04864489287137985 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.8471398949623108, + "style_target": 1, + "support_probability": 0.2826533019542694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02251652261839974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1931370198726654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38626325130462646, + 0.43578091263771057, + 0.8044914603233337, + 0.22731590270996094, + 0.04593680426478386 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.07688436657190323, + "style_target": 0, + "support_probability": 0.28484004735946655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21657882767860423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35001659393310547, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5808293223381042, + 0.9635259509086609, + 0.5021873712539673, + 0.4797629415988922, + 0.032909709960222244, + 0.0493820384144783 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6969449520111084, + "style_target": 1, + "support_probability": 0.25499555468559265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1783343989067819, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20137543976306915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5936638712882996, + 0.9228377342224121, + 0.49632638692855835, + 0.49109673500061035, + 0.0297827310860157, + 0.07094671577215195 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 0, + "style_probability": 0.5061641335487366, + "style_target": 0, + "support_probability": 0.22879652678966522, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.845853202254809, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016875075176358223, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43836015462875366, + 0.8173644542694092, + 0.9608510136604309, + 0.06743845343589783, + 0.033961836248636246, + 0.01085673552006483, + 0.052246131002902985, + 0.021023858338594437, + 0.029474342241883278 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.7851459980010986, + "style_target": 1, + "support_probability": 0.93619704246521, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.644222770529468, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02311038039624691, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.509725034236908, + 0.8202799558639526, + 0.9449582695960999, + 0.07901515811681747, + 0.03030337393283844, + 0.025651883333921432 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.4164731502532959, + "style_target": 0, + "support_probability": 0.9266211986541748, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014939549305644282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8915122747421265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8777215480804443, + 0.25810298323631287, + 0.2383449524641037, + 0.19764423370361328, + 0.03851206600666046, + 0.03993307426571846, + 0.03295905143022537, + 0.030322231352329254, + 0.06456887722015381, + 0.09178324788808823 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.8997697234153748, + "style_target": 1, + "support_probability": 0.006136056035757065, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010047586705030458, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5009516477584839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9423576593399048, + 0.273190975189209, + 0.1858544498682022, + 0.2497333139181137, + 0.04572923481464386, + 0.04925382137298584, + 0.2681301534175873, + 0.05365404486656189, + 0.02418246679008007, + 0.06267078220844269 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.17038589715957642, + "style_target": 0, + "support_probability": 0.035470690578222275, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2388877839588214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21747341752052307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3745371103286743, + 0.6056146621704102, + 0.15364906191825867, + 0.07540389150381088, + 0.036866407841444016 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 0, + "style_probability": 0.2610447406768799, + "style_target": 1, + "support_probability": 0.45282095670700073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4459419529430434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1799648255109787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3602297306060791, + 0.6877366304397583, + 0.1696818619966507, + 0.06286453455686569, + 0.011393364518880844 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 1, + "style_probability": 0.3453002870082855, + "style_target": 0, + "support_probability": 0.8242180943489075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02342266379490282, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.959976315498352, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4877183735370636, + 0.5907850861549377, + 0.13118669390678406, + 0.5044738054275513, + 0.04025636240839958, + 0.0765128880739212 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 0, + "style_probability": 0.31211400032043457, + "style_target": 1, + "support_probability": 0.012657450512051582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026229330301424016, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9539434909820557, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5101074576377869, + 0.6348746418952942, + 0.09973170608282089, + 0.5278220772743225, + 0.04948185756802559, + 0.0789608508348465 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 0, + "style_probability": 0.6561598181724548, + "style_target": 0, + "support_probability": 0.015418889001011848, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.12103681524597082, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6722760200500488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.560274064540863, + 0.5966487526893616, + 0.7469143867492676, + 0.32598206400871277, + 0.03551140055060387, + 0.04545210301876068, + 0.02310827188193798, + 0.0324246846139431, + 0.031786106526851654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 0, + "style_probability": 0.34952184557914734, + "style_target": 1, + "support_probability": 0.227921724319458, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0035095204839267926, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.85940021276474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4099441170692444, + 0.7904027700424194, + 0.8301671147346497, + 0.32206055521965027, + 0.05056634172797203, + 0.12839193642139435, + 0.03373933583498001, + 0.037003714591264725, + 0.07063211500644684 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 1, + "style_probability": 0.0437951385974884, + "style_target": 0, + "support_probability": 0.05915610119700432, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5499898977873129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06769262254238129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44808709621429443, + 0.3501493036746979, + 0.44808709621429443, + 0.9786125421524048, + 0.021461686119437218, + 0.04424130544066429, + 0.03391917422413826, + 0.017161741852760315, + 0.041088033467531204, + 0.03134562075138092 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 0, + "style_probability": 0.41053324937820435, + "style_target": 1, + "support_probability": 0.7669573426246643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5385674482657896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1358070820569992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.46640440821647644, + 0.38389915227890015, + 0.46640440821647644, + 0.9712342023849487, + 0.014804556034505367, + 0.05643503740429878, + 0.015875035896897316, + 0.01592879183590412, + 0.021557642146945, + 0.02990032732486725, + 0.02762429416179657 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 0, + "style_probability": 0.689909815788269, + "style_target": 0, + "support_probability": 0.5521159768104553, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0617868507817548, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31789344549179077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5338917970657349, + 0.8266302347183228, + 0.29517602920532227, + 0.16025635600090027, + 0.06057145819067955 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.8061705827713013, + "style_target": 1, + "support_probability": 0.023494482040405273, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033725722324630024, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42795753479003906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5391147136688232, + 0.8084385991096497, + 0.29602542519569397, + 0.17134711146354675, + 0.031841129064559937, + 0.09633872658014297, + 0.11616809666156769, + 0.07137531787157059, + 0.11755383759737015, + 0.10527313500642776, + 0.08886296302080154 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.4297941029071808, + "style_target": 0, + "support_probability": 0.022394811734557152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03789873839203011, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.641107976436615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22929321229457855, + 0.351007342338562, + 0.4998031556606293, + 0.5393679738044739, + 0.021863088011741638, + 0.01863953471183777 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.8250036835670471, + "style_target": 1, + "support_probability": 0.015190325677394867, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016560800141358584, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.780959963798523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19514130055904388, + 0.2847691476345062, + 0.3704066276550293, + 0.4583653211593628, + 0.03024192899465561, + 0.11729411780834198 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.07016605883836746, + "style_target": 0, + "support_probability": 0.014547818340361118, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002619836599806427, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9883628487586975, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35713985562324524, + 0.31802302598953247, + 0.40471866726875305, + 0.6142594814300537, + 0.021627800539135933, + 0.08370315283536911, + 0.09335391968488693, + 0.032331209629774094, + 0.05200309306383133 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 1, + "style_probability": 0.708273708820343, + "style_target": 1, + "support_probability": 0.0005566937034018338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0055690925586514934, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9641805291175842, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37378790974617004, + 0.3899672031402588, + 0.44897204637527466, + 0.6026150584220886, + 0.22585248947143555, + 0.09635747969150543, + 0.08320454508066177, + 0.03911355510354042, + 0.07696212083101273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 0, + "style_probability": 0.5557665228843689, + "style_target": 0, + "support_probability": 0.0012783738784492016, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.45067706425422094, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2703027129173279, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2832438349723816, + 0.154267355799675, + 0.43658414483070374, + 0.2832438349723816, + 0.15157705545425415, + 0.030098402872681618, + 0.016337944194674492, + 0.01053005550056696, + 0.024713309481739998, + 0.009492117911577225, + 0.011403272859752178 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.9505809545516968, + "style_target": 1, + "support_probability": 0.8716434836387634, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3521283977165143, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06592487543821335, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2732136845588684, + 0.11240541934967041, + 0.44926539063453674, + 0.2732136845588684, + 0.008187917992472649, + 0.051949333399534225, + 0.014663124457001686, + 0.004512071143835783, + 0.052093569189310074 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.23486338555812836, + "style_target": 0, + "support_probability": 0.967552661895752, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5676461241698018, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.017858220264315605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16466714441776276, + 0.9026145339012146, + 0.16535140573978424, + 0.20723220705986023, + 0.04761712998151779 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 0, + "style_probability": 0.4087573289871216, + "style_target": 1, + "support_probability": 0.921794056892395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5281768015585878, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010790725238621235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12549574673175812, + 0.9129483103752136, + 0.1598912477493286, + 0.2247212827205658, + 0.018789414316415787, + 0.04268280044198036, + 0.040904734283685684, + 0.1335359364748001, + 0.04228822514414787, + 0.01092046219855547, + 0.06777075678110123 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 1, + "style_probability": 0.3449214994907379, + "style_target": 0, + "support_probability": 0.9243299961090088, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3002834985219143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3268047869205475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3225960433483124, + 0.49115630984306335, + 0.21960997581481934, + 0.8938956260681152, + 0.2278980016708374, + 0.04656656086444855 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.5422479510307312, + "style_target": 1, + "support_probability": 0.3191584348678589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12427352637150932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11175072193145752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21035291254520416, + 0.5423448085784912, + 0.11488997936248779, + 0.9725326895713806, + 0.025254875421524048, + 0.08910457789897919 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.1546594500541687, + "style_target": 0, + "support_probability": 0.6299366354942322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01021192141370687, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9078081846237183, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15894532203674316, + 0.6327330470085144, + 0.1705554872751236, + 0.5954663157463074, + 0.07718914747238159, + 0.08720655739307404, + 0.2633529007434845, + 0.2203965187072754, + 0.06604033708572388 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 0, + "style_probability": 0.3425865173339844, + "style_target": 1, + "support_probability": 0.025776151567697525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18172245058860462, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5595203638076782, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1741207391023636, + 0.7738403081893921, + 0.10650088638067245, + 0.7778884768486023, + 0.13465933501720428, + 0.10020028799772263, + 0.09264884144067764, + 0.10229319334030151, + 0.2152303010225296 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 0, + "style_probability": 0.8686211705207825, + "style_target": 0, + "support_probability": 0.1311902403831482, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4005240352304889, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21741026639938354, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15845829248428345, + 0.09339048713445663, + 0.9402020573616028, + 0.10344529896974564, + 0.02247241511940956, + 0.05035915598273277, + 0.013511864468455315, + 0.0175229012966156, + 0.021430740132927895, + 0.07187224179506302, + 0.03307168558239937 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 1, + "style_probability": 0.578684389591217, + "style_target": 1, + "support_probability": 0.5971436500549316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21871175080489305, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.257632315158844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1292688399553299, + 0.0760432779788971, + 0.9343223571777344, + 0.09792356938123703, + 0.01846064068377018, + 0.018712913617491722, + 0.02582978457212448, + 0.028796106576919556, + 0.025412140414118767 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 1, + "style_probability": 0.2795141935348511, + "style_target": 0, + "support_probability": 0.4036508798599243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22213931303702109, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5409888029098511, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3410707712173462, + 0.6870096325874329, + 0.11735717207193375, + 0.5461038947105408, + 0.04521060362458229 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 0, + "style_probability": 0.47570309042930603, + "style_target": 1, + "support_probability": 0.24247430264949799, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04802281016676849, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.829940676689148, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3727317452430725, + 0.6137035489082336, + 0.1277768313884735, + 0.4834728538990021, + 0.08644520491361618 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 0, + "style_probability": 0.6793062686920166, + "style_target": 0, + "support_probability": 0.039307501167058945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19741932872164233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10043057054281235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22963663935661316, + 0.9372374415397644, + 0.2952667772769928, + 0.3494810461997986, + 0.019737031310796738, + 0.0744139701128006 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.4823693335056305, + "style_target": 1, + "support_probability": 0.31956717371940613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22885459260758353, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1863522082567215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18099345266819, + 0.9390147924423218, + 0.2614406645298004, + 0.29197052121162415, + 0.012000931426882744, + 0.15858638286590576 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.7539138793945312, + "style_target": 0, + "support_probability": 0.23406435549259186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02084859332910804, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.950575590133667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6760385036468506, + 0.3136928081512451, + 0.4407269060611725, + 0.2571670114994049, + 0.048328787088394165, + 0.05025867000222206, + 0.03712152689695358, + 0.05605516582727432, + 0.04079066962003708 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 0, + "style_probability": 0.47241419553756714, + "style_target": 1, + "support_probability": 0.0057552894577383995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013809553009192855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6555483341217041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6501719355583191, + 0.34298115968704224, + 0.7055222988128662, + 0.1644706130027771, + 0.04340798407793045, + 0.08242826163768768, + 0.03537546843290329, + 0.015534477308392525, + 0.07682015746831894 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 1, + "style_probability": 0.28826412558555603, + "style_target": 0, + "support_probability": 0.060134727507829666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2543754451783586, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5175332427024841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5946497321128845, + 0.5946497321128845, + 0.4352167546749115, + 0.5172936320304871, + 0.03780759498476982, + 0.05059673637151718, + 0.013530309312045574, + 0.023645475506782532, + 0.021686473861336708, + 0.011378888040781021, + 0.04408714547753334 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 0, + "style_probability": 0.36371365189552307, + "style_target": 1, + "support_probability": 0.6696891784667969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2588764917361619, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7116451263427734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.475384920835495, + 0.475384920835495, + 0.32969558238983154, + 0.6018435955047607, + 0.025164349004626274, + 0.03222688287496567, + 0.018124958500266075, + 0.017213059589266777, + 0.03136895224452019, + 0.009991184808313847, + 0.022721050307154655 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 1, + "style_probability": 0.43429338932037354, + "style_target": 0, + "support_probability": 0.6807153820991516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8178514580602007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027584142982959747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9339366555213928, + 0.5467997193336487, + 0.16575229167938232, + 0.26602283120155334, + 0.04633704572916031 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.8401814103126526, + "style_target": 1, + "support_probability": 0.8989557027816772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5824582150264103, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026033638045191765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9212729930877686, + 0.4599192142486572, + 0.18196623027324677, + 0.25523796677589417, + 0.0589258186519146, + 0.030648229643702507, + 0.047786835581064224, + 0.01771296188235283, + 0.02677934430539608, + 0.16488958895206451, + 0.025031449273228645 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.3117404580116272, + "style_target": 0, + "support_probability": 0.8990558385848999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016744589794665855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9565051198005676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3584587574005127, + 0.3548957109451294, + 0.4070506989955902, + 0.5616961121559143, + 0.06353525817394257, + 0.17218872904777527 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.8017942309379578, + "style_target": 1, + "support_probability": 0.0001922810188261792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005996328893833714, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9225874543190002, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4266970157623291, + 0.3333420157432556, + 0.4022814631462097, + 0.5264233946800232, + 0.057133641093969345, + 0.06168213114142418 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.1646103709936142, + "style_target": 0, + "support_probability": 0.0003611157007981092, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6287190164556008, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1107901930809021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9310856461524963, + 0.3557094931602478, + 0.3013821840286255, + 0.6613989472389221, + 0.05229458957910538, + 0.05184834823012352, + 0.024245277047157288, + 0.015147393569350243, + 0.058187663555145264 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 1, + "style_probability": 0.8781400918960571, + "style_target": 1, + "support_probability": 0.21545179188251495, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3427303082974427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06485748291015625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8854329586029053, + 0.3247728645801544, + 0.28409209847450256, + 0.7417173981666565, + 0.015056595206260681, + 0.02724481001496315, + 0.011072834022343159, + 0.021791791543364525, + 0.011187226511538029 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 0, + "style_probability": 0.6530377268791199, + "style_target": 0, + "support_probability": 0.323367178440094, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42945867793920994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20581084489822388, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3877289593219757, + 0.1487969160079956, + 0.9252206087112427, + 0.19275741279125214, + 0.030617142096161842, + 0.06790225952863693, + 0.05953915789723396, + 0.03374326974153519, + 0.03147965297102928, + 0.11400437355041504 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.8833198547363281, + "style_target": 1, + "support_probability": 0.13492056727409363, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16487204806525022, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08226826786994934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4845733344554901, + 0.1607820689678192, + 0.9366711974143982, + 0.2253546565771103, + 0.03674840182065964, + 0.06973867863416672, + 0.027010779827833176, + 0.03223384544253349, + 0.09074438363313675 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.3922808766365051, + "style_target": 0, + "support_probability": 0.21562005579471588, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.28707910343013937, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3259522616863251, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45277276635169983, + 0.405391126871109, + 0.8336896896362305, + 0.2699967622756958, + 0.12676912546157837 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.582731306552887, + "style_target": 1, + "support_probability": 0.2612798511981964, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007824419554858356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4871825873851776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5256714224815369, + 0.2578502893447876, + 0.8603681325912476, + 0.2571730613708496, + 0.1056845486164093 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.049649421125650406, + "style_target": 0, + "support_probability": 0.1092931404709816, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.031117990742013244, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6629489660263062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1246272474527359, + 0.35169151425361633, + 0.549751877784729, + 0.18938207626342773, + 0.06609857827425003, + 0.04345203936100006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 1, + "style_probability": 0.7499269843101501, + "style_target": 1, + "support_probability": 0.010328688658773899, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017918639819762272, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6041821241378784, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1466447412967682, + 0.3670543432235718, + 0.516211748123169, + 0.15649262070655823, + 0.0162897277623415, + 0.045874807983636856 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 0, + "style_probability": 0.5016336441040039, + "style_target": 0, + "support_probability": 0.016039161011576653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0304780278048978, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.88694828748703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19646036624908447, + 0.7124424576759338, + 0.4757496416568756, + 0.7459151744842529, + 0.02693231590092182, + 0.05164388194680214, + 0.07693581283092499, + 0.059182919561862946, + 0.055330790579319 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.8359089493751526, + "style_target": 1, + "support_probability": 0.015212384052574635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011848673763719628, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8943541049957275, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19162288308143616, + 0.6694045066833496, + 0.475818932056427, + 0.7700767517089844, + 0.04125852882862091, + 0.08250762522220612 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.2815730571746826, + "style_target": 0, + "support_probability": 0.0080675994977355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8281553747678676, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015357227064669132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9929049611091614, + 0.4099753499031067, + 0.04965215548872948, + 0.1368238776922226, + 0.13394325971603394, + 0.048026151955127716, + 0.025031432509422302, + 0.020829439163208008, + 0.1789982169866562, + 0.37075966596603394, + 0.19762864708900452 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.7931438088417053, + "style_target": 1, + "support_probability": 0.8017029166221619, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25464825378194583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01682761311531067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9956522583961487, + 0.36018744111061096, + 0.041526224464178085, + 0.1470402479171753, + 0.029047507792711258, + 0.01824037916958332, + 0.028565706685185432, + 0.04999389871954918, + 0.038246482610702515, + 0.07227393239736557 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.1090831309556961, + "style_target": 0, + "support_probability": 0.8509628772735596, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5396852673162933, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13616164028644562, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4704998731613159, + 0.09711640328168869, + 0.24287553131580353, + 0.8808948397636414, + 0.03227803856134415 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.7371968030929565, + "style_target": 1, + "support_probability": 0.38164612650871277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36898352006556273, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11788500100374222, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4336112141609192, + 0.10323129594326019, + 0.2453060746192932, + 0.8523913025856018, + 0.016989028081297874 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.3262138068675995, + "style_target": 0, + "support_probability": 0.6163975596427917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8295551139866136, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2313946634531021, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07450733333826065, + 0.14495110511779785, + 0.8187434673309326, + 0.9606387615203857, + 0.011745297349989414, + 0.05847589299082756 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 1, + "style_probability": 0.9751059412956238, + "style_target": 1, + "support_probability": 0.7410690188407898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3708069846838097, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3533715009689331, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0948491096496582, + 0.2058190405368805, + 0.8403377532958984, + 0.9463618397712708, + 0.02482200786471367 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 0, + "style_probability": 0.703510582447052, + "style_target": 0, + "support_probability": 0.5866090059280396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.980871499962252, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.001313709421083331, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9987152814865112, + 0.8606716990470886, + 0.05266546830534935, + 0.054478295147418976, + 0.008126203902065754, + 0.022500930353999138, + 0.011108233593404293, + 0.010119506157934666, + 0.025984136387705803 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.9214475750923157, + "style_target": 1, + "support_probability": 0.9983348250389099, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39757616331174916, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0013659109827131033, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9998536109924316, + 0.8439055681228638, + 0.08725511282682419, + 0.06407911330461502, + 0.02834611013531685, + 0.01659872569143772, + 0.03683328256011009, + 0.01712150312960148, + 0.043613117188215256 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.04121178388595581, + "style_target": 0, + "support_probability": 0.9971139430999756, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025177736802929718, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9933090209960938, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31552013754844666, + 0.22229665517807007, + 0.31552013754844666, + 0.9393104910850525, + 0.09856516122817993, + 0.03712765499949455, + 0.0783223807811737, + 0.06231767311692238, + 0.13920730352401733, + 0.03902721032500267, + 0.06101701781153679 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 1, + "style_probability": 0.7302008867263794, + "style_target": 1, + "support_probability": 0.0009370003244839609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014964784372774124, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9412399530410767, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2690425217151642, + 0.20791670680046082, + 0.2690425217151642, + 0.9356975555419922, + 0.1577468365430832, + 0.16435988247394562, + 0.10919443517923355, + 0.07643984258174896, + 0.3077006936073303, + 0.2495935559272766, + 0.09794645011425018 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 0, + "style_probability": 0.5337525606155396, + "style_target": 0, + "support_probability": 0.009968928061425686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13261811981706256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15303462743759155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3246246576309204, + 0.9270493388175964, + 0.5690770745277405, + 0.5313339233398438, + 0.03402402997016907 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 0, + "style_probability": 0.49248769879341125, + "style_target": 1, + "support_probability": 0.03689257800579071, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18685928682254704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09218606352806091, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28102415800094604, + 0.9406949877738953, + 0.47747355699539185, + 0.5555629134178162, + 0.00817791372537613 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 0, + "style_probability": 0.9210836291313171, + "style_target": 0, + "support_probability": 0.019562501460313797, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025252359455997506, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9125761389732361, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07623102515935898, + 0.08801794052124023, + 0.43643566966056824, + 0.07623102515935898, + 0.08275040984153748, + 0.06665223091840744 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 0, + "style_probability": 0.23597103357315063, + "style_target": 1, + "support_probability": 0.049839094281196594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.042850384497832766, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7713333964347839, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17569592595100403, + 0.16258621215820312, + 0.7217187881469727, + 0.17569592595100403, + 0.0268696378916502, + 0.05941001698374748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 1, + "style_probability": 0.3072608709335327, + "style_target": 0, + "support_probability": 0.2024441808462143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16897546274735314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3025834858417511, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21538636088371277, + 0.4425450265407562, + 0.9123551249504089, + 0.31875473260879517, + 0.05398012325167656, + 0.03006206452846527, + 0.043545156717300415, + 0.04524422436952591, + 0.07850484549999237 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 1, + "style_probability": 0.5014774799346924, + "style_target": 1, + "support_probability": 0.17409716546535492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29243546884187616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1259525865316391, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2084726244211197, + 0.42976003885269165, + 0.9147146344184875, + 0.25755611062049866, + 0.024441873654723167, + 0.0861588716506958, + 0.025173073634505272, + 0.019917331635951996, + 0.03001164086163044 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 0, + "style_probability": 0.6905722618103027, + "style_target": 0, + "support_probability": 0.4196056127548218, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08915479234266961, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9009314775466919, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5910277366638184, + 0.7471393942832947, + 0.7999098896980286, + 0.5910277366638184, + 0.023478897288441658, + 0.02850242331624031, + 0.029996702447533607, + 0.03594035655260086, + 0.021725798025727272, + 0.010572289116680622, + 0.0724196806550026 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.46604853868484497, + "style_target": 1, + "support_probability": 0.08090108633041382, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09453545638723893, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9413777589797974, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6046118140220642, + 0.7729749083518982, + 0.7923323512077332, + 0.6046118140220642, + 0.022718049585819244, + 0.026579927653074265, + 0.02176632359623909, + 0.04979129135608673, + 0.24778492748737335, + 0.035268768668174744, + 0.014067237265408039 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.7243890762329102, + "style_target": 0, + "support_probability": 0.05846291407942772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6567382900683457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13479596376419067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9304171204566956, + 0.18160897493362427, + 0.056814320385456085, + 0.4683401584625244, + 0.04516299441456795 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.8115496635437012, + "style_target": 1, + "support_probability": 0.7489619255065918, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040648090888764886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26457351446151733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9126771092414856, + 0.18354792892932892, + 0.07048847526311874, + 0.4610103964805603, + 0.03991149738430977 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.09837523847818375, + "style_target": 0, + "support_probability": 0.5275824069976807, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6140364076693968, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.26668134331703186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6636623740196228, + 0.2037050873041153, + 0.9246854782104492, + 0.1787354052066803, + 0.012296360917389393, + 0.052969641983509064 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 1, + "style_probability": 0.7935320138931274, + "style_target": 1, + "support_probability": 0.5747114419937134, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36774762507456427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2709093987941742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6587613821029663, + 0.29303184151649475, + 0.9231039881706238, + 0.18353965878486633, + 0.03317366540431976, + 0.038177285343408585 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 1, + "style_probability": 0.47116535902023315, + "style_target": 0, + "support_probability": 0.43481823801994324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12836397542124495, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5705651640892029, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34254515171051025, + 0.5216588973999023, + 0.24482713639736176, + 0.9683037400245667, + 0.07844801247119904, + 0.03371144458651543, + 0.027431074529886246, + 0.05271535366773605, + 0.19342242181301117 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.7626175284385681, + "style_target": 1, + "support_probability": 0.13351106643676758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03397071611416105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5952964425086975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3320998251438141, + 0.5148134827613831, + 0.25122731924057007, + 0.9428709745407104, + 0.06088630482554436, + 0.08592087775468826 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.17711873352527618, + "style_target": 0, + "support_probability": 0.08588415384292603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029297968344336543, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8769400715827942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7443763613700867, + 0.45466339588165283, + 0.7664046287536621, + 0.5013219714164734, + 0.17261844873428345, + 0.06952857226133347, + 0.04590421915054321, + 0.05420294404029846, + 0.012878966517746449, + 0.026647062972187996, + 0.11256834864616394 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8842438459396362, + "style_target": 1, + "support_probability": 0.0062735071405768394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006066852168015363, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.833972156047821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7128015160560608, + 0.46009859442710876, + 0.6718883514404297, + 0.4363577961921692, + 0.1291196197271347, + 0.050896529108285904, + 0.147071972489357, + 0.03348764777183533, + 0.03453806787729263, + 0.055439334362745285 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.15477396547794342, + "style_target": 0, + "support_probability": 0.018812835216522217, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6219902664725088, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.055993784219026566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7150512933731079, + 0.9927075505256653, + 0.34871625900268555, + 0.4586576521396637, + 0.07906842231750488 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.3379085063934326, + "style_target": 1, + "support_probability": 0.9221019148826599, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8326418366186921, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03072344698011875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7405491471290588, + 0.9888449311256409, + 0.38108834624290466, + 0.5510122776031494, + 0.03254867345094681 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.6253817677497864, + "style_target": 0, + "support_probability": 0.9390028119087219, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003898704996254181, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.981063723564148, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1550479680299759, + 0.3478167653083801, + 0.24962535500526428, + 0.32346782088279724, + 0.029890703037381172, + 0.039836082607507706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.3189437687397003, + "style_target": 1, + "support_probability": 8.763669029576704e-05, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005421520336449478, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8939623236656189, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19261126220226288, + 0.3036060035228729, + 0.1989356279373169, + 0.34366944432258606, + 0.2025500386953354, + 0.0559106282889843 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.7872684001922607, + "style_target": 0, + "support_probability": 0.0022500085178762674, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00485678166062373, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9289263486862183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5019817352294922, + 0.22411292791366577, + 0.1088605523109436, + 0.4978337287902832, + 0.03536398336291313, + 0.02266085520386696, + 0.02743867039680481, + 0.015070553869009018, + 0.03329485282301903 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 0, + "style_probability": 0.2298618108034134, + "style_target": 1, + "support_probability": 0.004595811013132334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009339675002764622, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8915596604347229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4984769821166992, + 0.14232204854488373, + 0.09637432545423508, + 0.5092777013778687, + 0.014401525259017944, + 0.07912585884332657, + 0.01539676170796156, + 0.01914575695991516, + 0.052162304520606995 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 1, + "style_probability": 0.2496708780527115, + "style_target": 0, + "support_probability": 0.03524598106741905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5512563723240884, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24876923859119415, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.13371020555496216, + 0.2316746711730957, + 0.887408971786499, + 0.17349426448345184, + 0.07467468082904816, + 0.02548305131494999, + 0.060261741280555725, + 0.055245015770196915, + 0.0382087342441082, + 0.08070456236600876 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 1, + "style_probability": 0.5465072393417358, + "style_target": 1, + "support_probability": 0.7350479960441589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5064321165775233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3250313699245453, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14158384501934052, + 0.2722448408603668, + 0.8346999883651733, + 0.1877678632736206, + 0.01607619784772396, + 0.018966400995850563, + 0.019358564168214798, + 0.014972957782447338, + 0.050309713929891586, + 0.01291961595416069, + 0.05509128421545029 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 0, + "style_probability": 0.7104650139808655, + "style_target": 0, + "support_probability": 0.6187857985496521, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7632260879153296, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.034553393721580505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23837335407733917, + 0.576392412185669, + 0.33400958776474, + 0.9368665218353271, + 0.14065147936344147 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 1, + "style_probability": 0.5935138463973999, + "style_target": 1, + "support_probability": 0.9173218011856079, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8877808528150216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.02038940042257309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25190725922584534, + 0.5932071805000305, + 0.29255256056785583, + 0.9687278866767883, + 0.16409917175769806 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 0, + "style_probability": 0.9009943604469299, + "style_target": 0, + "support_probability": 0.9322360754013062, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5684565705959242, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022677885368466377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9886184930801392, + 0.24799516797065735, + 0.7827370166778564, + 0.39574646949768066, + 0.029093336313962936, + 0.1180647760629654 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 0, + "style_probability": 0.45816805958747864, + "style_target": 1, + "support_probability": 0.8272526264190674, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3059907976813645, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03626296669244766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9918158650398254, + 0.23797118663787842, + 0.7826759219169617, + 0.37979304790496826, + 0.0334295928478241 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 1, + "style_probability": 0.21705354750156403, + "style_target": 0, + "support_probability": 0.6416497230529785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004003600588443868, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9840120077133179, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2169777899980545, + 0.3315769135951996, + 0.31668493151664734, + 0.14502833783626556, + 0.11510900408029556, + 0.03116188570857048, + 0.03953177109360695, + 0.048505477607250214, + 0.058288730680942535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 0, + "style_probability": 0.39901459217071533, + "style_target": 1, + "support_probability": 0.007743528578430414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012527921470073258, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9690203666687012, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12756672501564026, + 0.33225777745246887, + 0.17524708807468414, + 0.09981843829154968, + 0.05002519488334656, + 0.12445085495710373, + 0.12433231621980667, + 0.028651893138885498, + 0.051224786788225174 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 1, + "style_probability": 0.10925961285829544, + "style_target": 0, + "support_probability": 0.02227480709552765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12146827573458081, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6492013931274414, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4282039701938629, + 0.21655379235744476, + 0.7842525839805603, + 0.8054890036582947, + 0.01452264841645956, + 0.016222920268774033, + 0.0663968026638031, + 0.03252199664711952, + 0.03350646048784256, + 0.04539962112903595, + 0.019382091239094734 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.41522544622421265, + "style_target": 1, + "support_probability": 0.25211232900619507, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01259225261394009, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.963739812374115, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.278224915266037, + 0.2225075662136078, + 0.707611083984375, + 0.8269903659820557, + 0.028123511001467705, + 0.03501642122864723, + 0.026587510481476784, + 0.06592418998479843, + 0.061740510165691376, + 0.15331178903579712 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.8128438591957092, + "style_target": 0, + "support_probability": 0.013392300345003605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8495011485462497, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025813302025198936, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9575091004371643, + 0.5075963735580444, + 0.8802912831306458, + 0.1090972051024437, + 0.11408174782991409 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.8989928960800171, + "style_target": 1, + "support_probability": 0.7752175331115723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41922979401975374, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05655869469046593, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.954981803894043, + 0.4839194416999817, + 0.8750373721122742, + 0.09317909181118011, + 0.041905131191015244 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.41591188311576843, + "style_target": 0, + "support_probability": 0.6434110999107361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22531008172482406, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4871971905231476, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18305449187755585, + 0.10864594578742981, + 0.5725272297859192, + 0.18305449187755585, + 0.022311238572001457, + 0.03329935669898987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 1, + "style_probability": 0.7710989117622375, + "style_target": 1, + "support_probability": 0.190983384847641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06536811968784441, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7272278070449829, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20063328742980957, + 0.1280406266450882, + 0.44800466299057007, + 0.20063328742980957, + 0.0511486753821373, + 0.03653072193264961 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 0, + "style_probability": 0.5437127351760864, + "style_target": 0, + "support_probability": 0.10525141656398773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036756720365969796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7467954158782959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2703045904636383, + 0.8098621964454651, + 0.2263602912425995, + 0.696812093257904, + 0.037204302847385406, + 0.11588854342699051, + 0.041807107627391815, + 0.07045464217662811, + 0.1489575356245041 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.7452318072319031, + "style_target": 1, + "support_probability": 0.0625290721654892, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02083839036358381, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8022194504737854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27721747756004333, + 0.8016824722290039, + 0.23755331337451935, + 0.6925468444824219, + 0.06785047799348831, + 0.12403100728988647 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.23834961652755737, + "style_target": 0, + "support_probability": 0.02656145952641964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009343740011867195, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9950748085975647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10533063113689423, + 0.22703273594379425, + 0.6692541241645813, + 0.14576761424541473, + 0.025554988533258438, + 0.12515954673290253, + 0.061715587973594666, + 0.08787611126899719, + 0.06964459270238876, + 0.06898432970046997, + 0.05630125850439072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.6407781839370728, + "style_target": 1, + "support_probability": 0.0006634254241362214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0002401424161400307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9951175451278687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11970721930265427, + 0.2133413404226303, + 0.6148763298988342, + 0.1302420198917389, + 0.06439010798931122, + 0.055290061980485916, + 0.1568869948387146, + 0.07333832234144211, + 0.06187782809138298, + 0.08132574707269669 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.13545501232147217, + "style_target": 0, + "support_probability": 0.000702385907061398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7546383830770612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05185646191239357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16021116077899933, + 0.3591184914112091, + 0.5953342318534851, + 0.946477472782135, + 0.01955024152994156 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.6411476731300354, + "style_target": 1, + "support_probability": 0.9604793787002563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10848312616810235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07857420295476913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19362975656986237, + 0.301347941160202, + 0.5605618953704834, + 0.9809753894805908, + 0.06048930063843727 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.06865673512220383, + "style_target": 0, + "support_probability": 0.9008862972259521, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.48607043145911855, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2201170027256012, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21209241449832916, + 0.10222678631544113, + 0.9282405376434326, + 0.11026788502931595, + 0.14309461414813995, + 0.037434838712215424 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 0, + "style_probability": 0.42726993560791016, + "style_target": 1, + "support_probability": 0.6984288096427917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7820799991825892, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09024955332279205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21761415898799896, + 0.060313306748867035, + 0.9641759991645813, + 0.05607513710856438, + 0.03908742591738701, + 0.02585413306951523 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 0, + "style_probability": 0.6986695528030396, + "style_target": 0, + "support_probability": 0.8442705273628235, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6118075148116526, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08432158082723618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4772554337978363, + 0.9485305547714233, + 0.06945361196994781, + 0.11636760085821152, + 0.015611491166055202, + 0.028594817966222763, + 0.03005620464682579, + 0.036240436136722565, + 0.03875866159796715 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 0, + "style_probability": 0.364445298910141, + "style_target": 1, + "support_probability": 0.8836498260498047, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4422623029500752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06563765555620193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5696677565574646, + 0.9608895182609558, + 0.07893817126750946, + 0.11572872847318649, + 0.054774146527051926, + 0.029888544231653214 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 1, + "style_probability": 0.13944664597511292, + "style_target": 0, + "support_probability": 0.8570876121520996, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.051505830301944666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.624978244304657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15042859315872192, + 0.8029860854148865, + 0.13977587223052979, + 0.18979240953922272, + 0.03657398745417595, + 0.07484152168035507, + 0.025990767404437065, + 0.041245412081480026, + 0.04953009635210037, + 0.067562535405159, + 0.05593137443065643 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 0, + "style_probability": 0.3389640748500824, + "style_target": 1, + "support_probability": 0.1348980963230133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008362603265927512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4339216351509094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15253609418869019, + 0.9302796721458435, + 0.10199706256389618, + 0.24768401682376862, + 0.0905369371175766, + 0.07497835904359818, + 0.27097374200820923, + 0.04962253198027611, + 0.03883888199925423, + 0.08048874139785767 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 1, + "style_probability": 0.0752047672867775, + "style_target": 0, + "support_probability": 0.1739293932914734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.510214046173639, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14802615344524384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11187843233346939, + 0.3389222025871277, + 0.837601363658905, + 0.11187843233346939, + 0.045727651566267014 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 1, + "style_probability": 0.5612301230430603, + "style_target": 1, + "support_probability": 0.5876508951187134, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45850801670542396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2560775876045227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11161915212869644, + 0.2811723053455353, + 0.7760691046714783, + 0.11161915212869644, + 0.05865782871842384 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 0, + "style_probability": 0.9337072372436523, + "style_target": 0, + "support_probability": 0.23988401889801025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3210312324530337, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3603171408176422, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6303601861000061, + 0.6207504272460938, + 0.4834015369415283, + 0.8079006671905518, + 0.03186855465173721, + 0.014155320823192596 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.7633146643638611, + "style_target": 1, + "support_probability": 0.3462456464767456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22917881608086746, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19202587008476257, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6415417790412903, + 0.6363388895988464, + 0.39880985021591187, + 0.7617003917694092, + 0.031473178416490555 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.391141414642334, + "style_target": 0, + "support_probability": 0.3686178922653198, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.43031312751849266, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12084466218948364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9562833309173584, + 0.36565902829170227, + 0.2578853368759155, + 0.36565902829170227, + 0.024940215051174164, + 0.03913886845111847, + 0.14645035564899445, + 0.13731792569160461, + 0.0407804474234581 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 0, + "style_probability": 0.43391481041908264, + "style_target": 1, + "support_probability": 0.6392860412597656, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12337506578969319, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05157691612839699, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9778557419776917, + 0.2917044162750244, + 0.1598210632801056, + 0.2917044162750244, + 0.04255157336592674, + 0.09504596143960953, + 0.030111433938145638, + 0.023893533274531364, + 0.09998351335525513 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 1, + "style_probability": 0.04970826581120491, + "style_target": 0, + "support_probability": 0.9088611006736755, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.039996217545672, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8610178232192993, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3132086396217346, + 0.3132086396217346, + 0.7634764909744263, + 0.39963629841804504, + 0.042808808386325836, + 0.020858129486441612, + 0.010850229300558567, + 0.11699645221233368, + 0.02633107639849186, + 0.02888280153274536 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.4321187138557434, + "style_target": 1, + "support_probability": 0.03652703016996384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012912954715547357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9198669791221619, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2292885035276413, + 0.2292885035276413, + 0.7685331106185913, + 0.32651734352111816, + 0.03405923768877983, + 0.02070876955986023, + 0.04356014356017113, + 0.03595447540283203, + 0.06555592268705368, + 0.06283953785896301, + 0.02806985005736351 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.6408342719078064, + "style_target": 0, + "support_probability": 0.013426480814814568, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5088829782117951, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.693289041519165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24720187485218048, + 0.06631778180599213, + 0.5587178468704224, + 0.9852907657623291, + 0.029441457241773605 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.798244059085846, + "style_target": 1, + "support_probability": 0.3751395344734192, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05614044500393749, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6445990204811096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.185278981924057, + 0.07460910826921463, + 0.5605360269546509, + 0.9914548397064209, + 0.08267071843147278 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.16148439049720764, + "style_target": 0, + "support_probability": 0.33267977833747864, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7877073428706611, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11707565933465958, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1198497787117958, + 0.5008550882339478, + 0.9868038296699524, + 0.1198497787117958, + 0.04736890643835068, + 0.05216436833143234 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 1, + "style_probability": 0.6642175316810608, + "style_target": 1, + "support_probability": 0.7937588691711426, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7204289285301114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0767291933298111, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1339842826128006, + 0.4686824083328247, + 0.9748339653015137, + 0.1339842826128006, + 0.007772364187985659, + 0.031458280980587006 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 0, + "style_probability": 0.5969544649124146, + "style_target": 0, + "support_probability": 0.8804196715354919, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015278716224825684, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9949277639389038, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10209401696920395, + 0.41117405891418457, + 0.13814766705036163, + 0.7182444930076599, + 0.04928072169423103, + 0.07172521948814392, + 0.1519239842891693, + 0.08782533556222916, + 0.07515881955623627 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.7615597248077393, + "style_target": 1, + "support_probability": 0.0007165580173023045, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0038100652239783836, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9970519542694092, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10659117996692657, + 0.43908193707466125, + 0.17536969482898712, + 0.6955041885375977, + 0.12978141009807587, + 0.11489058285951614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.3571842908859253, + "style_target": 0, + "support_probability": 0.000984717858955264, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14189506165753832, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7763311266899109, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2630326747894287, + 0.8984189629554749, + 0.8729028105735779, + 0.2734799087047577, + 0.0782659500837326, + 0.041017986834049225, + 0.024677488952875137, + 0.06601379811763763, + 0.06086762249469757, + 0.03532334789633751 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.9407609105110168, + "style_target": 1, + "support_probability": 0.044729214161634445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012339003394106754, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24630317091941833, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2633865773677826, + 0.9359473586082458, + 0.9162878394126892, + 0.24029284715652466, + 0.07023140788078308, + 0.04230448603630066, + 0.3600190281867981, + 0.10345815122127533, + 0.04262333735823631, + 0.08920302987098694 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.07164430618286133, + "style_target": 0, + "support_probability": 0.15196622908115387, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5324517468319694, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.047523338347673416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2367790788412094, + 0.9917884469032288, + 0.12409812211990356, + 0.05336453765630722, + 0.014111503027379513 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.33054882287979126, + "style_target": 1, + "support_probability": 0.7993061542510986, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7027838440984702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0662362352013588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2750001549720764, + 0.9883639812469482, + 0.14083482325077057, + 0.10386203229427338, + 0.07140813022851944 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.6302049160003662, + "style_target": 0, + "support_probability": 0.705260694026947, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03326960370921992, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9365589022636414, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5888497233390808, + 0.448292076587677, + 0.48339372873306274, + 0.6621799468994141, + 0.06609281152486801, + 0.015642695128917694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.4410722851753235, + "style_target": 1, + "support_probability": 0.043157365173101425, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11253090144433996, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9074191451072693, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6351820230484009, + 0.40302774310112, + 0.4282291531562805, + 0.5865845084190369, + 0.02307172305881977, + 0.04273590072989464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.8696273565292358, + "style_target": 0, + "support_probability": 0.06613979488611221, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01611686661978338, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8348536491394043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29421207308769226, + 0.22415748238563538, + 0.20781539380550385, + 0.5510753393173218, + 0.08277968317270279, + 0.30203214287757874, + 0.20706099271774292, + 0.16451576352119446, + 0.16616520285606384 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 0, + "style_probability": 0.38252338767051697, + "style_target": 1, + "support_probability": 0.02813871018588543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01748468007820673, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23388543725013733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.515860915184021, + 0.19347260892391205, + 0.28176987171173096, + 0.6491678357124329, + 0.10542179644107819, + 0.11309879273176193, + 0.12807303667068481, + 0.03937678411602974, + 0.10181058943271637 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 1, + "style_probability": 0.17158421874046326, + "style_target": 0, + "support_probability": 0.11474408954381943, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6910513282922477, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.018251465633511543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9436327219009399, + 0.5552806258201599, + 0.4020403027534485, + 0.5209246277809143, + 0.016068408265709877, + 0.04826967418193817, + 0.010972573421895504, + 0.05488845705986023, + 0.02143547125160694, + 0.021936587989330292, + 0.01685292273759842 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 0, + "style_probability": 0.2834831774234772, + "style_target": 1, + "support_probability": 0.9787667989730835, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7444813124713153, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07118259370326996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9266031980514526, + 0.5784893035888672, + 0.4955957233905792, + 0.5844985842704773, + 0.10006871819496155, + 0.04336757957935333, + 0.05576173588633537, + 0.02570628933608532, + 0.026562826707959175, + 0.02909565158188343, + 0.04673158377408981 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 1, + "style_probability": 0.47765013575553894, + "style_target": 0, + "support_probability": 0.9579991102218628, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6445442025022253, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.039434030652046204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31196996569633484, + 0.25685685873031616, + 0.9886150360107422, + 0.15456773340702057, + 0.03128897398710251 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6863992810249329, + "style_target": 1, + "support_probability": 0.41655316948890686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35024924534102836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04125513136386871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3126455247402191, + 0.2785388231277466, + 0.9841020703315735, + 0.19310599565505981, + 0.07257124036550522, + 0.048694316297769547, + 0.04057902842760086, + 0.043495938181877136, + 0.03280557692050934, + 0.04140099138021469, + 0.04811468720436096 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.3133748173713684, + "style_target": 0, + "support_probability": 0.4415739178657532, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.872457754723782, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04698792099952698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18314507603645325, + 0.44115689396858215, + 0.9858130812644958, + 0.24886535108089447, + 0.015038009732961655, + 0.062447674572467804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.7893933653831482, + "style_target": 1, + "support_probability": 0.9324091076850891, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23201129638010412, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07345077395439148, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19509078562259674, + 0.49785691499710083, + 0.991533100605011, + 0.2204141467809677, + 0.04737551137804985, + 0.08961521834135056 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.059150248765945435, + "style_target": 0, + "support_probability": 0.9155763983726501, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6277902256915283, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3237329125404358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.759000301361084, + 0.19417162239551544, + 0.07534497231245041, + 0.8830393552780151, + 0.045041970908641815, + 0.019872218370437622, + 0.028389738872647285, + 0.0198721531778574, + 0.0232642013579607 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 1, + "style_probability": 0.7831505537033081, + "style_target": 1, + "support_probability": 0.5623052716255188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5586979636730346, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3627433180809021, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7211158275604248, + 0.16531100869178772, + 0.08268191665410995, + 0.8748834133148193, + 0.03720363229513168, + 0.022427603602409363, + 0.021993326023221016, + 0.05624830722808838, + 0.015353758819401264 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 0, + "style_probability": 0.575169563293457, + "style_target": 0, + "support_probability": 0.7359595894813538, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16417774578661895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6705177426338196, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5462484955787659, + 0.1783100664615631, + 0.908835232257843, + 0.5462484955787659, + 0.06875002384185791, + 0.03860551491379738, + 0.07580156624317169, + 0.02895672619342804, + 0.09936586767435074, + 0.02537575736641884, + 0.025911647826433182 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.9175502061843872, + "style_target": 1, + "support_probability": 0.09041997790336609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18657741350996032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4678654074668884, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5362170338630676, + 0.16244444251060486, + 0.923687756061554, + 0.5362170338630676, + 0.14752605557441711, + 0.0849093571305275, + 0.05002498999238014, + 0.04752250388264656, + 0.03385357931256294 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.37724360823631287, + "style_target": 0, + "support_probability": 0.2612052261829376, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5399969635835283, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6464113593101501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7824147343635559, + 0.06742704659700394, + 0.4420057237148285, + 0.2191701978445053, + 0.020549984648823738 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.8736955523490906, + "style_target": 1, + "support_probability": 0.5496063828468323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028156522280905167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6698482036590576, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.79694002866745, + 0.06013234332203865, + 0.30359649658203125, + 0.1563532054424286, + 0.04587162658572197 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.09995979070663452, + "style_target": 0, + "support_probability": 0.41842904686927795, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9367006737587574, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01641397550702095, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4101:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14284932613372803, + 0.9901626110076904, + 0.27720266580581665, + 0.14284932613372803, + 0.02483580820262432, + 0.03512497991323471 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 1, + "style_probability": 0.6900779008865356, + "style_target": 1, + "support_probability": 0.988646388053894, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8384967600537471, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054921556264162064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4101:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19150394201278687, + 0.9802699089050293, + 0.24471326172351837, + 0.19150394201278687, + 0.048574551939964294, + 0.25182345509529114 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 1, + "style_probability": 0.48131757974624634, + "style_target": 0, + "support_probability": 0.9607616662979126, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8658723095610156, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0205007866024971, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9955396056175232, + 0.48233333230018616, + 0.18065163493156433, + 0.2694416344165802, + 0.026478296145796776, + 0.012638788670301437, + 0.015248943120241165, + 0.012453258968889713, + 0.014946608804166317 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.8017299771308899, + "style_target": 1, + "support_probability": 0.9180237054824829, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38826663068804185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05878829583525658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9938481450080872, + 0.4554827809333801, + 0.18065245449543, + 0.26755213737487793, + 0.03174251317977905, + 0.017112702131271362 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.2860235869884491, + "style_target": 0, + "support_probability": 0.7864331603050232, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3410374847483159, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08609700202941895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19882452487945557, + 0.9727950096130371, + 0.6186944246292114, + 0.1047457754611969, + 0.008061278611421585, + 0.03033524565398693, + 0.009488504379987717, + 0.009648125618696213, + 0.011629786342382431, + 0.01759718358516693 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 0, + "style_probability": 0.47133588790893555, + "style_target": 1, + "support_probability": 0.5126147270202637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06753154879984402, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11114297807216644, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15858115255832672, + 0.9837160110473633, + 0.6800938844680786, + 0.0979471281170845, + 0.022188624367117882, + 0.019061140716075897, + 0.024002376943826675, + 0.027438560500741005, + 0.02567298524081707, + 0.02276475541293621 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 1, + "style_probability": 0.10453174263238907, + "style_target": 0, + "support_probability": 0.5238925814628601, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7829354333055615, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.099044069647789, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3403923213481903, + 0.3793947994709015, + 0.4486779272556305, + 0.9778168797492981, + 0.03236829489469528 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 1, + "style_probability": 0.8447507619857788, + "style_target": 1, + "support_probability": 0.8156703114509583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5448930650159765, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1317368745803833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3219583332538605, + 0.3916952311992645, + 0.4027545154094696, + 0.9671354293823242, + 0.053330279886722565 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 1, + "style_probability": 0.3864603042602539, + "style_target": 0, + "support_probability": 0.7956249713897705, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.836575954978763, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04913103207945824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32893481850624084, + 0.9711368083953857, + 0.1698748767375946, + 0.2601105272769928, + 0.028511662036180496, + 0.03325725719332695 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.8014993667602539, + "style_target": 1, + "support_probability": 0.7155444622039795, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5180480326849812, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.041344575583934784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27521252632141113, + 0.9759993553161621, + 0.18674145638942719, + 0.28639331459999084, + 0.025054657831788063 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.11183777451515198, + "style_target": 0, + "support_probability": 0.8664060831069946, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2990715577893224, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07527462393045425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14185528457164764, + 0.1607297956943512, + 0.9650832414627075, + 0.06736104935407639, + 0.09173234552145004, + 0.026153942570090294, + 0.06446629762649536, + 0.033891577273607254, + 0.04533449932932854 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.7431240081787109, + "style_target": 1, + "support_probability": 0.11635693162679672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05978569291621152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02023230493068695, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1306568682193756, + 0.1947222501039505, + 0.9730237722396851, + 0.06301309168338776, + 0.02584773488342762, + 0.04764818027615547, + 0.03761908784508705, + 0.02071843296289444, + 0.04435288533568382 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.16849540174007416, + "style_target": 0, + "support_probability": 0.15129831433296204, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8493195516170345, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004599243868142366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2650962173938751, + 0.6074605584144592, + 0.11783145368099213, + 0.9999011754989624, + 0.017717542126774788, + 0.02780340053141117, + 0.08431574702262878, + 0.15844477713108063, + 0.04786865785717964, + 0.010749442502856255, + 0.022693980485200882 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 1, + "style_probability": 0.6963603496551514, + "style_target": 1, + "support_probability": 0.9741244316101074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6532788165392014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008047551847994328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.317047655582428, + 0.6029393076896667, + 0.13619859516620636, + 0.9992214441299438, + 0.08717606216669083, + 0.029382994398474693, + 0.03458482399582863, + 0.01905578374862671, + 0.02513183280825615, + 0.05191656947135925, + 0.04232921078801155 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 0, + "style_probability": 0.599472165107727, + "style_target": 0, + "support_probability": 0.9332872629165649, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6927935568262793, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1546623855829239, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24391324818134308, + 0.9330416321754456, + 0.20085974037647247, + 0.412765234708786, + 0.04242951050400734 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.39961928129196167, + "style_target": 1, + "support_probability": 0.9462684392929077, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8551939206319408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.053685661405324936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27887266874313354, + 0.9663990139961243, + 0.1477987915277481, + 0.2284228503704071, + 0.011236419901251793 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.6353837251663208, + "style_target": 0, + "support_probability": 0.9489060044288635, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5687782052371064, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08144024759531021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16284295916557312, + 0.08852587640285492, + 0.16284295916557312, + 0.795157790184021, + 0.020221956074237823, + 0.02023111842572689 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 1, + "style_probability": 0.5056601762771606, + "style_target": 1, + "support_probability": 0.7806826829910278, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23787599252313996, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15710361301898956, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13785779476165771, + 0.09427227824926376, + 0.13785779476165771, + 0.7957145571708679, + 0.03920997679233551 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 1, + "style_probability": 0.3258380591869354, + "style_target": 0, + "support_probability": 0.4189106523990631, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6989291476037421, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01155264861881733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38886815309524536, + 0.1448315531015396, + 0.8808501958847046, + 0.25888791680336, + 0.01739608682692051, + 0.02361726015806198, + 0.023403646424412727, + 0.025978561490774155, + 0.021280214190483093 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 0, + "style_probability": 0.4489676058292389, + "style_target": 1, + "support_probability": 0.9346675872802734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27762843773020157, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01216975785791874, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.537746012210846, + 0.15610353648662567, + 0.9316595792770386, + 0.3376319706439972, + 0.02467947266995907, + 0.02950451336801052, + 0.03973766043782234, + 0.028380855917930603, + 0.06117275357246399 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 1, + "style_probability": 0.1486579030752182, + "style_target": 0, + "support_probability": 0.905889093875885, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5308799992342889, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015102388337254524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2981785833835602, + 0.45246681571006775, + 0.256074458360672, + 0.9226197004318237, + 0.04585493728518486, + 0.03670181334018707, + 0.013589958660304546, + 0.02521868236362934, + 0.02794426679611206, + 0.036592740565538406, + 0.02322673611342907 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 1, + "style_probability": 0.6856938004493713, + "style_target": 1, + "support_probability": 0.5190622806549072, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44648952936510783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03746764361858368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2636896073818207, + 0.4107514023780823, + 0.24350030720233917, + 0.888290524482727, + 0.030383693054318428, + 0.01011430099606514, + 0.009380720555782318, + 0.012491392903029919, + 0.023912357166409492, + 0.029366862028837204, + 0.022560471668839455 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 0, + "style_probability": 0.9149858951568604, + "style_target": 0, + "support_probability": 0.16950976848602295, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4693347506724048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.055212195962667465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29898911714553833, + 0.993757963180542, + 0.2894980013370514, + 0.501022219657898, + 0.007880173623561859 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 0, + "style_probability": 0.39284548163414, + "style_target": 1, + "support_probability": 0.7531614303588867, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2070805797776089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13824601471424103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22794996201992035, + 0.9978652596473694, + 0.2072928249835968, + 0.5093782544136047, + 0.02881404012441635 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 1, + "style_probability": 0.16771632432937622, + "style_target": 0, + "support_probability": 0.8105350136756897, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19492893559548233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5412495136260986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26869192719459534, + 0.7528396248817444, + 0.17169560492038727, + 0.5042077302932739, + 0.017711784690618515, + 0.01870374009013176 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.4814247786998749, + "style_target": 1, + "support_probability": 0.18533679842948914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02272724408116167, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8718060255050659, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27979326248168945, + 0.6647735238075256, + 0.20705831050872803, + 0.5037955641746521, + 0.04632199555635452, + 0.03994196653366089 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.6350985765457153, + "style_target": 0, + "support_probability": 0.010099612176418304, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6883253284832915, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014090575277805328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4186:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39060354232788086, + 0.11467093229293823, + 0.9975138902664185, + 0.18788576126098633, + 0.04662056267261505, + 0.06684160232543945, + 0.048851560801267624, + 0.05784428119659424, + 0.045465003699064255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.18475066125392914, + "style_target": 1, + "support_probability": 0.9458630681037903, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.918162043639045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014638466760516167, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4186:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4476170241832733, + 0.11246999353170395, + 0.998767614364624, + 0.21305210888385773, + 0.08206647634506226, + 0.01289718970656395, + 0.04490792378783226, + 0.03073432855308056, + 0.03466510772705078 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.7715187668800354, + "style_target": 0, + "support_probability": 0.9423943161964417, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.27912163831768105, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31013378500938416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3954235017299652, + 0.08834104239940643, + 0.6983052492141724, + 0.862554669380188, + 0.037860870361328125, + 0.09915567189455032, + 0.07595640420913696, + 0.041252296417951584, + 0.0317862443625927, + 0.018533194437623024, + 0.032447557896375656 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 0, + "style_probability": 0.3487968146800995, + "style_target": 1, + "support_probability": 0.47757217288017273, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0934284543674296, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3802450895309448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28607213497161865, + 0.0385553278028965, + 0.7193097472190857, + 0.9020476937294006, + 0.015308912843465805, + 0.05802232772111893, + 0.059058062732219696, + 0.026781519874930382, + 0.013403451070189476, + 0.042523931711912155 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 1, + "style_probability": 0.15299823880195618, + "style_target": 0, + "support_probability": 0.6941700577735901, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7322331185529536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.035199426114559174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9642471671104431, + 0.0641849935054779, + 0.10307930409908295, + 0.4093593955039978, + 0.01457060594111681 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 1, + "style_probability": 0.8005645275115967, + "style_target": 1, + "support_probability": 0.820986270904541, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6770067557267188, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04326669126749039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9358916282653809, + 0.08747848123311996, + 0.16055402159690857, + 0.5367118120193481, + 0.07547130435705185 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 0, + "style_probability": 0.5712430477142334, + "style_target": 0, + "support_probability": 0.8771228194236755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015937891892239337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9423311352729797, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25560444593429565, + 0.08282341808080673, + 0.9018059372901917, + 0.23509155213832855, + 0.04694444313645363, + 0.027030494064092636 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.7555583119392395, + "style_target": 1, + "support_probability": 0.01212669163942337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026866533337509563, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8724133968353271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31351521611213684, + 0.07879000902175903, + 0.9011910557746887, + 0.25834742188453674, + 0.2639397382736206 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.3385542035102844, + "style_target": 0, + "support_probability": 0.03298931568861008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17774726566478932, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6752095818519592, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17927630245685577, + 0.1939544826745987, + 0.5454918742179871, + 0.44335344433784485, + 0.01916509121656418, + 0.019372187554836273, + 0.17318738996982574, + 0.010544860735535622, + 0.005815276876091957 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.7704509496688843, + "style_target": 1, + "support_probability": 0.07459372282028198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.037929447016169474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5360400080680847, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15671120584011078, + 0.13105127215385437, + 0.4517630934715271, + 0.2748391926288605, + 0.03512692078948021, + 0.031709056347608566, + 0.03239186853170395, + 0.009358113631606102, + 0.02817744016647339 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.1360304206609726, + "style_target": 0, + "support_probability": 0.1520782858133316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058507072604971296, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5378996729850769, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39505735039711, + 0.48899105191230774, + 0.17088305950164795, + 0.08752009272575378, + 0.09238084405660629, + 0.013507308438420296, + 0.020589936524629593, + 0.02373802289366722, + 0.029543008655309677, + 0.05272678658366203, + 0.01764506660401821 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 1, + "style_probability": 0.5953360199928284, + "style_target": 1, + "support_probability": 0.07453898340463638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11457192162345615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40009862184524536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.47688817977905273, + 0.5112389922142029, + 0.16490907967090607, + 0.1229286938905716, + 0.009567312896251678, + 0.026882732287049294, + 0.018259486183524132, + 0.012623332440853119, + 0.007096088957041502, + 0.02700306847691536 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 0, + "style_probability": 0.5378731489181519, + "style_target": 0, + "support_probability": 0.10435345023870468, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3430054878299228, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07911624759435654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14520718157291412, + 0.23852984607219696, + 0.30411967635154724, + 0.9465782642364502, + 0.07645370066165924 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.42780762910842896, + "style_target": 1, + "support_probability": 0.6148309707641602, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8349715514488936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.03580278903245926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11871079355478287, + 0.2016105353832245, + 0.3199971914291382, + 0.9586905837059021, + 0.08281271159648895 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.9572574496269226, + "style_target": 0, + "support_probability": 0.8520888090133667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015759029545788013, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40847644209861755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14582186937332153, + 0.8487978577613831, + 0.5044366717338562, + 0.4096243977546692, + 0.027191245928406715, + 0.05001189559698105 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 0, + "style_probability": 0.3921653628349304, + "style_target": 1, + "support_probability": 0.00841500237584114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002702384262982106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6522697806358337, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06600374728441238, + 0.8251824378967285, + 0.4935430884361267, + 0.30134889483451843, + 0.126201331615448, + 0.382746160030365 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 1, + "style_probability": 0.17280590534210205, + "style_target": 0, + "support_probability": 0.010712874121963978, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4440426834666216, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17923206090927124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27131587266921997, + 0.5731850266456604, + 0.5081456899642944, + 0.5731850266456604, + 0.017883770167827606, + 0.021735165268182755, + 0.03748919069766998, + 0.02293098345398903, + 0.017394758760929108 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.44688865542411804, + "style_target": 1, + "support_probability": 0.8607475161552429, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5250327566056241, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20028899610042572, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29172006249427795, + 0.5674659609794617, + 0.5266920328140259, + 0.5674659609794617, + 0.031394027173519135, + 0.0055402484722435474, + 0.020555131137371063, + 0.027058687061071396, + 0.010330979712307453 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.6098533868789673, + "style_target": 0, + "support_probability": 0.9405975341796875, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13065583106499115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004653325304389, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29573628306388855, + 0.12741528451442719, + 0.29573628306388855, + 0.9876744151115417, + 0.019884034991264343, + 0.04815627261996269, + 0.04259561747312546, + 0.039655208587646484, + 0.0073530832305550575, + 0.004476564470678568, + 0.03502672538161278 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.1577076017856598, + "style_target": 1, + "support_probability": 0.2698000967502594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30867640491800585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0097125843167305, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2943170666694641, + 0.11517710238695145, + 0.2943170666694641, + 0.9886411428451538, + 0.00884082168340683, + 0.01612747646868229, + 0.020441154018044472, + 0.011936496943235397, + 0.015463952906429768, + 0.01898181438446045, + 0.024468066170811653 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.7997716665267944, + "style_target": 0, + "support_probability": 0.1873881071805954, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8371351313962082, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.030587265267968178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26166224479675293, + 0.18779560923576355, + 0.07152427732944489, + 0.9958510398864746, + 0.010674801655113697 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.7673488259315491, + "style_target": 1, + "support_probability": 0.9080374836921692, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30049243268330167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.039458148181438446, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2689252495765686, + 0.1964903026819229, + 0.06463708728551865, + 0.9972358345985413, + 0.044083546847105026 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.2283688485622406, + "style_target": 0, + "support_probability": 0.8867893815040588, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.684321976713811, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08817068487405777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9132711291313171, + 0.2006993293762207, + 0.3479563891887665, + 0.47174355387687683, + 0.13206470012664795, + 0.028911415487527847 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.8667604327201843, + "style_target": 1, + "support_probability": 0.6967005729675293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34684858057891566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2418217808008194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8871062397956848, + 0.20727892220020294, + 0.3229665160179138, + 0.4101468026638031, + 0.10478140413761139, + 0.053149573504924774 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.376064270734787, + "style_target": 0, + "support_probability": 0.5979974269866943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1055089408701427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.901446521282196, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19434522092342377, + 0.23011010885238647, + 0.9632043242454529, + 0.18574927747249603, + 0.03556009382009506, + 0.0429903008043766, + 0.04287344217300415, + 0.12424266338348389, + 0.04142840951681137 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.8098006248474121, + "style_target": 1, + "support_probability": 0.05781736224889755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15722292300576324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7203922271728516, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17306703329086304, + 0.19851352274417877, + 0.9652748107910156, + 0.20916526019573212, + 0.04973415285348892, + 0.12548954784870148 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.3752439320087433, + "style_target": 0, + "support_probability": 0.20749129354953766, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7601507632708409, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022547941654920578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1643199473619461, + 0.7682856321334839, + 0.2353854477405548, + 0.9938302636146545, + 0.004041521344333887, + 0.00833901111036539, + 0.011875811032950878, + 0.008898026309907436, + 0.018852349370718002, + 0.04564395546913147, + 0.015262311324477196 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 1, + "style_probability": 0.5325130820274353, + "style_target": 1, + "support_probability": 0.9508708119392395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3514365295227724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041256312280893326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16478417813777924, + 0.6964844465255737, + 0.2116934359073639, + 0.9902903437614441, + 0.0333084911108017, + 0.02130173332989216, + 0.035129278898239136, + 0.017200062051415443, + 0.011950621381402016, + 0.02773445099592209 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 1, + "style_probability": 0.3368145227432251, + "style_target": 0, + "support_probability": 0.8722561597824097, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.680681563993053, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3385593891143799, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8239702582359314, + 0.812923789024353, + 0.5006136894226074, + 0.8239702582359314, + 0.04552921652793884 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 1, + "style_probability": 0.5470520853996277, + "style_target": 1, + "support_probability": 0.9791859984397888, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08984352508819089, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2832186818122864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8590282797813416, + 0.8643711805343628, + 0.31782668828964233, + 0.8590282797813416, + 0.04651966691017151 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 1, + "style_probability": 0.06279332935810089, + "style_target": 0, + "support_probability": 0.9127240180969238, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001729682224565612, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9905068874359131, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1949751377105713, + 0.45246681571006775, + 0.4079057574272156, + 0.22235162556171417, + 0.06561998277902603, + 0.15592773258686066 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 0, + "style_probability": 0.4185415506362915, + "style_target": 1, + "support_probability": 0.0010284582385793328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018805869176668235, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.893637478351593, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17730385065078735, + 0.5199190974235535, + 0.43435996770858765, + 0.23040378093719482, + 0.15939347445964813, + 0.23323571681976318 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 0, + "style_probability": 0.8858861327171326, + "style_target": 0, + "support_probability": 0.0166627187281847, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20965495879805165, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20765988528728485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34087467193603516, + 0.7116782069206238, + 0.5321522951126099, + 0.34087467193603516, + 0.0345209501683712, + 0.039788927882909775, + 0.04573996365070343, + 0.0668547973036766, + 0.011506825685501099 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 0, + "style_probability": 0.41758543252944946, + "style_target": 1, + "support_probability": 0.36888933181762695, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26174133277355593, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09278237819671631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32972532510757446, + 0.7393302917480469, + 0.521679162979126, + 0.32972532510757446, + 0.04117139056324959, + 0.07585436850786209 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 1, + "style_probability": 0.36045581102371216, + "style_target": 0, + "support_probability": 0.45726969838142395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006890955767042082, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8833385109901428, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19545245170593262, + 0.47692471742630005, + 0.09129467606544495, + 0.1743381768465042, + 0.029508113861083984, + 0.050135232508182526, + 0.03476755693554878, + 0.05629364773631096, + 0.020321916788816452, + 0.029176779091358185 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 1, + "style_probability": 0.6175404787063599, + "style_target": 1, + "support_probability": 0.003795524826273322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009529947871915977, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8369525671005249, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16963443160057068, + 0.4304681122303009, + 0.10974035412073135, + 0.1639375239610672, + 0.028311222791671753, + 0.045813415199518204, + 0.04261058568954468, + 0.043546319007873535, + 0.01805780455470085, + 0.09249869734048843 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 0, + "style_probability": 0.5723369121551514, + "style_target": 0, + "support_probability": 0.01604614593088627, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8594700093149209, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3450703024864197, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6322839856147766, + 0.09510394930839539, + 0.07815800607204437, + 0.988149881362915, + 0.027443911880254745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 1, + "style_probability": 0.6953467726707458, + "style_target": 1, + "support_probability": 0.8871331810951233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5752380682337823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2369529902935028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6958695650100708, + 0.1068853959441185, + 0.08774081617593765, + 0.9858806729316711, + 0.04333610460162163, + 0.01560507994145155, + 0.020926279947161674, + 0.024791501462459564, + 0.02150796167552471, + 0.027912694960832596, + 0.0625024139881134 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 1, + "style_probability": 0.40769994258880615, + "style_target": 0, + "support_probability": 0.8695179224014282, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8950512200444983, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08483676612377167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38264748454093933, + 0.9807079434394836, + 0.27362343668937683, + 0.23587702214717865, + 0.08921705931425095, + 0.04743729159235954 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.8319025039672852, + "style_target": 1, + "support_probability": 0.9491797685623169, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1501079682262435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07918375730514526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3397383689880371, + 0.9916442036628723, + 0.26099956035614014, + 0.3172881603240967, + 0.03265244886279106, + 0.0882391706109047 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.032125264406204224, + "style_target": 0, + "support_probability": 0.9494496583938599, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8529992957880467, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05235978588461876, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8720389008522034, + 0.954351007938385, + 0.0687626302242279, + 0.8720389008522034, + 0.05282433703541756, + 0.04722144082188606, + 0.059688545763492584, + 0.05555138736963272, + 0.02973969653248787 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 1, + "style_probability": 0.8694452047348022, + "style_target": 1, + "support_probability": 0.8547778129577637, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5371448913953809, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21006052196025848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8405929207801819, + 0.9427723288536072, + 0.11222686618566513, + 0.8405929207801819, + 0.13203243911266327, + 0.03652250021696091, + 0.024928851053118706, + 0.026324840262532234, + 0.06776721775531769 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 0, + "style_probability": 0.7142800688743591, + "style_target": 0, + "support_probability": 0.5377944111824036, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04061263904370932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6172398328781128, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44001930952072144, + 0.5119935870170593, + 0.44001930952072144, + 0.8780896067619324, + 0.03754742443561554, + 0.017139647156000137, + 0.0552920438349247, + 0.03964298591017723, + 0.08635411411523819, + 0.04014558345079422 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.6885924935340881, + "style_target": 1, + "support_probability": 0.019127866253256798, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05042625118585267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43546542525291443, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4421462416648865, + 0.4975094199180603, + 0.4421462416648865, + 0.8835797309875488, + 0.09133193641901016, + 0.03507890924811363, + 0.009755224920809269, + 0.09380392730236053, + 0.020854415372014046 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.38782933354377747, + "style_target": 0, + "support_probability": 0.04172690585255623, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1252785764396964, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.48203858733177185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09085582196712494, + 0.4379781484603882, + 0.2442811280488968, + 0.6061562895774841, + 0.05970969423651695 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.862997829914093, + "style_target": 1, + "support_probability": 0.05737369507551193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004960397969318415, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.45432350039482117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07868945598602295, + 0.37855446338653564, + 0.18229657411575317, + 0.5151892304420471, + 0.036123957484960556 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.04369726777076721, + "style_target": 0, + "support_probability": 0.08811501413583755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06081759384433477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7619071006774902, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8290102481842041, + 0.09341821074485779, + 0.3795458972454071, + 0.5271309614181519, + 0.05978471040725708, + 0.0982581079006195 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 1, + "style_probability": 0.7894192934036255, + "style_target": 1, + "support_probability": 0.056974999606609344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0649541188341944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5280794501304626, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7664445042610168, + 0.12512001395225525, + 0.41642430424690247, + 0.5893353819847107, + 0.08675962686538696, + 0.06743025779724121 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 1, + "style_probability": 0.4901331961154938, + "style_target": 0, + "support_probability": 0.10885287076234818, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01461136495029592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5599251985549927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17469081282615662, + 0.9075219035148621, + 0.16586963832378387, + 0.37020760774612427, + 0.037486571818590164, + 0.04926368221640587, + 0.059119708836078644, + 0.033731136471033096, + 0.059341948479413986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.7615495324134827, + "style_target": 1, + "support_probability": 0.003903050674125552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009426600279931942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47103872895240784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22979708015918732, + 0.8799035549163818, + 0.1421302706003189, + 0.4238913953304291, + 0.03207701817154884, + 0.046329744160175323 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.441855251789093, + "style_target": 0, + "support_probability": 0.0014111713971942663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029100889620230698, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9018285274505615, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17185737192630768, + 0.4189400374889374, + 0.9174615740776062, + 0.7774788737297058, + 0.05951590836048126, + 0.11499251425266266, + 0.051771003752946854, + 0.10513458400964737, + 0.11949526518583298, + 0.15129879117012024 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.8058096170425415, + "style_target": 1, + "support_probability": 0.019363421946763992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00460090241622727, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7213827967643738, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17785857617855072, + 0.3876202404499054, + 0.9372826218605042, + 0.8198372721672058, + 0.08910097181797028, + 0.05765655264258385, + 0.09106554090976715, + 0.07275184988975525, + 0.048982616513967514, + 0.0626804530620575 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.07882686704397202, + "style_target": 0, + "support_probability": 0.03484759479761124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1478181409183494, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.25389477610588074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29619553685188293, + 0.2396632581949234, + 0.2076929807662964, + 0.8837076425552368, + 0.0830715149641037 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.5273578763008118, + "style_target": 1, + "support_probability": 0.09086254239082336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015171071744552525, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.17596444487571716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1348477602005005, + 0.24164342880249023, + 0.17558857798576355, + 0.9123806953430176, + 0.08578279614448547 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.10877872258424759, + "style_target": 0, + "support_probability": 0.0770040899515152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6841576267609224, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13461890816688538, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8253708481788635, + 0.7427347302436829, + 0.4231410026550293, + 0.612204372882843, + 0.03481632471084595, + 0.012938023544847965 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 1, + "style_probability": 0.5374755859375, + "style_target": 1, + "support_probability": 0.9767816066741943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.698267151495874, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.18383345007896423, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7742172479629517, + 0.7993280291557312, + 0.36856916546821594, + 0.5306827425956726, + 0.06053653731942177, + 0.02683868445456028 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 0, + "style_probability": 0.9286174178123474, + "style_target": 0, + "support_probability": 0.8940219283103943, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5722874269498903, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3028008043766022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6155070662498474, + 0.5935404896736145, + 0.750339686870575, + 0.7555146813392639, + 0.02552444115281105, + 0.046444281935691833, + 0.02078571543097496, + 0.19810928404331207, + 0.014182809740304947 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 0, + "style_probability": 0.4841657876968384, + "style_target": 1, + "support_probability": 0.882091760635376, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4711833165660365, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.35415083169937134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6065224409103394, + 0.5489406585693359, + 0.792914628982544, + 0.7594243288040161, + 0.034406088292598724, + 0.033851489424705505 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 1, + "style_probability": 0.2656049430370331, + "style_target": 0, + "support_probability": 0.7573056221008301, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6805778930100798, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026204871013760567, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3580993711948395, + 0.9312518835067749, + 0.18556876480579376, + 0.28989511728286743, + 0.0086208526045084, + 0.014548595994710922, + 0.00713830953463912, + 0.013362167403101921, + 0.021355358883738518, + 0.027001552283763885, + 0.014817623421549797 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 0, + "style_probability": 0.3146595060825348, + "style_target": 1, + "support_probability": 0.9375948905944824, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029677658878635153, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06245854124426842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24985161423683167, + 0.9513339996337891, + 0.3137619197368622, + 0.2679298222064972, + 0.02191629447042942, + 0.07920844852924347, + 0.038217298686504364, + 0.04582485929131508, + 0.017775585874915123, + 0.045028991997241974 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 1, + "style_probability": 0.03644614666700363, + "style_target": 0, + "support_probability": 0.6999978423118591, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.448580934853001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09956362098455429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4202214777469635, + 0.1625005602836609, + 0.8552712202072144, + 0.07010594755411148, + 0.014863928779959679 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 1, + "style_probability": 0.5032180547714233, + "style_target": 1, + "support_probability": 0.6361744403839111, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6639340381531813, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16362231969833374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3935197591781616, + 0.18491867184638977, + 0.9046115875244141, + 0.06589667499065399, + 0.011784704402089119 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 0, + "style_probability": 0.8499881625175476, + "style_target": 0, + "support_probability": 0.440572589635849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7008076461803787, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06404847651720047, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9934552311897278, + 0.3184548318386078, + 0.41480231285095215, + 0.2592870891094208, + 0.030218277126550674, + 0.09203453361988068 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.5606107115745544, + "style_target": 1, + "support_probability": 0.8559054732322693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5486754267081959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04521910846233368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9955821633338928, + 0.326096773147583, + 0.36013340950012207, + 0.22798797488212585, + 0.038922861218452454 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.4452340304851532, + "style_target": 0, + "support_probability": 0.8608083724975586, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7115301863637158, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02188611403107643, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9377423524856567, + 0.4418720602989197, + 0.15565091371536255, + 0.1674846112728119, + 0.011041585355997086, + 0.012233003973960876, + 0.018179764971137047, + 0.026112306863069534, + 0.02397223562002182 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 0, + "style_probability": 0.198073148727417, + "style_target": 1, + "support_probability": 0.9155870079994202, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.35983295115851593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026499606668949127, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9859297871589661, + 0.28748929500579834, + 0.28636258840560913, + 0.14633670449256897, + 0.059959813952445984, + 0.04915415123105049, + 0.028448479250073433, + 0.023845326155424118, + 0.06360267847776413 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 1, + "style_probability": 0.04912401735782623, + "style_target": 0, + "support_probability": 0.9476794004440308, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015723345442059975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2597969174385071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19885383546352386, + 0.34101009368896484, + 0.2836894392967224, + 0.13681334257125854, + 0.011545364744961262, + 0.024144291877746582, + 0.01795852929353714, + 0.032710060477256775, + 0.028949521481990814, + 0.0267865601927042, + 0.040288347750902176 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.21167053282260895, + "style_target": 1, + "support_probability": 0.019432086497545242, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020950333322305007, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.412781685590744, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.252851277589798, + 0.35740089416503906, + 0.253984659910202, + 0.12304018437862396, + 0.029903164133429527, + 0.013284352608025074, + 0.02409467287361622, + 0.00943676196038723, + 0.027573931962251663, + 0.026204660534858704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.8460000157356262, + "style_target": 0, + "support_probability": 0.00863255187869072, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04379962421547952, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4092840850353241, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.333484411239624, + 0.24865402281284332, + 0.29175466299057007, + 0.2767247259616852, + 0.015476793050765991 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 0, + "style_probability": 0.40065497159957886, + "style_target": 1, + "support_probability": 0.1302129328250885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019455309645929612, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36427178978919983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33292528986930847, + 0.18958871066570282, + 0.30614912509918213, + 0.3269171118736267, + 0.04227477312088013, + 0.07391733676195145, + 0.16302815079689026, + 0.08892753720283508, + 0.03805526718497276, + 0.031449805945158005 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 1, + "style_probability": 0.27260759472846985, + "style_target": 0, + "support_probability": 0.03860153257846832, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.463800044360702, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05669933184981346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1187605932354927, + 0.2946653664112091, + 0.1187605932354927, + 0.9760034084320068, + 0.03346576541662216, + 0.048911456018686295 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 1, + "style_probability": 0.6051303744316101, + "style_target": 1, + "support_probability": 0.695654571056366, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09186262006739954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15948544442653656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11539498716592789, + 0.26856786012649536, + 0.11539498716592789, + 0.9868487119674683, + 0.03488514944911003, + 0.05390021204948425 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 1, + "style_probability": 0.22836558520793915, + "style_target": 0, + "support_probability": 0.7206403017044067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004446908658849108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7926995158195496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31459131836891174, + 0.4917576313018799, + 0.4820249080657959, + 0.7535240054130554, + 0.03203309699892998, + 0.014157949015498161, + 0.1796962022781372, + 0.022278834134340286, + 0.026701048016548157 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 1, + "style_probability": 0.6212785243988037, + "style_target": 1, + "support_probability": 0.0009781179251149297, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004754534678059052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7178294658660889, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2805282771587372, + 0.633160412311554, + 0.46986642479896545, + 0.8171250820159912, + 0.018663566559553146, + 0.021406089887022972, + 0.019476456567645073, + 0.03135887533426285, + 0.010639236308634281 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 0, + "style_probability": 0.7584725618362427, + "style_target": 0, + "support_probability": 0.0006488513899967074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003858131908746952, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8368269801139832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7446774244308472, + 0.21032290160655975, + 0.17928893864154816, + 0.3308687210083008, + 0.1216658353805542, + 0.09770897030830383, + 0.08817312121391296, + 0.08537687361240387, + 0.1295032948255539, + 0.07838655263185501, + 0.05677036568522453 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.612617552280426, + "style_target": 1, + "support_probability": 0.0006484103505499661, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00889573962028354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8432662487030029, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7749847173690796, + 0.18079398572444916, + 0.2238059788942337, + 0.3403419256210327, + 0.04085732623934746, + 0.11049871146678925, + 0.061355166137218475, + 0.18811973929405212, + 0.06783285737037659 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.4722822904586792, + "style_target": 0, + "support_probability": 0.0012002651346847415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008078366191177513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9613780379295349, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16232244670391083, + 0.7441903352737427, + 0.13657502830028534, + 0.37588489055633545, + 0.11479587107896805 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.6683206558227539, + "style_target": 1, + "support_probability": 0.006000003311783075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009764626128574922, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9425257444381714, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22132867574691772, + 0.6739743947982788, + 0.20377624034881592, + 0.36042147874832153, + 0.02850097231566906 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.37625938653945923, + "style_target": 0, + "support_probability": 0.023288121446967125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17398446023446518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13898058235645294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.05261709913611412, + 0.5669354796409607, + 0.25845396518707275, + 0.917477011680603, + 0.04938948154449463, + 0.03150920197367668 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 1, + "style_probability": 0.5988245010375977, + "style_target": 1, + "support_probability": 0.14736612141132355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09570085122224223, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0647931694984436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.05080229789018631, + 0.5787650346755981, + 0.25187280774116516, + 0.8915606141090393, + 0.15357117354869843 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 1, + "style_probability": 0.17733921110630035, + "style_target": 0, + "support_probability": 0.18679159879684448, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4772949481617896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5080223679542542, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6715123057365417, + 0.5652064085006714, + 0.3280586898326874, + 0.9871516227722168, + 0.10498714447021484, + 0.08961436152458191, + 0.029586737975478172, + 0.04239504784345627, + 0.06420756876468658 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.8322124481201172, + "style_target": 1, + "support_probability": 0.3045596182346344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024394545390108154, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4512614905834198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6987003087997437, + 0.5376335382461548, + 0.3279692232608795, + 0.9902473092079163, + 0.05333389341831207, + 0.09534671157598495, + 0.08433816581964493, + 0.0395113080739975, + 0.17416946589946747 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.09204903990030289, + "style_target": 0, + "support_probability": 0.16506385803222656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.890126173723752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02760866843163967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9681413769721985, + 0.6174505352973938, + 0.11578651517629623, + 0.32392171025276184, + 0.013552066870033741, + 0.0825105756521225, + 0.07276173681020737, + 0.0647776797413826, + 0.01827245019376278, + 0.021630898118019104, + 0.04344181343913078 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 1, + "style_probability": 0.9246343970298767, + "style_target": 1, + "support_probability": 0.9199443459510803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7924180893995018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04209107160568237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9584364891052246, + 0.6184842586517334, + 0.12419608235359192, + 0.41131269931793213, + 0.02326241321861744, + 0.029568426311016083, + 0.07108928263187408, + 0.015961218625307083, + 0.021706879138946533, + 0.04858773201704025 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 0, + "style_probability": 0.6410689949989319, + "style_target": 0, + "support_probability": 0.8991197943687439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4551716782586398, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2366359531879425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13031654059886932, + 0.9234610795974731, + 0.23615077137947083, + 0.4646485149860382, + 0.19715838134288788 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.9138738512992859, + "style_target": 1, + "support_probability": 0.11897800117731094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0591392478735815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4103981852531433, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11397559940814972, + 0.919486403465271, + 0.18848681449890137, + 0.5056901574134827, + 0.1355513334274292, + 0.1159023642539978, + 0.321763277053833, + 0.19376923143863678, + 0.30608606338500977, + 0.2606567144393921 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.3304605185985565, + "style_target": 0, + "support_probability": 0.031733881682157516, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7830529442137819, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02099561132490635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5064167976379395, + 0.23325558006763458, + 0.9620442390441895, + 0.1623537838459015, + 0.010863112285733223, + 0.015177319757640362 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.696413516998291, + "style_target": 1, + "support_probability": 0.8308830857276917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07109073960695253, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.035823747515678406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44779860973358154, + 0.1699371486902237, + 0.9647493958473206, + 0.09299052506685257, + 0.021702226251363754, + 0.05468924716114998 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.06183105334639549, + "style_target": 0, + "support_probability": 0.5147255659103394, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9574707596767181, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004328726790845394, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4459:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2358972430229187, + 0.42506685853004456, + 0.2345210611820221, + 0.9983780384063721, + 0.01920096017420292, + 0.024043425917625427, + 0.027771897614002228, + 0.016689084470272064, + 0.017975440248847008 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 1, + "style_probability": 0.9158108234405518, + "style_target": 1, + "support_probability": 0.9853256344795227, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8660250598840239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.00345448381267488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4459:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20357872545719147, + 0.4481160640716553, + 0.2151806354522705, + 0.9969248175621033, + 0.038169074803590775, + 0.034827593713998795, + 0.022023363038897514, + 0.01915433257818222, + 0.05128788948059082 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 0, + "style_probability": 0.6285527348518372, + "style_target": 0, + "support_probability": 0.9881717562675476, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5079634080211258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1630164533853531, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9401274919509888, + 0.25203737616539, + 0.16067272424697876, + 0.2516671419143677, + 0.025557974353432655, + 0.018017984926700592, + 0.03090645745396614, + 0.02815401181578636, + 0.01942874863743782, + 0.06479372829198837 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 1, + "style_probability": 0.9354719519615173, + "style_target": 1, + "support_probability": 0.32981714606285095, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36868171018084817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20794211328029633, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9327991604804993, + 0.2716164290904999, + 0.17926675081253052, + 0.2473558932542801, + 0.04455361142754555, + 0.022556375712156296, + 0.02486215904355049, + 0.022715354338288307, + 0.027991941198706627 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 0, + "style_probability": 0.5990410447120667, + "style_target": 0, + "support_probability": 0.3362062871456146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1678050293207241, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2926190197467804, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5948585271835327, + 0.6496401429176331, + 0.5711266398429871, + 0.1322094202041626, + 0.04220512509346008 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 0, + "style_probability": 0.49471771717071533, + "style_target": 1, + "support_probability": 0.28375107049942017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02847054239134747, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8130843043327332, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5839417576789856, + 0.584301769733429, + 0.5641217827796936, + 0.11194638907909393, + 0.032844800502061844, + 0.03745804354548454, + 0.015274147503077984, + 0.0196105744689703, + 0.09646781533956528, + 0.04272683709859848 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 1, + "style_probability": 0.3191487789154053, + "style_target": 0, + "support_probability": 0.03929249569773674, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018527048610857962, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7573184967041016, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.572227954864502, + 0.15692532062530518, + 0.44775450229644775, + 0.15692532062530518, + 0.025897860527038574, + 0.05973777547478676 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 0, + "style_probability": 0.4566192030906677, + "style_target": 1, + "support_probability": 0.02697066217660904, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0024470596003731892, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7607747912406921, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6093689799308777, + 0.13552673161029816, + 0.41353505849838257, + 0.13552673161029816, + 0.013406692072749138, + 0.040477193892002106 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 1, + "style_probability": 0.09891572594642639, + "style_target": 0, + "support_probability": 0.028516443446278572, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.05890763469578268, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2870613634586334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11867395043373108, + 0.1118142232298851, + 0.9288110136985779, + 0.11867395043373108, + 0.02822120487689972, + 0.018622633069753647, + 0.019351378083229065, + 0.01776925101876259, + 0.061647333204746246 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 0, + "style_probability": 0.38439175486564636, + "style_target": 1, + "support_probability": 0.052515946328639984, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1414999977038731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32992932200431824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16208407282829285, + 0.15601591765880585, + 0.9562907218933105, + 0.16208407282829285, + 0.011288967914879322, + 0.021862909197807312, + 0.011593224480748177, + 0.025940900668501854, + 0.033244889229536057 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 0, + "style_probability": 0.8149505853652954, + "style_target": 0, + "support_probability": 0.04284448176622391, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4184171673139715, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06281828135251999, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3625167906284332, + 0.9628644585609436, + 0.4977909028530121, + 0.3173748850822449, + 0.0361151359975338, + 0.030823608860373497, + 0.02635599486529827, + 0.06007922813296318, + 0.03157849609851837, + 0.18443922698497772 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 1, + "style_probability": 0.5996609330177307, + "style_target": 1, + "support_probability": 0.7099699378013611, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49420344312961884, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03912292793393135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42864230275154114, + 0.9854394197463989, + 0.6247940063476562, + 0.29166868329048157, + 0.0524425245821476, + 0.058028072118759155, + 0.04601249471306801, + 0.07372027635574341, + 0.05530468747019768 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 1, + "style_probability": 0.4375403821468353, + "style_target": 0, + "support_probability": 0.9082491993904114, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8737491779501148, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04495847597718239, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9783576726913452, + 0.2144446074962616, + 0.7248250842094421, + 0.19987095892429352, + 0.07758823782205582 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 1, + "style_probability": 0.6717668771743774, + "style_target": 1, + "support_probability": 0.9279344081878662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8443539430995066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.055757518857717514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9645216464996338, + 0.21974721550941467, + 0.696222186088562, + 0.31865164637565613, + 0.030871573835611343 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 1, + "style_probability": 0.4973391890525818, + "style_target": 0, + "support_probability": 0.9607819318771362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.699912540565826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011538859456777573, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18748469650745392, + 0.9933017492294312, + 0.29946592450141907, + 0.4284476339817047, + 0.022870805114507675, + 0.06186952069401741 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 1, + "style_probability": 0.7239717245101929, + "style_target": 1, + "support_probability": 0.564683198928833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24603960763754723, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.042709141969680786, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19883346557617188, + 0.9837251305580139, + 0.3379393219947815, + 0.40364956855773926, + 0.09489487111568451 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 1, + "style_probability": 0.28748613595962524, + "style_target": 0, + "support_probability": 0.24134673178195953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4324345619058218, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6628440022468567, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42112991213798523, + 0.3978922665119171, + 0.9876412749290466, + 0.3113255798816681, + 0.012358685955405235, + 0.007925664074718952, + 0.021617842838168144, + 0.033678993582725525, + 0.09721651673316956 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.7478348016738892, + "style_target": 1, + "support_probability": 0.16791442036628723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040679464498213636, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7676990628242493, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3422233760356903, + 0.313968688249588, + 0.9889499545097351, + 0.2798410654067993, + 0.027479127049446106, + 0.03163333237171173, + 0.026480572298169136, + 0.012378077022731304, + 0.06443540006875992 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.23597244918346405, + "style_target": 0, + "support_probability": 0.07921189069747925, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6915848814991006, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10666976124048233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9831588268280029, + 0.5753809213638306, + 0.42778295278549194, + 0.47453105449676514, + 0.00755529198795557, + 0.03483948856592178, + 0.026343168690800667, + 0.029952265322208405, + 0.05616455152630806, + 0.01624486595392227, + 0.0658029317855835 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 0, + "style_probability": 0.49257907271385193, + "style_target": 1, + "support_probability": 0.8594096899032593, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7543650006746364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11550849676132202, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9665697813034058, + 0.6028964519500732, + 0.5226659774780273, + 0.48149698972702026, + 0.036858510226011276, + 0.060474943369627, + 0.09270927309989929, + 0.04167648404836655, + 0.02031364105641842, + 0.07900787144899368 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 0, + "style_probability": 0.5188851952552795, + "style_target": 0, + "support_probability": 0.8742708563804626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08467892931317565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.622945249080658, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30735787749290466, + 0.5587591528892517, + 0.8408487439155579, + 0.2845692038536072, + 0.08887553215026855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 1, + "style_probability": 0.7940855026245117, + "style_target": 1, + "support_probability": 0.04335189238190651, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03441051893562072, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8737185001373291, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3423647880554199, + 0.5183138251304626, + 0.8229751586914062, + 0.25345635414123535, + 0.030060285702347755, + 0.03854898363351822, + 0.05576866865158081, + 0.08223380893468857, + 0.1810595542192459, + 0.026665883138775826, + 0.033809173852205276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 1, + "style_probability": 0.4873743951320648, + "style_target": 0, + "support_probability": 0.013214655220508575, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7592118936887983, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06294809281826019, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5799648761749268, + 0.3725602328777313, + 0.8567283749580383, + 0.5857419967651367, + 0.05745343491435051, + 0.014002395793795586 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.7310161590576172, + "style_target": 1, + "support_probability": 0.9655389189720154, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11678633645534775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08397871255874634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4258771538734436, + 0.2818719744682312, + 0.8610374331474304, + 0.43761587142944336, + 0.015582587569952011, + 0.018969209864735603 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.02886785939335823, + "style_target": 0, + "support_probability": 0.9338304996490479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0569923222621251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4077732264995575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1577012687921524, + 0.1577012687921524, + 0.41844722628593445, + 0.8242046236991882, + 0.023672817274928093, + 0.010790307074785233, + 0.027484923601150513, + 0.06600315868854523, + 0.016370384022593498 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.7591851353645325, + "style_target": 1, + "support_probability": 0.014178029261529446, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032050888575098836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34590595960617065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1519894152879715, + 0.1519894152879715, + 0.41304636001586914, + 0.7600942850112915, + 0.03485732898116112, + 0.03363046422600746, + 0.012522022239863873, + 0.022521479055285454, + 0.05004328116774559 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.42514580488204956, + "style_target": 0, + "support_probability": 0.022707846015691757, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006070771993521774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9226881265640259, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2570444941520691, + 0.5186854004859924, + 0.2154541164636612, + 0.7278590202331543, + 0.07616350799798965, + 0.024823224171996117, + 0.029535748064517975, + 0.029374970123171806, + 0.05560990795493126, + 0.05456556752324104, + 0.09304890781641006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.7522855401039124, + "style_target": 1, + "support_probability": 0.004040602128952742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007399496279493263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9560052752494812, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3082360029220581, + 0.5357223749160767, + 0.26515862345695496, + 0.7490739226341248, + 0.04558544605970383, + 0.07303542643785477, + 0.039551522582769394, + 0.051859721541404724, + 0.07818975299596786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.41501152515411377, + "style_target": 0, + "support_probability": 0.002427944215014577, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6312082392886539, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.026981394737958908, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19344031810760498, + 0.5119569301605225, + 0.19344031810760498, + 0.9968084692955017, + 0.0785258337855339 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 1, + "style_probability": 0.5673856735229492, + "style_target": 1, + "support_probability": 0.8617177605628967, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6308132364962528, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015171597711741924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14982789754867554, + 0.617175281047821, + 0.14982789754867554, + 0.9987494945526123, + 0.0690046176314354, + 0.0651361420750618, + 0.07314389944076538, + 0.12363588809967041, + 0.06541281938552856, + 0.06403651833534241, + 0.09735611826181412 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 1, + "style_probability": 0.40139442682266235, + "style_target": 0, + "support_probability": 0.9108365774154663, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7617608947028107, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07419395446777344, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3064444363117218, + 0.41612115502357483, + 0.2229204773902893, + 0.963039219379425, + 0.01862664707005024, + 0.010500891134142876 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 0, + "style_probability": 0.45229867100715637, + "style_target": 1, + "support_probability": 0.9805281758308411, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1575505142995399, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12226010113954544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.341939240694046, + 0.42440828680992126, + 0.11331583559513092, + 0.9759558439254761, + 0.029192768037319183, + 0.08835780620574951 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 1, + "style_probability": 0.09994801133871078, + "style_target": 0, + "support_probability": 0.9092659950256348, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04435243288792868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7732048034667969, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15707021951675415, + 0.03342828154563904, + 0.4944705665111542, + 0.7945187091827393, + 0.07147957384586334, + 0.022731227800250053, + 0.04630652070045471, + 0.05181797966361046, + 0.04996722564101219 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.40366625785827637, + "style_target": 1, + "support_probability": 0.09392387419939041, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13059296517866872, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5776233673095703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2866128385066986, + 0.023119527846574783, + 0.6076274514198303, + 0.8583777546882629, + 0.023776020854711533, + 0.1037185937166214, + 0.1851384937763214, + 0.28623563051223755, + 0.12835794687271118 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.7417464852333069, + "style_target": 0, + "support_probability": 0.24231354892253876, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18327583140199022, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05914865434169769, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12834104895591736, + 0.7533389925956726, + 0.33516064286231995, + 0.30745404958724976, + 0.005440293811261654, + 0.023846102878451347, + 0.011172155849635601, + 0.004878323990851641, + 0.03235539421439171, + 0.011541496962308884, + 0.00506116496399045 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 0, + "style_probability": 0.37496039271354675, + "style_target": 1, + "support_probability": 0.2456793636083603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10412134797181213, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07053692638874054, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16827580332756042, + 0.7698628902435303, + 0.3974587321281433, + 0.3413141071796417, + 0.019949475303292274, + 0.03010386787354946, + 0.03171417489647865, + 0.01899983361363411, + 0.051536791026592255 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 1, + "style_probability": 0.22932912409305573, + "style_target": 0, + "support_probability": 0.1676226705312729, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3931026387776853, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3810243606567383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9175095558166504, + 0.5484862923622131, + 0.15704487264156342, + 0.6012998819351196, + 0.018260115757584572 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 0, + "style_probability": 0.4026561975479126, + "style_target": 1, + "support_probability": 0.6449886560440063, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1523264771947608, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34288114309310913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9627171754837036, + 0.6492465138435364, + 0.22525626420974731, + 0.7743637561798096, + 0.07997794449329376 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 1, + "style_probability": 0.23290634155273438, + "style_target": 0, + "support_probability": 0.5715619325637817, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6237098661133939, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1033530905842781, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6753550171852112, + 0.48030349612236023, + 0.8212074041366577, + 0.9695544242858887, + 0.049264419823884964, + 0.03902612254023552 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 0, + "style_probability": 0.48249951004981995, + "style_target": 1, + "support_probability": 0.8761236667633057, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7214099458321163, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12468776851892471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6493260264396667, + 0.3587115406990051, + 0.8043941259384155, + 0.9829094409942627, + 0.0370713472366333, + 0.027867890894412994 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 0, + "style_probability": 0.5556300282478333, + "style_target": 0, + "support_probability": 0.795345664024353, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7131542477568401, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011875767260789871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2871544361114502, + 0.10816686600446701, + 0.9587528705596924, + 0.283846378326416, + 0.05608946084976196, + 0.032606203109025955, + 0.03221333771944046, + 0.057029418647289276, + 0.03696475550532341 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 0, + "style_probability": 0.3385610580444336, + "style_target": 1, + "support_probability": 0.9544373154640198, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5418745129642152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.015975747257471085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24523323774337769, + 0.12123964726924896, + 0.9515615701675415, + 0.24636274576187134, + 0.02512820065021515, + 0.016947176307439804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 1, + "style_probability": 0.22624970972537994, + "style_target": 0, + "support_probability": 0.9310547709465027, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05268092994394554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9488571882247925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38595321774482727, + 0.48945072293281555, + 0.9289330244064331, + 0.1848861128091812, + 0.0602586530148983, + 0.03374739736318588, + 0.0932050421833992, + 0.04876239597797394, + 0.07045901566743851, + 0.04807422310113907 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.568671703338623, + "style_target": 1, + "support_probability": 0.05094294250011444, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007050861459723323, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9679480791091919, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40845736861228943, + 0.32711121439933777, + 0.9152597188949585, + 0.23387721180915833, + 0.08544852584600449, + 0.06148245558142662, + 0.28125956654548645, + 0.058159247040748596, + 0.07985098659992218, + 0.23356370627880096 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.18772031366825104, + "style_target": 0, + "support_probability": 0.08451907336711884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10396682114209559, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.659788191318512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3841428756713867, + 0.4075178802013397, + 0.9680389165878296, + 0.5266426205635071, + 0.13401782512664795 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.37143123149871826, + "style_target": 1, + "support_probability": 0.05867793411016464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02889669648440151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9109632968902588, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39852410554885864, + 0.367887407541275, + 0.9678522944450378, + 0.48994964361190796, + 0.09645494073629379 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.8014855980873108, + "style_target": 0, + "support_probability": 0.009393171407282352, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5511641748631718, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.006593065336346626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35597535967826843, + 0.2075045257806778, + 0.0343262255191803, + 0.9816309809684753, + 0.02324790693819523, + 0.03817865997552872 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 0, + "style_probability": 0.18692760169506073, + "style_target": 1, + "support_probability": 0.9896916747093201, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05346663010421324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.023236632347106934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5295501947402954, + 0.17670272290706635, + 0.09624084085226059, + 0.9819859266281128, + 0.023014379665255547, + 0.09195537120103836 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 1, + "style_probability": 0.019224928691983223, + "style_target": 0, + "support_probability": 0.9858507513999939, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6046594224178641, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031258851289749146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34721124172210693, + 0.21093861758708954, + 0.1025070920586586, + 0.9813975095748901, + 0.04405582696199417, + 0.08944409340620041, + 0.04872941970825195, + 0.03337636962532997, + 0.10820160061120987 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 0, + "style_probability": 0.2458900511264801, + "style_target": 1, + "support_probability": 0.9384663105010986, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6578939491821801, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.049072202295064926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34995484352111816, + 0.17599280178546906, + 0.128662109375, + 0.9692748785018921, + 0.02881224825978279, + 0.017703060060739517, + 0.03943905234336853, + 0.047950245440006256, + 0.09355057030916214 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 1, + "style_probability": 0.38143792748451233, + "style_target": 0, + "support_probability": 0.9130301475524902, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23540312411648592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12749864161014557, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.819736123085022, + 0.038652461022138596, + 0.6375350952148438, + 0.267924040555954, + 0.03569532930850983, + 0.036907512694597244, + 0.08367075026035309, + 0.060040563344955444, + 0.0358419343829155, + 0.029206976294517517, + 0.0673191100358963 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.4345338046550751, + "style_target": 1, + "support_probability": 0.14662852883338928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38436639402017775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12720948457717896, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8072510957717896, + 0.03908347710967064, + 0.5703884959220886, + 0.27055543661117554, + 0.022518906742334366, + 0.030027365311980247, + 0.02556619420647621, + 0.02125779539346695, + 0.01580779440701008, + 0.028114695101976395, + 0.01848652772605419 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.8294486403465271, + "style_target": 0, + "support_probability": 0.13637858629226685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013173678776932467, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9314036965370178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5606367588043213, + 0.04739997535943985, + 0.1788690686225891, + 0.22210532426834106, + 0.03186001628637314 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 1, + "style_probability": 0.6361358761787415, + "style_target": 1, + "support_probability": 0.009615398943424225, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013399831985271221, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9396732449531555, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.511201024055481, + 0.04532783105969429, + 0.2105524241924286, + 0.2592775821685791, + 0.032649219036102295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 1, + "style_probability": 0.11569823324680328, + "style_target": 0, + "support_probability": 0.008592440746724606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.774823565549405, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020450713112950325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10574303567409515, + 0.8963521122932434, + 0.479677677154541, + 0.287557989358902, + 0.03646615520119667, + 0.004427826963365078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.8704392910003662, + "style_target": 1, + "support_probability": 0.8975446820259094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14558242979302594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11570238322019577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09887370467185974, + 0.7685678601264954, + 0.39721623063087463, + 0.22701464593410492, + 0.1221141368150711, + 0.02563585713505745 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.42941224575042725, + "style_target": 0, + "support_probability": 0.193727508187294, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6264740557266587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23580637574195862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3928377628326416, + 0.946583092212677, + 0.21093648672103882, + 0.19501547515392303, + 0.0057509359903633595, + 0.011080177500844002, + 0.01730729639530182, + 0.044971827417612076, + 0.005674571264535189 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 1, + "style_probability": 0.8816776871681213, + "style_target": 1, + "support_probability": 0.5166882872581482, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3292057455584426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21712569892406464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4018601179122925, + 0.9452823996543884, + 0.2040841281414032, + 0.17111942172050476, + 0.060889095067977905, + 0.08344363421201706 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 1, + "style_probability": 0.46930745244026184, + "style_target": 0, + "support_probability": 0.5225940346717834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06553584775010814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3501545488834381, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1345537155866623, + 0.12119875848293304, + 0.21449317038059235, + 0.22939665615558624, + 0.03663301840424538, + 0.1446923464536667, + 0.015279126353561878, + 0.03172294422984123, + 0.04478432983160019, + 0.08683682233095169, + 0.1801854819059372 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.7744333148002625, + "style_target": 1, + "support_probability": 0.07208630442619324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00041810444995262326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9346228241920471, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12681877613067627, + 0.09997137635946274, + 0.2003428190946579, + 0.17706210911273956, + 0.1519128829240799, + 0.09150970727205276, + 0.0750708281993866, + 0.09434597939252853, + 0.10780699551105499, + 0.09071709960699081 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.08651558309793472, + "style_target": 0, + "support_probability": 0.007483158726245165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4991574997600846, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3509021997451782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1841648519039154, + 0.8949105739593506, + 0.0917123407125473, + 0.14044395089149475, + 0.04976573586463928 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.5658487677574158, + "style_target": 1, + "support_probability": 0.6282453536987305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024500400353727844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3414086699485779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3349226415157318, + 0.9032329320907593, + 0.13027937710285187, + 0.08524526655673981, + 0.09261512011289597 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.06419917196035385, + "style_target": 0, + "support_probability": 0.46400371193885803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2244781424059579, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7736192941665649, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09600744396448135, + 0.06056661531329155, + 0.6426648497581482, + 0.5964857339859009, + 0.04218917340040207, + 0.03295193240046501 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 1, + "style_probability": 0.6259209513664246, + "style_target": 1, + "support_probability": 0.16826142370700836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04525003050287974, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8959353566169739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08126845210790634, + 0.05199538171291351, + 0.5781883001327515, + 0.6228477358818054, + 0.03717755898833275, + 0.06983993947505951 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 0, + "style_probability": 0.8203654885292053, + "style_target": 0, + "support_probability": 0.02668697200715542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1983481519487542, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23772336542606354, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26414453983306885, + 0.2657424211502075, + 0.9742166996002197, + 0.07850494235754013, + 0.0742044746875763, + 0.026523081585764885, + 0.04754159227013588, + 0.06379403918981552, + 0.04470293968915939 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 1, + "style_probability": 0.5432686805725098, + "style_target": 1, + "support_probability": 0.12054285407066345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1909225186452637, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1783035397529602, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30365675687789917, + 0.26165133714675903, + 0.9840126037597656, + 0.08250804990530014, + 0.10122963786125183, + 0.4370766580104828 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 1, + "style_probability": 0.3853592574596405, + "style_target": 0, + "support_probability": 0.1463797390460968, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5771758724093632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.053330279886722565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2997196316719055, + 0.20818524062633514, + 0.059414200484752655, + 0.9764233231544495, + 0.03704310208559036, + 0.04255213961005211, + 0.08296708017587662, + 0.09270858019590378, + 0.14774325489997864, + 0.04088005796074867 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.5900825262069702, + "style_target": 1, + "support_probability": 0.7870206832885742, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37615492772317155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0362008698284626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.228494331240654, + 0.1730690598487854, + 0.04751067981123924, + 0.9937841892242432, + 0.09528151899576187, + 0.07247896492481232, + 0.05179410055279732, + 0.048411786556243896, + 0.043862953782081604, + 0.16180098056793213 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.16436399519443512, + "style_target": 0, + "support_probability": 0.9203969240188599, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.834229324118019, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.010619369335472584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23485344648361206, + 0.1847594529390335, + 0.9235451817512512, + 0.15313035249710083, + 0.07016811519861221 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.9450384378433228, + "style_target": 1, + "support_probability": 0.9660121202468872, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5433803583382855, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008003611117601395, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19246318936347961, + 0.13153010606765747, + 0.8960007429122925, + 0.1293788105249405, + 0.009162070229649544, + 0.009325101040303707, + 0.02120840549468994, + 0.06500207632780075, + 0.0380203016102314, + 0.012500638142228127, + 0.2740269899368286 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.42110520601272583, + "style_target": 0, + "support_probability": 0.9141644835472107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07235782418837639, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2968588173389435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7862749695777893, + 0.1656419336795807, + 0.22858044505119324, + 0.20559543371200562, + 0.05698718875646591, + 0.025121258571743965 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 1, + "style_probability": 0.7337965965270996, + "style_target": 1, + "support_probability": 0.031034409999847412, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008436282021135286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5731054544448853, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7775645852088928, + 0.14622023701667786, + 0.17371401190757751, + 0.19990380108356476, + 0.019232802093029022, + 0.057128649204969406 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 1, + "style_probability": 0.1488054245710373, + "style_target": 0, + "support_probability": 0.020298505201935768, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49665841769428415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009224876761436462, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36969542503356934, + 0.1713491976261139, + 0.37110817432403564, + 0.9861406087875366, + 0.01705876737833023, + 0.0059545706026256084, + 0.015698866918683052, + 0.032490577548742294, + 0.045178983360528946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 1, + "style_probability": 0.6488118767738342, + "style_target": 1, + "support_probability": 0.4510887563228607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40131674126897304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016901280730962753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35916686058044434, + 0.2124948501586914, + 0.3267360031604767, + 0.9721823930740356, + 0.014608084224164486, + 0.00705338828265667, + 0.01767522655427456, + 0.01085631549358368, + 0.018612658604979515 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 0, + "style_probability": 0.5564876794815063, + "style_target": 0, + "support_probability": 0.4585031569004059, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8324184346350472, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041362639516592026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21751511096954346, + 0.968421459197998, + 0.03227103129029274, + 0.15049953758716583, + 0.01956510730087757, + 0.021858427673578262, + 0.011864799074828625, + 0.028290947899222374, + 0.019296161830425262, + 0.025987256318330765, + 0.04781521484255791 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.854422390460968, + "style_target": 1, + "support_probability": 0.9207820296287537, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5018816079566266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06924978643655777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17677165567874908, + 0.9618675112724304, + 0.03930671885609627, + 0.16403593122959137, + 0.08439644426107407, + 0.04805602878332138, + 0.026429064571857452, + 0.05027944594621658, + 0.046099208295345306 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.2529248297214508, + "style_target": 0, + "support_probability": 0.903204619884491, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7538302565210315, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01920965500175953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9049220085144043, + 0.31975966691970825, + 0.22498096525669098, + 0.31975966691970825, + 0.08774497359991074 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 1, + "style_probability": 0.6307703852653503, + "style_target": 1, + "support_probability": 0.9902516603469849, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5598093842722485, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03067265823483467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9053178429603577, + 0.34301096200942993, + 0.18174734711647034, + 0.34301096200942993, + 0.013679579831659794, + 0.05051516368985176, + 0.059549666941165924, + 0.013701509684324265, + 0.022306466475129128, + 0.01769992522895336, + 0.014884351752698421 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 1, + "style_probability": 0.3854818046092987, + "style_target": 0, + "support_probability": 0.9785374402999878, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5354228411707845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.018558476120233536, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5182788968086243, + 0.262732595205307, + 0.10523945093154907, + 0.9763666391372681, + 0.09235106408596039, + 0.14978374540805817 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 0, + "style_probability": 0.3325342833995819, + "style_target": 1, + "support_probability": 0.8858315944671631, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.055576425859921195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.045494407415390015, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5993722677230835, + 0.2500763237476349, + 0.1277339905500412, + 0.9933890104293823, + 0.06943492591381073, + 0.09568139165639877 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 1, + "style_probability": 0.0549582876265049, + "style_target": 0, + "support_probability": 0.7212201356887817, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6885055109577181, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010370109230279922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9296196699142456, + 0.06025564670562744, + 0.30513331294059753, + 0.7211779356002808, + 0.00594642199575901, + 0.0063115740194916725, + 0.020626772195100784, + 0.03784933686256409, + 0.14111579954624176 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 1, + "style_probability": 0.5125833749771118, + "style_target": 1, + "support_probability": 0.9505852460861206, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8431831588492164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009054360911250114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9567990899085999, + 0.062012556940317154, + 0.3196893036365509, + 0.7359315156936646, + 0.023304684087634087, + 0.020322728902101517, + 0.02300005964934826, + 0.026579175144433975, + 0.021550189703702927 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 0, + "style_probability": 0.8535611629486084, + "style_target": 0, + "support_probability": 0.9170597791671753, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16376272464840216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17044232785701752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5186601281166077, + 0.2975042462348938, + 0.9563045501708984, + 0.15799425542354584, + 0.10338245332241058, + 0.02298567444086075, + 0.03235916048288345, + 0.027680138126015663, + 0.027588581666350365, + 0.0980270579457283, + 0.0542881153523922 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.6531308889389038, + "style_target": 1, + "support_probability": 0.09929158538579941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16766873259816645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20017345249652863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5550269484519958, + 0.25183969736099243, + 0.9702370166778564, + 0.12283068895339966, + 0.06999120861291885, + 0.03945353254675865, + 0.09502822160720825, + 0.08387983590364456, + 0.07787363976240158 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.44698649644851685, + "style_target": 0, + "support_probability": 0.09664499759674072, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9040448641149557, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.013519767671823502, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4723:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06300598382949829, + 0.2651388645172119, + 0.39492326974868774, + 0.9970909357070923, + 0.020470790565013885 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 1, + "style_probability": 0.8028189539909363, + "style_target": 1, + "support_probability": 0.932528555393219, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7080283850457619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.044604819267988205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07973576337099075, + 0.21343377232551575, + 0.34306949377059937, + 0.9875285625457764, + 0.04761636629700661 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 0, + "style_probability": 0.6352384090423584, + "style_target": 0, + "support_probability": 0.7562973499298096, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23585888100002547, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.787411093711853, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9420941472053528, + 0.2432275265455246, + 0.2571554183959961, + 0.7926271557807922, + 0.08188703656196594, + 0.058195289224386215 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 1, + "style_probability": 0.604849636554718, + "style_target": 1, + "support_probability": 0.2920936048030853, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08571134779389675, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8315039277076721, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9186791181564331, + 0.2629033923149109, + 0.31194156408309937, + 0.7320844531059265, + 0.0954902246594429 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 1, + "style_probability": 0.24912746250629425, + "style_target": 0, + "support_probability": 0.1643190234899521, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01993646685363636, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9733614325523376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11991558969020844, + 0.36565548181533813, + 0.2114618718624115, + 0.4129277765750885, + 0.13439252972602844, + 0.018372900784015656, + 0.044357556849718094, + 0.09141330420970917, + 0.1256999522447586 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.9204052090644836, + "style_target": 1, + "support_probability": 0.0055399807170033455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009571807510975239, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9776409268379211, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08757439255714417, + 0.3055247664451599, + 0.2945726215839386, + 0.5442084670066833, + 0.03139637038111687, + 0.17201459407806396, + 0.11273866146802902, + 0.03536591678857803, + 0.06814344972372055 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.20181290805339813, + "style_target": 0, + "support_probability": 0.0015766758006066084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3500887477082033, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1739458292722702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9675067663192749, + 0.25734251737594604, + 0.11123000830411911, + 0.3103995621204376, + 0.026817388832569122, + 0.028504550457000732, + 0.02629566378891468, + 0.022292794659733772, + 0.06939365714788437, + 0.0739055722951889 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.7834279537200928, + "style_target": 1, + "support_probability": 0.3239781856536865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22083291414029754, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07562652975320816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9778274893760681, + 0.22920580208301544, + 0.09034687280654907, + 0.38651031255722046, + 0.023966986685991287, + 0.036308404058218, + 0.016673417761921883, + 0.031038662418723106, + 0.18125608563423157, + 0.05389263853430748, + 0.07808609306812286 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.3625785708427429, + "style_target": 0, + "support_probability": 0.45989859104156494, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6445318178413283, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19400620460510254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34222909808158875, + 0.953169584274292, + 0.42404744029045105, + 0.34222909808158875, + 0.048598166555166245 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.754447877407074, + "style_target": 1, + "support_probability": 0.6927575469017029, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3299009658905619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15571719408035278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34179356694221497, + 0.943457841873169, + 0.42909184098243713, + 0.34179356694221497, + 0.03942986950278282, + 0.04051705077290535, + 0.025126706808805466, + 0.0707286149263382, + 0.04314688965678215, + 0.05326914042234421, + 0.04566948860883713 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.3070780038833618, + "style_target": 0, + "support_probability": 0.7571258544921875, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7429393861948603, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05932534113526344, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28229787945747375, + 0.2434714138507843, + 0.9904600381851196, + 0.39437058568000793, + 0.03067963756620884, + 0.019460268318653107 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.8883313536643982, + "style_target": 1, + "support_probability": 0.6833091378211975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09131510660636286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07749368995428085, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2589753568172455, + 0.23654572665691376, + 0.9923292994499207, + 0.3048766851425171, + 0.011914893053472042, + 0.0391404815018177 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.23334699869155884, + "style_target": 0, + "support_probability": 0.6277066469192505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12352808905157452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17093533277511597, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14998044073581696, + 0.9685404300689697, + 0.04066748544573784, + 0.4221266210079193, + 0.045909155160188675, + 0.016585562378168106, + 0.038966838270425797, + 0.043966036289930344, + 0.041640352457761765 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 1, + "style_probability": 0.8445896506309509, + "style_target": 1, + "support_probability": 0.03906964138150215, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.328000365597768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.051948562264442444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.151109516620636, + 0.9677619338035583, + 0.04496258497238159, + 0.3242570459842682, + 0.019651172682642937, + 0.020867325365543365, + 0.019370203837752342, + 0.05233830213546753, + 0.027597693726420403 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 1, + "style_probability": 0.4116397202014923, + "style_target": 0, + "support_probability": 0.4585878252983093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4471414751730444, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6541122198104858, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16117244958877563, + 0.1735941469669342, + 0.4961084723472595, + 0.6138931512832642, + 0.015430092811584473, + 0.008380314335227013, + 0.026480847969651222, + 0.013602889142930508, + 0.014646591618657112, + 0.02478320151567459, + 0.013153621926903725 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.8853632807731628, + "style_target": 1, + "support_probability": 0.4990779459476471, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09310416326088196, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.793480396270752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16287249326705933, + 0.16558393836021423, + 0.4445216953754425, + 0.7182724475860596, + 0.055749811232089996, + 0.02458033338189125, + 0.08531887829303741, + 0.016906043514609337, + 0.07235777378082275 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.26226893067359924, + "style_target": 0, + "support_probability": 0.2488928735256195, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7438778728963753, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024842282757163048, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8528902530670166, + 0.3599393367767334, + 0.08555131405591965, + 0.2327444702386856, + 0.0031068662647157907 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.8202357888221741, + "style_target": 1, + "support_probability": 0.9457637667655945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028232464184443404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061784595251083374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.898266077041626, + 0.34018534421920776, + 0.05261693149805069, + 0.22454538941383362, + 0.01832086779177189 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.021432630717754364, + "style_target": 0, + "support_probability": 0.7818178534507751, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3911635508014401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.290414035320282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5988770127296448, + 0.9886179566383362, + 0.45120376348495483, + 0.1837717443704605, + 0.020418254658579826, + 0.013130727224051952 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 1, + "style_probability": 0.7451202869415283, + "style_target": 1, + "support_probability": 0.14540031552314758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3604228200886646, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15571129322052002, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5129538774490356, + 0.9731931686401367, + 0.49270984530448914, + 0.26517465710639954, + 0.015090824104845524, + 0.03941473737359047 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 0, + "style_probability": 0.6342437863349915, + "style_target": 0, + "support_probability": 0.2804965674877167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9068149070798308, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009767895564436913, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4790:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17454536259174347, + 0.4618947505950928, + 0.9997809529304504, + 0.3455968499183655, + 0.015108156017959118, + 0.011799123138189316, + 0.011083443649113178, + 0.03121473826467991, + 0.022390024736523628 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 1, + "style_probability": 0.7192806601524353, + "style_target": 1, + "support_probability": 0.9521971940994263, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6398574118200835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.009921254590153694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4790:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19657453894615173, + 0.45828574895858765, + 0.9993959665298462, + 0.33052095770835876, + 0.04017093777656555, + 0.20821410417556763 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 1, + "style_probability": 0.31757012009620667, + "style_target": 0, + "support_probability": 0.9385684728622437, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05565858586410335, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8010056018829346, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2816818654537201, + 0.7464732527732849, + 0.2932550311088562, + 0.2816818654537201, + 0.10552527010440826, + 0.07205335050821304, + 0.056544139981269836, + 0.06701948493719101, + 0.04762142524123192, + 0.07408543676137924, + 0.04916999861598015 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.844923734664917, + "style_target": 1, + "support_probability": 0.04524152725934982, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00814401423892887, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6333522200584412, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2753857970237732, + 0.8068369626998901, + 0.27469831705093384, + 0.2753857970237732, + 0.16128969192504883, + 0.03151596710085869, + 0.43518632650375366, + 0.11659664660692215, + 0.05769708752632141, + 0.14744025468826294 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.09573773294687271, + "style_target": 0, + "support_probability": 0.066341333091259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012346556238247836, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9320045113563538, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.542366623878479, + 0.8255709409713745, + 0.5203356146812439, + 0.43974849581718445, + 0.058818962424993515 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 0, + "style_probability": 0.40806156396865845, + "style_target": 1, + "support_probability": 0.0047143204137682915, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08589623403033164, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7353854775428772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44150176644325256, + 0.7748304605484009, + 0.5098723769187927, + 0.36050790548324585, + 0.02364782989025116 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 0, + "style_probability": 0.5071690082550049, + "style_target": 0, + "support_probability": 0.05362692102789879, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.10320112696783035, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3413724899291992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6145089864730835, + 0.13988876342773438, + 0.21480782330036163, + 0.13988876342773438, + 0.0449339859187603, + 0.07375931739807129 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 0, + "style_probability": 0.2771022319793701, + "style_target": 1, + "support_probability": 0.20931215584278107, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18632778894433003, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4455038607120514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6065467596054077, + 0.1280035525560379, + 0.2308339923620224, + 0.1280035525560379, + 0.01917165331542492, + 0.04515712708234787 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 0, + "style_probability": 0.7608731389045715, + "style_target": 0, + "support_probability": 0.152902290225029, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7729069006796113, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.00514945387840271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23541122674942017, + 0.0708586648106575, + 0.9987673759460449, + 0.23541122674942017, + 0.011753104627132416, + 0.14728300273418427, + 0.035021860152482986, + 0.028704151511192322, + 0.03865719959139824 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 1, + "style_probability": 0.5988336801528931, + "style_target": 1, + "support_probability": 0.9739799499511719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5118929533799276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010516847483813763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1493796706199646, + 0.05744748190045357, + 0.9996212720870972, + 0.1493796706199646, + 0.034984517842531204, + 0.11861888319253922, + 0.03267408162355423, + 0.01951349899172783, + 0.05599559098482132 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 1, + "style_probability": 0.43300798535346985, + "style_target": 0, + "support_probability": 0.9377672076225281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05835653141148124, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41567742824554443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3605972230434418, + 0.9145258069038391, + 0.629449188709259, + 0.5626961588859558, + 0.056378453969955444, + 0.03255375847220421, + 0.017089907079935074, + 0.20660987496376038, + 0.08300142735242844, + 0.019521012902259827, + 0.013476476073265076 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.3056609034538269, + "style_target": 1, + "support_probability": 0.043575502932071686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07771501375224307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4678352475166321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47274571657180786, + 0.9014800190925598, + 0.5698592662811279, + 0.6040517687797546, + 0.015014168806374073, + 0.030302463099360466, + 0.13583625853061676, + 0.015476335771381855, + 0.0283981841057539, + 0.02165479212999344, + 0.010772706009447575 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.5421615839004517, + "style_target": 0, + "support_probability": 0.08263202011585236, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8454831019567317, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.024837424978613853, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.053090400993824005, + 0.2743828594684601, + 0.4597073793411255, + 0.9327743649482727, + 0.017571309581398964 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.9171218276023865, + "style_target": 1, + "support_probability": 0.9909948706626892, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49415340203747643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13391441106796265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05712001398205757, + 0.22458775341510773, + 0.42719578742980957, + 0.905012309551239, + 0.04036279022693634, + 0.01040599588304758, + 0.05196897312998772, + 0.0759410560131073, + 0.03814609721302986, + 0.014647349715232849, + 0.013624340295791626 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.3430452048778534, + "style_target": 0, + "support_probability": 0.8741819262504578, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.559465114652877, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2416411191225052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25509050488471985, + 0.3284047544002533, + 0.9720380902290344, + 0.5132479071617126, + 0.010671422816812992, + 0.01086315419524908 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.735490083694458, + "style_target": 1, + "support_probability": 0.4468502104282379, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022314282304273066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3771416246891022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3181413412094116, + 0.4493674337863922, + 0.9726219177246094, + 0.5960544347763062, + 0.020413005724549294, + 0.030638322234153748 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.08055975288152695, + "style_target": 0, + "support_probability": 0.28643015027046204, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03220954125040242, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6519304513931274, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2521461546421051, + 0.2453719675540924, + 0.22572846710681915, + 0.3546830713748932, + 0.04378814995288849, + 0.07294672727584839, + 0.020297903567552567, + 0.042452290654182434, + 0.028533345088362694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 1, + "style_probability": 0.780705988407135, + "style_target": 1, + "support_probability": 0.01566040702164173, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005701411193346741, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8713334798812866, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4099600911140442, + 0.3112870156764984, + 0.2567378282546997, + 0.37130028009414673, + 0.02754811756312847, + 0.041142918169498444, + 0.030235109850764275, + 0.018120024353265762, + 0.039420560002326965 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 1, + "style_probability": 0.44528961181640625, + "style_target": 0, + "support_probability": 0.003151585813611746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7938875725457918, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044457290321588516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3584913909435272, + 0.2313566505908966, + 0.9935489296913147, + 0.179879292845726, + 0.04593513533473015, + 0.03701258450746536, + 0.021970558911561966, + 0.0784773901104927, + 0.08277297765016556, + 0.3373393416404724, + 0.05924902483820915 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 1, + "style_probability": 0.8068372011184692, + "style_target": 1, + "support_probability": 0.7397904396057129, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3251335670352464, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1281207799911499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43447694182395935, + 0.2669857442378998, + 0.9856429696083069, + 0.19700263440608978, + 0.027874669060111046, + 0.041316594928503036, + 0.48342031240463257, + 0.04812372103333473, + 0.2895592749118805 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 1, + "style_probability": 0.4615200459957123, + "style_target": 0, + "support_probability": 0.35176733136177063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3984463696599086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3579758405685425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2725197374820709, + 0.19629250466823578, + 0.9859277009963989, + 0.17117837071418762, + 0.03658701851963997 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.7310662269592285, + "style_target": 1, + "support_probability": 0.2849169969558716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027710689824983332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.49229416251182556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2832670509815216, + 0.14230434596538544, + 0.992254376411438, + 0.14475151896476746, + 0.16069738566875458 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.11812079697847366, + "style_target": 0, + "support_probability": 0.20255166292190552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9196584305640401, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034032877534627914, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4897:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20373189449310303, + 0.9946240782737732, + 0.19736593961715698, + 0.22583016753196716, + 0.03260492533445358, + 0.15664774179458618 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 1, + "style_probability": 0.8821861743927002, + "style_target": 1, + "support_probability": 0.9229347109794617, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7913182496156088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03166818991303444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4897:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2230297178030014, + 0.9888021945953369, + 0.14435246586799622, + 0.19006235897541046, + 0.04786556214094162, + 0.0699792131781578 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 0, + "style_probability": 0.6452952027320862, + "style_target": 0, + "support_probability": 0.8909366726875305, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10702241960422007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.800499439239502, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5835193991661072, + 0.8119720816612244, + 0.4343856871128082, + 0.5835193991661072, + 0.041943322867155075, + 0.15020667016506195, + 0.03823968395590782, + 0.015932979062199593, + 0.09239725768566132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.8156667351722717, + "style_target": 1, + "support_probability": 0.03023059479892254, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1398032339860501, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5848905444145203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.591223418712616, + 0.8658463954925537, + 0.4250008761882782, + 0.591223418712616, + 0.0759894922375679, + 0.1058308407664299 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.356185644865036, + "style_target": 0, + "support_probability": 0.10414019227027893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33885017143261265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15951503813266754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.07915829122066498, + 0.1802096962928772, + 0.38030868768692017, + 0.9509811997413635, + 0.03490800783038139, + 0.011839890852570534, + 0.02337896265089512, + 0.02619568072259426, + 0.04179643839597702, + 0.1137700229883194 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.7449808716773987, + "style_target": 1, + "support_probability": 0.130606546998024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03150914645250246, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14336955547332764, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.06062554195523262, + 0.1607869267463684, + 0.38758450746536255, + 0.9482433199882507, + 0.016468916088342667, + 0.02499273233115673, + 0.05303904041647911, + 0.040560413151979446, + 0.030818890780210495, + 0.11889412254095078 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.13695362210273743, + "style_target": 0, + "support_probability": 0.13334299623966217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005931781658246468, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9482144117355347, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38481971621513367, + 0.09736690670251846, + 0.8412226438522339, + 0.3328167498111725, + 0.11609134078025818 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 0, + "style_probability": 0.20737507939338684, + "style_target": 1, + "support_probability": 0.0037600460927933455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007479657872076092, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.940353512763977, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45282819867134094, + 0.09203696250915527, + 0.8041061758995056, + 0.26800018548965454, + 0.07886917144060135 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 1, + "style_probability": 0.3710992634296417, + "style_target": 0, + "support_probability": 0.006912644952535629, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.041248434008602765, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3381734788417816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3848493695259094, + 0.41599464416503906, + 0.5561474561691284, + 0.7813819050788879, + 0.02623700350522995, + 0.05437743291258812 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.19956612586975098, + "style_target": 1, + "support_probability": 0.04293467104434967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07227222061412666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3290375769138336, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43465369939804077, + 0.43086013197898865, + 0.5249595642089844, + 0.8239370584487915, + 0.030939703807234764, + 0.03420136123895645 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.725220263004303, + "style_target": 0, + "support_probability": 0.06025512143969536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1685327693586755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6222358345985413, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8707557320594788, + 0.22170209884643555, + 0.29847458004951477, + 0.4041725695133209, + 0.09722202271223068, + 0.054988376796245575, + 0.05387139692902565, + 0.11317076534032822, + 0.09596536308526993 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 0, + "style_probability": 0.4202950596809387, + "style_target": 1, + "support_probability": 0.24323762953281403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028282343354434906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5346726179122925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8643863797187805, + 0.18929560482501984, + 0.2927473783493042, + 0.44924768805503845, + 0.05887215584516525, + 0.03883979842066765, + 0.0777752473950386, + 0.11931537836790085, + 0.11387534439563751 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 1, + "style_probability": 0.13430675864219666, + "style_target": 0, + "support_probability": 0.24212391674518585, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.10693591339362278, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2742846608161926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8789840340614319, + 0.46044883131980896, + 0.2584826648235321, + 0.46044883131980896, + 0.06712724268436432, + 0.037532057613134384, + 0.04339823126792908, + 0.018657179549336433, + 0.025622593238949776, + 0.03202066570520401 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 0, + "style_probability": 0.49793741106987, + "style_target": 1, + "support_probability": 0.11220935732126236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2972720569838377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18695001304149628, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8702462911605835, + 0.5334799885749817, + 0.2897901237010956, + 0.5334799885749817, + 0.055730100721120834, + 0.03682728484272957, + 0.03404265269637108, + 0.10491877049207687, + 0.02216259390115738, + 0.012896813452243805 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 0, + "style_probability": 0.6482387185096741, + "style_target": 0, + "support_probability": 0.31947559118270874, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5843460341348685, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4404166638851166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36147433519363403, + 0.48023539781570435, + 0.7541905641555786, + 0.36147433519363403, + 0.027286315336823463 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.8899592161178589, + "style_target": 1, + "support_probability": 0.7081366777420044, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17893126483428975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5016198754310608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3947870135307312, + 0.4455481767654419, + 0.7157984972000122, + 0.3947870135307312, + 0.023967646062374115, + 0.015383705496788025, + 0.02781522087752819, + 0.2973346412181854, + 0.016050249338150024, + 0.03589877858757973, + 0.07023503631353378 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.22777730226516724, + "style_target": 0, + "support_probability": 0.46898332238197327, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.19222985235562365, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29861509799957275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9273090362548828, + 0.1520942747592926, + 0.10126250982284546, + 0.20000357925891876, + 0.1275632232427597, + 0.04082265868782997 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.8923704624176025, + "style_target": 1, + "support_probability": 0.05214663967490196, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06811342530502143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19409945607185364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9446222186088562, + 0.08033058047294617, + 0.05388224497437477, + 0.10517914593219757, + 0.04363540932536125, + 0.1333366334438324 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.12907084822654724, + "style_target": 0, + "support_probability": 0.666977047920227, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7300026890565817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08303382247686386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13648828864097595, + 0.14189451932907104, + 0.944687008857727, + 0.13648828864097595, + 0.012634588405489922, + 0.016490943729877472, + 0.008994312025606632, + 0.030483823269605637, + 0.026621993631124496 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 1, + "style_probability": 0.9285244345664978, + "style_target": 1, + "support_probability": 0.7021930813789368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6396679070698102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04403248801827431, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1415002942085266, + 0.15454775094985962, + 0.9026631116867065, + 0.1415002942085266, + 0.009676140733063221, + 0.010849646292626858, + 0.025051897391676903, + 0.06579789519309998, + 0.09987524151802063 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 0, + "style_probability": 0.7001567482948303, + "style_target": 0, + "support_probability": 0.8195233941078186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011412074508007018, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8012231588363647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14970441162586212, + 0.18019825220108032, + 0.5068843960762024, + 0.46766749024391174, + 0.03191402554512024, + 0.03147168830037117, + 0.021481774747371674, + 0.03577972576022148, + 0.010042375884950161, + 0.019581276923418045, + 0.04047307372093201 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 1, + "style_probability": 0.6901582479476929, + "style_target": 1, + "support_probability": 0.004405539017170668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008882794975731316, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8195609450340271, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1369895339012146, + 0.24027247726917267, + 0.5434665083885193, + 0.4784090518951416, + 0.07749916613101959, + 0.0745835155248642, + 0.10944437980651855, + 0.06257878988981247, + 0.037059519439935684 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 1, + "style_probability": 0.423724502325058, + "style_target": 0, + "support_probability": 0.003579748095944524, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5438541151620209, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30740803480148315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9027106165885925, + 0.23071426153182983, + 0.43928420543670654, + 0.379162073135376, + 0.017650216817855835 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.760918378829956, + "style_target": 1, + "support_probability": 0.46648773550987244, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010087215233770704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6886605024337769, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9007737636566162, + 0.35114410519599915, + 0.51854407787323, + 0.375395804643631, + 0.06779184192419052 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.060833197087049484, + "style_target": 0, + "support_probability": 0.1238887757062912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9046746190094649, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015056907199323177, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4941:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16069954633712769, + 0.9944630265235901, + 0.34035301208496094, + 0.16069954633712769, + 0.00997632835060358, + 0.01570827141404152 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 1, + "style_probability": 0.8405571579933167, + "style_target": 1, + "support_probability": 0.8984054923057556, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.865276105551573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01399162132292986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4941:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16252849996089935, + 0.9828184247016907, + 0.3059312403202057, + 0.16252849996089935, + 0.018650580197572708, + 0.1125393807888031 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 0, + "style_probability": 0.6924552321434021, + "style_target": 0, + "support_probability": 0.9396267533302307, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26965017020597887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35311126708984375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5421102643013, + 0.5421102643013, + 0.3497929871082306, + 0.10560032725334167, + 0.05269999802112579, + 0.04093442112207413, + 0.03765701502561569, + 0.0366143062710762, + 0.028097743168473244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.8744062185287476, + "style_target": 1, + "support_probability": 0.265612930059433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03930098836201594, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7121161222457886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6356471180915833, + 0.6356471180915833, + 0.3621462881565094, + 0.12404339015483856, + 0.055071279406547546, + 0.06423819810152054 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.4157475531101227, + "style_target": 0, + "support_probability": 0.060721464455127716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9123938185497315, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006436028517782688, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4953:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.269125759601593, + 0.9858249425888062, + 0.23566360771656036, + 0.361028790473938, + 0.009322644211351871, + 0.023952167481184006, + 0.021789301186800003, + 0.021014073863625526, + 0.019145622849464417, + 0.015754010528326035 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.7840593457221985, + "style_target": 1, + "support_probability": 0.973564088344574, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09443356751135829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014866296201944351, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4953:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2357010543346405, + 0.9813801646232605, + 0.11938472837209702, + 0.27392008900642395, + 0.021215742453932762, + 0.03097458742558956, + 0.038481634110212326, + 0.0466986708343029, + 0.017935367301106453, + 0.039629265666007996 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.039883293211460114, + "style_target": 0, + "support_probability": 0.9141707420349121, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15376090969988374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30988651514053345, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.875931978225708, + 0.4721073806285858, + 0.24198010563850403, + 0.3042980432510376, + 0.052559953182935715 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 0, + "style_probability": 0.3104832172393799, + "style_target": 1, + "support_probability": 0.11795169115066528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24212403363517881, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3778127431869507, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8506647944450378, + 0.45878979563713074, + 0.2857011556625366, + 0.3691227436065674, + 0.05707651749253273 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 0, + "style_probability": 0.6385073065757751, + "style_target": 0, + "support_probability": 0.11777875572443008, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3333911909484, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04171160236001015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8528137803077698, + 0.14754128456115723, + 0.31500908732414246, + 0.5411967039108276, + 0.028248010203242302, + 0.06967240571975708 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.2204996645450592, + "style_target": 1, + "support_probability": 0.6223278045654297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6266342699309423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10139481723308563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8382108807563782, + 0.13132677972316742, + 0.26045140624046326, + 0.4999164342880249, + 0.040672417730093, + 0.053380075842142105 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.8364077210426331, + "style_target": 0, + "support_probability": 0.5551093816757202, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05481806763188018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7309331297874451, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7671559453010559, + 0.14027616381645203, + 0.3492736220359802, + 0.08545918017625809, + 0.03150251507759094, + 0.024565916508436203, + 0.033381327986717224, + 0.07771028578281403, + 0.05773821473121643 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 0, + "style_probability": 0.36486583948135376, + "style_target": 1, + "support_probability": 0.020398326218128204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005808946673573824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8306981921195984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8503834009170532, + 0.19357125461101532, + 0.3909863829612732, + 0.08090535551309586, + 0.030102990567684174, + 0.017614370211958885, + 0.07111051678657532, + 0.021142365410923958, + 0.05735011771321297 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 1, + "style_probability": 0.10771752148866653, + "style_target": 0, + "support_probability": 0.012939236126840115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6142581618933354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011575834825634956, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9698188304901123, + 0.052185021340847015, + 0.04438314959406853, + 0.24878504872322083, + 0.015130926854908466, + 0.05761495977640152, + 0.07155252248048782, + 0.04493960365653038, + 0.018914878368377686, + 0.029502185061573982, + 0.21293650567531586 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.2990418076515198, + "style_target": 1, + "support_probability": 0.8802239298820496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6870923227590459, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021308327093720436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9513012766838074, + 0.0544726587831974, + 0.04453250393271446, + 0.18783296644687653, + 0.023189520463347435, + 0.028148280456662178, + 0.025891246274113655, + 0.03366267308592796, + 0.007795355748385191, + 0.05126633122563362, + 0.15382030606269836 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.5290454030036926, + "style_target": 0, + "support_probability": 0.8121832609176636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5013781539972975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3346707224845886, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9365023970603943, + 0.17670133709907532, + 0.045463792979717255, + 0.1525866985321045, + 0.07423828542232513 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 1, + "style_probability": 0.9190446734428406, + "style_target": 1, + "support_probability": 0.3656659424304962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18545837709076785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3848299980163574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9185361862182617, + 0.16217097640037537, + 0.043208058923482895, + 0.13405653834342957, + 0.04252315312623978, + 0.051291752606630325, + 0.04155764356255531, + 0.036537837237119675, + 0.091666579246521, + 0.18618282675743103, + 0.0988946333527565 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 0, + "style_probability": 0.5274653434753418, + "style_target": 0, + "support_probability": 0.20954900979995728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46377601077167757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.029738644137978554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.360856831073761, + 0.10022889822721481, + 0.2996048033237457, + 0.8715996742248535, + 0.08056525141000748, + 0.030800925567746162 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.7937750816345215, + "style_target": 1, + "support_probability": 0.33217546343803406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021571045093206835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07209047675132751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3893793225288391, + 0.1468610167503357, + 0.38160592317581177, + 0.8989456295967102, + 0.05484017729759216, + 0.09893340617418289 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.039885517209768295, + "style_target": 0, + "support_probability": 0.3284514546394348, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16949483076329785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009812879376113415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34938815236091614, + 0.9846937656402588, + 0.14235198497772217, + 0.2523721754550934, + 0.15111000835895538, + 0.039971936494112015, + 0.10203460603952408, + 0.031162122264504433, + 0.057010672986507416 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 1, + "style_probability": 0.6368347406387329, + "style_target": 1, + "support_probability": 0.08817564696073532, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31138775933342405, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011511307209730148, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34799543023109436, + 0.9802533388137817, + 0.1711384356021881, + 0.21873614192008972, + 0.12002374231815338, + 0.0865386351943016, + 0.11996471136808395, + 0.36883699893951416, + 0.20693641901016235 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 1, + "style_probability": 0.47918835282325745, + "style_target": 0, + "support_probability": 0.31937384605407715, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8415699596506908, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010502244345843792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4662282466888428, + 0.17049579322338104, + 0.6920405030250549, + 0.9632936716079712, + 0.016148000955581665, + 0.0061168852262198925, + 0.02701752260327339, + 0.009534023702144623, + 0.02584066428244114, + 0.012775656767189503, + 0.008876820094883442 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 1, + "style_probability": 0.7781229615211487, + "style_target": 1, + "support_probability": 0.9573357105255127, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5053538516329509, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041581086814403534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45858895778656006, + 0.1758812665939331, + 0.6967645883560181, + 0.9497437477111816, + 0.05009578540921211, + 0.03405512124300003, + 0.03598710894584656, + 0.05049571767449379, + 0.14815682172775269 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 1, + "style_probability": 0.43839144706726074, + "style_target": 0, + "support_probability": 0.8476038575172424, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4278238060305971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7576967477798462, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16453322768211365, + 0.9972957968711853, + 0.4122580289840698, + 0.21223866939544678, + 0.05219721794128418 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.7414916157722473, + "style_target": 1, + "support_probability": 0.19706404209136963, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04855789095550411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9068495631217957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12159621715545654, + 0.9987927675247192, + 0.372305303812027, + 0.2394055873155594, + 0.06039990857243538 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.11303110420703888, + "style_target": 0, + "support_probability": 0.11402277648448944, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07313290039478883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.38732144236564636, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34621462225914, + 0.11973990499973297, + 0.08881419152021408, + 0.9747253060340881, + 0.05923585221171379, + 0.22133547067642212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 1, + "style_probability": 0.6974835395812988, + "style_target": 1, + "support_probability": 0.03401079773902893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14161983130428907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1686355173587799, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3635673522949219, + 0.1455460488796234, + 0.11341611295938492, + 0.9581639170646667, + 0.043463487178087234, + 0.01235539186745882 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 0, + "style_probability": 0.5321822166442871, + "style_target": 0, + "support_probability": 0.11527758836746216, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6440506806334865, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.050440385937690735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.912115752696991, + 0.060284387320280075, + 0.12315807491540909, + 0.060284387320280075, + 0.0172894150018692, + 0.011643930338323116, + 0.008556177839636803, + 0.01842292957007885, + 0.08530646562576294 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.7832232713699341, + "style_target": 1, + "support_probability": 0.6452054977416992, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32849753009484267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06090352684259415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8804025053977966, + 0.06022941321134567, + 0.14402684569358826, + 0.06022941321134567, + 0.03386459872126579, + 0.03551480546593666 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.377766877412796, + "style_target": 0, + "support_probability": 0.5682364702224731, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31643893856796806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2807467579841614, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05142764374613762, + 0.19516213238239288, + 0.7675756812095642, + 0.04890007898211479, + 0.0316523015499115, + 0.011928620748221874, + 0.01583487167954445, + 0.020041929557919502, + 0.00936774630099535, + 0.02994461916387081, + 0.021742111071944237 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.8473934531211853, + "style_target": 1, + "support_probability": 0.2554566264152527, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006100984772685378, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6858136653900146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.04425215348601341, + 0.15447397530078888, + 0.5995631814002991, + 0.046328578144311905, + 0.02003183588385582, + 0.03902343660593033, + 0.013315345160663128, + 0.049587182700634, + 0.03296801820397377, + 0.060637492686510086 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.07621198147535324, + "style_target": 0, + "support_probability": 0.07376902550458908, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8629458422943301, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0074928863905370235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9968956708908081, + 0.727466344833374, + 0.7348756194114685, + 0.2601383924484253, + 0.07396511733531952 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.5067098736763, + "style_target": 1, + "support_probability": 0.9879488945007324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7264024666241511, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0022953038569539785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9998886585235596, + 0.6996423602104187, + 0.6274007558822632, + 0.2325648069381714, + 0.035264138132333755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.10350858420133591, + "style_target": 0, + "support_probability": 0.996830403804779, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6391382817044113, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09251080453395844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2649235427379608, + 0.36903584003448486, + 0.9050953984260559, + 0.28510943055152893, + 0.03091512806713581, + 0.03240504860877991 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 1, + "style_probability": 0.6562708616256714, + "style_target": 1, + "support_probability": 0.58650803565979, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7126082869882785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12949171662330627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3150927424430847, + 0.3966991901397705, + 0.9233166575431824, + 0.3579341471195221, + 0.03159624710679054, + 0.051033422350883484 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 0, + "style_probability": 0.8760927319526672, + "style_target": 0, + "support_probability": 0.4553443193435669, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2852024384720596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07568199932575226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28380098938941956, + 0.20320747792720795, + 0.4560113549232483, + 0.9511033892631531, + 0.08926849812269211, + 0.019301652908325195, + 0.023119430989027023, + 0.023765403777360916, + 0.027328182011842728 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 1, + "style_probability": 0.6300633549690247, + "style_target": 1, + "support_probability": 0.12373349070549011, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23290550358572207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13180996477603912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30131518840789795, + 0.192801371216774, + 0.47237229347229004, + 0.9575657248497009, + 0.02457435615360737, + 0.058167021721601486 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 0, + "style_probability": 0.5647212862968445, + "style_target": 0, + "support_probability": 0.07555379718542099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012592703812995789, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8329371809959412, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20488178730010986, + 0.8893587589263916, + 0.4879528284072876, + 0.25508347153663635, + 0.036931075155735016, + 0.06401710212230682, + 0.04064062610268593, + 0.15266017615795135, + 0.08340177685022354, + 0.06151045486330986, + 0.10242221504449844 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 0, + "style_probability": 0.4483729600906372, + "style_target": 1, + "support_probability": 0.010585090145468712, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01007582258429899, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6192058324813843, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09234455227851868, + 0.8794717192649841, + 0.6684119701385498, + 0.1448856145143509, + 0.026708079501986504, + 0.045321621000766754, + 0.02392716333270073, + 0.029816223308444023, + 0.08385114371776581, + 0.13060137629508972 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 1, + "style_probability": 0.09655372053384781, + "style_target": 0, + "support_probability": 0.1306331902742386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058527803219394416, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5901074409484863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2130582183599472, + 0.3425617516040802, + 0.3428567051887512, + 0.34673377871513367, + 0.3114083409309387 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.8951305747032166, + "style_target": 1, + "support_probability": 0.07412775605916977, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004593054371119325, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7037854194641113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.26061052083969116, + 0.29917746782302856, + 0.32820773124694824, + 0.2912921607494354, + 0.035195767879486084, + 0.03928495571017265, + 0.08113463968038559, + 0.09295440465211868, + 0.03141763433814049, + 0.045894935727119446, + 0.07209812104701996, + 0.04712700843811035 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.34598714113235474, + "style_target": 0, + "support_probability": 0.010114343836903572, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33355767861206387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11214345693588257, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40085363388061523, + 0.3779970705509186, + 0.17018380761146545, + 0.9656445384025574, + 0.1602778434753418, + 0.0692429170012474 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.7845367789268494, + "style_target": 1, + "support_probability": 0.17477811872959137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044095920210354966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16208378970623016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2639232873916626, + 0.3565555512905121, + 0.18813298642635345, + 0.9789303541183472, + 0.11295140534639359, + 0.12438612431287766 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.0746689960360527, + "style_target": 0, + "support_probability": 0.24608145654201508, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.880149522278614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017344795167446136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20550581812858582, + 0.20550581812858582, + 0.9943795800209045, + 0.36477965116500854, + 0.030280526727437973, + 0.0366922952234745, + 0.03776903077960014, + 0.04282886162400246, + 0.1426001638174057 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 1, + "style_probability": 0.8182716965675354, + "style_target": 1, + "support_probability": 0.8063167333602905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6941400473915234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01238955371081829, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20312058925628662, + 0.20312058925628662, + 0.9864889979362488, + 0.4620581269264221, + 0.10497444868087769, + 0.04028110206127167, + 0.05375782400369644, + 0.24149152636528015, + 0.04420945793390274 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 1, + "style_probability": 0.44094792008399963, + "style_target": 0, + "support_probability": 0.713262140750885, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8439413442674834, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11488679051399231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28118234872817993, + 0.18519634008407593, + 0.17727527022361755, + 0.9583274126052856, + 0.01834685355424881, + 0.073629230260849, + 0.03674028441309929, + 0.03429139778017998, + 0.014470853842794895, + 0.0291006900370121, + 0.0836077630519867 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 1, + "style_probability": 0.9137391448020935, + "style_target": 1, + "support_probability": 0.7917807102203369, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6002350627913694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.128774031996727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3107883930206299, + 0.19636499881744385, + 0.15924383699893951, + 0.9327404499053955, + 0.15741309523582458, + 0.042036112397909164, + 0.1069021224975586, + 0.05779430270195007, + 0.12635375559329987 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 0, + "style_probability": 0.5129883289337158, + "style_target": 0, + "support_probability": 0.8324667811393738, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.056249422243363906, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6857345104217529, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15344461798667908, + 0.1592598259449005, + 0.7354958057403564, + 0.07203652709722519, + 0.01553365495055914 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.6933334469795227, + "style_target": 1, + "support_probability": 0.01996348612010479, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00490903410498145, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6665287017822266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15037114918231964, + 0.14567384123802185, + 0.7246243953704834, + 0.07415811717510223, + 0.06130650267004967 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.1752033531665802, + "style_target": 0, + "support_probability": 0.016553595662117004, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8202396419140549, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011876299045979977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5585325956344604, + 0.9986388087272644, + 0.04291832819581032, + 0.21786409616470337, + 0.022639917209744453, + 0.015917368233203888 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 1, + "style_probability": 0.6576095223426819, + "style_target": 1, + "support_probability": 0.9466112852096558, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8046114251785861, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008851829916238785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5560250878334045, + 0.9967589974403381, + 0.05468617007136345, + 0.1923372894525528, + 0.019549217075109482, + 0.03445126488804817 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 1, + "style_probability": 0.42196303606033325, + "style_target": 0, + "support_probability": 0.9789623618125916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5802641031178185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06831619888544083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4990357458591461, + 0.120240218937397, + 0.9185449481010437, + 0.4990357458591461, + 0.007431871723383665, + 0.02063646912574768, + 0.02663514018058777, + 0.019450630992650986, + 0.00643857242539525 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 1, + "style_probability": 0.8357210159301758, + "style_target": 1, + "support_probability": 0.2989845275878906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2515324506579428, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09292062371969223, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5412600636482239, + 0.14849241077899933, + 0.882821261882782, + 0.5412600636482239, + 0.03467867523431778, + 0.0381089523434639 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 1, + "style_probability": 0.46906644105911255, + "style_target": 0, + "support_probability": 0.24754102528095245, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8370760183552424, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016532819718122482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2784770429134369, + 0.7061488032341003, + 0.4120241105556488, + 0.9532484412193298, + 0.02977917529642582, + 0.03794379532337189, + 0.009617147967219353, + 0.032626062631607056, + 0.06746455281972885, + 0.04819173738360405, + 0.015654798597097397 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.8422022461891174, + "style_target": 1, + "support_probability": 0.8586676716804504, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09250106032471804, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03228544816374779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25609245896339417, + 0.5760774612426758, + 0.43028944730758667, + 0.9636098146438599, + 0.051127318292856216, + 0.036689624190330505, + 0.11709562689065933, + 0.03596305847167969, + 0.04261985793709755, + 0.07077034562826157 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.11927536875009537, + "style_target": 0, + "support_probability": 0.5727618932723999, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002278187964502, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9512745141983032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1984698623418808, + 0.2154923379421234, + 0.1769154965877533, + 0.05065033957362175, + 0.02271563932299614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 1, + "style_probability": 0.5187393426895142, + "style_target": 1, + "support_probability": 0.003653752151876688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011025534075981113, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9803885221481323, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13025662302970886, + 0.24555566906929016, + 0.15874189138412476, + 0.07154156267642975, + 0.027037912979722023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 1, + "style_probability": 0.3527583181858063, + "style_target": 0, + "support_probability": 0.0016746623441576958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5711589784809225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06656293570995331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9883576035499573, + 0.21400028467178345, + 0.6995369791984558, + 0.24981345236301422, + 0.12233787029981613, + 0.029704244807362556 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 1, + "style_probability": 0.793701708316803, + "style_target": 1, + "support_probability": 0.3573903739452362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4632822509417621, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04864445701241493, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9845032691955566, + 0.2671056091785431, + 0.6892707347869873, + 0.29012200236320496, + 0.05033646896481514 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 0, + "style_probability": 0.5254693627357483, + "style_target": 0, + "support_probability": 0.454571008682251, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9288408926736319, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.00730712478980422, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:5153:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2772926390171051, + 0.9909054040908813, + 0.5094477534294128, + 0.2772926390171051, + 0.013217763975262642, + 0.03669150546193123, + 0.01681520789861679, + 0.07463417947292328, + 0.019974181428551674 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.9190548658370972, + "style_target": 1, + "support_probability": 0.926072895526886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4401592848747515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008939116261899471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5153:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3624623417854309, + 0.9941794872283936, + 0.3600460886955261, + 0.3624623417854309, + 0.01335690077394247, + 0.03165183961391449, + 0.07401368767023087, + 0.021219564601778984, + 0.051673416048288345 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.24467071890830994, + "style_target": 0, + "support_probability": 0.8981356024742126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5103774191823245, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28508874773979187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19878153502941132, + 0.3768903911113739, + 0.4422498643398285, + 0.6770769357681274, + 0.014079530723392963, + 0.021612903103232384, + 0.014040002599358559, + 0.03964311629533768, + 0.022968294098973274, + 0.1169755682349205, + 0.010207878425717354 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 1, + "style_probability": 0.9067637324333191, + "style_target": 1, + "support_probability": 0.382972776889801, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07574590648521491, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6262363195419312, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23050515353679657, + 0.3227238357067108, + 0.42890965938568115, + 0.5081011056900024, + 0.014217720367014408, + 0.011517477221786976, + 0.012690787203609943, + 0.021582992747426033, + 0.02507816068828106, + 0.03573383018374443, + 0.020056234672665596 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 0, + "style_probability": 0.5153313875198364, + "style_target": 0, + "support_probability": 0.10389601439237595, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8722924617825871, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.015879185870289803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1402502954006195, + 0.30775249004364014, + 0.9599958658218384, + 0.3452594578266144, + 0.010097723454236984 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.9047470092773438, + "style_target": 1, + "support_probability": 0.9504775404930115, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5543531022945005, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016996681690216064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12128850817680359, + 0.2995787560939789, + 0.9604328870773315, + 0.23845931887626648, + 0.04678541421890259, + 0.04370054602622986, + 0.0471857413649559, + 0.04559464752674103, + 0.022612664848566055, + 0.041516222059726715, + 0.045529529452323914 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.4145490527153015, + "style_target": 0, + "support_probability": 0.926101803779602, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0020201647906744205, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9551092386245728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23076148331165314, + 0.427228182554245, + 0.40414443612098694, + 0.2092527151107788, + 0.024715114384889603, + 0.01213123369961977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.7813202142715454, + "style_target": 1, + "support_probability": 0.0008033904596231878, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00023342592405766174, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9553669095039368, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2226715385913849, + 0.36560577154159546, + 0.3571120500564575, + 0.181441530585289, + 0.022185541689395905, + 0.032746367156505585 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.21748299896717072, + "style_target": 0, + "support_probability": 0.00032559040118940175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05702112708125046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8158553242683411, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8577916026115417, + 0.6023510098457336, + 0.07873224467039108, + 0.38525259494781494, + 0.14700263738632202, + 0.07208767533302307, + 0.08299505710601807, + 0.045658010989427567, + 0.02660265751183033 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.8972893357276917, + "style_target": 1, + "support_probability": 0.03556099534034729, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.054188889192742375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7507036328315735, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8905437588691711, + 0.6626089215278625, + 0.09964390844106674, + 0.3213436007499695, + 0.03642747923731804, + 0.06067812442779541, + 0.03843914717435837, + 0.05263930559158325, + 0.05135282501578331 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.419796884059906, + "style_target": 0, + "support_probability": 0.060104988515377045, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023073043713672015, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9577438235282898, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5078077912330627, + 0.6848068833351135, + 0.3347453773021698, + 0.3174261450767517, + 0.034684810787439346, + 0.06797930598258972, + 0.11911638081073761, + 0.09701983630657196, + 0.08149248361587524, + 0.025022994726896286, + 0.026803994551301003 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.8177309036254883, + "style_target": 1, + "support_probability": 0.017719153314828873, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01785403509867921, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9376519918441772, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5052916407585144, + 0.6251323223114014, + 0.36547139286994934, + 0.32090315222740173, + 0.3081904351711273, + 0.043337609618902206, + 0.029354901984333992, + 0.04540155455470085, + 0.09635275602340698 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.35800454020500183, + "style_target": 0, + "support_probability": 0.018590260297060013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013252174193050517, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8215394616127014, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3620523512363434, + 0.09447859972715378, + 0.4920026659965515, + 0.4845476448535919, + 0.13454216718673706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 0, + "style_probability": 0.49744877219200134, + "style_target": 1, + "support_probability": 0.017054667696356773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011671536184184789, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8778636455535889, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34151381254196167, + 0.12391389906406403, + 0.46934327483177185, + 0.5096393823623657, + 0.08249060064554214, + 0.08651626110076904, + 0.08890609443187714, + 0.08399767428636551, + 0.021406738087534904, + 0.3090863525867462, + 0.08543488383293152 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 1, + "style_probability": 0.3128224313259125, + "style_target": 0, + "support_probability": 0.02304268442094326, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6098774173455574, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24895794689655304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9120778441429138, + 0.15560665726661682, + 0.24611975252628326, + 0.18307799100875854, + 0.09414631873369217, + 0.08299021422863007 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.5125330686569214, + "style_target": 1, + "support_probability": 0.7711860537528992, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1510689770269915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.159610778093338, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9776543974876404, + 0.044569820165634155, + 0.21067258715629578, + 0.09075602889060974, + 0.03488462045788765, + 0.05342596769332886 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.13699902594089508, + "style_target": 0, + "support_probability": 0.8334980010986328, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13605512031953917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22654490172863007, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31236639618873596, + 0.24741004407405853, + 0.9722453951835632, + 0.5083978772163391, + 0.0416032113134861, + 0.07713717967271805, + 0.0299850981682539, + 0.09632577002048492, + 0.09506969153881073 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 1, + "style_probability": 0.6556293368339539, + "style_target": 1, + "support_probability": 0.07301221042871475, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09701476080597407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15681129693984985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3035477101802826, + 0.20008766651153564, + 0.9919270277023315, + 0.3336550295352936, + 0.024846581742167473, + 0.02291715331375599, + 0.026182303205132484, + 0.0269326139241457, + 0.08559020608663559 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 0, + "style_probability": 0.5306525230407715, + "style_target": 0, + "support_probability": 0.07206294685602188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4172758587119311, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2532730996608734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9734892845153809, + 0.18368059396743774, + 0.0645204484462738, + 0.2122301310300827, + 0.035126522183418274, + 0.02013438567519188, + 0.032853756099939346, + 0.037008631974458694, + 0.013825323432683945, + 0.14005255699157715, + 0.030382396653294563 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.6442558169364929, + "style_target": 1, + "support_probability": 0.26902422308921814, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2714388621194589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28983786702156067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9745675921440125, + 0.17589551210403442, + 0.044482700526714325, + 0.23379860818386078, + 0.10981821268796921, + 0.09955834597349167, + 0.046768561005592346, + 0.2657369077205658, + 0.05052170529961586 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.4486957788467407, + "style_target": 0, + "support_probability": 0.20856666564941406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36649763619775455, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5026598572731018, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5060544610023499, + 0.6141986846923828, + 0.3810877203941345, + 0.5060544610023499, + 0.05454874411225319 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 1, + "style_probability": 0.9439399242401123, + "style_target": 1, + "support_probability": 0.4254334270954132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4934874176106982, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19999812543392181, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5987797379493713, + 0.6662912964820862, + 0.31209123134613037, + 0.5987797379493713, + 0.12023768573999405 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 1, + "style_probability": 0.4168732166290283, + "style_target": 0, + "support_probability": 0.7431929707527161, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7787329464478275, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023801164701581, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9855151176452637, + 0.3009765148162842, + 0.3628494143486023, + 0.9071414470672607, + 0.017814749851822853, + 0.018996529281139374 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.5756369233131409, + "style_target": 1, + "support_probability": 0.9771590828895569, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6995268562777368, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019856436178088188, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9803229570388794, + 0.3656602203845978, + 0.4120582640171051, + 0.9160641431808472, + 0.0622396320104599 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.3965548276901245, + "style_target": 0, + "support_probability": 0.9779912233352661, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0077258469132628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9313214421272278, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3017389476299286, + 0.8981645107269287, + 0.12342730164527893, + 0.37884828448295593, + 0.04812668636441231, + 0.041979704052209854, + 0.12038197368383408, + 0.07054083794355392, + 0.12000022083520889 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.8119676113128662, + "style_target": 1, + "support_probability": 0.004819643683731556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0024083409039426505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.912613570690155, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3256901204586029, + 0.9061754941940308, + 0.15011925995349884, + 0.42256301641464233, + 0.0640447810292244, + 0.11494863033294678, + 0.08072831481695175, + 0.07028571516275406, + 0.1152244582772255 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.21118341386318207, + "style_target": 0, + "support_probability": 0.007977222092449665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5888456482476171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1707753986120224, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3080073595046997, + 0.29864582419395447, + 0.9624583721160889, + 0.604814887046814, + 0.04137808829545975, + 0.0198794174939394, + 0.018764108419418335, + 0.06567688286304474, + 0.05857163667678833, + 0.07857025414705276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 1, + "style_probability": 0.8653034567832947, + "style_target": 1, + "support_probability": 0.37506619095802307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18477627751115866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32951316237449646, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2554120123386383, + 0.26226505637168884, + 0.950486958026886, + 0.49060869216918945, + 0.05715474858880043, + 0.027650801464915276, + 0.06059158220887184, + 0.05814983695745468, + 0.02320815809071064, + 0.21551570296287537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 1, + "style_probability": 0.47324129939079285, + "style_target": 0, + "support_probability": 0.20348608493804932, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7790768866118487, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034072354435920715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9129680395126343, + 0.25137943029403687, + 0.45233863592147827, + 0.25137943029403687, + 0.00621706061065197 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.7113264799118042, + "style_target": 1, + "support_probability": 0.9508290886878967, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4656114090598945, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11186317354440689, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8959249258041382, + 0.31658491492271423, + 0.44888076186180115, + 0.31658491492271423, + 0.02770300954580307, + 0.024254709482192993, + 0.1210285946726799, + 0.03563077375292778, + 0.07735318690538406, + 0.04252837598323822 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.3151615262031555, + "style_target": 0, + "support_probability": 0.8632871508598328, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.273922873263283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17811180651187897, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2722001075744629, + 0.21801824867725372, + 0.17866896092891693, + 0.8094193339347839, + 0.06795363128185272, + 0.03523930907249451 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.9285265803337097, + "style_target": 1, + "support_probability": 0.07098671793937683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02684922698527143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1334337741136551, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.279754102230072, + 0.20448827743530273, + 0.12063895910978317, + 0.8689779043197632, + 0.08544408529996872, + 0.17129932343959808 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.3184371590614319, + "style_target": 0, + "support_probability": 0.04004264622926712, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7859272569862706, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03986131772398949, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09745217859745026, + 0.18416406214237213, + 0.10493063926696777, + 0.9998784065246582, + 0.029581598937511444, + 0.06447887420654297, + 0.03325488045811653, + 0.06298047304153442, + 0.03530653193593025 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 1, + "style_probability": 0.7185825705528259, + "style_target": 1, + "support_probability": 0.6097639203071594, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47094867000222074, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06135031208395958, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11833183467388153, + 0.258668452501297, + 0.11279325932264328, + 0.9976450800895691, + 0.05233531445264816, + 0.019407175481319427, + 0.06448142230510712, + 0.05076863616704941, + 0.06380351632833481 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 0, + "style_probability": 0.6183983087539673, + "style_target": 0, + "support_probability": 0.3561975657939911, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.478166999236739, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6694250106811523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3285384774208069, + 0.9159528017044067, + 0.12208986282348633, + 0.3285384774208069, + 0.03395438194274902, + 0.028866281732916832, + 0.023324424400925636, + 0.07426239550113678, + 0.02824964001774788, + 0.047287508845329285, + 0.07129817456007004 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.8701521754264832, + "style_target": 1, + "support_probability": 0.29082244634628296, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.032067977632551724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8825499415397644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.280849814414978, + 0.8826746344566345, + 0.1097450852394104, + 0.280849814414978, + 0.026216043159365654, + 0.07964461296796799, + 0.05129270628094673, + 0.057287152856588364, + 0.06388895958662033 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.20317015051841736, + "style_target": 0, + "support_probability": 0.05196492001414299, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041450484684743695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9065184593200684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17648623883724213, + 0.4248717725276947, + 0.863537073135376, + 0.2820991575717926, + 0.09453162550926208 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.6428771018981934, + "style_target": 1, + "support_probability": 0.013308669440448284, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008264718473338834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.95253586769104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13486407697200775, + 0.46842828392982483, + 0.7858463525772095, + 0.28212040662765503, + 0.09127100557088852, + 0.11161144077777863, + 0.048122331500053406, + 0.15130139887332916, + 0.05530355125665665, + 0.09972458332777023, + 0.05736063793301582 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.4629875719547272, + "style_target": 0, + "support_probability": 0.003630609717220068, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5508529944167584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.053292375057935715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9269993305206299, + 0.10607688128948212, + 0.16958998143672943, + 0.10607688128948212, + 0.06509160250425339, + 0.022273242473602295 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.7176466584205627, + "style_target": 1, + "support_probability": 0.6765004992485046, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033504974417295656, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2838621139526367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.896958589553833, + 0.15215100347995758, + 0.18557660281658173, + 0.15215100347995758, + 0.02804550714790821, + 0.05683430656790733 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.08630000799894333, + "style_target": 0, + "support_probability": 0.4212965667247772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007227118776316746, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8383893966674805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1787327080965042, + 0.21388627588748932, + 0.7499039769172668, + 0.1787327080965042, + 0.060456935316324234, + 0.06801913678646088, + 0.0272205863147974, + 0.01817353256046772, + 0.04023280367255211 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 1, + "style_probability": 0.5764592289924622, + "style_target": 1, + "support_probability": 0.0027768630534410477, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030030606295292455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7818830013275146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20323249697685242, + 0.18879744410514832, + 0.7944058775901794, + 0.20323249697685242, + 0.027958806604146957, + 0.08168013393878937, + 0.01374555192887783, + 0.044792529195547104, + 0.026731090620160103 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 0, + "style_probability": 0.8253526091575623, + "style_target": 0, + "support_probability": 0.0058435555547475815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07358431878861069, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34874850511550903, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.222984179854393, + 0.9344414472579956, + 0.3313823342323303, + 0.24618014693260193, + 0.021805930882692337, + 0.07924625277519226, + 0.11782136559486389, + 0.05036956071853638, + 0.020589392632246017, + 0.034845076501369476 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 1, + "style_probability": 0.7011427879333496, + "style_target": 1, + "support_probability": 0.03220042213797569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07976805788589983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5452632308006287, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21681712567806244, + 0.9357290863990784, + 0.24682661890983582, + 0.17611825466156006, + 0.080300472676754, + 0.03128914535045624, + 0.08777870237827301, + 0.052862200886011124, + 0.06513730436563492 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 0, + "style_probability": 0.516466498374939, + "style_target": 0, + "support_probability": 0.020482255145907402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.68493536540864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2570425570011139, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2036626935005188, + 0.5243707299232483, + 0.9637950658798218, + 0.16374526917934418, + 0.02942359261214733 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 1, + "style_probability": 0.6004438996315002, + "style_target": 1, + "support_probability": 0.896807849407196, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7982768083458234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2592276334762573, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28926175832748413, + 0.6297280788421631, + 0.9521961212158203, + 0.2060435265302658, + 0.01403671596199274 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 0, + "style_probability": 0.7924143075942993, + "style_target": 0, + "support_probability": 0.9313985109329224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2296270196351912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05022565647959709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9475632905960083, + 0.4143465459346771, + 0.6200816035270691, + 0.5626406073570251, + 0.0725800171494484, + 0.05381375178694725 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.4541589319705963, + "style_target": 1, + "support_probability": 0.38163065910339355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6437121299718084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024226777255535126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9652153849601746, + 0.4494011402130127, + 0.6004276275634766, + 0.5329087972640991, + 0.20642273128032684, + 0.10052686929702759 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.8051462769508362, + "style_target": 0, + "support_probability": 0.5981993079185486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04436665763027983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6458213329315186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8980852365493774, + 0.0813210979104042, + 0.14811381697654724, + 0.13715723156929016, + 0.07711485028266907, + 0.08216989785432816, + 0.059613555669784546, + 0.11270730197429657, + 0.08204986155033112 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 0, + "style_probability": 0.18745802342891693, + "style_target": 1, + "support_probability": 0.029104769229888916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018372730982366703, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7760787010192871, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8726685643196106, + 0.0661805272102356, + 0.12913161516189575, + 0.1685025691986084, + 0.05557532608509064, + 0.1510683000087738, + 0.07324758917093277, + 0.03929503262042999, + 0.08655834943056107 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 1, + "style_probability": 0.05331101268529892, + "style_target": 0, + "support_probability": 0.036961331963539124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2007800738028508, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15565010905265808, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13008911907672882, + 0.17485599219799042, + 0.959933340549469, + 0.28322166204452515, + 0.03780490905046463, + 0.02963865175843239, + 0.03261043503880501, + 0.0720161497592926, + 0.0701977089047432, + 0.06137000396847725, + 0.07881506532430649 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.41721075773239136, + "style_target": 1, + "support_probability": 0.19125141203403473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3069845895374702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2005959153175354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15754695236682892, + 0.19801059365272522, + 0.9611234664916992, + 0.3372988700866699, + 0.033542606979608536, + 0.01888241246342659, + 0.011026225052773952, + 0.023956449702382088, + 0.014323047362267971, + 0.028125589713454247 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.6396705508232117, + "style_target": 0, + "support_probability": 0.2597395181655884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4871678823812928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01206539198756218, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9736517071723938, + 0.24652236700057983, + 0.1720196008682251, + 0.33477941155433655, + 0.3091951012611389 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 1, + "style_probability": 0.5664509534835815, + "style_target": 1, + "support_probability": 0.44206464290618896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39541822857497677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03119487687945366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9814491868019104, + 0.18414564430713654, + 0.10536830127239227, + 0.2728602886199951, + 0.05017269402742386 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 0, + "style_probability": 0.6588988304138184, + "style_target": 0, + "support_probability": 0.21538017690181732, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04812486527721527, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6666719913482666, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20956024527549744, + 0.5580459237098694, + 0.1391378492116928, + 0.1538315862417221, + 0.02142551727592945, + 0.08543192595243454 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 0, + "style_probability": 0.4610186219215393, + "style_target": 1, + "support_probability": 0.06398455053567886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013248886504220554, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7270808219909668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2734079957008362, + 0.5714800953865051, + 0.1475277990102768, + 0.1756957471370697, + 0.1219068318605423 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 1, + "style_probability": 0.17522956430912018, + "style_target": 0, + "support_probability": 0.031734950840473175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5406842732737719, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08569558709859848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13892170786857605, + 0.25336921215057373, + 0.4803885221481323, + 0.9662945866584778, + 0.06792785972356796, + 0.02492683008313179, + 0.41200804710388184, + 0.33047279715538025, + 0.04921318590641022 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.5813915133476257, + "style_target": 1, + "support_probability": 0.6371311545372009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09761684015710485, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054910916835069656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.06849672645330429, + 0.22482652962207794, + 0.3597169518470764, + 0.9744927287101746, + 0.04775625094771385, + 0.04342489689588547, + 0.04101074859499931, + 0.036249980330467224, + 0.06616606563329697 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.0574486069381237, + "style_target": 0, + "support_probability": 0.9189226627349854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2648299921334265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22803166508674622, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9809429049491882, + 0.21014873683452606, + 0.3348751366138458, + 0.07491973042488098, + 0.026730697602033615, + 0.017482737079262733, + 0.05091165378689766, + 0.0452527217566967, + 0.028897766023874283, + 0.02214704267680645, + 0.0338168665766716 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 0, + "style_probability": 0.4926971197128296, + "style_target": 1, + "support_probability": 0.22323769330978394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5670241088804, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3298254609107971, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9968357682228088, + 0.1749987155199051, + 0.2453145682811737, + 0.044498592615127563, + 0.07535868138074875, + 0.03558160364627838, + 0.030070582404732704, + 0.035076387226581573, + 0.06986744701862335, + 0.02341562695801258, + 0.045503903180360794 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 0, + "style_probability": 0.8622310757637024, + "style_target": 0, + "support_probability": 0.43838897347450256, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.31584142436423085, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20750968158245087, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4626375138759613, + 0.4626375138759613, + 0.6572597026824951, + 0.45593106746673584, + 0.03056127019226551 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 1, + "style_probability": 0.6354336142539978, + "style_target": 1, + "support_probability": 0.2772168219089508, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04422368110911154, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4619370698928833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37430474162101746, + 0.37430474162101746, + 0.6442046761512756, + 0.34755757451057434, + 0.03131227195262909, + 0.05739830434322357, + 0.02847682125866413, + 0.026087522506713867, + 0.018285078927874565, + 0.023433487862348557, + 0.04949706420302391 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 1, + "style_probability": 0.3436741530895233, + "style_target": 0, + "support_probability": 0.05753536894917488, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004097135749100501, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5769561529159546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4664897918701172, + 0.38201263546943665, + 0.2838452160358429, + 0.45527273416519165, + 0.10910572856664658, + 0.04488435760140419 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 0, + "style_probability": 0.46904125809669495, + "style_target": 1, + "support_probability": 0.0012766047148033977, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004635220169096875, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9125461578369141, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40691420435905457, + 0.2732689082622528, + 0.18300293385982513, + 0.4529281556606293, + 0.06541483104228973, + 0.15710872411727905 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 1, + "style_probability": 0.12040466070175171, + "style_target": 0, + "support_probability": 0.001687748241238296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00760076094411617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.948877215385437, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36270052194595337, + 0.18675114214420319, + 0.36270052194595337, + 0.7163308262825012, + 0.014112306758761406, + 0.012503334321081638, + 0.024681424722075462, + 0.08714675903320312, + 0.05900043249130249 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 1, + "style_probability": 0.5941647291183472, + "style_target": 1, + "support_probability": 0.0021305466070771217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008831267210734971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9090980887413025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2978556752204895, + 0.1750485599040985, + 0.2978556752204895, + 0.8051734566688538, + 0.044777922332286835, + 0.1009737104177475, + 0.01810869388282299, + 0.039813339710235596, + 0.023294826969504356 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 0, + "style_probability": 0.8160039782524109, + "style_target": 0, + "support_probability": 0.0015868737827986479, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13421701604080144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27321144938468933, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.811623215675354, + 0.31939396262168884, + 0.2790691554546356, + 0.31939396262168884, + 0.02591867372393608, + 0.027497176080942154, + 0.011613267473876476, + 0.010191944427788258, + 0.01822434552013874, + 0.04309611767530441, + 0.03868423402309418 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 1, + "style_probability": 0.5470596551895142, + "style_target": 1, + "support_probability": 0.07572649419307709, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08305727480975596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4497159719467163, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8232385516166687, + 0.29127106070518494, + 0.2614215612411499, + 0.29127106070518494, + 0.043446093797683716, + 0.03056483343243599, + 0.00979364663362503, + 0.0406048409640789, + 0.07778863608837128 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 1, + "style_probability": 0.4949636459350586, + "style_target": 0, + "support_probability": 0.034109000116586685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08995881620602923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4427293837070465, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2131005972623825, + 0.46465015411376953, + 0.9027296304702759, + 0.38447853922843933, + 0.05202552303671837 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 0, + "style_probability": 0.335214227437973, + "style_target": 1, + "support_probability": 0.13921090960502625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006211911776384649, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7418188452720642, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2807191014289856, + 0.3297926187515259, + 0.9102041721343994, + 0.18247979879379272, + 0.08103214204311371 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 1, + "style_probability": 0.05375898629426956, + "style_target": 0, + "support_probability": 0.06428772956132889, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7429686635232573, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.050888653844594955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7251796722412109, + 0.3335481584072113, + 0.5633853673934937, + 0.9708771705627441, + 0.09269163012504578, + 0.016282256692647934 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 1, + "style_probability": 0.70319002866745, + "style_target": 1, + "support_probability": 0.8504260778427124, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7834566090663704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04401838779449463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7224041223526001, + 0.3428477346897125, + 0.515467643737793, + 0.9800066351890564, + 0.04631680250167847, + 0.041360028088092804 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 0, + "style_probability": 0.8327928781509399, + "style_target": 0, + "support_probability": 0.8331637382507324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08558355650011949, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11706013232469559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4881143271923065, + 0.19585387408733368, + 0.4550863802433014, + 0.9345551133155823, + 0.016477461904287338, + 0.0639960765838623, + 0.05971008166670799, + 0.0889701396226883, + 0.04547048360109329 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 1, + "style_probability": 0.5686882734298706, + "style_target": 1, + "support_probability": 0.02903900295495987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1769086401316504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.046845074743032455, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47693145275115967, + 0.1997380405664444, + 0.45372703671455383, + 0.9604095816612244, + 0.020105231553316116, + 0.021582769230008125 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 0, + "style_probability": 0.5088775157928467, + "style_target": 0, + "support_probability": 0.08830954134464264, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04478393566303808, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5117279291152954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29146119952201843, + 0.5871501564979553, + 0.6636655926704407, + 0.30046185851097107, + 0.02599162608385086, + 0.02472751960158348, + 0.01831548660993576, + 0.04615970700979233, + 0.48451992869377136, + 0.15211807191371918, + 0.020075784996151924 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 1, + "style_probability": 0.5239569544792175, + "style_target": 1, + "support_probability": 0.056895606219768524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019975093664058472, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7104184031486511, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21014545857906342, + 0.6776388883590698, + 0.5968358516693115, + 0.23689159750938416, + 0.02366117760539055, + 0.029823044314980507, + 0.025407109409570694, + 0.0370318777859211, + 0.03467991575598717, + 0.05207856744527817 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 1, + "style_probability": 0.07907924056053162, + "style_target": 0, + "support_probability": 0.0708475112915039, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7317098371468695, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12949718534946442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6909881234169006, + 0.15882955491542816, + 0.46990063786506653, + 0.9413975477218628, + 0.03321414440870285 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.8040417432785034, + "style_target": 1, + "support_probability": 0.8973492383956909, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4309700376149665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06707942485809326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6176425218582153, + 0.19243356585502625, + 0.4271301031112671, + 0.9400386810302734, + 0.043909940868616104, + 0.01690104231238365, + 0.013277092017233372, + 0.04409315437078476, + 0.027777260169386864, + 0.042240265756845474, + 0.02181747555732727 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.36274927854537964, + "style_target": 0, + "support_probability": 0.9270868301391602, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8647167962260112, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007743642199784517, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9582700729370117, + 0.22345054149627686, + 0.7389376759529114, + 0.22345054149627686, + 0.009682023897767067, + 0.026111861690878868 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.7412493824958801, + "style_target": 1, + "support_probability": 0.9590861797332764, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19739079319587266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016248555853962898, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9324765801429749, + 0.23268477618694305, + 0.6738784909248352, + 0.23268477618694305, + 0.019057026132941246, + 0.05743535980582237 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.06003078073263168, + "style_target": 0, + "support_probability": 0.9253084659576416, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9451255502738194, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.003708313452079892, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:5393:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5869789719581604, + 0.9937042593955994, + 0.27718859910964966, + 0.24237753450870514, + 0.013622591271996498, + 0.009388101287186146, + 0.023951036855578423, + 0.017327142879366875, + 0.12680907547473907 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.8723082542419434, + "style_target": 1, + "support_probability": 0.993069052696228, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7590894613997288, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01264724787324667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4320838153362274, + 0.9819785952568054, + 0.20519527792930603, + 0.21533863246440887, + 0.010984533466398716, + 0.016244715079665184, + 0.013524048030376434, + 0.007239028811454773, + 0.007982532493770123 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.37217020988464355, + "style_target": 0, + "support_probability": 0.9834132194519043, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8613947261572372, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.035071924328804016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45132631063461304, + 0.9670922160148621, + 0.0732932910323143, + 0.45132631063461304, + 0.05439450964331627, + 0.01648796536028385, + 0.032805267721414566, + 0.014617985114455223, + 0.06401512026786804, + 0.013628179207444191, + 0.048320919275283813 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 1, + "style_probability": 0.8721493482589722, + "style_target": 1, + "support_probability": 0.9118995666503906, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5014136454302358, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09506778419017792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49012109637260437, + 0.9462260603904724, + 0.0960470587015152, + 0.49012109637260437, + 0.09616647660732269, + 0.04733054339885712, + 0.03756682202219963, + 0.018691057339310646, + 0.015368335880339146 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 0, + "style_probability": 0.5487549901008606, + "style_target": 0, + "support_probability": 0.6993553638458252, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.897404037137591, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.011972058564424515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.357846200466156, + 0.9948505759239197, + 0.38865843415260315, + 0.357846200466156, + 0.04117247089743614 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.6773319244384766, + "style_target": 1, + "support_probability": 0.9855480790138245, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.435706546567199, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.014482316561043262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44245290756225586, + 0.9972994923591614, + 0.42917805910110474, + 0.44245290756225586, + 0.05685678869485855 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.15605193376541138, + "style_target": 0, + "support_probability": 0.9844686388969421, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8565231031781181, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04595610871911049, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9984261989593506, + 0.5445026755332947, + 0.5573667287826538, + 0.06625448912382126, + 0.04776157811284065, + 0.04446663334965706 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 1, + "style_probability": 0.9160385131835938, + "style_target": 1, + "support_probability": 0.8377491235733032, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6137729792996263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03908948600292206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9974699020385742, + 0.5310373902320862, + 0.4596957862377167, + 0.08658900856971741, + 0.032263077795505524, + 0.10396162420511246 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 0, + "style_probability": 0.5054585933685303, + "style_target": 0, + "support_probability": 0.8940428495407104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36695367614991503, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2035120725631714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09130842238664627, + 0.09130842238664627, + 0.9125850796699524, + 0.06679652631282806, + 0.029583940282464027, + 0.02092868462204933, + 0.01414396706968546, + 0.02175687626004219, + 0.020836351439356804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.5648115873336792, + "style_target": 1, + "support_probability": 0.41341081261634827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1453869205130785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3413950800895691, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07479899376630783, + 0.07479899376630783, + 0.8852591514587402, + 0.09806394577026367, + 0.03866313770413399, + 0.054004017263650894 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.2096705436706543, + "style_target": 0, + "support_probability": 0.2526620626449585, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.49897076826548437, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1703002154827118, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2964569926261902, + 0.5993636846542358, + 0.2757311761379242, + 0.8367531299591064, + 0.06740910559892654, + 0.017998838797211647, + 0.02692740596830845, + 0.042947396636009216, + 0.00852842628955841, + 0.05087681859731674, + 0.04289858415722847 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.9102506637573242, + "style_target": 1, + "support_probability": 0.327847421169281, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07028606985957367, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22277730703353882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23482072353363037, + 0.5581852793693542, + 0.2310754954814911, + 0.8801924586296082, + 0.040147483348846436, + 0.04460551217198372, + 0.040778834372758865, + 0.033500801771879196, + 0.027686240151524544, + 0.08792829513549805 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.13992910087108612, + "style_target": 0, + "support_probability": 0.3963339626789093, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00908109745604257, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5053908228874207, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1127009317278862, + 0.44508737325668335, + 0.10044021904468536, + 0.5706382989883423, + 0.24447494745254517 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 0, + "style_probability": 0.23073603212833405, + "style_target": 1, + "support_probability": 0.02834593877196312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013533770604593293, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.590717077255249, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08012915402650833, + 0.4932648539543152, + 0.12865084409713745, + 0.6534963250160217, + 0.13965795934200287 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 1, + "style_probability": 0.09381725639104843, + "style_target": 0, + "support_probability": 0.05986796319484711, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7082406624470365, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06449864059686661, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19873200356960297, + 0.9040769338607788, + 0.2606562674045563, + 0.17490015923976898, + 0.03263886272907257, + 0.09743158519268036 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 1, + "style_probability": 0.7372802495956421, + "style_target": 1, + "support_probability": 0.8860297203063965, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.804116807500454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.030751487240195274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3742521107196808, + 0.8982853293418884, + 0.4084707200527191, + 0.15003259479999542, + 0.015674378722906113, + 0.02425745129585266 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 0, + "style_probability": 0.9721649289131165, + "style_target": 0, + "support_probability": 0.9527031183242798, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.026818034278155232, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9284669756889343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09508252888917923, + 0.5933887958526611, + 0.25412309169769287, + 0.27732163667678833, + 0.034095603972673416, + 0.03351777419447899, + 0.0451543815433979, + 0.0353611595928669, + 0.031221013516187668 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 0, + "style_probability": 0.4331915080547333, + "style_target": 1, + "support_probability": 0.03481220826506615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15638102967383397, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5431371927261353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.0730755403637886, + 0.7278029918670654, + 0.21292506158351898, + 0.18254922330379486, + 0.0241605956107378, + 0.03308163955807686 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 1, + "style_probability": 0.29231515526771545, + "style_target": 0, + "support_probability": 0.34835973381996155, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.026792005002607998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6932210326194763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16302700340747833, + 0.877452552318573, + 0.5062745809555054, + 0.12283274531364441, + 0.10272686928510666, + 0.03241428732872009, + 0.05081403627991676, + 0.026037633419036865, + 0.06811736524105072, + 0.07060917466878891, + 0.06173853948712349 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.6095713973045349, + "style_target": 1, + "support_probability": 0.010549040511250496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.042727385882724706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2493521124124527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1903984695672989, + 0.9255600571632385, + 0.4963781535625458, + 0.05895458906888962, + 0.07816470414400101, + 0.04437756910920143, + 0.068710096180439, + 0.06716743111610413, + 0.030718974769115448, + 0.06808259338140488 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.2268265336751938, + "style_target": 0, + "support_probability": 0.20050500333309174, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8290949160263322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06515652686357498, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9949212670326233, + 0.3942318260669708, + 0.3618067502975464, + 0.15514439344406128, + 0.05099398270249367 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.8197928071022034, + "style_target": 1, + "support_probability": 0.6774009466171265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3567374044964098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07133661210536957, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9936723709106445, + 0.3705439567565918, + 0.3842928111553192, + 0.18685925006866455, + 0.03423978388309479, + 0.06864213198423386, + 0.1576106697320938, + 0.03647768869996071, + 0.04201600328087807, + 0.12788918614387512, + 0.03407393768429756 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.23550523817539215, + "style_target": 0, + "support_probability": 0.6675894260406494, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.774933976504579, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.022411342710256577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5665341019630432, + 0.6905063390731812, + 0.8779510259628296, + 0.9565358757972717, + 0.009550482966005802, + 0.021669387817382812 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.7135897874832153, + "style_target": 1, + "support_probability": 0.9182287454605103, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.168908611737261, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04531631991267204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4567626118659973, + 0.716123640537262, + 0.8484861254692078, + 0.9683465361595154, + 0.020432017743587494, + 0.06750275194644928 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.125559002161026, + "style_target": 0, + "support_probability": 0.8837749361991882, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5994746693322359, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04484710469841957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30068057775497437, + 0.7330384850502014, + 0.9455628395080566, + 0.30068057775497437, + 0.022419974207878113, + 0.07667217403650284, + 0.04018804058432579, + 0.02615812048316002, + 0.050773292779922485 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 1, + "style_probability": 0.758342981338501, + "style_target": 1, + "support_probability": 0.450530469417572, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2794640119317744, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12863275408744812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29360371828079224, + 0.6260110139846802, + 0.9006688594818115, + 0.29360371828079224, + 0.2632025480270386, + 0.05916222929954529, + 0.027687780559062958, + 0.021003032103180885, + 0.06192644685506821 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 1, + "style_probability": 0.46555042266845703, + "style_target": 0, + "support_probability": 0.39519163966178894, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7547863975124626, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06695369631052017, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34958621859550476, + 0.532684862613678, + 0.41494202613830566, + 0.984889805316925, + 0.017290078103542328, + 0.023365579545497894, + 0.07569313049316406, + 0.016857672482728958, + 0.0180452149361372, + 0.014012464322149754, + 0.03629551827907562 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.6117556095123291, + "style_target": 1, + "support_probability": 0.8819868564605713, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43286302117703307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10685370117425919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3365769386291504, + 0.5846813321113586, + 0.3984948396682739, + 0.970769464969635, + 0.033056244254112244, + 0.0380992591381073, + 0.12083283811807632, + 0.015817055478692055, + 0.11687254160642624 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.23061607778072357, + "style_target": 0, + "support_probability": 0.7741340398788452, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8273883254552723, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12839855253696442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9574950337409973, + 0.24252595007419586, + 0.3559514582157135, + 0.56048184633255, + 0.03153912350535393 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.8956849575042725, + "style_target": 1, + "support_probability": 0.7103990912437439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2596446554751421, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17591431736946106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9625347852706909, + 0.2358352243900299, + 0.3007194995880127, + 0.6031566262245178, + 0.042632076889276505 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.2314791977405548, + "style_target": 0, + "support_probability": 0.6032190918922424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007201350734672463, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6100447177886963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6910627484321594, + 0.2528807818889618, + 0.23792722821235657, + 0.09489376097917557, + 0.008479014970362186, + 0.06286770105361938 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 1, + "style_probability": 0.7673892974853516, + "style_target": 1, + "support_probability": 0.0014487642329186201, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014862253249951805, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4168316125869751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7192245721817017, + 0.24925437569618225, + 0.22968856990337372, + 0.11585625261068344, + 0.04404450207948685, + 0.10226648300886154 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 0, + "style_probability": 0.5874125957489014, + "style_target": 0, + "support_probability": 0.007168494630604982, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5033377250302554, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17525197565555573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2485535591840744, + 0.1623714417219162, + 0.36356765031814575, + 0.9271056056022644, + 0.017486775293946266, + 0.014890966936945915, + 0.01535861287266016, + 0.012831967324018478, + 0.01714629866182804 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 0, + "style_probability": 0.4883933365345001, + "style_target": 1, + "support_probability": 0.8871277570724487, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48920699619193636, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1737493872642517, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3515685498714447, + 0.2594986855983734, + 0.4489738345146179, + 0.9239253997802734, + 0.029528861865401268, + 0.008855977095663548 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 1, + "style_probability": 0.38757258653640747, + "style_target": 0, + "support_probability": 0.9231722950935364, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.742527041013988, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3652750849723816, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15099765360355377, + 0.34457018971443176, + 0.8757705688476562, + 0.45628172159194946, + 0.02412959188222885, + 0.008879639208316803, + 0.035331301391124725, + 0.01422199048101902, + 0.04922393336892128, + 0.01782553270459175, + 0.07540455460548401 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.951961874961853, + "style_target": 1, + "support_probability": 0.7705988883972168, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0404423608349996, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3551008403301239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1266394555568695, + 0.18799357116222382, + 0.9338510036468506, + 0.3299337923526764, + 0.01930917054414749, + 0.02630702219903469, + 0.013868479989469051, + 0.04517713189125061, + 0.016600990667939186, + 0.034134529531002045 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.06205396726727486, + "style_target": 0, + "support_probability": 0.6014258861541748, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3416016558863773, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13462543487548828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07682105898857117, + 0.2645227909088135, + 0.5080537796020508, + 0.9772836565971375, + 0.049610186368227005 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.29464980959892273, + "style_target": 1, + "support_probability": 0.5877716541290283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6675851370276655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07529056072235107, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0709267258644104, + 0.22523826360702515, + 0.4109340012073517, + 0.9739557504653931, + 0.039719730615615845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.502166211605072, + "style_target": 0, + "support_probability": 0.7744319438934326, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6450265868543476, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05630955472588539, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5553:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24380826950073242, + 0.9942392110824585, + 0.18632207810878754, + 0.2773006558418274, + 0.06320653110742569, + 0.02331511676311493 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 1, + "style_probability": 0.5627158880233765, + "style_target": 1, + "support_probability": 0.8517813086509705, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9029113077896147, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09839457273483276, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:5553:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24716001749038696, + 0.994775652885437, + 0.19968253374099731, + 0.30334344506263733, + 0.028991257771849632, + 0.02439914084970951 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 0, + "style_probability": 0.9481451511383057, + "style_target": 0, + "support_probability": 0.7409639358520508, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1639549951674152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12947487831115723, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8479279279708862, + 0.29826128482818604, + 0.15297874808311462, + 0.29826128482818604, + 0.04469313472509384, + 0.025045650079846382, + 0.027747921645641327, + 0.08700639009475708, + 0.019067775458097458 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 0, + "style_probability": 0.32296398282051086, + "style_target": 1, + "support_probability": 0.2041129171848297, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0174448586185377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30168282985687256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7368078231811523, + 0.16232112050056458, + 0.1083717793226242, + 0.16232112050056458, + 0.015079056844115257, + 0.02622385323047638, + 0.06477437168359756, + 0.028123889118433, + 0.04426092281937599 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 1, + "style_probability": 0.05330988019704819, + "style_target": 0, + "support_probability": 0.28117406368255615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2742624460068519, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07729765772819519, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30143243074417114, + 0.6184483170509338, + 0.9694637656211853, + 0.30143243074417114, + 0.008772140368819237, + 0.037109412252902985, + 0.18276990950107574, + 0.02265702746808529, + 0.021346641704440117, + 0.06260357797145844, + 0.011366557329893112 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 0, + "style_probability": 0.416835755109787, + "style_target": 1, + "support_probability": 0.33706027269363403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21021644468837217, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.462275892496109, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3217954933643341, + 0.5867477655410767, + 0.9593047499656677, + 0.3217954933643341, + 0.0356503501534462, + 0.04431344196200371, + 0.01873559132218361, + 0.03453582897782326, + 0.03643478453159332, + 0.05423599109053612, + 0.01169173140078783 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 0, + "style_probability": 0.6483188271522522, + "style_target": 0, + "support_probability": 0.1232219859957695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5884030807107259, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13109301030635834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8758318424224854, + 0.5592069029808044, + 0.3680611252784729, + 0.5116135478019714, + 0.04620153456926346 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.9255461692810059, + "style_target": 1, + "support_probability": 0.3427148759365082, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12204129513766904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23564259707927704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.882760226726532, + 0.5312857031822205, + 0.254200279712677, + 0.5090606808662415, + 0.04410029947757721, + 0.02163933590054512, + 0.04437694326043129, + 0.07015304267406464, + 0.05095744505524635, + 0.02024896629154682 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.46741148829460144, + "style_target": 0, + "support_probability": 0.13989031314849854, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.875300379208311, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010024935938417912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.364122211933136, + 0.9909049868583679, + 0.3281336724758148, + 0.14508265256881714, + 0.0310793649405241, + 0.033455848693847656 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.8081183433532715, + "style_target": 1, + "support_probability": 0.9398600459098816, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17897889615808207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01919221505522728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3023466467857361, + 0.988339900970459, + 0.26863741874694824, + 0.14102119207382202, + 0.02728458121418953, + 0.0453631617128849 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.115724578499794, + "style_target": 0, + "support_probability": 0.8549292087554932, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6323960278942518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061445750296115875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6391336917877197, + 0.9387633204460144, + 0.20676523447036743, + 0.42845863103866577, + 0.11401285231113434, + 0.02199587970972061, + 0.061493393033742905, + 0.023875225335359573, + 0.023496659472584724 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 1, + "style_probability": 0.9018188714981079, + "style_target": 1, + "support_probability": 0.5523239970207214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.174152191162409, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11412480473518372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6128437519073486, + 0.9013065695762634, + 0.266163170337677, + 0.3270508944988251, + 0.037233881652355194, + 0.03971891105175018, + 0.024520350620150566, + 0.03706331551074982, + 0.05562916770577431 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 1, + "style_probability": 0.39162594079971313, + "style_target": 0, + "support_probability": 0.22555097937583923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0398742286340364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9292360544204712, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3973652720451355, + 0.26461994647979736, + 0.7540867924690247, + 0.1637123078107834, + 0.04224414750933647, + 0.0907822921872139, + 0.12235122919082642, + 0.04593253508210182, + 0.10256043076515198, + 0.0688164085149765, + 0.039156604558229446 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.7934274077415466, + "style_target": 1, + "support_probability": 0.017043352127075195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031525506877385964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8324109315872192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3267166316509247, + 0.3230663239955902, + 0.8206183910369873, + 0.18404044210910797, + 0.06437981128692627, + 0.026530161499977112, + 0.027319982647895813, + 0.06991631537675858, + 0.0494583360850811 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.3132413923740387, + "style_target": 0, + "support_probability": 0.022817479446530342, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10858638383767415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.41945162415504456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2615686058998108, + 0.8477176427841187, + 0.08447688072919846, + 0.15299631655216217, + 0.03923766687512398 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 1, + "style_probability": 0.6435513496398926, + "style_target": 1, + "support_probability": 0.04812048748135567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019512638748876186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4805566370487213, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27819135785102844, + 0.8523703813552856, + 0.07794732600450516, + 0.14353716373443604, + 0.061208125203847885 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 1, + "style_probability": 0.2123483568429947, + "style_target": 0, + "support_probability": 0.04539595544338226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006280833394729042, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9658463001251221, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38422515988349915, + 0.14180976152420044, + 0.1137668639421463, + 0.44834718108177185, + 0.042432669550180435, + 0.07734936475753784 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 1, + "style_probability": 0.635507345199585, + "style_target": 1, + "support_probability": 5.5548436648678035e-05, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006347236617978296, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9655706286430359, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47568878531455994, + 0.17219214141368866, + 0.13962817192077637, + 0.4942256808280945, + 0.029315689578652382, + 0.029434630647301674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 0, + "style_probability": 0.5856921672821045, + "style_target": 0, + "support_probability": 3.5029133869102225e-05, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17776787809735378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07389804720878601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13000991940498352, + 0.36204153299331665, + 0.7580283880233765, + 0.10800687223672867, + 0.008159096352756023, + 0.016746651381254196, + 0.01701495237648487, + 0.03604007512331009, + 0.011363358236849308 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.8526502251625061, + "style_target": 1, + "support_probability": 0.09623967111110687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10363803489941913, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03825525939464569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1424783170223236, + 0.3786237835884094, + 0.7177764773368835, + 0.0961117371916771, + 0.009144348092377186, + 0.014561605639755726 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.3739106059074402, + "style_target": 0, + "support_probability": 0.13815636932849884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11039390556999444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30039045214653015, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26410406827926636, + 0.524376630783081, + 0.9390137791633606, + 0.0725746899843216, + 0.0662326067686081, + 0.019481845200061798, + 0.019971806555986404, + 0.14689236879348755, + 0.0274394191801548, + 0.022368431091308594, + 0.044105879962444305 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.8855656981468201, + "style_target": 1, + "support_probability": 0.021683312952518463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012816440762354381, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4524805247783661, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20036767423152924, + 0.5450288653373718, + 0.9290289282798767, + 0.06431853771209717, + 0.07262314110994339, + 0.022890059277415276, + 0.034354016184806824, + 0.04606027156114578, + 0.04483139142394066, + 0.06514610350131989 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.18755066394805908, + "style_target": 0, + "support_probability": 0.03218445926904678, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32696423448687234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10644704848527908, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9645783305168152, + 0.4863426387310028, + 0.3478218913078308, + 0.2853521406650543, + 0.0354776568710804 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 0, + "style_probability": 0.3436115086078644, + "style_target": 1, + "support_probability": 0.6685088276863098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41826852231826195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0477217435836792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9687566161155701, + 0.4335918426513672, + 0.3107975125312805, + 0.25647422671318054, + 0.031608015298843384 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 1, + "style_probability": 0.4771028757095337, + "style_target": 0, + "support_probability": 0.8278394341468811, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4655289911265825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05321795120835304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8689881563186646, + 0.19682511687278748, + 0.1451406329870224, + 0.10380221903324127, + 0.021988261491060257, + 0.04721086472272873 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.3653538227081299, + "style_target": 1, + "support_probability": 0.5387740731239319, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6282777033159118, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10575208067893982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8757717609405518, + 0.22693422436714172, + 0.13789668679237366, + 0.13894525170326233, + 0.029114989563822746, + 0.035513296723365784 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.929419219493866, + "style_target": 0, + "support_probability": 0.36000755429267883, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5749131898603368, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15570181608200073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9042474627494812, + 0.8025637269020081, + 0.07497533410787582, + 0.1460937112569809, + 0.051131367683410645, + 0.05532315745949745, + 0.06279945373535156, + 0.009377534501254559, + 0.1471540629863739 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 0, + "style_probability": 0.42656832933425903, + "style_target": 1, + "support_probability": 0.9072262644767761, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13797439304448744, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09463342279195786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9482973217964172, + 0.819801390171051, + 0.09258829802274704, + 0.2232605218887329, + 0.058383431285619736, + 0.10314829647541046, + 0.052239593118429184, + 0.046071890741586685, + 0.07613574713468552 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 1, + "style_probability": 0.05881445109844208, + "style_target": 0, + "support_probability": 0.9535849094390869, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4399439003145885, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32548433542251587, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27138203382492065, + 0.9060367941856384, + 0.1584436446428299, + 0.10799169540405273, + 0.06984634697437286, + 0.04426806792616844, + 0.031660858541727066, + 0.016544019803404808, + 0.013708638027310371, + 0.029964903369545937 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 0, + "style_probability": 0.1882348358631134, + "style_target": 1, + "support_probability": 0.9066953063011169, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04102500170225332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.925392210483551, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2273920178413391, + 0.8132393956184387, + 0.16695362329483032, + 0.09121102839708328, + 0.02674635872244835, + 0.03130493313074112, + 0.040391240268945694, + 0.0114694619551301, + 0.0627029538154602, + 0.02812318503856659, + 0.07712075114250183 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 1, + "style_probability": 0.279333233833313, + "style_target": 0, + "support_probability": 0.1018962413072586, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.506146565840727, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13576047122478485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.04890841618180275, + 0.2992675006389618, + 0.3938278555870056, + 0.9049696326255798, + 0.0321677029132843 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.7064858078956604, + "style_target": 1, + "support_probability": 0.5851336121559143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2693511079294493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1737206131219864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.039894554764032364, + 0.24231521785259247, + 0.35734596848487854, + 0.9019700288772583, + 0.05122877284884453, + 0.13719668984413147, + 0.04764487221837044, + 0.10281588137149811, + 0.017831886187195778, + 0.06802483648061752, + 0.14714808762073517 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.18984733521938324, + "style_target": 0, + "support_probability": 0.5114332437515259, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.731043133223254, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0767323300242424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4792250394821167, + 0.9911419153213501, + 0.5333603024482727, + 0.4621255397796631, + 0.03029206022620201, + 0.012975100427865982 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.6005864143371582, + "style_target": 1, + "support_probability": 0.8175665140151978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25510350540640897, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13674437999725342, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45340874791145325, + 0.9944844841957092, + 0.4457501769065857, + 0.5008861422538757, + 0.01106585469096899, + 0.02301018126308918 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.22622622549533844, + "style_target": 0, + "support_probability": 0.7193005084991455, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02189227024075361, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8949169516563416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5125066041946411, + 0.253814697265625, + 0.41732531785964966, + 0.253814697265625, + 0.03717595338821411, + 0.04751281440258026, + 0.0629974827170372, + 0.014184488914906979, + 0.10786689072847366 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 1, + "style_probability": 0.7532870173454285, + "style_target": 1, + "support_probability": 0.011763460002839565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008276423854642582, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9607097506523132, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5228422284126282, + 0.22234365344047546, + 0.35732635855674744, + 0.22234365344047546, + 0.050549354404211044, + 0.04080712050199509, + 0.011472003534436226, + 0.01478290744125843, + 0.06713057309389114 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 1, + "style_probability": 0.4777248203754425, + "style_target": 0, + "support_probability": 0.005667123477905989, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.44717361797862687, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16667191684246063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1553242951631546, + 0.960071325302124, + 0.4767211377620697, + 0.2189263552427292, + 0.01666993275284767, + 0.02915586531162262, + 0.023416634649038315, + 0.05727820098400116, + 0.031118446961045265, + 0.01906392350792885 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.781974196434021, + "style_target": 1, + "support_probability": 0.2739116847515106, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10090584612021443, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26106855273246765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1847972422838211, + 0.9596346616744995, + 0.5507868528366089, + 0.22624902427196503, + 0.020642133429646492, + 0.02289639040827751, + 0.10097382962703705, + 0.41266492009162903, + 0.28226974606513977 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.20779941976070404, + "style_target": 0, + "support_probability": 0.13247376680374146, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6739663975163549, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06343353539705276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9986146688461304, + 0.2605454921722412, + 0.5420597195625305, + 0.5076297521591187, + 0.10439242422580719 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 1, + "style_probability": 0.8768940567970276, + "style_target": 1, + "support_probability": 0.48752474784851074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.304169379197283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061023373156785965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9995154142379761, + 0.31859540939331055, + 0.5912729501724243, + 0.42838725447654724, + 0.044152069836854935 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 1, + "style_probability": 0.421006977558136, + "style_target": 0, + "support_probability": 0.5936757326126099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026593377495449915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6649832129478455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16651564836502075, + 0.8829337358474731, + 0.2249492108821869, + 0.21387748420238495, + 0.1478140950202942, + 0.15081146359443665 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 1, + "style_probability": 0.8062079548835754, + "style_target": 1, + "support_probability": 0.013615704141557217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04883465408168466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7173247933387756, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20498988032341003, + 0.8071346282958984, + 0.13931085169315338, + 0.23534658551216125, + 0.024410413578152657, + 0.012174361385405064 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 0, + "style_probability": 0.7231854200363159, + "style_target": 0, + "support_probability": 0.023648707196116447, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8379646984782819, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026196179911494255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3543020784854889, + 0.9685680270195007, + 0.20382775366306305, + 0.09051652252674103, + 0.008024217560887337, + 0.01042092964053154, + 0.025385357439517975, + 0.03228551521897316, + 0.07787889242172241 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.7542698383331299, + "style_target": 1, + "support_probability": 0.884811520576477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4196928219728857, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03360668569803238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3926219344139099, + 0.9666310548782349, + 0.26194295287132263, + 0.09464339166879654, + 0.02020340971648693, + 0.04758818820118904 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.2462547868490219, + "style_target": 0, + "support_probability": 0.8310351967811584, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18018981365438386, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44190627336502075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7499110102653503, + 0.27051231265068054, + 0.1311594396829605, + 0.5443387031555176, + 0.01540366094559431, + 0.033445388078689575, + 0.00724615016952157, + 0.013031273148953915, + 0.04927409812808037, + 0.024413367733359337, + 0.019192157313227654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.7272133827209473, + "style_target": 1, + "support_probability": 0.14021682739257812, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009773741359433416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4414868950843811, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7717410922050476, + 0.36885279417037964, + 0.1800367832183838, + 0.5481294989585876, + 0.03812514990568161, + 0.028479214757680893, + 0.06648889183998108, + 0.02390015684068203, + 0.010174209251999855, + 0.03047388419508934 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.07924390584230423, + "style_target": 0, + "support_probability": 0.16886800527572632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7724593073539776, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19342325627803802, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4709682762622833, + 0.7018123865127563, + 0.9239003658294678, + 0.7102396488189697, + 0.04996664822101593 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 1, + "style_probability": 0.9029029011726379, + "style_target": 1, + "support_probability": 0.6936033964157104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2912133361911637, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33272483944892883, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5982367396354675, + 0.6252321600914001, + 0.8898759484291077, + 0.6251787543296814, + 0.07994993031024933 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 0, + "style_probability": 0.6006873846054077, + "style_target": 0, + "support_probability": 0.23254507780075073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7918369517483576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02475542016327381, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43201905488967896, + 0.993340790271759, + 0.30502286553382874, + 0.25778478384017944, + 0.06113072857260704, + 0.06942566484212875 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.8759562373161316, + "style_target": 1, + "support_probability": 0.7664433717727661, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3639590692170551, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02417045086622238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40096646547317505, + 0.991283655166626, + 0.2990724444389343, + 0.257767915725708, + 0.03599991649389267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.3539770543575287, + "style_target": 0, + "support_probability": 0.6872193217277527, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002711296997226679, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9951443076133728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3249634802341461, + 0.2618933320045471, + 0.4016250669956207, + 0.3249634802341461, + 0.02390921488404274, + 0.04894950985908508, + 0.07099475711584091, + 0.09515983611345291, + 0.05090099573135376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.7323929667472839, + "style_target": 1, + "support_probability": 0.0011553795775398612, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006545229199005305, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9958916902542114, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3478248715400696, + 0.16910794377326965, + 0.2699431777000427, + 0.3478248715400696, + 0.027837874367833138, + 0.13894984126091003, + 0.1358768492937088, + 0.06693321466445923, + 0.087409608066082 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.28820881247520447, + "style_target": 0, + "support_probability": 0.001312563894316554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03116518755323638, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5526922941207886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5249212980270386, + 0.3405619263648987, + 0.15201961994171143, + 0.12441027909517288, + 0.1236337199807167, + 0.04566723108291626, + 0.031461309641599655, + 0.044050294905900955, + 0.07074324041604996, + 0.10672637075185776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 1, + "style_probability": 0.8067364692687988, + "style_target": 1, + "support_probability": 0.008211763575673103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0066589132718395064, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4982088804244995, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5956553816795349, + 0.2496066391468048, + 0.16475443542003632, + 0.10057418793439865, + 0.05261312052607536, + 0.042790766805410385, + 0.07641660422086716, + 0.07851573824882507, + 0.06977178901433945, + 0.14169365167617798, + 0.07506197690963745 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 1, + "style_probability": 0.4575783312320709, + "style_target": 0, + "support_probability": 0.005256798584014177, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020611388679815677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.802845299243927, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9343677759170532, + 0.2574385404586792, + 0.328299343585968, + 0.2518700957298279, + 0.03528796136379242 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.3614034056663513, + "style_target": 1, + "support_probability": 0.025746185332536697, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05146846663835694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8335092663764954, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9446805119514465, + 0.26127082109451294, + 0.2923123240470886, + 0.25861111283302307, + 0.1105523407459259 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.8784528374671936, + "style_target": 0, + "support_probability": 0.024290813133120537, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.09777705976007105, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6273273825645447, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3339459002017975, + 0.9542600512504578, + 0.22987902164459229, + 0.3504663109779358, + 0.10253093391656876, + 0.02978629246354103 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 0, + "style_probability": 0.4479387402534485, + "style_target": 1, + "support_probability": 0.17813196778297424, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05042016504086616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5137377977371216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41364115476608276, + 0.9662685394287109, + 0.20646275579929352, + 0.3359196186065674, + 0.01391447801142931, + 0.03791813552379608 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 1, + "style_probability": 0.15087810158729553, + "style_target": 0, + "support_probability": 0.41423115134239197, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.3581062265740586, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22941303253173828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.974157452583313, + 0.1857559084892273, + 0.3588411808013916, + 0.14401890337467194, + 0.014820796437561512, + 0.006907186470925808, + 0.040856730192899704, + 0.0555574968457222, + 0.021408721804618835 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 0, + "style_probability": 0.4706449508666992, + "style_target": 1, + "support_probability": 0.18136924505233765, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47337257487619017, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20511159300804138, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9644955992698669, + 0.258446604013443, + 0.37489718198776245, + 0.13894639909267426, + 0.013737409375607967, + 0.029452605172991753, + 0.02730153128504753, + 0.02388625033199787, + 0.011216317303478718 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 1, + "style_probability": 0.48994243144989014, + "style_target": 0, + "support_probability": 0.33027026057243347, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33494071751800564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17830684781074524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3086167275905609, + 0.2198144495487213, + 0.13534514605998993, + 0.938721239566803, + 0.01129082590341568, + 0.021203001961112022, + 0.017003536224365234, + 0.009138555265963078, + 0.0322258435189724, + 0.04260317608714104, + 0.009786694310605526 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.29369205236434937, + "style_target": 1, + "support_probability": 0.4191721975803375, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2970614596042314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1882881373167038, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26447710394859314, + 0.1975868046283722, + 0.09875908493995667, + 0.9310188889503479, + 0.011573761701583862, + 0.016656259074807167, + 0.012179678305983543, + 0.006474292371422052, + 0.014813189394772053, + 0.02090759202837944, + 0.011462289839982986 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.6159785985946655, + "style_target": 0, + "support_probability": 0.2621915340423584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01008214933545094, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23869377374649048, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22897547483444214, + 0.3989694118499756, + 0.6083121299743652, + 0.5225788950920105, + 0.02990128844976425 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 0, + "style_probability": 0.3721771538257599, + "style_target": 1, + "support_probability": 0.008722225204110146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008045028219894326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46199074387550354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22194865345954895, + 0.4183242619037628, + 0.6089516282081604, + 0.5802990198135376, + 0.06817162781953812, + 0.1630190908908844, + 0.041958920657634735, + 0.10643146932125092, + 0.1295526623725891, + 0.06412758678197861 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 1, + "style_probability": 0.3151650130748749, + "style_target": 0, + "support_probability": 0.002433396875858307, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7686488806124951, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.013104954734444618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9341281652450562, + 0.6570121049880981, + 0.12243489176034927, + 0.32191741466522217, + 0.030587775632739067, + 0.012015250511467457 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.5957813858985901, + "style_target": 1, + "support_probability": 0.9452810287475586, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5978420473010927, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016397293657064438, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9582328200340271, + 0.6046814322471619, + 0.09827455878257751, + 0.2887442708015442, + 0.017484309151768684, + 0.0385417677462101 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.3397827446460724, + "style_target": 0, + "support_probability": 0.8980486989021301, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014736102568521824, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7963467836380005, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5860850214958191, + 0.32548895478248596, + 0.5001917481422424, + 0.6477919816970825, + 0.04251744598150253, + 0.03246089816093445, + 0.046363815665245056, + 0.07555422931909561, + 0.0791582465171814 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 1, + "style_probability": 0.583824098110199, + "style_target": 1, + "support_probability": 0.005177288316190243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10277355601637764, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29488715529441833, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38615190982818604, + 0.45183324813842773, + 0.5071368217468262, + 0.5795656442642212, + 0.03105034865438938, + 0.04719296842813492, + 0.03850594535470009, + 0.05366192385554314, + 0.09244303405284882 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 0, + "style_probability": 0.9122655391693115, + "style_target": 0, + "support_probability": 0.02198941446840763, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0100456363394586, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9447938203811646, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.399583637714386, + 0.7067630290985107, + 0.3174196481704712, + 0.6663458943367004, + 0.060702234506607056, + 0.026362886652350426, + 0.064365454018116, + 0.04762262478470802, + 0.20233826339244843, + 0.045381173491477966, + 0.029537690803408623 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 1, + "style_probability": 0.5157341361045837, + "style_target": 1, + "support_probability": 0.005477289669215679, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007524004325325058, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9658787846565247, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4405558407306671, + 0.6867792010307312, + 0.24778050184249878, + 0.7118442058563232, + 0.10439838469028473, + 0.03226427733898163, + 0.09856223315000534, + 0.06066352128982544, + 0.1109328344464302 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 1, + "style_probability": 0.4017947316169739, + "style_target": 0, + "support_probability": 0.0024129648227244616, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8489545756926555, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0060489666648209095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9698994755744934, + 0.5250278115272522, + 0.6512620449066162, + 0.29258838295936584, + 0.026335757225751877 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 1, + "style_probability": 0.612208366394043, + "style_target": 1, + "support_probability": 0.9963561296463013, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1398227859817662, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05192704126238823, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9527941942214966, + 0.4990396499633789, + 0.6532517075538635, + 0.18163250386714935, + 0.11275910586118698 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 1, + "style_probability": 0.05807911232113838, + "style_target": 0, + "support_probability": 0.9677382707595825, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007907117617573556, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9627193808555603, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43418437242507935, + 0.8863846659660339, + 0.3838181495666504, + 0.19475293159484863, + 0.054996244609355927, + 0.10043740272521973 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 1, + "style_probability": 0.5916925668716431, + "style_target": 1, + "support_probability": 0.003331188578158617, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4233763643807138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5026382803916931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5287834405899048, + 0.9033514261245728, + 0.3298639953136444, + 0.1695873886346817, + 0.032877665013074875, + 0.01414042990654707 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 0, + "style_probability": 0.9276160001754761, + "style_target": 0, + "support_probability": 0.12790895998477936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007641872473120442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9854423999786377, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2587956190109253, + 0.7723425626754761, + 0.2536361813545227, + 0.40649405121803284, + 0.03175267204642296, + 0.016412382945418358, + 0.05555742233991623, + 0.05392808839678764, + 0.02917514555156231 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 0, + "style_probability": 0.406414657831192, + "style_target": 1, + "support_probability": 0.005912506487220526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026297666329593548, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9625527858734131, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2412133663892746, + 0.7601895332336426, + 0.20031176507472992, + 0.42914634943008423, + 0.027978990226984024, + 0.16763390600681305 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 1, + "style_probability": 0.2782157063484192, + "style_target": 0, + "support_probability": 0.02311118319630623, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23524184686042388, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060407813638448715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.376162588596344, + 0.36009088158607483, + 0.4548743963241577, + 0.9390670657157898, + 0.018780941143631935, + 0.012065164744853973, + 0.02455553226172924, + 0.02549927495419979, + 0.10141442716121674, + 0.0421513207256794, + 0.02286992035806179 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.6905888915061951, + "style_target": 1, + "support_probability": 0.23454463481903076, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04692761263860705, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09328888356685638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3331388831138611, + 0.4206675887107849, + 0.34141817688941956, + 0.9485876560211182, + 0.021204154938459396, + 0.015677468851208687, + 0.033541254699230194, + 0.037060726433992386, + 0.015170770697295666, + 0.03142102435231209 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.33400118350982666, + "style_target": 0, + "support_probability": 0.11850793659687042, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6209188129796661, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07324542105197906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2253073751926422, + 0.8392820358276367, + 0.44525906443595886, + 0.20455069839954376, + 0.016706356778740883 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.8800164461135864, + "style_target": 1, + "support_probability": 0.39772751927375793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06730431733992201, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11501374840736389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23555195331573486, + 0.8120642900466919, + 0.3391861617565155, + 0.18244366347789764, + 0.022928260266780853, + 0.03937466815114021, + 0.02532191202044487, + 0.02884429320693016, + 0.04295362904667854, + 0.025709329172968864, + 0.04212609678506851 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.43504512310028076, + "style_target": 0, + "support_probability": 0.1121666207909584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20953741676987098, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5828466415405273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5916033387184143, + 0.41731324791908264, + 0.5849874019622803, + 0.2825729548931122, + 0.009179371409118176, + 0.010612097568809986 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.7545309066772461, + "style_target": 1, + "support_probability": 0.09002763777971268, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022931430395143948, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6141084432601929, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6109095811843872, + 0.40127870440483093, + 0.5655620694160461, + 0.2502070665359497, + 0.020485281944274902, + 0.06768115609884262 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.15677089989185333, + "style_target": 0, + "support_probability": 0.08936721086502075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6767236114465782, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06809371709823608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2366422414779663, + 0.9827896356582642, + 0.3064785897731781, + 0.5066344141960144, + 0.028682924807071686, + 0.020351899787783623, + 0.0038680604193359613, + 0.010843691416084766, + 0.009587621316313744 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 1, + "style_probability": 0.646143913269043, + "style_target": 1, + "support_probability": 0.5368911623954773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7046056778014129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034268420189619064, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25720295310020447, + 0.9857432246208191, + 0.2571418285369873, + 0.511059582233429, + 0.010319826193153858, + 0.00691831624135375, + 0.009646044112741947, + 0.035126958042383194, + 0.02423236146569252 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 1, + "style_probability": 0.44666966795921326, + "style_target": 0, + "support_probability": 0.7855436205863953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6205335142783815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15747034549713135, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6026797890663147, + 0.20702360570430756, + 0.06798925995826721, + 0.9587013721466064, + 0.015913328155875206, + 0.010893050581216812, + 0.031009798869490623, + 0.06912773102521896, + 0.007461198139935732, + 0.016020944342017174, + 0.01757880114018917 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.8203378915786743, + "style_target": 1, + "support_probability": 0.46318885684013367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.349338172298752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16212198138237, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6586107611656189, + 0.2098626047372818, + 0.07738150656223297, + 0.9483584761619568, + 0.02304909937083721, + 0.031964559108018875, + 0.04674285277724266, + 0.01897570677101612, + 0.015288269147276878 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.21033838391304016, + "style_target": 0, + "support_probability": 0.5513594150543213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8022400002793312, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11212930828332901, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9547892212867737, + 0.1461978554725647, + 0.6002322435379028, + 0.5843143463134766, + 0.01684853993356228 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.8461528420448303, + "style_target": 1, + "support_probability": 0.9082293510437012, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017579903895168374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4610900580883026, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8967235684394836, + 0.1436065286397934, + 0.453962117433548, + 0.45779648423194885, + 0.10229922831058502 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.04179009795188904, + "style_target": 0, + "support_probability": 0.43038269877433777, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5580021811835564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03174322471022606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4190729260444641, + 0.794994592666626, + 0.2934033274650574, + 0.4190729260444641, + 0.014146978966891766, + 0.017785420641303062 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 1, + "style_probability": 0.8019064664840698, + "style_target": 1, + "support_probability": 0.6799854040145874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1080240115741888, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13500069081783295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43647563457489014, + 0.6635891199111938, + 0.2849810719490051, + 0.43647563457489014, + 0.010224243625998497, + 0.046763285994529724 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 0, + "style_probability": 0.601428747177124, + "style_target": 0, + "support_probability": 0.18736645579338074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8037060667550192, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3422156870365143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3098301291465759, + 0.09941258281469345, + 0.9886808395385742, + 0.8522152900695801, + 0.01611585170030594, + 0.011623939499258995, + 0.022978756576776505, + 0.009071040898561478, + 0.036405015736818314 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.6777749061584473, + "style_target": 1, + "support_probability": 0.8705251216888428, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5349733629456848, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3265717625617981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2535701394081116, + 0.10120702534914017, + 0.9826280474662781, + 0.8281002044677734, + 0.04387957230210304, + 0.058939531445503235 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.16708730161190033, + "style_target": 0, + "support_probability": 0.874405562877655, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5694548351157412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11807064712047577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6453520655632019, + 0.1975744664669037, + 0.24851232767105103, + 0.6197540760040283, + 0.03970888629555702, + 0.031135665252804756, + 0.013126416131854057, + 0.03515242040157318, + 0.013227078132331371, + 0.010366766713559628 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.934752345085144, + "style_target": 1, + "support_probability": 0.8108635544776917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05060462759033986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11059566587209702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6128212213516235, + 0.13374640047550201, + 0.2201451063156128, + 0.6302717328071594, + 0.031076163053512573, + 0.027117768302559853, + 0.02561004087328911, + 0.023617573082447052, + 0.03750751167535782, + 0.07279885560274124 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.08047761023044586, + "style_target": 0, + "support_probability": 0.7484580278396606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004580379543483278, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9875465035438538, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5840399861335754, + 0.33096110820770264, + 0.3177931606769562, + 0.7297567129135132, + 0.2169826477766037 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.8536763787269592, + "style_target": 1, + "support_probability": 0.004914160817861557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01992370907397446, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8801151514053345, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5294471383094788, + 0.3521718382835388, + 0.44486305117607117, + 0.6262857913970947, + 0.3022615611553192 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.4543164372444153, + "style_target": 0, + "support_probability": 0.05421445146203041, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6384111157693816, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3659376800060272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31062182784080505, + 0.34905490279197693, + 0.20789940655231476, + 0.8718083500862122, + 0.018280506134033203, + 0.07516373693943024 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.8026379346847534, + "style_target": 1, + "support_probability": 0.7564237713813782, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16981843158658194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4626414477825165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2747085392475128, + 0.28849267959594727, + 0.21301423013210297, + 0.8450298309326172, + 0.0470401868224144 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.29792502522468567, + "style_target": 0, + "support_probability": 0.47069379687309265, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8679086441255456, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03770483285188675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12012035399675369, + 0.06139971688389778, + 0.14931152760982513, + 0.9801786541938782, + 0.01712433621287346, + 0.021202275529503822, + 0.011726267635822296, + 0.01274397037923336, + 0.10628648102283478 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.8541107773780823, + "style_target": 1, + "support_probability": 0.950643002986908, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1198151408244855, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05292641371488571, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10011115670204163, + 0.05564728379249573, + 0.11105924844741821, + 0.9837026596069336, + 0.03777093440294266, + 0.019136693328619003, + 0.033240169286727905, + 0.0373125821352005, + 0.07792062312364578 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.09829940646886826, + "style_target": 0, + "support_probability": 0.8641072511672974, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31161639250598056, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6414172053337097, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6662818193435669, + 0.2760624885559082, + 0.6662818193435669, + 0.4561375677585602, + 0.013051318004727364, + 0.012348806485533714, + 0.02879580669105053, + 0.04084485396742821, + 0.005148242227733135, + 0.009728807024657726, + 0.024694709107279778 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 1, + "style_probability": 0.8864424824714661, + "style_target": 1, + "support_probability": 0.2782118618488312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12882370810002136, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7857758402824402, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6255660653114319, + 0.32865995168685913, + 0.6255660653114319, + 0.3822386860847473, + 0.021008936688303947, + 0.015602598898112774, + 0.011042465455830097, + 0.021650033071637154, + 0.02985793724656105, + 0.02756238728761673, + 0.021082300692796707 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 1, + "style_probability": 0.4649631679058075, + "style_target": 0, + "support_probability": 0.32178664207458496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3348959403511389, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14662735164165497, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3186887800693512, + 0.38023534417152405, + 0.5389436483383179, + 0.9019357562065125, + 0.07271325588226318 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.28003644943237305, + "style_target": 1, + "support_probability": 0.5131206512451172, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6696270375375883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11653503775596619, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32335054874420166, + 0.40057989954948425, + 0.5030456185340881, + 0.9108400940895081, + 0.03069547936320305 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.7063153386116028, + "style_target": 0, + "support_probability": 0.6556556820869446, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008123667768691861, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8937801718711853, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2564137876033783, + 0.4519159197807312, + 0.4629586935043335, + 0.3145381808280945, + 0.02276933379471302, + 0.05579591542482376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 0, + "style_probability": 0.3324192762374878, + "style_target": 1, + "support_probability": 0.004012036137282848, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018914952802735025, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7929999232292175, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21623378992080688, + 0.34914103150367737, + 0.4007333815097809, + 0.29828956723213196, + 0.021742479875683784, + 0.0493643581867218 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 1, + "style_probability": 0.20627957582473755, + "style_target": 0, + "support_probability": 0.012645270675420761, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0185732082638491, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.764859676361084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26233094930648804, + 0.14153306186199188, + 0.17529675364494324, + 0.7498652935028076, + 0.03964245319366455, + 0.022405890747904778, + 0.018722116947174072, + 0.01608091965317726, + 0.019207488745450974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.377571702003479, + "style_target": 1, + "support_probability": 0.007635099813342094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027868226670484057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6951620578765869, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2724539637565613, + 0.1364586055278778, + 0.2003570795059204, + 0.8155786395072937, + 0.037024643272161484, + 0.038059335201978683, + 0.016385793685913086, + 0.06862574815750122, + 0.038423869758844376 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.5338122844696045, + "style_target": 0, + "support_probability": 0.014606972225010395, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7173214135289923, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017320794984698296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.299136221408844, + 0.3303929567337036, + 0.9496915936470032, + 0.299136221408844, + 0.012815126217901707, + 0.01125222910195589, + 0.03692861646413803, + 0.014885623939335346, + 0.01882798969745636, + 0.03574857860803604 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.33847692608833313, + "style_target": 1, + "support_probability": 0.9911624193191528, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8858792743428452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007382949814200401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3027704954147339, + 0.32100751996040344, + 0.964053213596344, + 0.3027704954147339, + 0.014170307666063309, + 0.020819013938307762, + 0.005058221984654665, + 0.006422148086130619, + 0.04223054647445679, + 0.03058846853673458, + 0.01697714254260063 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.8399282693862915, + "style_target": 0, + "support_probability": 0.9939374923706055, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1593749103328179, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45300963521003723, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2566588819026947, + 0.9591614007949829, + 0.36731815338134766, + 0.6065241694450378, + 0.05152730271220207 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 1, + "style_probability": 0.7444459795951843, + "style_target": 1, + "support_probability": 0.0460478812456131, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10911205656497544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34335675835609436, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2806110978126526, + 0.9549526572227478, + 0.31933411955833435, + 0.655268132686615, + 0.018295040354132652, + 0.0564563013613224, + 0.06439530849456787, + 0.0290233101695776, + 0.06510056555271149, + 0.027847379446029663, + 0.095619335770607 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 0, + "style_probability": 0.5526379346847534, + "style_target": 0, + "support_probability": 0.0435299351811409, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8253447415353532, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03140263631939888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27452605962753296, + 0.947598397731781, + 0.177658811211586, + 0.3693525493144989, + 0.015762796625494957, + 0.04701030254364014 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 0, + "style_probability": 0.4845717251300812, + "style_target": 1, + "support_probability": 0.9913722276687622, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2912814654459217, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0539545975625515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22641974687576294, + 0.9738633036613464, + 0.123222716152668, + 0.3441373407840729, + 0.01235449593514204, + 0.024704385548830032 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 1, + "style_probability": 0.07183663547039032, + "style_target": 0, + "support_probability": 0.9758110046386719, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5130557913989311, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17015597224235535, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4098050892353058, + 0.2769007682800293, + 0.12240016460418701, + 0.9147909879684448, + 0.024024683982133865, + 0.043299026787281036, + 0.03181225061416626, + 0.020285267382860184, + 0.04427735507488251 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 1, + "style_probability": 0.5628654360771179, + "style_target": 1, + "support_probability": 0.759596586227417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7730726129402943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05838887393474579, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3699615001678467, + 0.2630258798599243, + 0.06875331699848175, + 0.9256004691123962, + 0.056029994040727615, + 0.01338116079568863, + 0.01647195965051651, + 0.010830906219780445, + 0.06683950126171112 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 0, + "style_probability": 0.8139342069625854, + "style_target": 0, + "support_probability": 0.8868129849433899, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43045592441603103, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11931643635034561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35803699493408203, + 0.2633560299873352, + 0.9691749811172485, + 0.2716946601867676, + 0.018376192077994347, + 0.021449493244290352, + 0.017148364335298538, + 0.050886694341897964, + 0.018526529893279076, + 0.023237766698002815, + 0.07613198459148407 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.6571935415267944, + "style_target": 1, + "support_probability": 0.3994397819042206, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3986634495466268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10088387131690979, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3666045069694519, + 0.2958025336265564, + 0.9790385365486145, + 0.23500971496105194, + 0.009149659425020218, + 0.054593849927186966, + 0.012827769853174686, + 0.026523912325501442, + 0.015636734664440155 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.4132422208786011, + "style_target": 0, + "support_probability": 0.6076706051826477, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8894184488423635, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04889499768614769, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9865516424179077, + 0.3935443162918091, + 0.41211989521980286, + 0.2593189477920532, + 0.052704498171806335 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 1, + "style_probability": 0.9079039096832275, + "style_target": 1, + "support_probability": 0.9350967407226562, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8011490951284512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038164425641298294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.983725368976593, + 0.3062678277492523, + 0.4201321005821228, + 0.2806098163127899, + 0.03558221831917763 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 0, + "style_probability": 0.5000372529029846, + "style_target": 0, + "support_probability": 0.9641397595405579, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5697036858228214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23392947018146515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32536381483078003, + 0.5275319814682007, + 0.6607862710952759, + 0.32536381483078003, + 0.015059481374919415, + 0.06369174271821976 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.9359920620918274, + "style_target": 1, + "support_probability": 0.7946580052375793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.217456044142909, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6011604070663452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2497108280658722, + 0.43222352862358093, + 0.5963048338890076, + 0.2497108280658722, + 0.10317578166723251 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.3629515767097473, + "style_target": 0, + "support_probability": 0.608110249042511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4358311531908861, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06315165758132935, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11783923953771591, + 0.627871036529541, + 0.9803631901741028, + 0.16807039082050323, + 0.012399564497172832, + 0.018584489822387695, + 0.02008192427456379, + 0.02163403108716011, + 0.046875905245542526 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 1, + "style_probability": 0.7013701796531677, + "style_target": 1, + "support_probability": 0.2789599299430847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13115109521359422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10819734632968903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08849885314702988, + 0.658706545829773, + 0.9774777889251709, + 0.16596359014511108, + 0.011717474088072777, + 0.017378229647874832, + 0.008388969115912914, + 0.013349300250411034, + 0.021635491400957108 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 1, + "style_probability": 0.36528170108795166, + "style_target": 0, + "support_probability": 0.21700437366962433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16379981370912497, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08157157152891159, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19753167033195496, + 0.4533478915691376, + 0.33878275752067566, + 0.8613646626472473, + 0.03280794620513916, + 0.025699442252516747, + 0.019796546548604965, + 0.011987144127488136, + 0.0070250676944851875, + 0.005934893619269133, + 0.030067475512623787 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 1, + "style_probability": 0.794420599937439, + "style_target": 1, + "support_probability": 0.1086396649479866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07795116036382015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17391610145568848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25766950845718384, + 0.46371662616729736, + 0.331755667924881, + 0.8162577152252197, + 0.02436155267059803, + 0.032645661383867264, + 0.07126283645629883, + 0.0653756782412529, + 0.03846793994307518, + 0.010035423561930656, + 0.029884031042456627 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 1, + "style_probability": 0.4888525903224945, + "style_target": 0, + "support_probability": 0.06797443330287933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15568832157266146, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8454410433769226, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32201480865478516, + 0.40641748905181885, + 0.225820392370224, + 0.3269985020160675, + 0.14172853529453278 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.9244554042816162, + "style_target": 1, + "support_probability": 0.03745640814304352, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006330723332902697, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8777117729187012, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30860018730163574, + 0.3899089992046356, + 0.21504434943199158, + 0.3294382095336914, + 0.04936233162879944, + 0.06858230382204056, + 0.03829237446188927, + 0.07674597948789597, + 0.09856384247541428, + 0.05880481004714966, + 0.06926088035106659 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.41053780913352966, + "style_target": 0, + "support_probability": 0.005152957513928413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4951225501739499, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4218018651008606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6991451382637024, + 0.568014919757843, + 0.3076367974281311, + 0.16565275192260742, + 0.020478228107094765, + 0.06167029216885567 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.9628807902336121, + "style_target": 1, + "support_probability": 0.3784792125225067, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0048754513199321175, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5826737284660339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6232887506484985, + 0.3138722777366638, + 0.19720354676246643, + 0.0785566195845604, + 0.039626363664865494, + 0.21136650443077087 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.029015081003308296, + "style_target": 0, + "support_probability": 0.20549532771110535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30786940289231496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22198083996772766, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39702048897743225, + 0.08539048582315445, + 0.39702048897743225, + 0.8117626905441284, + 0.011225612834095955, + 0.011236335150897503, + 0.021018588915467262, + 0.010748986154794693, + 0.14601965248584747 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 1, + "style_probability": 0.7243168950080872, + "style_target": 1, + "support_probability": 0.13389204442501068, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09717839590590162, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2591629922389984, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3609204888343811, + 0.08669637888669968, + 0.3609204888343811, + 0.7283541560173035, + 0.04631439968943596, + 0.022942975163459778, + 0.017160939052700996, + 0.06735233962535858, + 0.012606073170900345 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 1, + "style_probability": 0.36819592118263245, + "style_target": 0, + "support_probability": 0.10717909783124924, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003036481985717429, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8713735938072205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43328583240509033, + 0.29553118348121643, + 0.42954471707344055, + 0.6148474216461182, + 0.041805826127529144, + 0.05926552414894104, + 0.07159940898418427, + 0.03982129693031311, + 0.006601199973374605, + 0.021257061511278152, + 0.04270300269126892 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.6885538697242737, + "style_target": 1, + "support_probability": 0.0005336554604582489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008617196059785379, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7636052370071411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5487344861030579, + 0.3553817570209503, + 0.49119946360588074, + 0.6481612324714661, + 0.07160811126232147, + 0.03364063799381256, + 0.09796332567930222, + 0.11344749480485916, + 0.030837031081318855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.38486745953559875, + "style_target": 0, + "support_probability": 0.0014689622912555933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.348394130009702, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19073528051376343, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.817478358745575, + 0.8099237084388733, + 0.9623512625694275, + 0.2108934223651886, + 0.03713461011648178 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 0, + "style_probability": 0.3568744957447052, + "style_target": 1, + "support_probability": 0.49326473474502563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04282058103619931, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3011845648288727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7919855713844299, + 0.7773289680480957, + 0.9564496874809265, + 0.18302814662456512, + 0.04681950807571411 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 1, + "style_probability": 0.04747290909290314, + "style_target": 0, + "support_probability": 0.5764821767807007, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018951300954264436, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8981988430023193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2146240770816803, + 0.08379281312227249, + 0.19337643682956696, + 0.5776775479316711, + 0.03303723782300949, + 0.054601576179265976 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.2595002353191376, + "style_target": 1, + "support_probability": 0.022869642823934555, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010149695679557823, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9266085028648376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19024373590946198, + 0.09271019697189331, + 0.2236129492521286, + 0.5344176888465881, + 0.08524368703365326, + 0.047571297734975815 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.5455020070075989, + "style_target": 0, + "support_probability": 0.010704346001148224, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3558718576661721, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4204869568347931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41550105810165405, + 0.35530486702919006, + 0.8018962144851685, + 0.8052012324333191, + 0.008854897692799568, + 0.029698830097913742, + 0.07700671255588531, + 0.04360807687044144, + 0.024548247456550598 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.3125675320625305, + "style_target": 1, + "support_probability": 0.6655795574188232, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7056622965335466, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20694103837013245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4204387068748474, + 0.34256890416145325, + 0.817741870880127, + 0.8067908883094788, + 0.02031240239739418, + 0.025767339393496513, + 0.010576434433460236, + 0.02838536910712719, + 0.02928324230015278 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.8976821303367615, + "style_target": 0, + "support_probability": 0.8865424394607544, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0730073106901303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2261883169412613, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6898030638694763, + 0.23791365325450897, + 0.2721713185310364, + 0.9033937454223633, + 0.0553034171462059, + 0.011368517763912678, + 0.029439477249979973, + 0.047772735357284546, + 0.24948260188102722, + 0.04727492853999138, + 0.08166445046663284 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 0, + "style_probability": 0.3134983479976654, + "style_target": 1, + "support_probability": 0.06212916225194931, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006366062007578066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.755708634853363, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6237677931785583, + 0.15025007724761963, + 0.21872848272323608, + 0.7806039452552795, + 0.147790789604187, + 0.12379884719848633, + 0.045529868453741074, + 0.11655053496360779, + 0.1432131975889206, + 0.2283402979373932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 1, + "style_probability": 0.16583560407161713, + "style_target": 0, + "support_probability": 0.020747389644384384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017318299342394088, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9645651578903198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4240727126598358, + 0.11366192251443863, + 0.5707600712776184, + 0.21438458561897278, + 0.05166468024253845 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 1, + "style_probability": 0.5066173076629639, + "style_target": 1, + "support_probability": 0.010855860076844692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013126176277437898, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9799398183822632, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3378305435180664, + 0.18816661834716797, + 0.5086939334869385, + 0.19604486227035522, + 0.10978151857852936 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 1, + "style_probability": 0.10034845024347305, + "style_target": 0, + "support_probability": 0.00816207192838192, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0360852825217616, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8486860990524292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6437111496925354, + 0.2504573166370392, + 0.05478117614984512, + 0.2504573166370392, + 0.015732552856206894, + 0.033372484147548676 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.4498787820339203, + "style_target": 1, + "support_probability": 0.05628129467368126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37364622191031077, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5783985257148743, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7431042790412903, + 0.19440145790576935, + 0.0428910069167614, + 0.19440145790576935, + 0.02966919355094433, + 0.041126422584056854 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.9279184937477112, + "style_target": 0, + "support_probability": 0.19822122156620026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3231025913792353, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18167594075202942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3732638657093048, + 0.08386792242527008, + 0.7769930958747864, + 0.3732638657093048, + 0.00924568809568882, + 0.04553176462650299, + 0.01583397388458252, + 0.022434473037719727, + 0.026479046791791916 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 0, + "style_probability": 0.3370842933654785, + "style_target": 1, + "support_probability": 0.28738218545913696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18561731124551734, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13384126126766205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3560495972633362, + 0.0794035792350769, + 0.763652503490448, + 0.3560495972633362, + 0.03885788097977638, + 0.055615417659282684 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 1, + "style_probability": 0.24603654444217682, + "style_target": 0, + "support_probability": 0.16520819067955017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06963156662186593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8697534203529358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14037060737609863, + 0.15447169542312622, + 0.7525994181632996, + 0.23971670866012573, + 0.026887746527791023, + 0.02335396595299244, + 0.048032406717538834, + 0.02160395123064518, + 0.03226981684565544, + 0.025523409247398376, + 0.04234115779399872 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 0, + "style_probability": 0.43690967559814453, + "style_target": 1, + "support_probability": 0.11297636479139328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004241307482743955, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.667630672454834, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09657233208417892, + 0.250447154045105, + 0.76862633228302, + 0.20634213089942932, + 0.02822592854499817, + 0.03355870768427849, + 0.03920508176088333, + 0.04759592562913895, + 0.03748283162713051, + 0.04100475460290909 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 1, + "style_probability": 0.04642308130860329, + "style_target": 0, + "support_probability": 0.0791521966457367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20284780690824952, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.37411317229270935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24417702853679657, + 0.7019830942153931, + 0.9301197528839111, + 0.4704788625240326, + 0.021075019612908363 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.7967870831489563, + "style_target": 1, + "support_probability": 0.20565682649612427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027771602444178232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8948294520378113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27158069610595703, + 0.6558912992477417, + 0.8864406943321228, + 0.4464793801307678, + 0.0861763134598732, + 0.09754206985235214, + 0.02480209246277809, + 0.028959302231669426, + 0.062203094363212585, + 0.075269415974617, + 0.14297202229499817 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.24164606630802155, + "style_target": 0, + "support_probability": 0.029356492683291435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010597037346111676, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.49253639578819275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5734788775444031, + 0.5971260666847229, + 0.9670774936676025, + 0.2918475270271301, + 0.012668462470173836, + 0.07533638179302216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.8606030941009521, + "style_target": 1, + "support_probability": 0.0011966166784986854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003046237186795454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45074185729026794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5879197716712952, + 0.5811281204223633, + 0.9738419055938721, + 0.24552300572395325, + 0.04552663490176201, + 0.07155665010213852 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.3926551043987274, + "style_target": 0, + "support_probability": 0.001746120979078114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0036493094433768373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9764421582221985, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3801107406616211, + 0.13087597489356995, + 0.39103463292121887, + 0.7636141777038574, + 0.028912333771586418, + 0.11205881088972092, + 0.041597262024879456, + 0.05394968017935753, + 0.07549747824668884 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 1, + "style_probability": 0.8106217980384827, + "style_target": 1, + "support_probability": 0.0009059009025804698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033097347396485208, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9520394206047058, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3827444314956665, + 0.17534980177879333, + 0.4722760021686554, + 0.7091810703277588, + 0.052128683775663376, + 0.022615604102611542, + 0.01487935334444046, + 0.059748124331235886, + 0.03431529551744461 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 0, + "style_probability": 0.5273223519325256, + "style_target": 0, + "support_probability": 0.001227973378263414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.035656174100818165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8846174478530884, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.07829930633306503, + 0.9312378168106079, + 0.07124967873096466, + 0.15682606399059296, + 0.08991523087024689, + 0.10564611107110977, + 0.04197588935494423, + 0.10323767364025116, + 0.06085743382573128, + 0.053483668714761734, + 0.0479680672287941 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.635221004486084, + "style_target": 1, + "support_probability": 0.009577982127666473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07701318874862334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6884387135505676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.0808941200375557, + 0.9262855648994446, + 0.07332587987184525, + 0.18562529981136322, + 0.14515908062458038, + 0.053825464099645615, + 0.0314825102686882, + 0.06646208465099335, + 0.0443987175822258 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.2825523614883423, + "style_target": 0, + "support_probability": 0.027286114171147346, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7250011036979762, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.007129457779228687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14374250173568726, + 0.2013091742992401, + 0.3722698390483856, + 0.9994639754295349, + 0.03091232106089592 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.7478797435760498, + "style_target": 1, + "support_probability": 0.7453664541244507, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21400063765406002, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.015787968412041664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14010931551456451, + 0.20293313264846802, + 0.4226103723049164, + 0.9994589686393738, + 0.07515542209148407 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.21588966250419617, + "style_target": 0, + "support_probability": 0.6480650901794434, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27579224802482677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11354001611471176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14326277375221252, + 0.9446497559547424, + 0.07561612874269485, + 0.20005910098552704, + 0.07455027848482132, + 0.02219114825129509 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 0, + "style_probability": 0.4620349705219269, + "style_target": 1, + "support_probability": 0.5359174013137817, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.365148971866877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08479463309049606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1810840219259262, + 0.9114454984664917, + 0.095100998878479, + 0.3176282048225403, + 0.037329450249671936, + 0.05157649889588356 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 1, + "style_probability": 0.44468918442726135, + "style_target": 0, + "support_probability": 0.5895015001296997, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3760971252947059, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07492658495903015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7963363528251648, + 0.35068926215171814, + 0.17783410847187042, + 0.9775201678276062, + 0.008925342932343483, + 0.00883792620152235, + 0.01845068857073784, + 0.19959430396556854, + 0.08260615915060043 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.8657439947128296, + "style_target": 1, + "support_probability": 0.14285773038864136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26615227077468495, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03875415027141571, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8306329846382141, + 0.2957365810871124, + 0.15346340835094452, + 0.9785584211349487, + 0.12573468685150146, + 0.09308276325464249 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.3502703905105591, + "style_target": 0, + "support_probability": 0.2578158378601074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05283529845016146, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9255149364471436, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09809992462396622, + 0.7578299641609192, + 0.4487903416156769, + 0.1305050551891327, + 0.03621010482311249, + 0.019616514444351196, + 0.017871607095003128, + 0.020976854488253593, + 0.029147153720259666, + 0.13227210938930511, + 0.09200754016637802 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.8557694554328918, + "style_target": 1, + "support_probability": 0.03562096878886223, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007283682577881579, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25199028849601746, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09148385375738144, + 0.8383787274360657, + 0.2631218135356903, + 0.11043660342693329, + 0.053906816989183426, + 0.024227706715464592, + 0.14740633964538574, + 0.06025657430291176, + 0.030103052034974098, + 0.05545497685670853 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.07101359218358994, + "style_target": 0, + "support_probability": 0.19329756498336792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30837271703890984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13291221857070923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.206149622797966, + 0.3286362588405609, + 0.14570960402488708, + 0.9436048269271851, + 0.026343032717704773 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 0, + "style_probability": 0.48190411925315857, + "style_target": 1, + "support_probability": 0.4859205484390259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019003286961378284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1743047535419464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22443225979804993, + 0.41920822858810425, + 0.0971091240644455, + 0.9548234343528748, + 0.10393579304218292 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 1, + "style_probability": 0.08524934202432632, + "style_target": 0, + "support_probability": 0.22472254931926727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6432806754986254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03671437129378319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9388694763183594, + 0.20753270387649536, + 0.2731027901172638, + 0.21510683000087738, + 0.0590587854385376, + 0.028431441634893417 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.4973513185977936, + "style_target": 1, + "support_probability": 0.9024990797042847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6360813809788027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04023633152246475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9548309445381165, + 0.28552016615867615, + 0.254586786031723, + 0.23536726832389832, + 0.025743715465068817, + 0.019862301647663116 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.6961695551872253, + "style_target": 0, + "support_probability": 0.797656774520874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011413543036130215, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9206695556640625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7844348549842834, + 0.3293822407722473, + 0.47726044058799744, + 0.3293822407722473, + 0.023892292752861977, + 0.06659738719463348, + 0.06293100863695145, + 0.07730429619550705, + 0.04957550764083862 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 1, + "style_probability": 0.6785847544670105, + "style_target": 1, + "support_probability": 0.0030982650350779295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02839987121382752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8903676867485046, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7864248156547546, + 0.3113243281841278, + 0.5230247378349304, + 0.3113243281841278, + 0.11732739210128784 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 0, + "style_probability": 0.5745776295661926, + "style_target": 0, + "support_probability": 0.006758208852261305, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5735181111260277, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02322469837963581, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3257211446762085, + 0.058809634298086166, + 0.1819121390581131, + 0.9937050938606262, + 0.04246777296066284, + 0.12727610766887665, + 0.06683871895074844, + 0.05253339931368828, + 0.06057951971888542, + 0.10826165229082108 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 0, + "style_probability": 0.48574814200401306, + "style_target": 1, + "support_probability": 0.8335924744606018, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17639029694989564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02068648859858513, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2804214060306549, + 0.07089436799287796, + 0.17129945755004883, + 0.9979451298713684, + 0.04533049836754799, + 0.05578349530696869, + 0.11371667683124542, + 0.12628836929798126, + 0.014330597594380379, + 0.034094829112291336 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 1, + "style_probability": 0.13198316097259521, + "style_target": 0, + "support_probability": 0.7901608347892761, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1335922011895433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34376034140586853, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9448228478431702, + 0.23640763759613037, + 0.1650426983833313, + 0.2799678146839142, + 0.06006839498877525 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.2992497682571411, + "style_target": 1, + "support_probability": 0.1632675975561142, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15171800642645636, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37703782320022583, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.953819751739502, + 0.27698394656181335, + 0.18303856253623962, + 0.21006853878498077, + 0.033342842012643814 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.7639831304550171, + "style_target": 0, + "support_probability": 0.09730710089206696, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4301475835126686, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.037659212946891785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9618435502052307, + 0.40357571840286255, + 0.18265098333358765, + 0.40357571840286255, + 0.03554435819387436, + 0.05661873519420624 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 0, + "style_probability": 0.4142427444458008, + "style_target": 1, + "support_probability": 0.5893219709396362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25448307122019415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07442222535610199, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9830693006515503, + 0.3218933641910553, + 0.14300382137298584, + 0.3218933641910553, + 0.022354425862431526, + 0.027406787499785423 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 1, + "style_probability": 0.2997054159641266, + "style_target": 0, + "support_probability": 0.5592522025108337, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.003091753061513015, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9971035122871399, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5142360925674438, + 0.3787792921066284, + 0.23783397674560547, + 0.17310969531536102, + 0.07447835057973862, + 0.07536522299051285, + 0.06457973271608353, + 0.06048409640789032, + 0.07997269928455353 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 0, + "style_probability": 0.43301084637641907, + "style_target": 1, + "support_probability": 0.0008493661880493164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001786310816306509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9960758090019226, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5367786288261414, + 0.4769739806652069, + 0.2422356754541397, + 0.20210279524326324, + 0.039872486144304276, + 0.03479374945163727, + 0.02996753714978695, + 0.04717288166284561, + 0.0918557196855545 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 1, + "style_probability": 0.4701308012008667, + "style_target": 0, + "support_probability": 0.0007784520275890827, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6222675284695149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08762451261281967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9937418103218079, + 0.3316766917705536, + 0.34099146723747253, + 0.2382415235042572, + 0.02222246304154396, + 0.03109431080520153, + 0.1187768280506134, + 0.11483138054609299, + 0.056987836956977844, + 0.06273060292005539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.4454634487628937, + "style_target": 1, + "support_probability": 0.8597760796546936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8710674633944143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07496050000190735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9964620471000671, + 0.34277603030204773, + 0.4217602014541626, + 0.2619960606098175, + 0.04761422425508499, + 0.06554467976093292, + 0.037761472165584564, + 0.034307047724723816, + 0.06852983683347702, + 0.056231800466775894, + 0.030855748802423477 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.8960452675819397, + "style_target": 0, + "support_probability": 0.8768231868743896, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6576352827599766, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0347127765417099, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2973741292953491, + 0.2973741292953491, + 0.9337424635887146, + 0.2574760317802429, + 0.03022719919681549 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 0, + "style_probability": 0.25347548723220825, + "style_target": 1, + "support_probability": 0.9686928391456604, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2494581445466425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06506618857383728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24054892361164093, + 0.24054892361164093, + 0.9703966975212097, + 0.11765405535697937, + 0.06452424824237823 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 1, + "style_probability": 0.08275715261697769, + "style_target": 0, + "support_probability": 0.9501370787620544, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20358922753600392, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14093205332756042, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3567603826522827, + 0.32359540462493896, + 0.6915296316146851, + 0.6360857486724854, + 0.10133550316095352, + 0.029345713555812836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 0, + "style_probability": 0.15001438558101654, + "style_target": 1, + "support_probability": 0.6534280180931091, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1892539649840344, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2139906883239746, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28896284103393555, + 0.1945231705904007, + 0.6928467154502869, + 0.5259630680084229, + 0.01957695558667183, + 0.12184611707925797 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 1, + "style_probability": 0.2974105775356293, + "style_target": 0, + "support_probability": 0.4609392583370209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11056092362342529, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7345610857009888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5977318286895752, + 0.7965542078018188, + 0.12472036480903625, + 0.6079495549201965, + 0.022136464715003967, + 0.03327750042080879, + 0.04426348954439163, + 0.05415874347090721, + 0.06490825861692429 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 1, + "style_probability": 0.5155963897705078, + "style_target": 1, + "support_probability": 0.07470870763063431, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24319584695471422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5312725305557251, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6389276385307312, + 0.8290042877197266, + 0.12875531613826752, + 0.6796488165855408, + 0.032054729759693146, + 0.09274018555879593, + 0.019100848585367203, + 0.028881823644042015, + 0.08931896835565567 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 0, + "style_probability": 0.8532229065895081, + "style_target": 0, + "support_probability": 0.14811784029006958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020841942683962826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5871920585632324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.39345669746398926, + 0.5679860711097717, + 0.5832760334014893, + 0.16271406412124634, + 0.03227691352367401, + 0.05076232925057411, + 0.055902402848005295, + 0.05905873328447342, + 0.21311381459236145, + 0.1786637306213379 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 0, + "style_probability": 0.3831051290035248, + "style_target": 1, + "support_probability": 0.013853292912244797, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0027529532161176917, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5484773516654968, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3617129623889923, + 0.5271583199501038, + 0.5572992563247681, + 0.08938919752836227, + 0.05022929608821869, + 0.033083125948905945, + 0.07688909024000168, + 0.08046568185091019, + 0.025109028443694115, + 0.030773138627409935 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 1, + "style_probability": 0.11189643293619156, + "style_target": 0, + "support_probability": 0.02349252998828888, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8435666253595765, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07649213820695877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15332505106925964, + 0.629542350769043, + 0.9736043214797974, + 0.4901904761791229, + 0.03372746706008911 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.831885576248169, + "style_target": 1, + "support_probability": 0.9000676274299622, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5286173763294488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14336532354354858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1298137605190277, + 0.6529097557067871, + 0.94224613904953, + 0.511096715927124, + 0.1114666610956192 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.34636151790618896, + "style_target": 0, + "support_probability": 0.8130970597267151, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1937064574738385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.829014241695404, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33751818537712097, + 0.4009692966938019, + 0.1797204166650772, + 0.8989610075950623, + 0.04696959629654884, + 0.025416174903512 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.8067061901092529, + "style_target": 1, + "support_probability": 0.0328892283141613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07238910696303238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7596091032028198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36778295040130615, + 0.4389115869998932, + 0.22726641595363617, + 0.9125460386276245, + 0.03881417214870453, + 0.11389931291341782, + 0.053844042122364044, + 0.02261224202811718, + 0.08989004045724869, + 0.0656660869717598 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.4071500301361084, + "style_target": 0, + "support_probability": 0.03045191429555416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019414740395831468, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9132460355758667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.06307903677225113, + 0.6534587144851685, + 0.09553767740726471, + 0.45509082078933716, + 0.01308625191450119, + 0.0177199337631464, + 0.017861314117908478, + 0.025036711245775223, + 0.015983521938323975 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.7245720624923706, + "style_target": 1, + "support_probability": 0.011878973804414272, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00481135605999603, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.774367094039917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.05229315161705017, + 0.6705484390258789, + 0.08710835129022598, + 0.5326154232025146, + 0.039109181612730026, + 0.08583081513643265, + 0.024217061698436737, + 0.018864382058382034, + 0.033797360956668854 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.11313904821872711, + "style_target": 0, + "support_probability": 0.04072914645075798, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.936413455317297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04946865886449814, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6260:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.993790328502655, + 0.5530275106430054, + 0.0948859378695488, + 0.3011389970779419, + 0.06893883645534515, + 0.013367136009037495, + 0.02115786448121071, + 0.015797670930624008, + 0.04377995431423187, + 0.0287710539996624 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 1, + "style_probability": 0.9188514947891235, + "style_target": 1, + "support_probability": 0.9008400440216064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8072713259277032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03714011237025261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6260:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.991205632686615, + 0.5700010657310486, + 0.0819980576634407, + 0.3369370400905609, + 0.022904306650161743, + 0.032573919743299484, + 0.022027375176548958, + 0.03127351775765419, + 0.024724122136831284, + 0.014179189689457417, + 0.08290426433086395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 0, + "style_probability": 0.5813325047492981, + "style_target": 0, + "support_probability": 0.936099648475647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8878403740794578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.02799234166741371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28506356477737427, + 0.4678804874420166, + 0.9783188700675964, + 0.28506356477737427, + 0.012794348411262035 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.726337730884552, + "style_target": 1, + "support_probability": 0.9200153350830078, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6901115743371289, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017001409083604813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28981855511665344, + 0.5138092637062073, + 0.9756645560264587, + 0.28981855511665344, + 0.013793055899441242, + 0.012718354351818562, + 0.03607283905148506, + 0.05289933830499649, + 0.03445953130722046 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.2827662229537964, + "style_target": 0, + "support_probability": 0.8959878087043762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6467853016782025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06860673427581787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06698763370513916, + 0.24518051743507385, + 0.285955011844635, + 0.9861262440681458, + 0.01598445326089859, + 0.014280645176768303 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.8749818801879883, + "style_target": 1, + "support_probability": 0.6578317880630493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13871537232713962, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17233115434646606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.03945455327630043, + 0.3111165165901184, + 0.3513294458389282, + 0.9931460618972778, + 0.04534504562616348, + 0.036433905363082886 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.19852399826049805, + "style_target": 0, + "support_probability": 0.6545408964157104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3719477785595494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2362709641456604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28019705414772034, + 0.12805619835853577, + 0.5586137771606445, + 0.19597291946411133, + 0.015107410959899426, + 0.0258123017847538, + 0.017191050574183464, + 0.02459941804409027, + 0.017040971666574478 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.9033731818199158, + "style_target": 1, + "support_probability": 0.29720860719680786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10823637084695296, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2717803418636322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31116700172424316, + 0.16087637841701508, + 0.6540820002555847, + 0.2158626914024353, + 0.040580544620752335, + 0.038804247975349426, + 0.036754123866558075, + 0.043751996010541916, + 0.025280267000198364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.41197916865348816, + "style_target": 0, + "support_probability": 0.1320141702890396, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6254189866674659, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09262688457965851, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36101260781288147, + 0.05799397826194763, + 0.5935667157173157, + 0.9221547842025757, + 0.011789730750024319, + 0.017483416944742203, + 0.017610996961593628, + 0.009114235639572144, + 0.030152728781104088, + 0.00658429553732276 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 0, + "style_probability": 0.4156717360019684, + "style_target": 1, + "support_probability": 0.9504255652427673, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3901344752168551, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11261887848377228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34025976061820984, + 0.07543925195932388, + 0.6306689381599426, + 0.8876957893371582, + 0.006687875837087631, + 0.022821752354502678 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 1, + "style_probability": 0.18115925788879395, + "style_target": 0, + "support_probability": 0.9147909879684448, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02972433552412634, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9411879777908325, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21319089829921722, + 0.5878074765205383, + 0.39971059560775757, + 0.2526223063468933, + 0.0548839271068573 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.8445225358009338, + "style_target": 1, + "support_probability": 0.018124839290976524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00408403195258518, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9227182865142822, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1680745780467987, + 0.5642863512039185, + 0.4645342528820038, + 0.2302861511707306, + 0.03897694870829582 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.17101214826107025, + "style_target": 0, + "support_probability": 0.023437662050127983, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20118207372611202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4595654308795929, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5962703227996826, + 0.6045989990234375, + 0.26431015133857727, + 0.9877090454101562, + 0.17160995304584503, + 0.11733175069093704 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 1, + "style_probability": 0.8648361563682556, + "style_target": 1, + "support_probability": 0.04198117181658745, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21023012901117194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.328269898891449, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5018308758735657, + 0.46007704734802246, + 0.21575459837913513, + 0.9807437062263489, + 0.042857252061367035, + 0.09288109838962555 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 0, + "style_probability": 0.5275369882583618, + "style_target": 0, + "support_probability": 0.07678952068090439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4107746310551377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22787897288799286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44000476598739624, + 0.123322494328022, + 0.996618390083313, + 0.1021488681435585, + 0.07323440164327621, + 0.04990161210298538, + 0.0876801460981369, + 0.23270371556282043, + 0.44649040699005127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.8021547794342041, + "style_target": 1, + "support_probability": 0.3666393756866455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2667435747497267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21518100798130035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4199988543987274, + 0.17831012606620789, + 0.9947419166564941, + 0.10741395503282547, + 0.061177872121334076 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.32034093141555786, + "style_target": 0, + "support_probability": 0.4291193187236786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5526556055479928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2135850042104721, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25616198778152466, + 0.8703998923301697, + 0.5973577499389648, + 0.26947540044784546, + 0.034595657140016556, + 0.01215881947427988, + 0.017232801765203476, + 0.029061036184430122, + 0.017363179475069046, + 0.014535115100443363, + 0.016562355682253838 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.8824943900108337, + "style_target": 1, + "support_probability": 0.3680889904499054, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10223208435883249, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3963834047317505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16361504793167114, + 0.9135786294937134, + 0.6711074113845825, + 0.2613438069820404, + 0.020980503410100937, + 0.011691642925143242, + 0.015424596145749092, + 0.04886668547987938, + 0.0400390625, + 0.05790570750832558 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.20014430582523346, + "style_target": 0, + "support_probability": 0.49276304244995117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013533021542496112, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6305249929428101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9676374793052673, + 0.27728068828582764, + 0.25523048639297485, + 0.27396050095558167, + 0.07924438267946243 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 1, + "style_probability": 0.6199622750282288, + "style_target": 1, + "support_probability": 0.010654013603925705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033547274670310295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27781665325164795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9700381755828857, + 0.33513209223747253, + 0.3307100534439087, + 0.3064575493335724, + 0.0995822623372078 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 1, + "style_probability": 0.2837187647819519, + "style_target": 0, + "support_probability": 0.0431780144572258, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7604820989988669, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.051352497190237045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9836190342903137, + 0.22250978648662567, + 0.02426709607243538, + 0.09746690094470978, + 0.006471437867730856, + 0.01926238462328911 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 0, + "style_probability": 0.467072457075119, + "style_target": 1, + "support_probability": 0.9533054232597351, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7020791690703349, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022383999079465866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9766786694526672, + 0.256508469581604, + 0.037954457104206085, + 0.14069309830665588, + 0.02372325211763382, + 0.027247576043009758, + 0.07353364676237106, + 0.028875967487692833, + 0.015331242233514786, + 0.07508835196495056, + 0.018939953297376633 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 1, + "style_probability": 0.29576462507247925, + "style_target": 0, + "support_probability": 0.9789227247238159, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8953974298061895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0068078055046498775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.050770290195941925, + 0.18308934569358826, + 0.09910369664430618, + 0.9906954765319824, + 0.07172755151987076, + 0.02932509407401085, + 0.01575997844338417, + 0.04066653177142143, + 0.017760256305336952 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.8128564357757568, + "style_target": 1, + "support_probability": 0.8878946304321289, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17261860966914702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016597380861639977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.05015489086508751, + 0.14383164048194885, + 0.11083193123340607, + 0.9878374338150024, + 0.03405466303229332, + 0.028564779087901115, + 0.05386007949709892, + 0.026459988206624985, + 0.05358216539025307 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.09542352706193924, + "style_target": 0, + "support_probability": 0.80515456199646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017348674132805855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3381870985031128, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35164475440979004, + 0.5783615112304688, + 0.6244471669197083, + 0.3633546233177185, + 0.05932489037513733, + 0.07055579870939255, + 0.06946246325969696, + 0.03123711608350277, + 0.03474033996462822, + 0.05271076038479805, + 0.016067713499069214 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 1, + "style_probability": 0.6965990662574768, + "style_target": 1, + "support_probability": 0.012830760329961777, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02389271399146847, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39770764112472534, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3392549753189087, + 0.5827500820159912, + 0.5951921939849854, + 0.36918729543685913, + 0.04275892302393913, + 0.04550287500023842, + 0.0983622744679451, + 0.0274218637496233, + 0.13393835723400116, + 0.24336470663547516 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 1, + "style_probability": 0.37651848793029785, + "style_target": 0, + "support_probability": 0.015041859820485115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7589368579562571, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3842976689338684, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28902381658554077, + 0.9697667956352234, + 0.5079606175422668, + 0.42115968465805054, + 0.04959431663155556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.8180139660835266, + "style_target": 1, + "support_probability": 0.8820061683654785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2869898206420755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5662373304367065, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24644988775253296, + 0.9662367105484009, + 0.3977295458316803, + 0.4184833765029907, + 0.06164272874593735, + 0.10984864830970764, + 0.17589707672595978, + 0.03128036484122276, + 0.04509812965989113 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.3975464999675751, + "style_target": 0, + "support_probability": 0.6199065446853638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03831535631143535, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7253740429878235, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4730171859264374, + 0.4047599136829376, + 0.14255917072296143, + 0.28362780809402466, + 0.09285587817430496, + 0.06787877529859543 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.71357661485672, + "style_target": 1, + "support_probability": 0.07985582947731018, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016095128325175334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9292902946472168, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4142106771469116, + 0.23063506186008453, + 0.1750710904598236, + 0.2803024351596832, + 0.07514072209596634, + 0.12389698624610901 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.11073889583349228, + "style_target": 0, + "support_probability": 0.041107069700956345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5660703090868802, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06990686058998108, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41232702136039734, + 0.9731317758560181, + 0.07585938274860382, + 0.1921757012605667, + 0.02247597463428974, + 0.13807721436023712, + 0.029874688014388084, + 0.08394568413496017, + 0.04133594036102295 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 1, + "style_probability": 0.8426142930984497, + "style_target": 1, + "support_probability": 0.3593379259109497, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3857477808627832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09110754728317261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41572999954223633, + 0.9725343585014343, + 0.09193927049636841, + 0.23861896991729736, + 0.0328088216483593, + 0.09451193362474442, + 0.04238371551036835, + 0.04423312097787857, + 0.038228943943977356 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 0, + "style_probability": 0.5292056202888489, + "style_target": 0, + "support_probability": 0.5208462476730347, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02671506964835899, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8658856749534607, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26634663343429565, + 0.2206243872642517, + 0.16195739805698395, + 0.5941869020462036, + 0.03923724591732025, + 0.059447214007377625, + 0.05153544992208481, + 0.09041140973567963, + 0.16408221423625946, + 0.054578714072704315 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.8778282999992371, + "style_target": 1, + "support_probability": 0.03508896753191948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004308772588912088, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9443112015724182, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32091695070266724, + 0.25112849473953247, + 0.18828216195106506, + 0.5026842355728149, + 0.17385563254356384, + 0.20671795308589935 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.2539428770542145, + "style_target": 0, + "support_probability": 0.004889253061264753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8136520479887979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06843540072441101, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9610174298286438, + 0.5349773168563843, + 0.413555771112442, + 0.5479018688201904, + 0.03090393729507923 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.888860821723938, + "style_target": 1, + "support_probability": 0.8913817405700684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07889734495641237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09630358964204788, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9777736663818359, + 0.5628232955932617, + 0.38462215662002563, + 0.40951234102249146, + 0.11146744340658188 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.04754272475838661, + "style_target": 0, + "support_probability": 0.8278260231018066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05856115216919596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.49365857243537903, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.04365914314985275, + 0.33819353580474854, + 0.6819562911987305, + 0.26350727677345276, + 0.01754354126751423, + 0.019301142543554306 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 1, + "style_probability": 0.8256789445877075, + "style_target": 1, + "support_probability": 0.02042531967163086, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03308582886730102, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.519601047039032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06417009979486465, + 0.2919786870479584, + 0.6317006349563599, + 0.3867630064487457, + 0.017660563811659813, + 0.0757332593202591 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 0, + "style_probability": 0.5593271255493164, + "style_target": 0, + "support_probability": 0.015119925141334534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04407554594288004, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37614983320236206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4074508547782898, + 0.8828173279762268, + 0.328911691904068, + 0.17285500466823578, + 0.0166353490203619, + 0.03529813885688782, + 0.04439298063516617, + 0.045512136071920395, + 0.061409369111061096 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.7209377288818359, + "style_target": 1, + "support_probability": 0.02683073654770851, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05035485523106024, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17126516997814178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4267595112323761, + 0.886982798576355, + 0.36972498893737793, + 0.17809879779815674, + 0.05583517998456955 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.3109184205532074, + "style_target": 0, + "support_probability": 0.05106295272707939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9288992406211072, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01737387292087078, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6419:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2063082456588745, + 0.5383577346801758, + 0.036690931767225266, + 0.9850043058395386, + 0.024538695812225342, + 0.010733183473348618, + 0.01813126914203167, + 0.04911298304796219, + 0.06029363349080086, + 0.03035631962120533 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.8907307386398315, + "style_target": 1, + "support_probability": 0.9891600012779236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5182982213128929, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020549466833472252, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21382735669612885, + 0.5826927423477173, + 0.030797958374023438, + 0.9956325888633728, + 0.042220037430524826, + 0.020257076248526573, + 0.038036592304706573, + 0.0208806823939085, + 0.029585812240839005, + 0.057538241147994995 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.11504736542701721, + "style_target": 0, + "support_probability": 0.991229772567749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07159442174448337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04811416193842888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24623346328735352, + 0.8869868516921997, + 0.0662260577082634, + 0.16155417263507843, + 0.04630022868514061 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 0, + "style_probability": 0.27223634719848633, + "style_target": 1, + "support_probability": 0.1617526113986969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012160268750577541, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08384737372398376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19461733102798462, + 0.9274641275405884, + 0.0719292163848877, + 0.1381056308746338, + 0.2593729794025421 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 1, + "style_probability": 0.09127713739871979, + "style_target": 0, + "support_probability": 0.13128863275051117, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5819560304850149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.030550962314009666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30887311697006226, + 0.16259799897670746, + 0.9903693795204163, + 0.40618470311164856, + 0.017297541722655296, + 0.036730244755744934 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 1, + "style_probability": 0.7195178270339966, + "style_target": 1, + "support_probability": 0.448879599571228, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9162442140311244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.029722226783633232, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35559192299842834, + 0.09979888796806335, + 0.9959349632263184, + 0.39996322989463806, + 0.05720393732190132, + 0.05426415428519249 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 0, + "style_probability": 0.9655972719192505, + "style_target": 0, + "support_probability": 0.624484658241272, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004374964519129131, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9893767833709717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6551181077957153, + 0.3750191926956177, + 0.38445302844047546, + 0.353206992149353, + 0.03257777914404869, + 0.07425111532211304, + 0.035670775920152664, + 0.023532062768936157, + 0.04591180756688118 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.5191910266876221, + "style_target": 1, + "support_probability": 0.0025002025067806244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003694047623773454, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.984084963798523, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5522201061248779, + 0.4387258291244507, + 0.37152963876724243, + 0.3899839520454407, + 0.058605849742889404 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.27288180589675903, + "style_target": 0, + "support_probability": 0.004275925923138857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03061605318592112, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7737613916397095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.416940838098526, + 0.9513044357299805, + 0.29689109325408936, + 0.23768502473831177, + 0.07480567693710327, + 0.1771380454301834, + 0.03531581908464432, + 0.029976841062307358, + 0.10788286477327347, + 0.04899309203028679, + 0.0316152423620224 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 1, + "style_probability": 0.5322394967079163, + "style_target": 1, + "support_probability": 0.016119111329317093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015540014171442879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31980428099632263, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38124850392341614, + 0.9726009368896484, + 0.26278528571128845, + 0.17722578346729279, + 0.027484165504574776, + 0.01889694482088089, + 0.03775184974074364, + 0.08613143116235733, + 0.0429769828915596, + 0.06623555719852448 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 1, + "style_probability": 0.1504238247871399, + "style_target": 0, + "support_probability": 0.0749516561627388, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6283818365383087, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005966917611658573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9928054213523865, + 0.3063349723815918, + 0.2712482511997223, + 0.2832948565483093, + 0.12598593533039093 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.3567107617855072, + "style_target": 1, + "support_probability": 0.9545392394065857, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.895183840250472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0032807961106300354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9960241317749023, + 0.24525494873523712, + 0.24130196869373322, + 0.22593045234680176, + 0.0164203904569149 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.8073640465736389, + "style_target": 0, + "support_probability": 0.9580603837966919, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7427163899990853, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.011446507647633553, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12867003679275513, + 0.1305471509695053, + 0.9981654286384583, + 0.10165497660636902, + 0.03683383762836456, + 0.04161854833364487 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 1, + "style_probability": 0.5388006567955017, + "style_target": 1, + "support_probability": 0.9807736277580261, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48391637048230984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.01858549937605858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10797049850225449, + 0.1591915637254715, + 0.9995607733726501, + 0.08586613833904266, + 0.02841358631849289, + 0.05401640012860298 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 1, + "style_probability": 0.24652718007564545, + "style_target": 0, + "support_probability": 0.9748688340187073, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004428155752802877, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8972985148429871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11299614608287811, + 0.27336835861206055, + 0.1555735319852829, + 0.16750885546207428, + 0.07705573737621307, + 0.0731063261628151, + 0.056498195976018906, + 0.12482082098722458, + 0.1408665031194687 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.3752360939979553, + "style_target": 1, + "support_probability": 0.0011149011552333832, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011175115864390807, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8824948668479919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13546137511730194, + 0.2777898907661438, + 0.13293859362602234, + 0.18863609433174133, + 0.04028589278459549, + 0.030988207086920738, + 0.03568093106150627, + 0.09426908195018768, + 0.037962980568408966 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.548812210559845, + "style_target": 0, + "support_probability": 0.00013966164260637015, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5331888450996746, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13496188819408417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14236055314540863, + 0.22622212767601013, + 0.9588249325752258, + 0.1701582819223404, + 0.041386678814888, + 0.053218111395835876, + 0.025758931413292885, + 0.0482671894133091, + 0.07344038039445877, + 0.09008398652076721, + 0.11433064937591553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 0, + "style_probability": 0.4504680037498474, + "style_target": 1, + "support_probability": 0.4513089656829834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5781657494847262, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2689586579799652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15345580875873566, + 0.21715235710144043, + 0.9530206918716431, + 0.17294061183929443, + 0.018938828259706497, + 0.032400477677583694, + 0.061882562935352325, + 0.037550024688243866, + 0.04516901820898056, + 0.03156452253460884 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 0, + "style_probability": 0.8695776462554932, + "style_target": 0, + "support_probability": 0.16015195846557617, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1565216588782624, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2769779562950134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5217848420143127, + 0.10551611334085464, + 0.2456611841917038, + 0.2814815938472748, + 0.023954100906848907 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 1, + "style_probability": 0.6813441514968872, + "style_target": 1, + "support_probability": 0.19669510424137115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07183666459228658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2870514690876007, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.545464277267456, + 0.08304400742053986, + 0.23077066242694855, + 0.2528240382671356, + 0.02025686204433441, + 0.07711213082075119, + 0.019688135012984276, + 0.07903433591127396, + 0.08336935937404633 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 0, + "style_probability": 0.5511175990104675, + "style_target": 0, + "support_probability": 0.06265267729759216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01677430202065566, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9857792854309082, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16004472970962524, + 0.20032984018325806, + 0.16004472970962524, + 0.5646308660507202, + 0.01630963757634163, + 0.051900532096624374 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 0, + "style_probability": 0.48101139068603516, + "style_target": 1, + "support_probability": 0.021663572639226913, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016707251201903011, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9711121916770935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10189840197563171, + 0.17019520699977875, + 0.10189840197563171, + 0.580832302570343, + 0.040484435856342316, + 0.1177668496966362 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 1, + "style_probability": 0.09741877764463425, + "style_target": 0, + "support_probability": 0.015923097729682922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015385866593206792, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8924155235290527, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5647619962692261, + 0.40248823165893555, + 0.2783351540565491, + 0.25027695298194885, + 0.04169835150241852, + 0.029591092839837074, + 0.027583211660385132, + 0.04253894090652466, + 0.06929757446050644 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 1, + "style_probability": 0.5431936383247375, + "style_target": 1, + "support_probability": 0.008299211971461773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026466068541994434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9293367266654968, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6532995700836182, + 0.36945438385009766, + 0.3361951410770416, + 0.25382137298583984, + 0.12544725835323334, + 0.05250172317028046, + 0.03398154303431511, + 0.013243100605905056, + 0.021883022040128708 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 0, + "style_probability": 0.7158481478691101, + "style_target": 0, + "support_probability": 0.00948222354054451, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009173696728676644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8765814304351807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11330877989530563, + 0.1861242800951004, + 0.30797022581100464, + 0.6990007162094116, + 0.05709931254386902, + 0.05698816105723381, + 0.029131276533007622, + 0.03282205015420914, + 0.02354232408106327, + 0.01423331443220377, + 0.0240321084856987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 1, + "style_probability": 0.558375895023346, + "style_target": 1, + "support_probability": 0.002986540785059333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006367680450231176, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9040192365646362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1198744922876358, + 0.15172331035137177, + 0.2984844148159027, + 0.6315221786499023, + 0.06135167181491852, + 0.06555970758199692 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 1, + "style_probability": 0.39077699184417725, + "style_target": 0, + "support_probability": 0.0015315578784793615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05258029904278523, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8900595903396606, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39747175574302673, + 0.2012891173362732, + 0.09890444576740265, + 0.2012891173362732, + 0.10750643163919449 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 1, + "style_probability": 0.8912432789802551, + "style_target": 1, + "support_probability": 0.024251990020275116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030387118671338413, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9154160618782043, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5502406358718872, + 0.20546859502792358, + 0.10866780579090118, + 0.20546859502792358, + 0.04001939296722412 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 0, + "style_probability": 0.6869441270828247, + "style_target": 0, + "support_probability": 0.018325844779610634, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7397283267878244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.019205782562494278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4529758393764496, + 0.4529758393764496, + 0.11207593977451324, + 0.9726572036743164, + 0.03830336779356003, + 0.06040545552968979 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.7485649585723877, + "style_target": 1, + "support_probability": 0.8497064709663391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44280039038192126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020122336223721504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4976353943347931, + 0.4976353943347931, + 0.12146930396556854, + 0.9658568501472473, + 0.029218805953860283, + 0.05744592100381851, + 0.00838717631995678, + 0.04688258096575737, + 0.04108100011944771, + 0.06649601459503174, + 0.025505708530545235 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.34387049078941345, + "style_target": 0, + "support_probability": 0.8784998655319214, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8722184639890577, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017477381974458694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10080846399068832, + 0.987542986869812, + 0.1554788053035736, + 0.2401372194290161, + 0.014007998630404472, + 0.015221943147480488, + 0.04773207753896713, + 0.03793230652809143, + 0.0960313081741333 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.6795077919960022, + "style_target": 1, + "support_probability": 0.9510893225669861, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2792325013255681, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02116764336824417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10275644809007645, + 0.9898311495780945, + 0.1333109736442566, + 0.266939640045166, + 0.018601322546601295, + 0.02324150875210762, + 0.03802196681499481, + 0.030176501721143723, + 0.06632380187511444 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.09580497443675995, + "style_target": 0, + "support_probability": 0.9531282782554626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04394039471888454, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48163411021232605, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5062441825866699, + 0.7812727689743042, + 0.7663094997406006, + 0.24544037878513336, + 0.042116399854421616, + 0.05945912003517151, + 0.06518895924091339, + 0.02683885209262371, + 0.04402415081858635, + 0.12538163363933563 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 1, + "style_probability": 0.7122284770011902, + "style_target": 1, + "support_probability": 0.01853100396692753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06754827302072453, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40174511075019836, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5402660369873047, + 0.7766188383102417, + 0.7640358805656433, + 0.2065175622701645, + 0.03959209471940994, + 0.18316809833049774, + 0.019687173888087273, + 0.0109397629275918, + 0.014189524576067924, + 0.053270116448402405, + 0.03518733009696007 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 1, + "style_probability": 0.3796183168888092, + "style_target": 0, + "support_probability": 0.047840435057878494, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.46612231016738553, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03452398627996445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21845893561840057, + 0.3759576678276062, + 0.8724080920219421, + 0.24509356915950775, + 0.029523294419050217 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.4569626748561859, + "style_target": 1, + "support_probability": 0.5363563895225525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6997844691875946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.021427320316433907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2248963862657547, + 0.34516581892967224, + 0.9246971011161804, + 0.27259916067123413, + 0.03384197875857353 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.7633563876152039, + "style_target": 0, + "support_probability": 0.6574246287345886, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8069734833742439, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03746900334954262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9919025301933289, + 0.374578595161438, + 0.12129511684179306, + 0.24176684021949768, + 0.1282380372285843, + 0.06198081746697426 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.504483699798584, + "style_target": 1, + "support_probability": 0.942430853843689, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7366687904428098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024549176916480064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9961665272712708, + 0.4441128671169281, + 0.12005016207695007, + 0.23193411529064178, + 0.03416700288653374, + 0.04442834481596947, + 0.035510607063770294, + 0.03857741504907608, + 0.022749103605747223, + 0.08660290390253067, + 0.056750375777482986 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.36987391114234924, + "style_target": 0, + "support_probability": 0.9766871929168701, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004407229539165909, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.874301016330719, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19473735988140106, + 0.3652292490005493, + 0.2579596936702728, + 0.3166273236274719, + 0.020104527473449707, + 0.04220201075077057, + 0.05340660735964775, + 0.027107419446110725, + 0.03559477999806404 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 0, + "style_probability": 0.4740673303604126, + "style_target": 1, + "support_probability": 0.001639503869228065, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002302896372176843, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8405115604400635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25344279408454895, + 0.23213189840316772, + 0.2645077705383301, + 0.23720760643482208, + 0.02255811169743538, + 0.017533592879772186, + 0.023634133860468864, + 0.012987295165657997, + 0.04167673736810684 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 1, + "style_probability": 0.11269634962081909, + "style_target": 0, + "support_probability": 0.011630559340119362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8010655828558949, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03394896164536476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9500692486763, + 0.5516769289970398, + 0.15306422114372253, + 0.4268316626548767, + 0.028295930474996567, + 0.02607874386012554, + 0.0522172749042511, + 0.020398594439029694, + 0.05410439521074295, + 0.0924297347664833 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 0, + "style_probability": 0.44810378551483154, + "style_target": 1, + "support_probability": 0.9696686267852783, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8841273993181573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02961241826415062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9668809771537781, + 0.5794335007667542, + 0.13237179815769196, + 0.40293627977371216, + 0.022035891190171242, + 0.026461860164999962, + 0.026981594040989876, + 0.006532462313771248, + 0.00726085901260376, + 0.07503753155469894, + 0.04262448474764824 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 0, + "style_probability": 0.6642497181892395, + "style_target": 0, + "support_probability": 0.9775223731994629, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050551388380788716, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6671556234359741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21241091191768646, + 0.5708368420600891, + 0.19945332407951355, + 0.21241091191768646, + 0.04370976239442825 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 0, + "style_probability": 0.3677337169647217, + "style_target": 1, + "support_probability": 0.15468615293502808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0049499262909502816, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.91823810338974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14377616345882416, + 0.5371739864349365, + 0.16375520825386047, + 0.14377616345882416, + 0.05449308082461357, + 0.03473784774541855, + 0.042731139808893204, + 0.039035502821207047, + 0.06536941230297089 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 1, + "style_probability": 0.20386414229869843, + "style_target": 0, + "support_probability": 0.0076473248191177845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2327445437174106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0731537938117981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14715930819511414, + 0.9928340911865234, + 0.14717116951942444, + 0.2644333243370056, + 0.014962447807192802, + 0.019327128306031227 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.5093934535980225, + "style_target": 1, + "support_probability": 0.26496177911758423, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04705878694550372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10094559192657471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2118464559316635, + 0.9978759288787842, + 0.061657220125198364, + 0.21562622487545013, + 0.013126237317919731, + 0.026108844205737114 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.1681133359670639, + "style_target": 0, + "support_probability": 0.4263656735420227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003110911838661939, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9097414612770081, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2949877679347992, + 0.14078019559383392, + 0.23452545702457428, + 0.5384942293167114, + 0.15287646651268005, + 0.0697665736079216, + 0.04585779085755348, + 0.1166902706027031, + 0.04702675715088844 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.2921403646469116, + "style_target": 1, + "support_probability": 0.004797197412699461, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00811607503131272, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8851222395896912, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4611523151397705, + 0.14608165621757507, + 0.257666677236557, + 0.6212441921234131, + 0.04456637054681778, + 0.05903683975338936, + 0.050516970455646515, + 0.02801705151796341, + 0.08582282811403275 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.6841088533401489, + "style_target": 0, + "support_probability": 0.01987385004758835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8211921061961547, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03455682098865509, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.936356782913208, + 0.2609274387359619, + 0.20306386053562164, + 0.09830722957849503, + 0.047127190977334976, + 0.04969155415892601, + 0.045152612030506134, + 0.05958567187190056, + 0.036560606211423874, + 0.12869159877300262 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.5718246698379517, + "style_target": 1, + "support_probability": 0.9967710375785828, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7846551569961808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01897026225924492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9605470299720764, + 0.23084473609924316, + 0.13939809799194336, + 0.09887108206748962, + 0.05538748949766159, + 0.05806905776262283 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.3747936189174652, + "style_target": 0, + "support_probability": 0.997337281703949, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19152416824473414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13079318404197693, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2999677360057831, + 0.2276921570301056, + 0.05185334011912346, + 0.973425030708313, + 0.014972577802836895 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 1, + "style_probability": 0.6895726919174194, + "style_target": 1, + "support_probability": 0.36301353573799133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4066320777514363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.04810531809926033, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3652477264404297, + 0.23580995202064514, + 0.0785650759935379, + 0.9682658314704895, + 0.007791901472955942 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 0, + "style_probability": 0.5227861404418945, + "style_target": 0, + "support_probability": 0.7190495729446411, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05542249397786731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33196204900741577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9831212759017944, + 0.17149145901203156, + 0.2978994846343994, + 0.4555067718029022, + 0.04525575041770935, + 0.08157527446746826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 1, + "style_probability": 0.8056718707084656, + "style_target": 1, + "support_probability": 0.030881205573678017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07526020814276185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14767736196517944, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9814963936805725, + 0.22080248594284058, + 0.2842392921447754, + 0.46887585520744324, + 0.05529214069247246, + 0.025989651679992676, + 0.07660306245088577, + 0.04147888720035553, + 0.06521617621183395, + 0.07720117270946503 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 1, + "style_probability": 0.3999974727630615, + "style_target": 0, + "support_probability": 0.09479504823684692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6407065871165748, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15055757761001587, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18588583171367645, + 0.9550811052322388, + 0.16578412055969238, + 0.5469644069671631, + 0.035189419984817505, + 0.03990572318434715, + 0.05905407667160034, + 0.05376674607396126, + 0.03287456929683685 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.8072474598884583, + "style_target": 1, + "support_probability": 0.527080774307251, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1111644733456511, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14422649145126343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1518453061580658, + 0.9584384560585022, + 0.20929892361164093, + 0.5746002793312073, + 0.03753365948796272, + 0.05142543464899063, + 0.03882365673780441, + 0.025274908170104027, + 0.12036305665969849 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.16703903675079346, + "style_target": 0, + "support_probability": 0.4974810481071472, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3392257947518925, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07284893840551376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12097704410552979, + 0.33999326825141907, + 0.1585521399974823, + 0.9679964780807495, + 0.00738433375954628, + 0.03342479467391968, + 0.014816809445619583, + 0.026210932061076164, + 0.032792892307043076, + 0.08272728323936462, + 0.0412958525121212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 1, + "style_probability": 0.849406361579895, + "style_target": 1, + "support_probability": 0.18559376895427704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19253419474274927, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11404541879892349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12017692625522614, + 0.35252177715301514, + 0.17020097374916077, + 0.9527275562286377, + 0.07801323384046555, + 0.025938333943486214, + 0.05608018487691879, + 0.029032954946160316, + 0.10586150735616684, + 0.13953152298927307, + 0.06907971203327179 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 0, + "style_probability": 0.5126828551292419, + "style_target": 0, + "support_probability": 0.2009669840335846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26676755758988013, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3287654221057892, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2903045117855072, + 0.38591885566711426, + 0.7484337091445923, + 0.2903045117855072, + 0.05836499109864235 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.885010302066803, + "style_target": 1, + "support_probability": 0.09307745099067688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08938567927490065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3070112466812134, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2935076653957367, + 0.4236573874950409, + 0.7562832832336426, + 0.2935076653957367, + 0.047074466943740845, + 0.036302924156188965, + 0.06480718404054642, + 0.02894582971930504, + 0.06346861273050308 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.3424636423587799, + "style_target": 0, + "support_probability": 0.08185365051031113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43773704622946286, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.387023389339447, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12332375347614288, + 0.10210582613945007, + 0.3986094295978546, + 0.6530303359031677, + 0.019921893253922462, + 0.019447674974799156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.7970402836799622, + "style_target": 1, + "support_probability": 0.4169257581233978, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04160106297573475, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5814754366874695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12102530896663666, + 0.060921330004930496, + 0.40910667181015015, + 0.6318069100379944, + 0.010012270882725716, + 0.024331355467438698 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.16594983637332916, + "style_target": 0, + "support_probability": 0.263248473405838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03550178714139137, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9727340936660767, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12011014670133591, + 0.14972956478595734, + 0.737784206867218, + 0.6481605768203735, + 0.057212527841329575, + 0.06696506589651108, + 0.0553581602871418, + 0.03166069835424423, + 0.03606247901916504 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 1, + "style_probability": 0.9310406446456909, + "style_target": 1, + "support_probability": 0.015997357666492462, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002747289233330579, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9858962893486023, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1607368439435959, + 0.12894879281520844, + 0.6894479393959045, + 0.5735401511192322, + 0.04922836273908615, + 0.042471494525671005, + 0.05582677572965622, + 0.05235026031732559, + 0.10594641417264938 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 1, + "style_probability": 0.3324168622493744, + "style_target": 0, + "support_probability": 0.0029369951225817204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07171750758653929, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9185711741447449, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6437240839004517, + 0.3649667799472809, + 0.30522456765174866, + 0.45818981528282166, + 0.1057610735297203, + 0.05525707080960274, + 0.05207562819123268, + 0.07521634548902512, + 0.01756698079407215, + 0.09103111177682877 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.908454179763794, + "style_target": 1, + "support_probability": 0.059447962790727615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.034742286420776286, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9131240844726562, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6887820959091187, + 0.40164193511009216, + 0.25255653262138367, + 0.4587708115577698, + 0.0332462415099144, + 0.12713804841041565 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.31712812185287476, + "style_target": 0, + "support_probability": 0.045223165303468704, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8457299824277843, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11338506639003754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9960641264915466, + 0.22446393966674805, + 0.11660873144865036, + 0.04227612540125847, + 0.029993481934070587 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 0, + "style_probability": 0.4515215754508972, + "style_target": 1, + "support_probability": 0.9147229790687561, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39779972463482915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21761082112789154, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9971265196800232, + 0.1883108913898468, + 0.10993103682994843, + 0.03873966634273529, + 0.03182030841708183 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 1, + "style_probability": 0.09783818572759628, + "style_target": 0, + "support_probability": 0.9070041179656982, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08993078341463165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8089726567268372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5769263505935669, + 0.16487501561641693, + 0.30186355113983154, + 0.1326521933078766, + 0.024913951754570007, + 0.03619306907057762 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 1, + "style_probability": 0.8702408075332642, + "style_target": 1, + "support_probability": 0.04754750803112984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05393218760996871, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8504309058189392, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6472018957138062, + 0.3047930598258972, + 0.30324143171310425, + 0.1454053372144699, + 0.08615562319755554, + 0.09753714501857758 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 0, + "style_probability": 0.5611394047737122, + "style_target": 0, + "support_probability": 0.08982338756322861, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34214485311659815, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5658115148544312, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3404978811740875, + 0.2677586078643799, + 0.23027867078781128, + 0.7322131991386414, + 0.1106175109744072, + 0.12541207671165466, + 0.038142938166856766, + 0.16670720279216766, + 0.10691723972558975 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.8470789194107056, + "style_target": 1, + "support_probability": 0.19580022990703583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08502898483820434, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.45098114013671875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3219869136810303, + 0.28155699372291565, + 0.24605657160282135, + 0.7517879009246826, + 0.33765509724617004 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.3336259424686432, + "style_target": 0, + "support_probability": 0.1027163416147232, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6412685039618834, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14708727598190308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10330446809530258, + 0.8873070478439331, + 0.1657256931066513, + 0.30480748414993286, + 0.09113143384456635, + 0.024446165189146996, + 0.021573109552264214, + 0.08330954611301422, + 0.020767197012901306, + 0.015469716861844063, + 0.021193113178014755 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.7126560807228088, + "style_target": 1, + "support_probability": 0.7665932178497314, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09918800633917858, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14602285623550415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.08135556429624557, + 0.9053522944450378, + 0.1142963171005249, + 0.23435018956661224, + 0.010911314748227596, + 0.03378979489207268, + 0.020922983065247536, + 0.025473378598690033, + 0.011665022000670433, + 0.02739103138446808 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.14147454500198364, + "style_target": 0, + "support_probability": 0.6776054501533508, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27602660313133315, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9153883457183838, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22640256583690643, + 0.17481987178325653, + 0.666408121585846, + 0.8447092175483704, + 0.07712302356958389 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 1, + "style_probability": 0.9080553650856018, + "style_target": 1, + "support_probability": 0.07291702926158905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06027825470294823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.82099848985672, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3208893835544586, + 0.16016635298728943, + 0.6605868339538574, + 0.8274138569831848, + 0.14511258900165558 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 0, + "style_probability": 0.5594810843467712, + "style_target": 0, + "support_probability": 0.06481482833623886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0540144030472062, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7993401288986206, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7782924771308899, + 0.22457650303840637, + 0.8518223762512207, + 0.41297632455825806, + 0.038878004997968674, + 0.11998311430215836 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.6925666332244873, + "style_target": 1, + "support_probability": 0.029906518757343292, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017613653211252942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8086235523223877, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.730591356754303, + 0.26151031255722046, + 0.805711030960083, + 0.3592906594276428, + 0.037992361932992935, + 0.04083406925201416, + 0.02108684740960598, + 0.06865137070417404, + 0.09413488954305649, + 0.02998054400086403, + 0.03647400438785553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.39397236704826355, + "style_target": 0, + "support_probability": 0.007895871996879578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2507746220205196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6252341866493225, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8677075505256653, + 0.19520945847034454, + 0.38979172706604004, + 0.5358622670173645, + 0.06988565623760223, + 0.014846385456621647, + 0.08026950061321259, + 0.024320043623447418, + 0.14695832133293152 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.790890097618103, + "style_target": 1, + "support_probability": 0.097858726978302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07091479867957111, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5027991533279419, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8670361042022705, + 0.18981006741523743, + 0.27081000804901123, + 0.5362427234649658, + 0.012076844461262226, + 0.13127385079860687, + 0.017425071448087692, + 0.00820735190063715, + 0.05588541179895401 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.21613769233226776, + "style_target": 0, + "support_probability": 0.2303808480501175, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8471982185110246, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005269195884466171, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3329438865184784, + 0.3002660274505615, + 0.7817807793617249, + 0.9885129928588867, + 0.02599387615919113, + 0.049814172089099884, + 0.04571555182337761, + 0.008729665540158749, + 0.054643414914608, + 0.018096530809998512, + 0.06500139832496643 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 0, + "style_probability": 0.39424610137939453, + "style_target": 1, + "support_probability": 0.9907468557357788, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7897372924307051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006384124513715506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43334805965423584, + 0.30978935956954956, + 0.7123101949691772, + 0.9749234318733215, + 0.013097414746880531, + 0.009840304031968117, + 0.016283195465803146, + 0.05234435945749283, + 0.04733331501483917, + 0.010286212898790836, + 0.034365952014923096 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 1, + "style_probability": 0.3631804585456848, + "style_target": 0, + "support_probability": 0.979722797870636, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32811609559152255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.35308048129081726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3375403881072998, + 0.9381510615348816, + 0.203829824924469, + 0.36964449286460876, + 0.09423889964818954 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 0, + "style_probability": 0.4916498363018036, + "style_target": 1, + "support_probability": 0.45213186740875244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7305293810740701, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16127558052539825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3504113554954529, + 0.9529220461845398, + 0.1817777454853058, + 0.43045830726623535, + 0.030600998550653458 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 0, + "style_probability": 0.8473227620124817, + "style_target": 0, + "support_probability": 0.7450087666511536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27332892370772116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034856367856264114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5213549733161926, + 0.7828083038330078, + 0.25687694549560547, + 0.044105738401412964, + 0.017830941826105118, + 0.02604650892317295 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 0, + "style_probability": 0.23322314023971558, + "style_target": 1, + "support_probability": 0.6871398687362671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028856634295980916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1341889202594757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35813823342323303, + 0.8987259268760681, + 0.14385248720645905, + 0.038249656558036804, + 0.012885503470897675, + 0.017749056220054626 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 1, + "style_probability": 0.07252953201532364, + "style_target": 0, + "support_probability": 0.5225527882575989, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09462116040732914, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5167989730834961, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35554274916648865, + 0.4002707302570343, + 0.3927530348300934, + 0.8918246626853943, + 0.03550688549876213, + 0.025180334225296974, + 0.04901280254125595, + 0.0625256896018982, + 0.10670682042837143 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 0, + "style_probability": 0.44034501910209656, + "style_target": 1, + "support_probability": 0.04670889303088188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15627877227256906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3391171991825104, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36107149720191956, + 0.311698317527771, + 0.32033923268318176, + 0.8657302260398865, + 0.029152508825063705, + 0.015941482037305832, + 0.09642869234085083, + 0.012048499658703804, + 0.05577123537659645 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 0, + "style_probability": 0.5662152767181396, + "style_target": 0, + "support_probability": 0.06440699100494385, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007820871267805656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3969476521015167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1547408401966095, + 0.6618888974189758, + 0.0924985408782959, + 0.18279606103897095, + 0.03659088537096977, + 0.040349677205085754, + 0.069790780544281, + 0.05990505963563919, + 0.22665861248970032, + 0.0316646583378315, + 0.0600743405520916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.42412975430488586, + "style_target": 1, + "support_probability": 0.0006349206669256091, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0027160899266184123, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6645020246505737, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14407704770565033, + 0.651308000087738, + 0.08206507563591003, + 0.17431332170963287, + 0.015621687285602093, + 0.032156169414520264, + 0.012378992512822151, + 0.05657963454723358, + 0.1136520504951477, + 0.04225059226155281, + 0.03310399875044823 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.8013233542442322, + "style_target": 0, + "support_probability": 0.00026989003526978195, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7557935071379124, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05555056408047676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.551703691482544, + 0.2226162552833557, + 0.99326092004776, + 0.27856236696243286, + 0.017604224383831024 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 0, + "style_probability": 0.47637850046157837, + "style_target": 1, + "support_probability": 0.9234093427658081, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36625810478821963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08366049826145172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5527923107147217, + 0.221540167927742, + 0.9931434392929077, + 0.2573995292186737, + 0.029622286558151245 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 1, + "style_probability": 0.19738221168518066, + "style_target": 0, + "support_probability": 0.8971850275993347, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4235344284022235, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4340638518333435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17729291319847107, + 0.8596382737159729, + 0.4179803431034088, + 0.5818665027618408, + 0.033061105757951736, + 0.05462368205189705 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 1, + "style_probability": 0.8650972247123718, + "style_target": 1, + "support_probability": 0.45099878311157227, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08205607503493492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.877971351146698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17091886699199677, + 0.7563429474830627, + 0.3517325818538666, + 0.4632449448108673, + 0.05042670667171478, + 0.131790429353714 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 1, + "style_probability": 0.43034154176712036, + "style_target": 0, + "support_probability": 0.13738279044628143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04400347233563096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7916333675384521, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2753314673900604, + 0.9185665845870972, + 0.4911326766014099, + 0.4514748752117157, + 0.07954473793506622, + 0.14139880239963531, + 0.04692072421312332, + 0.03822626546025276, + 0.11270851641893387 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.7725104093551636, + "style_target": 1, + "support_probability": 0.030748218297958374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07600143510167254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6416709423065186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25591006875038147, + 0.9053452014923096, + 0.5058033466339111, + 0.43519115447998047, + 0.12327374517917633 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.19488272070884705, + "style_target": 0, + "support_probability": 0.08246038109064102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9617793654845771, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0322747677564621, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6704:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9978251457214355, + 0.5196950435638428, + 0.16492699086666107, + 0.4469536244869232, + 0.05296420305967331, + 0.06217968091368675, + 0.03545667976140976, + 0.08982092887163162, + 0.0407116636633873, + 0.05415304750204086, + 0.03956631198525429 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.8545541763305664, + "style_target": 1, + "support_probability": 0.9909296631813049, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5276054226178748, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.043410662561655045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6704:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9996837377548218, + 0.4639381170272827, + 0.1389429271221161, + 0.42973431944847107, + 0.018732093274593353, + 0.04151538014411926, + 0.11069270223379135, + 0.05917266011238098, + 0.02793978899717331, + 0.05400901287794113 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.11082208901643753, + "style_target": 0, + "support_probability": 0.986935555934906, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5843520731669961, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08947340399026871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6478241086006165, + 0.4875239431858063, + 0.38275057077407837, + 0.33578062057495117, + 0.02674984559416771 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 1, + "style_probability": 0.9119592308998108, + "style_target": 1, + "support_probability": 0.9263888001441956, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33512986370694814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27685728669166565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6349655389785767, + 0.5804771184921265, + 0.398910790681839, + 0.3304920196533203, + 0.04089249297976494 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 1, + "style_probability": 0.4912671148777008, + "style_target": 0, + "support_probability": 0.7113960385322571, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17064645918402435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9161217212677002, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20431536436080933, + 0.6656372547149658, + 0.30372756719589233, + 0.6859115958213806, + 0.05261348560452461, + 0.046609267592430115 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.8269021511077881, + "style_target": 1, + "support_probability": 0.17477209866046906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010729228957393129, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9640908241271973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1838511824607849, + 0.6070618033409119, + 0.30158665776252747, + 0.7009797692298889, + 0.17369040846824646, + 0.19456130266189575, + 0.03783915191888809, + 0.06058722734451294, + 0.27645236253738403, + 0.0599517896771431, + 0.05961649864912033 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.19777151942253113, + "style_target": 0, + "support_probability": 0.021270545199513435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3799174426988745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3169938921928406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5884129405021667, + 0.8650054335594177, + 0.3201899230480194, + 0.1329902857542038, + 0.03400477394461632, + 0.013848981820046902, + 0.03393405303359032, + 0.010215272195637226, + 0.03284195438027382 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 1, + "style_probability": 0.7281994223594666, + "style_target": 1, + "support_probability": 0.2717548608779907, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02258079194392011, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5662047266960144, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5114043354988098, + 0.8444899320602417, + 0.2706868052482605, + 0.09688720107078552, + 0.027547260746359825, + 0.06520451605319977, + 0.031400930136442184, + 0.03217413276433945, + 0.1517380177974701 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 1, + "style_probability": 0.0888456478714943, + "style_target": 0, + "support_probability": 0.188350111246109, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1542426535804146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7867883443832397, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3148525655269623, + 0.3535906672477722, + 0.7908778190612793, + 0.2794272005558014, + 0.013671314343810081, + 0.02555771917104721, + 0.03506236895918846, + 0.019710838794708252, + 0.015308510512113571, + 0.028408192098140717, + 0.019578343257308006 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 1, + "style_probability": 0.8374788165092468, + "style_target": 1, + "support_probability": 0.05910364165902138, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22114004707469015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3765803873538971, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3264335095882416, + 0.36513179540634155, + 0.757062554359436, + 0.3185247480869293, + 0.02351335436105728, + 0.03848540410399437, + 0.02918725647032261, + 0.029217103496193886, + 0.05128123238682747, + 0.5486128330230713, + 0.01942235790193081 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 0, + "style_probability": 0.5421279668807983, + "style_target": 0, + "support_probability": 0.2749719023704529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6670904338164486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.014625053852796555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11579188704490662, + 0.9735550880432129, + 0.046425607055425644, + 0.11579188704490662, + 0.043645575642585754 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 1, + "style_probability": 0.9413447976112366, + "style_target": 1, + "support_probability": 0.6747043132781982, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32620601467087823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0071203699335455894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09779009222984314, + 0.9702918529510498, + 0.04156970605254173, + 0.09779009222984314, + 0.01816401816904545, + 0.012908656150102615, + 0.019072627648711205, + 0.03417195752263069, + 0.045806918293237686 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 0, + "style_probability": 0.5202995538711548, + "style_target": 0, + "support_probability": 0.6212638020515442, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012098582583522623, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9236310124397278, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2924574017524719, + 0.08066712319850922, + 0.16185496747493744, + 0.31932103633880615, + 0.04322131723165512, + 0.05751015618443489 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.7876999378204346, + "style_target": 1, + "support_probability": 0.0067572202533483505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018153561448684984, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9398390054702759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2663339078426361, + 0.09745924174785614, + 0.1308077573776245, + 0.23637829720973969, + 0.042825400829315186, + 0.06939589977264404 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.131752148270607, + "style_target": 0, + "support_probability": 0.01830693706870079, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3573800557521115, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7537540793418884, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6452798247337341, + 0.1472967118024826, + 0.194551020860672, + 0.40770095586776733, + 0.03383677825331688, + 0.08935955166816711, + 0.026938308030366898, + 0.03307696431875229, + 0.05074871703982353 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 1, + "style_probability": 0.9239895343780518, + "style_target": 1, + "support_probability": 0.10040277242660522, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.059792416912202884, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7782081365585327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6406244039535522, + 0.18756984174251556, + 0.11598014086484909, + 0.35231438279151917, + 0.01371376495808363, + 0.03264613822102547, + 0.04692709073424339, + 0.016665462404489517, + 0.038434531539678574 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 1, + "style_probability": 0.38854914903640747, + "style_target": 0, + "support_probability": 0.05525430664420128, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7833611668509979, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1948878914117813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1895454227924347, + 0.6498989462852478, + 0.9676434993743896, + 0.6031677722930908, + 0.04154590889811516, + 0.03321434184908867, + 0.04972642660140991, + 0.07654259353876114, + 0.053858961910009384, + 0.03439199924468994, + 0.058491919189691544 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 1, + "style_probability": 0.9141400456428528, + "style_target": 1, + "support_probability": 0.7665945291519165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5048576791353023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09845367819070816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19639736413955688, + 0.6263882517814636, + 0.9652774930000305, + 0.5342260003089905, + 0.06854455918073654, + 0.04722314700484276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 0, + "style_probability": 0.5236660838127136, + "style_target": 0, + "support_probability": 0.824264645576477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5727892559079595, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.013336512260138988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.926029622554779, + 0.09576893597841263, + 0.9968208074569702, + 0.926029622554779, + 0.02264602668583393 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.6578261256217957, + "style_target": 1, + "support_probability": 0.8560205698013306, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0344029593941874, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.043589841574430466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9279056191444397, + 0.09938057512044907, + 0.9969348907470703, + 0.9279056191444397, + 0.0929923877120018 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.07105698436498642, + "style_target": 0, + "support_probability": 0.7781233191490173, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4569124449911857, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06427042931318283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4884289503097534, + 0.9532648921012878, + 0.7051334381103516, + 0.09154224395751953, + 0.029237378388643265, + 0.03249841928482056 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 1, + "style_probability": 0.7743695974349976, + "style_target": 1, + "support_probability": 0.14052365720272064, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3502942149881574, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04322785511612892, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5065411329269409, + 0.9415470361709595, + 0.6555932760238647, + 0.1175428181886673, + 0.04514917731285095, + 0.01782665029168129 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 0, + "style_probability": 0.6556905508041382, + "style_target": 0, + "support_probability": 0.18550904095172882, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5844289326809085, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015378108248114586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3174956142902374, + 0.9976402521133423, + 0.34048736095428467, + 0.288255900144577, + 0.023310409858822823, + 0.024549951776862144, + 0.04766419529914856, + 0.02708231471478939, + 0.010645166039466858 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.5294628143310547, + "style_target": 1, + "support_probability": 0.8075051307678223, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.312083480917836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.015618673525750637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30096715688705444, + 0.9961047172546387, + 0.37800124287605286, + 0.30623432993888855, + 0.06705325841903687 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.23137541115283966, + "style_target": 0, + "support_probability": 0.7612622976303101, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5175397015186707, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2656109631061554, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9035362601280212, + 0.5471452474594116, + 0.4180198311805725, + 0.1983729898929596, + 0.02736607939004898, + 0.014962876215577126, + 0.05123244598507881, + 0.06652822345495224, + 0.08815126866102219, + 0.058258190751075745, + 0.01278641913086176 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.7784318923950195, + "style_target": 1, + "support_probability": 0.3942459225654602, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03368043791218693, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43032488226890564, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8937968611717224, + 0.42175471782684326, + 0.27094927430152893, + 0.1682967245578766, + 0.04571778327226639, + 0.03227726370096207, + 0.0671226903796196, + 0.07848338782787323, + 0.03463292121887207, + 0.01955850049853325 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.10369224101305008, + "style_target": 0, + "support_probability": 0.3593079745769501, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.565957034703422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08050622791051865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10969556868076324, + 0.9763403534889221, + 0.16152562201023102, + 0.10692416876554489, + 0.02431941032409668 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 1, + "style_probability": 0.6139772534370422, + "style_target": 1, + "support_probability": 0.5806853771209717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07558191793553763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14447809755802155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0796273946762085, + 0.9849599003791809, + 0.1165325939655304, + 0.05922192335128784, + 0.01355629414319992 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 1, + "style_probability": 0.1438034027814865, + "style_target": 0, + "support_probability": 0.47203660011291504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026200447889605377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9033234119415283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15854141116142273, + 0.30356112122535706, + 0.8295921683311462, + 0.3542948365211487, + 0.043557871133089066, + 0.025994762778282166 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 1, + "style_probability": 0.5083295702934265, + "style_target": 1, + "support_probability": 0.015395519323647022, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04001000206682328, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9238114356994629, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11247462779283524, + 0.3614140450954437, + 0.8387984037399292, + 0.30046361684799194, + 0.014521467499434948, + 0.037772003561258316 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 0, + "style_probability": 0.757495641708374, + "style_target": 0, + "support_probability": 0.019002346321940422, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0771862495399327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7557768225669861, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26474103331565857, + 0.1641213446855545, + 0.10044414550065994, + 0.8015603423118591, + 0.09987042099237442, + 0.032979391515254974, + 0.056963905692100525, + 0.031725477427244186, + 0.0554855652153492 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 1, + "style_probability": 0.5974891781806946, + "style_target": 1, + "support_probability": 0.031351830810308456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09962072900535661, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5762998461723328, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24796639382839203, + 0.1780930906534195, + 0.08131282776594162, + 0.886218249797821, + 0.15585428476333618 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 1, + "style_probability": 0.2539023160934448, + "style_target": 0, + "support_probability": 0.028024382889270782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02705158410579878, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8309967517852783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15707005560398102, + 0.16121181845664978, + 0.4819554388523102, + 0.12444768100976944, + 0.03403769060969353, + 0.14784282445907593, + 0.10705146938562393, + 0.015886519104242325, + 0.02014567144215107, + 0.02791418321430683, + 0.06715137511491776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 0, + "style_probability": 0.47491714358329773, + "style_target": 1, + "support_probability": 0.03141576424241066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00345354530888017, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8506879806518555, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09869536757469177, + 0.16489939391613007, + 0.44579070806503296, + 0.14176726341247559, + 0.01390688307583332, + 0.021226434037089348, + 0.011594230309128761, + 0.0374603196978569, + 0.01365708652883768, + 0.045409638434648514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 1, + "style_probability": 0.0945078581571579, + "style_target": 0, + "support_probability": 0.055439796298742294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9438458438392375, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.008759801276028156, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6841:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2120078057050705, + 0.9970188140869141, + 0.22137781977653503, + 0.23541221022605896, + 0.016314135864377022 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 1, + "style_probability": 0.7915309071540833, + "style_target": 1, + "support_probability": 0.9553227424621582, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7704327151690206, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004375148098915815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22947968542575836, + 0.995519757270813, + 0.19784747064113617, + 0.23363249003887177, + 0.015543371438980103, + 0.015743345022201538, + 0.009383242577314377, + 0.034515030682086945, + 0.013046356849372387 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 1, + "style_probability": 0.4882407486438751, + "style_target": 0, + "support_probability": 0.9558707475662231, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04014980687681867, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8679371476173401, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1866820901632309, + 0.18332473933696747, + 0.7350558042526245, + 0.28532111644744873, + 0.02754039503633976, + 0.03039700724184513 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.761880099773407, + "style_target": 1, + "support_probability": 0.014627602882683277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021157422866979067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8592427968978882, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12493009120225906, + 0.10946870595216751, + 0.7688291668891907, + 0.18220488727092743, + 0.013417153619229794, + 0.025207608938217163 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.1812518984079361, + "style_target": 0, + "support_probability": 0.054448310285806656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10598778468637766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6301237344741821, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9091665744781494, + 0.14268305897712708, + 0.0632767453789711, + 0.45243728160858154, + 0.20990629494190216, + 0.027801964432001114, + 0.13042396306991577, + 0.029607104137539864, + 0.02300802618265152 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 1, + "style_probability": 0.6953659057617188, + "style_target": 1, + "support_probability": 0.06863004714250565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27274048547051577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38682878017425537, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9000900387763977, + 0.24086450040340424, + 0.09806784987449646, + 0.6219702363014221, + 0.072999969124794, + 0.04411975294351578, + 0.02699069306254387, + 0.011036478914320469, + 0.054797396063804626 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 0, + "style_probability": 0.6657558679580688, + "style_target": 0, + "support_probability": 0.18681447207927704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16486264088064478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1954243779182434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:positive", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.43544936180114746, + 0.9903736114501953, + 0.10184428095817566, + 0.5343573093414307, + 0.02285773865878582, + 0.08856097608804703, + 0.13790132105350494, + 0.04345157369971275, + 0.07632748782634735, + 0.19941982626914978, + 0.04488757997751236, + 0.08601430803537369, + 0.07228005677461624, + 0.022374959662556648 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.6258108019828796, + "style_target": 1, + "support_probability": 0.12385927140712738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14909704564393442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2146301418542862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.49687114357948303, + 0.9880451560020447, + 0.11778724938631058, + 0.6056639552116394, + 0.04111245274543762, + 0.07846441864967346 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.25500136613845825, + "style_target": 0, + "support_probability": 0.14994704723358154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46198023215168893, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.26840758323669434, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1717996448278427, + 0.27661290764808655, + 0.29248228669166565, + 0.9811443090438843, + 0.01833494007587433 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.715708315372467, + "style_target": 1, + "support_probability": 0.24949462711811066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.046898092394862735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3202991187572479, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13834063708782196, + 0.24317890405654907, + 0.2879950702190399, + 0.9784767031669617, + 0.024567145854234695 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.10488413274288177, + "style_target": 0, + "support_probability": 0.19949446618556976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5833360178178788, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11613886803388596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9887750148773193, + 0.13016164302825928, + 0.330399751663208, + 0.1444777250289917, + 0.14083175361156464, + 0.06269471347332001 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 1, + "style_probability": 0.7917360663414001, + "style_target": 1, + "support_probability": 0.6089855432510376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2644653246422742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2494107037782669, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9561476111412048, + 0.19509966671466827, + 0.3484579920768738, + 0.13898156583309174, + 0.11548270285129547, + 0.0812462791800499 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 0, + "style_probability": 0.5755794048309326, + "style_target": 0, + "support_probability": 0.36352163553237915, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14507103976669122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4098008871078491, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.415365070104599, + 0.1472708135843277, + 0.5647150874137878, + 0.901176929473877, + 0.012891697697341442, + 0.02673977054655552, + 0.025812553241848946, + 0.07673221081495285, + 0.02700720727443695 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.6811097860336304, + "style_target": 1, + "support_probability": 0.10735823214054108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18262716369090415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2142682522535324, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4795585572719574, + 0.16727641224861145, + 0.6497148275375366, + 0.9088017344474792, + 0.01282405760139227 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.46700701117515564, + "style_target": 0, + "support_probability": 0.2143733650445938, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01033732446391246, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4735410213470459, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.350168377161026, + 0.4996708333492279, + 0.24160361289978027, + 0.5428997278213501, + 0.008967641741037369, + 0.013260933570563793, + 0.012756118550896645, + 0.014302732422947884, + 0.058139778673648834, + 0.01598905771970749 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.6297121047973633, + "style_target": 1, + "support_probability": 0.003651610342785716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001966205161953862, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6586876511573792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3188333511352539, + 0.34000638127326965, + 0.267831951379776, + 0.5591866374015808, + 0.01089849229902029, + 0.014602834358811378, + 0.009013581089675426, + 0.020309243351221085, + 0.009754911996424198, + 0.026148203760385513 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.13268138468265533, + "style_target": 0, + "support_probability": 0.004733322188258171, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5306382306021796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15645413100719452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27556124329566956, + 0.2785257399082184, + 0.9687678217887878, + 0.3386642634868622, + 0.0895705297589302 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.8157957792282104, + "style_target": 1, + "support_probability": 0.5378584265708923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032445150731554045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.740685224533081, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2990689277648926, + 0.28013867139816284, + 0.941634476184845, + 0.42104172706604004, + 0.05089709162712097 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.3560628294944763, + "style_target": 0, + "support_probability": 0.05573810636997223, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5842341657637689, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23498781025409698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39427605271339417, + 0.23683862388134003, + 0.726161539554596, + 0.39427605271339417, + 0.08257057517766953, + 0.1012701615691185 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.9229071140289307, + "style_target": 1, + "support_probability": 0.586105227470398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03012510381358348, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6686722040176392, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2947206199169159, + 0.21426919102668762, + 0.711515486240387, + 0.2947206199169159, + 0.034826263785362244, + 0.061324574053287506, + 0.03884822875261307, + 0.026470037177205086, + 0.0959787666797638, + 0.04524632543325424, + 0.04560079053044319 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.4304928183555603, + "style_target": 0, + "support_probability": 0.01435285434126854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25274792177713545, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09403661638498306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21657641232013702, + 0.20854397118091583, + 0.9651942849159241, + 0.18216939270496368, + 0.018094176426529884, + 0.04067648947238922, + 0.13765670359134674, + 0.3292822539806366, + 0.43252041935920715 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.7893097996711731, + "style_target": 1, + "support_probability": 0.1109938770532608, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07867700088142847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13244447112083435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27031174302101135, + 0.25068435072898865, + 0.9705444574356079, + 0.23815010488033295, + 0.06861157715320587, + 0.07523299753665924, + 0.06481235474348068, + 0.1040584072470665, + 0.13720911741256714 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.45792683959007263, + "style_target": 0, + "support_probability": 0.06880465894937515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11530761167472425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2829609513282776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.37897735834121704, + 0.649839460849762, + 0.37897735834121704, + 0.8953537940979004, + 0.07001647353172302, + 0.12351875007152557, + 0.05118399113416672, + 0.12418831884860992, + 0.12835069000720978, + 0.15057113766670227 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 1, + "style_probability": 0.9500402212142944, + "style_target": 1, + "support_probability": 0.06757111102342606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07213162784761384, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23567169904708862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.369392454624176, + 0.6500244140625, + 0.369392454624176, + 0.9112590551376343, + 0.07971522957086563, + 0.22292546927928925, + 0.03919055685400963, + 0.07828401029109955, + 0.05684997886419296, + 0.023362988606095314, + 0.07281044125556946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 1, + "style_probability": 0.4129806458950043, + "style_target": 0, + "support_probability": 0.1913706362247467, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26607344853787873, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10454359650611877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13995738327503204, + 0.0796542763710022, + 0.2311023324728012, + 0.7872498035430908, + 0.018026968464255333 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 1, + "style_probability": 0.6735799312591553, + "style_target": 1, + "support_probability": 0.47039708495140076, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15604102577112414, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.170759916305542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14103645086288452, + 0.09072547405958176, + 0.24955150485038757, + 0.7346594929695129, + 0.013690623454749584, + 0.027801034972071648, + 0.06522136926651001, + 0.025815822184085846, + 0.06104230880737305 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 1, + "style_probability": 0.3553166687488556, + "style_target": 0, + "support_probability": 0.45435720682144165, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5724820225964999, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1333332508802414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16264475882053375, + 0.9807425737380981, + 0.1269524097442627, + 0.5399553179740906, + 0.01448043156415224, + 0.012513039633631706 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.8390050530433655, + "style_target": 1, + "support_probability": 0.30997875332832336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.045333618782937325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20082980394363403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14182408154010773, + 0.9913143515586853, + 0.10973178595304489, + 0.50299471616745, + 0.024107275530695915, + 0.03693912923336029 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.10410432517528534, + "style_target": 0, + "support_probability": 0.36014702916145325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20027710956061115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.441932737827301, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5480872988700867, + 0.11350807547569275, + 0.9270898103713989, + 0.16672252118587494, + 0.04539107531309128, + 0.017205188050866127, + 0.023470796644687653, + 0.014755799435079098, + 0.012741300277411938 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 1, + "style_probability": 0.7084794044494629, + "style_target": 1, + "support_probability": 0.07373747229576111, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10894240028612696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4212194085121155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5937457084655762, + 0.08069153130054474, + 0.8965870141983032, + 0.14511732757091522, + 0.009210814721882343, + 0.015775466337800026, + 0.024201275780797005, + 0.020677395164966583, + 0.03320050239562988 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 1, + "style_probability": 0.371370792388916, + "style_target": 0, + "support_probability": 0.09495630115270615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3587701680187263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13814780116081238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.40447133779525757, + 0.28529444336891174, + 0.8415290117263794, + 0.7236608862876892, + 0.05075521767139435, + 0.03287986293435097, + 0.02595781534910202, + 0.034220144152641296, + 0.022167673334479332, + 0.02800546959042549, + 0.011969057843089104 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 1, + "style_probability": 0.9056263566017151, + "style_target": 1, + "support_probability": 0.06260301917791367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18586901394164942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11811177432537079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39083752036094666, + 0.2633964717388153, + 0.8435871601104736, + 0.6913019418716431, + 0.05226951465010643, + 0.11590983718633652 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 1, + "style_probability": 0.489304780960083, + "style_target": 0, + "support_probability": 0.12120989710092545, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.905347854499053, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.015225183218717575, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6943:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.497312992811203, + 0.201017826795578, + 0.497312992811203, + 0.984429121017456, + 0.032394640147686005 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.9208007454872131, + "style_target": 1, + "support_probability": 0.9083983302116394, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14717249850013242, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.030432673171162605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6943:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6191089749336243, + 0.15938687324523926, + 0.6191089749336243, + 0.9929266571998596, + 0.04887606203556061 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.05336664244532585, + "style_target": 0, + "support_probability": 0.9095677733421326, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3317454568098171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8286773562431335, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5819347500801086, + 0.15176527202129364, + 0.8731114864349365, + 0.5819347500801086, + 0.031235894188284874, + 0.05202479660511017 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 1, + "style_probability": 0.7607139348983765, + "style_target": 1, + "support_probability": 0.330737829208374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04638615432060219, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9758689999580383, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5420325398445129, + 0.2432669699192047, + 0.8742623329162598, + 0.5420325398445129, + 0.05210729315876961, + 0.025555642321705818 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 0, + "style_probability": 0.6118258237838745, + "style_target": 0, + "support_probability": 0.04154804348945618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4633148413196678, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3577367663383484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48011964559555054, + 0.5734186768531799, + 0.4413611888885498, + 0.6756940484046936, + 0.018390454351902008, + 0.02376730926334858, + 0.025901030749082565, + 0.04898057132959366, + 0.04618013650178909 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.9121816158294678, + "style_target": 1, + "support_probability": 0.3988288938999176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03087157726282823, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42902126908302307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37784868478775024, + 0.5231485366821289, + 0.3783026933670044, + 0.6019095182418823, + 0.061112552881240845 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.272938996553421, + "style_target": 0, + "support_probability": 0.06095018982887268, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8895435678192452, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022384801879525185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7186653017997742, + 0.05308736860752106, + 0.6027252078056335, + 0.9820680022239685, + 0.02568511664867401, + 0.13881415128707886, + 0.06429273635149002, + 0.012993047945201397, + 0.031797125935554504, + 0.019980154931545258 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.7819011211395264, + "style_target": 1, + "support_probability": 0.9780795574188232, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13676302712114904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02453511394560337, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6866816878318787, + 0.04569733515381813, + 0.5902093052864075, + 0.9808196425437927, + 0.03456416353583336, + 0.09869703650474548, + 0.05257725343108177, + 0.05653930827975273, + 0.01313507929444313, + 0.056959427893161774 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.04673360288143158, + "style_target": 0, + "support_probability": 0.9703283309936523, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7428258855610492, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15145434439182281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4075954556465149, + 0.19089244306087494, + 0.24273070693016052, + 0.9700114727020264, + 0.010619203560054302 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 0, + "style_probability": 0.4451943337917328, + "style_target": 1, + "support_probability": 0.9645143747329712, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11988457295929089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16299152374267578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31476032733917236, + 0.09535808116197586, + 0.09290866553783417, + 0.9956938028335571, + 0.07216547429561615 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 1, + "style_probability": 0.04591698944568634, + "style_target": 0, + "support_probability": 0.8983997106552124, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4953813957387901, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33159637451171875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3309752643108368, + 0.18095622956752777, + 0.9625183343887329, + 0.9056445956230164, + 0.02231432870030403, + 0.013383811339735985 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 0, + "style_probability": 0.40468865633010864, + "style_target": 1, + "support_probability": 0.8278460502624512, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.730919603533359, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1765221506357193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19977082312107086, + 0.10324037075042725, + 0.9893414378166199, + 0.9251528382301331, + 0.04408956691622734, + 0.09122065454721451 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 0, + "style_probability": 0.6052488088607788, + "style_target": 0, + "support_probability": 0.8702401518821716, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5946316206304826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3341015875339508, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7711189985275269, + 0.48539161682128906, + 0.07768141478300095, + 0.4548211097717285, + 0.014759334735572338, + 0.01864570751786232, + 0.007232825271785259, + 0.012405124492943287, + 0.005926845129579306 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.6427072882652283, + "style_target": 1, + "support_probability": 0.8294200301170349, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5376718395170184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2529691457748413, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7957718968391418, + 0.3843168318271637, + 0.07766612619161606, + 0.48052817583084106, + 0.01270682830363512 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.41521891951560974, + "style_target": 0, + "support_probability": 0.8683096170425415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002780087571627088, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7895169258117676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11282146722078323, + 0.15494059026241302, + 0.14427591860294342, + 0.14552044868469238, + 0.03651651740074158, + 0.0422002375125885, + 0.014112147502601147, + 0.039268653839826584, + 0.027884554117918015, + 0.021769551560282707, + 0.025759929791092873 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 0, + "style_probability": 0.3451344966888428, + "style_target": 1, + "support_probability": 0.005471004638820887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013410221325110955, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6476118564605713, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.07858677208423615, + 0.12791334092617035, + 0.20066913962364197, + 0.14847101271152496, + 0.03552248328924179, + 0.03753570839762688, + 0.016756074503064156, + 0.08334897458553314, + 0.051823705434799194, + 0.05770546942949295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 1, + "style_probability": 0.06630325317382812, + "style_target": 0, + "support_probability": 0.04587017744779587, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2021413797010178, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6486452221870422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26287856698036194, + 0.23590202629566193, + 0.16986005008220673, + 0.7540343403816223, + 0.02015608921647072 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.20240139961242676, + "style_target": 1, + "support_probability": 0.33594900369644165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.463431413812465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5869110226631165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23859119415283203, + 0.21240200102329254, + 0.15189912915229797, + 0.7600265741348267, + 0.10591689497232437 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.7250288724899292, + "style_target": 0, + "support_probability": 0.43452194333076477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16113129734562684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22287152707576752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6287392973899841, + 0.9270763993263245, + 0.32958894968032837, + 0.5234461426734924, + 0.01698964834213257, + 0.018085112795233727 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 0, + "style_probability": 0.21796439588069916, + "style_target": 1, + "support_probability": 0.16156864166259766, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13468238300388385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1184159517288208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5855427384376526, + 0.9697346091270447, + 0.3934483230113983, + 0.32546335458755493, + 0.01817225106060505, + 0.07780150324106216 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 1, + "style_probability": 0.17864933609962463, + "style_target": 0, + "support_probability": 0.5225700736045837, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22046263076424594, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46039101481437683, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32503145933151245, + 0.20211203396320343, + 0.019129334017634392, + 0.5611379742622375, + 0.013533237390220165, + 0.022430213168263435, + 0.00934519711881876, + 0.021998383104801178, + 0.03696979582309723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 0, + "style_probability": 0.19517241418361664, + "style_target": 1, + "support_probability": 0.6258309483528137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2604596300101022, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.564457356929779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38104456663131714, + 0.21527758240699768, + 0.04883947968482971, + 0.5072203874588013, + 0.014170687645673752, + 0.013917454518377781, + 0.027411794289946556, + 0.021474357694387436, + 0.020562907680869102 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 1, + "style_probability": 0.4560958445072174, + "style_target": 0, + "support_probability": 0.6003835201263428, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024640206066237534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9150126576423645, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4711492955684662, + 0.8660439252853394, + 0.16240237653255463, + 0.15685772895812988, + 0.12144305557012558, + 0.05435382202267647, + 0.0513734370470047, + 0.0633622258901596, + 0.039729297161102295, + 0.10135103017091751, + 0.09106051176786423 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.24910137057304382, + "style_target": 1, + "support_probability": 0.015417804941534996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023598571672758365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.943024218082428, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38576555252075195, + 0.8340484499931335, + 0.14742307364940643, + 0.12195499241352081, + 0.08621642738580704, + 0.04885410889983177, + 0.06393325328826904, + 0.29852721095085144, + 0.10689627379179001, + 0.06360593438148499, + 0.15519721806049347 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.7825034260749817, + "style_target": 0, + "support_probability": 0.018495969474315643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4827241182407416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6378216743469238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7851887345314026, + 0.7278512716293335, + 0.2602410316467285, + 0.35969528555870056, + 0.016071710735559464 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.8540679216384888, + "style_target": 1, + "support_probability": 0.62896329164505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012864628749030617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8480110168457031, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7241557836532593, + 0.7532358169555664, + 0.3020000755786896, + 0.2692495584487915, + 0.08015981316566467 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.12021845579147339, + "style_target": 0, + "support_probability": 0.2588185966014862, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7216621511210753, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10053539276123047, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5733624696731567, + 0.3031670153141022, + 0.9973318576812744, + 0.6837682723999023, + 0.014548036269843578, + 0.027751004323363304 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 1, + "style_probability": 0.7678103446960449, + "style_target": 1, + "support_probability": 0.6534324288368225, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5285459274637112, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21050797402858734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7241531610488892, + 0.2785336375236511, + 0.9869281649589539, + 0.7677896022796631, + 0.021021144464612007, + 0.017289364710450172 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 0, + "style_probability": 0.6350439786911011, + "style_target": 0, + "support_probability": 0.6123030185699463, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46832413206612766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5620481371879578, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30132102966308594, + 0.16643254458904266, + 0.30132102966308594, + 0.9501178860664368, + 0.05599576234817505, + 0.04889887943863869, + 0.07424338161945343, + 0.030598705634474754, + 0.03630012273788452 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.9159600138664246, + "style_target": 1, + "support_probability": 0.16142648458480835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16768444837414223, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.45516237616539, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3022391200065613, + 0.1817030906677246, + 0.3022391200065613, + 0.9310203790664673, + 0.11497725546360016 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.28979888558387756, + "style_target": 0, + "support_probability": 0.21094369888305664, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2258080851654503, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6824101209640503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5531752109527588, + 0.18889333307743073, + 0.6075403094291687, + 0.5262227058410645, + 0.01588333584368229, + 0.03332269936800003, + 0.014705395326018333, + 0.05212454870343208, + 0.02984360232949257, + 0.14202244579792023, + 0.03679624944925308 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.7781437039375305, + "style_target": 1, + "support_probability": 0.1508030742406845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009305902950246903, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5751802921295166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4629353880882263, + 0.10050202161073685, + 0.367485374212265, + 0.6050410270690918, + 0.051415231078863144, + 0.08049645274877548, + 0.062132541090250015, + 0.0315215140581131, + 0.03626616671681404, + 0.09167911857366562 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.05635538324713707, + "style_target": 0, + "support_probability": 0.3426862359046936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00481558999294465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8958356976509094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7163420915603638, + 0.7651488780975342, + 0.14588521420955658, + 0.7163420915603638, + 0.07233375310897827 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 1, + "style_probability": 0.6574392318725586, + "style_target": 1, + "support_probability": 0.0009124806965701282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012803689195546676, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7333780527114868, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7490841150283813, + 0.8224936127662659, + 0.16942059993743896, + 0.7490841150283813, + 0.03752782940864563 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 0, + "style_probability": 0.5098173022270203, + "style_target": 0, + "support_probability": 0.004020587541162968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06379154166447297, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.893881618976593, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.045684486627578735, + 0.7508520483970642, + 0.22189515829086304, + 0.014855126850306988, + 0.0990624725818634, + 0.04679499939084053 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.8391222953796387, + "style_target": 1, + "support_probability": 0.007806410081684589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008474640657081017, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8734058141708374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06404763460159302, + 0.6564003825187683, + 0.21764911711215973, + 0.021761033684015274, + 0.03216511383652687, + 0.04142342135310173, + 0.19070406258106232, + 0.07612573355436325, + 0.04333558306097984, + 0.09656066447496414, + 0.10054460167884827 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.2567082941532135, + "style_target": 0, + "support_probability": 0.0066171386279165745, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5183180440982923, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09750136733055115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.583061158657074, + 0.13836337625980377, + 0.9314979910850525, + 0.3721131682395935, + 0.06708253920078278, + 0.09599879384040833, + 0.24977096915245056, + 0.03192419931292534, + 0.08379120379686356 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.9188549518585205, + "style_target": 1, + "support_probability": 0.4089125990867615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11035997067533779, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031518034636974335, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.519835352897644, + 0.11816512048244476, + 0.9955483675003052, + 0.33823272585868835, + 0.08670905232429504, + 0.12030403316020966, + 0.06763166934251785, + 0.0438714437186718, + 0.06511154770851135 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.08526164293289185, + "style_target": 0, + "support_probability": 0.7419629693031311, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.659631348771643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22636087238788605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5072497725486755, + 0.14989112317562103, + 0.5072497725486755, + 0.9856797456741333, + 0.050146013498306274, + 0.05207718163728714, + 0.035583242774009705, + 0.02288961596786976, + 0.09493712335824966, + 0.03809763491153717, + 0.06430993974208832 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 1, + "style_probability": 0.8068411350250244, + "style_target": 1, + "support_probability": 0.6681032180786133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25418512020167366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4968982934951782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41750216484069824, + 0.22568073868751526, + 0.41750216484069824, + 0.970557689666748, + 0.06205180659890175, + 0.059420906007289886, + 0.08805662393569946, + 0.04207945987582207, + 0.04298427700996399, + 0.05358460918068886, + 0.06845326721668243 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 0, + "style_probability": 0.5976104736328125, + "style_target": 0, + "support_probability": 0.29341158270835876, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4400005407337285, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36090701818466187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7377105355262756, + 0.5554395318031311, + 0.6567977666854858, + 0.20624180138111115, + 0.029543336480855942 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 1, + "style_probability": 0.8610108494758606, + "style_target": 1, + "support_probability": 0.43796759843826294, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14448303003699756, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5937677025794983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6748536825180054, + 0.5078286528587341, + 0.7319932579994202, + 0.2152775377035141, + 0.0328826904296875, + 0.03326881676912308, + 0.040582604706287384, + 0.039562687277793884, + 0.12872307002544403 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 0, + "style_probability": 0.5009074211120605, + "style_target": 0, + "support_probability": 0.21397651731967926, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8211106120127714, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.028172023594379425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1909906268119812, + 0.5941859483718872, + 0.986172616481781, + 0.46957889199256897, + 0.015034661628305912, + 0.019962213933467865 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.691237211227417, + "style_target": 1, + "support_probability": 0.8008809685707092, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23790878250395894, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07422146201133728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1835743933916092, + 0.5414599776268005, + 0.9858099818229675, + 0.3493732511997223, + 0.011703395284712315, + 0.032584402710199356 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.16287069022655487, + "style_target": 0, + "support_probability": 0.6490778923034668, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.817290341687996, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.047658227384090424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3209778666496277, + 0.6248548626899719, + 0.9168605804443359, + 0.5150708556175232, + 0.010866095311939716, + 0.014736739918589592, + 0.028515122830867767, + 0.015372282825410366, + 0.007201141212135553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 1, + "style_probability": 0.7198237776756287, + "style_target": 1, + "support_probability": 0.9748556017875671, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6934663241019361, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06500986218452454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3538605868816376, + 0.6220146417617798, + 0.8785247802734375, + 0.5436794757843018, + 0.013399749994277954, + 0.006456343457102776, + 0.026787299662828445, + 0.01920200139284134, + 0.026353854686021805 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 1, + "style_probability": 0.419492244720459, + "style_target": 0, + "support_probability": 0.9685683846473694, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03784152552735859, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9217148423194885, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.715835690498352, + 0.46754685044288635, + 0.2396775782108307, + 0.7022161483764648, + 0.013915681280195713, + 0.03969327732920647, + 0.0138718131929636, + 0.040364209562540054, + 0.07483609020709991, + 0.03316822275519371, + 0.017888998612761497 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.7750217914581299, + "style_target": 1, + "support_probability": 0.030006838962435722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010472876202847075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9134028553962708, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6843915581703186, + 0.5375204682350159, + 0.25077593326568604, + 0.6903886198997498, + 0.056792084127664566, + 0.07569971680641174 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.24646633863449097, + "style_target": 0, + "support_probability": 0.020731057971715927, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.345732399341485, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14481644332408905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9935218691825867, + 0.11240522563457489, + 0.19631363451480865, + 0.17222543060779572, + 0.09975933283567429 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.6156722903251648, + "style_target": 1, + "support_probability": 0.14724718034267426, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05659353171200851, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28721609711647034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9915325045585632, + 0.15687137842178345, + 0.175912007689476, + 0.16039390861988068, + 0.08311620354652405 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.09670261293649673, + "style_target": 0, + "support_probability": 0.14605221152305603, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.64670251197164, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14845050871372223, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8382681608200073, + 0.5496030449867249, + 0.99601149559021, + 0.8382681608200073, + 0.023480335250496864, + 0.01854083314538002 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 1, + "style_probability": 0.7062029242515564, + "style_target": 1, + "support_probability": 0.5707124471664429, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8414657239540437, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.053249508142471313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8528033494949341, + 0.602353036403656, + 0.9923524260520935, + 0.8528033494949341, + 0.03234294429421425, + 0.03050152398645878 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 0, + "style_probability": 0.6800962090492249, + "style_target": 0, + "support_probability": 0.9207546710968018, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018257159984143616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9788219928741455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31346395611763, + 0.5328596830368042, + 0.9532265663146973, + 0.33882763981819153, + 0.10984483361244202, + 0.08116071671247482, + 0.07009117305278778, + 0.06096382066607475, + 0.07086938619613647 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.6574680209159851, + "style_target": 1, + "support_probability": 0.012369055300951004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017299491011524042, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9691832065582275, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2742893695831299, + 0.5420993566513062, + 0.9445151090621948, + 0.326709121465683, + 0.1492723971605301 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.2581677734851837, + "style_target": 0, + "support_probability": 0.010929345153272152, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7585929356387616, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044432688504457474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9846204519271851, + 0.39686283469200134, + 0.37199297547340393, + 0.6676764488220215, + 0.038967784494161606, + 0.028221620246767998, + 0.039829179644584656, + 0.03428442403674126, + 0.0805562287569046, + 0.03787870332598686, + 0.052690763026475906 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.7551337480545044, + "style_target": 1, + "support_probability": 0.8313273787498474, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018509476863916356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08605439215898514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9857136011123657, + 0.4074085056781769, + 0.4353026747703552, + 0.5437906384468079, + 0.10000134259462357, + 0.05547002702951431, + 0.06702922284603119, + 0.07953663915395737, + 0.04510808363556862, + 0.13133279979228973 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.037915971130132675, + "style_target": 0, + "support_probability": 0.5791966319084167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7083538871957543, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.050956059247255325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3235606849193573, + 0.26502686738967896, + 0.8023303747177124, + 0.3653516173362732, + 0.02495879866182804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 1, + "style_probability": 0.8647385239601135, + "style_target": 1, + "support_probability": 0.9469571113586426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5665916256454586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14170172810554504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2433246672153473, + 0.18063753843307495, + 0.7697525024414062, + 0.24312184751033783, + 0.07945458590984344 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 0, + "style_probability": 0.5939960479736328, + "style_target": 0, + "support_probability": 0.8565927743911743, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5386524562505649, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4146476984024048, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5881794691085815, + 0.6526966691017151, + 0.4520002603530884, + 0.22247157990932465, + 0.02216622233390808, + 0.046001434326171875 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.9373044967651367, + "style_target": 1, + "support_probability": 0.4990004003047943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12040007232537558, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23342767357826233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.564214825630188, + 0.7296295166015625, + 0.33525460958480835, + 0.2561289966106415, + 0.03983441740274429, + 0.27916625142097473, + 0.06874675303697586, + 0.024097023531794548, + 0.019351808354258537, + 0.0670531690120697, + 0.19276702404022217 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.18699491024017334, + "style_target": 0, + "support_probability": 0.27437692880630493, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6217681776946318, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045709576457738876, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9825918674468994, + 0.27982887625694275, + 0.6031837463378906, + 0.4177427887916565, + 0.040161628276109695, + 0.020055502653121948, + 0.015598086640238762, + 0.013663304969668388, + 0.034491751343011856 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.6835368275642395, + "style_target": 1, + "support_probability": 0.8286508917808533, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041701013502880906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04869838058948517, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9840925931930542, + 0.2942338287830353, + 0.5106382369995117, + 0.4790876805782318, + 0.028975162655115128, + 0.0118964659050107, + 0.03253239765763283, + 0.024282904341816902, + 0.05992210656404495 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.0955733135342598, + "style_target": 0, + "support_probability": 0.7241855263710022, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8904290602786376, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04001113399863243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30054962635040283, + 0.9924725294113159, + 0.32610735297203064, + 0.3797839283943176, + 0.02360503189265728, + 0.02460481971502304, + 0.02142876572906971, + 0.01321393996477127, + 0.010988917201757431, + 0.018889807164669037, + 0.023819833993911743 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 1, + "style_probability": 0.8510578870773315, + "style_target": 1, + "support_probability": 0.8914931416511536, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.83879125477437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05856921523809433, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3164777159690857, + 0.9860284924507141, + 0.3042334020137787, + 0.32472550868988037, + 0.019425660371780396, + 0.08021330833435059, + 0.07690095156431198, + 0.03124496340751648, + 0.040365491062402725, + 0.1183653324842453 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 0, + "style_probability": 0.6693595051765442, + "style_target": 0, + "support_probability": 0.9346023201942444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36002345636338007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2756912112236023, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8509862422943115, + 0.18481101095676422, + 0.05864155665040016, + 0.7214235067367554, + 0.01399741880595684 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 0, + "style_probability": 0.4901580512523651, + "style_target": 1, + "support_probability": 0.5056709051132202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1155408126006233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7648899555206299, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7976049184799194, + 0.2253633439540863, + 0.057631831616163254, + 0.6587541103363037, + 0.01956293173134327 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 0, + "style_probability": 0.6965210437774658, + "style_target": 0, + "support_probability": 0.07597040385007858, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4037579009370802, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.015856832265853882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13277409970760345, + 0.13570483028888702, + 0.06267813593149185, + 0.964431643486023, + 0.01804182305932045, + 0.023750949651002884 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 0, + "style_probability": 0.3073291778564453, + "style_target": 1, + "support_probability": 0.8360944390296936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45153042931697535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006969538517296314, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11412733048200607, + 0.08903028070926666, + 0.06365089118480682, + 0.9774723649024963, + 0.01356814056634903, + 0.02727436274290085, + 0.015723541378974915, + 0.03297242894768715, + 0.03891918808221817, + 0.06144207715988159, + 0.050320811569690704 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 1, + "style_probability": 0.21834982931613922, + "style_target": 0, + "support_probability": 0.9118608236312866, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8565128702429045, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02128712274134159, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12128283828496933, + 0.5631169080734253, + 0.9956396818161011, + 0.06797236204147339, + 0.013533058576285839, + 0.010490343905985355, + 0.0101915979757905, + 0.02225174941122532, + 0.013694037683308125 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.7599667906761169, + "style_target": 1, + "support_probability": 0.9878851771354675, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5801754995351218, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006669569294899702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07629034668207169, + 0.4919026792049408, + 0.9994127750396729, + 0.0407949723303318, + 0.024146603420376778, + 0.01683300919830799, + 0.030365068465471268, + 0.010246793739497662, + 0.0256274975836277 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.162112757563591, + "style_target": 0, + "support_probability": 0.9955314993858337, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0455601517060252, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6222015023231506, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8377979397773743, + 0.1096944808959961, + 0.49870505928993225, + 0.7123282551765442, + 0.0472574420273304, + 0.06004628539085388, + 0.026337049901485443, + 0.04909501224756241, + 0.02992958016693592, + 0.14197374880313873, + 0.06858673691749573 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 1, + "style_probability": 0.5006512403488159, + "style_target": 1, + "support_probability": 0.0231497660279274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10204987055929712, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40082600712776184, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8510317206382751, + 0.08595924079418182, + 0.5167421698570251, + 0.7037568688392639, + 0.03131268545985222, + 0.014171892777085304, + 0.023323750123381615, + 0.014954092912375927, + 0.022880597040057182, + 0.019176429137587547, + 0.036658208817243576 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 1, + "style_probability": 0.4469482898712158, + "style_target": 0, + "support_probability": 0.0844893828034401, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.060606357435176325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15832434594631195, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4791330099105835, + 0.16032426059246063, + 0.4791330099105835, + 0.8616997003555298, + 0.03966302424669266 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 0, + "style_probability": 0.46480920910835266, + "style_target": 1, + "support_probability": 0.07647890597581863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04410564261830386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4077339470386505, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4756644368171692, + 0.23684686422348022, + 0.4756644368171692, + 0.8536275029182434, + 0.046008143573999405, + 0.020915741100907326, + 0.033715877681970596, + 0.015677668154239655, + 0.019450925290584564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 1, + "style_probability": 0.41891178488731384, + "style_target": 0, + "support_probability": 0.02370372973382473, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.13843525168868184, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5794834494590759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2789871394634247, + 0.16509364545345306, + 0.4191811680793762, + 0.8677940964698792, + 0.021743737161159515, + 0.021711092442274094 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.5806102156639099, + "style_target": 1, + "support_probability": 0.2638644576072693, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04166094303318285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.464443564414978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17009902000427246, + 0.07492347806692123, + 0.20036213099956512, + 0.954319179058075, + 0.012804747559130192, + 0.034829795360565186 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.18355782330036163, + "style_target": 0, + "support_probability": 0.5467661619186401, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0522065384782912, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6119757294654846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25718140602111816, + 0.48922306299209595, + 0.12060640752315521, + 0.08740031719207764, + 0.023298894986510277, + 0.10171236842870712, + 0.07919375598430634, + 0.037474241107702255, + 0.0632752776145935 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.3653123676776886, + "style_target": 1, + "support_probability": 0.09964273869991302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23095745871939738, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37761977314949036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2544366121292114, + 0.5492194890975952, + 0.1015864834189415, + 0.11207254976034164, + 0.016529308632016182, + 0.005480514373630285, + 0.008154918439686298, + 0.015412021428346634, + 0.008552304469048977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.8075924515724182, + "style_target": 0, + "support_probability": 0.33894163370132446, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6613529221795031, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07296633720397949, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.39092469215393066, + 0.23040302097797394, + 0.9300847053527832, + 0.17737451195716858, + 0.009809669107198715, + 0.022964410483837128, + 0.06971702724695206, + 0.06011054292321205, + 0.04563426598906517, + 0.0231376551091671 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 1, + "style_probability": 0.5079509615898132, + "style_target": 1, + "support_probability": 0.9111784100532532, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6694831027879552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.024315569549798965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3681009113788605, + 0.2917756140232086, + 0.9609735608100891, + 0.1780560314655304, + 0.05931820720434189, + 0.02578754723072052 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 1, + "style_probability": 0.362112432718277, + "style_target": 0, + "support_probability": 0.954903781414032, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21822778682454685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.017972957342863083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4901256263256073, + 0.8990294933319092, + 0.4961027204990387, + 0.0347600057721138, + 0.043619561940431595 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 0, + "style_probability": 0.20724456012248993, + "style_target": 1, + "support_probability": 0.4883100390434265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25872296856176236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.024688053876161575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5643645524978638, + 0.8815230131149292, + 0.5023332238197327, + 0.036325603723526, + 0.03128957748413086 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 1, + "style_probability": 0.30332425236701965, + "style_target": 0, + "support_probability": 0.5281369686126709, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5476309337822406, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0587681420147419, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.05511835962533951, + 0.36540383100509644, + 0.9122960567474365, + 0.22977791726589203, + 0.08768922835588455, + 0.04831864684820175 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.3443925678730011, + "style_target": 1, + "support_probability": 0.8717714548110962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7857370484365923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05617837980389595, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.04977136105298996, + 0.38341978192329407, + 0.9062487483024597, + 0.2775067687034607, + 0.048765283077955246, + 0.024888264015316963 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.9025753736495972, + "style_target": 0, + "support_probability": 0.8553503155708313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30019818164468315, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4896719753742218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09119047969579697, + 0.18702122569084167, + 0.029299981892108917, + 0.843406617641449, + 0.0800035297870636, + 0.0863538533449173, + 0.05744442343711853, + 0.05422789603471756, + 0.0519094355404377 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.5456352233886719, + "style_target": 1, + "support_probability": 0.15950854122638702, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2215271469037819, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2613813877105713, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.061115242540836334, + 0.15999282896518707, + 0.03820957615971565, + 0.8346883654594421, + 0.07521315664052963, + 0.08347715437412262, + 0.05532478168606758, + 0.028069064021110535, + 0.0588550940155983 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.2330111712217331, + "style_target": 0, + "support_probability": 0.4735683500766754, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6668675933090136, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05814865604043007, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12165526300668716, + 0.4861879348754883, + 0.1050003245472908, + 0.9693218469619751, + 0.03233008459210396, + 0.008579505607485771, + 0.058205291628837585, + 0.02339763008058071, + 0.029299791902303696, + 0.04178228974342346, + 0.030398646369576454 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.3735520839691162, + "style_target": 1, + "support_probability": 0.955690324306488, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7841914053167688, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08388987183570862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1551484316587448, + 0.4023156464099884, + 0.10661572217941284, + 0.9437500238418579, + 0.011941307224333286, + 0.16097673773765564, + 0.07494240254163742, + 0.02432722970843315, + 0.0099917221814394, + 0.010272275656461716, + 0.030333423987030983 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.6880682706832886, + "style_target": 0, + "support_probability": 0.9331380724906921, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6890766983592331, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10361568629741669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3540908694267273, + 0.6220580339431763, + 0.9653400182723999, + 0.1378011554479599, + 0.05255027487874031 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.7024988532066345, + "style_target": 1, + "support_probability": 0.693467378616333, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5442617424733175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04307974874973297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4413678050041199, + 0.6318426728248596, + 0.9669222831726074, + 0.11409024149179459, + 0.034852173179388046, + 0.05213785171508789, + 0.05519909784197807, + 0.02894235961139202, + 0.08277682960033417 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.34413942694664, + "style_target": 0, + "support_probability": 0.8454267978668213, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9364125548352636, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.01023179292678833, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:7259:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7870146632194519, + 0.4585570991039276, + 0.14353181421756744, + 0.9962677359580994, + 0.015695132315158844, + 0.05160282552242279 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.807195246219635, + "style_target": 1, + "support_probability": 0.9882877469062805, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06916786045149136, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03821507841348648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7259:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7237163782119751, + 0.4323984682559967, + 0.13747334480285645, + 0.9945958256721497, + 0.03541143983602524, + 0.08267223834991455 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.027829622849822044, + "style_target": 0, + "support_probability": 0.9400454163551331, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8154878772678295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028617575764656067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.600170910358429, + 0.9553502798080444, + 0.34431540966033936, + 0.29956406354904175, + 0.08517765253782272, + 0.27547571063041687, + 0.030407242476940155, + 0.03465937823057175, + 0.032866060733795166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 1, + "style_probability": 0.8838691115379333, + "style_target": 1, + "support_probability": 0.8264185190200806, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6432698484676369, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.048528093844652176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5704668164253235, + 0.9257977604866028, + 0.30206331610679626, + 0.30894380807876587, + 0.05594725161790848, + 0.07653098553419113, + 0.027564069256186485, + 0.014370984397828579, + 0.03335270285606384 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 0, + "style_probability": 0.6967301964759827, + "style_target": 0, + "support_probability": 0.674626886844635, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.548745309303907, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05895638093352318, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5863020420074463, + 0.5358807444572449, + 0.5912429094314575, + 0.43407002091407776, + 0.016211850568652153, + 0.013274486176669598, + 0.034009065479040146, + 0.14672672748565674, + 0.012049424462020397, + 0.01743902824819088, + 0.03283635526895523 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.8743447661399841, + "style_target": 1, + "support_probability": 0.9152493476867676, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.241176120515337, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14128756523132324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4775291681289673, + 0.4969278872013092, + 0.556419312953949, + 0.36795544624328613, + 0.03164283186197281, + 0.05229693278670311 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.22098670899868011, + "style_target": 0, + "support_probability": 0.7295697927474976, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018885215792414797, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7612906694412231, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18180228769779205, + 0.41267356276512146, + 0.3186063766479492, + 0.25099584460258484, + 0.054092977195978165 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 0, + "style_probability": 0.3742501735687256, + "style_target": 1, + "support_probability": 0.041243601590394974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004910298812352772, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9140819907188416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1428968459367752, + 0.2639925181865692, + 0.2765015661716461, + 0.24253425002098083, + 0.13628697395324707, + 0.06938937306404114, + 0.041631266474723816, + 0.049612872302532196, + 0.12018140405416489 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 1, + "style_probability": 0.14130550622940063, + "style_target": 0, + "support_probability": 0.009578637778759003, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.542777787525762, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03787793219089508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5071709156036377, + 0.9771000146865845, + 0.23447160422801971, + 0.37634748220443726, + 0.15355773270130157, + 0.02355685457587242 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 0, + "style_probability": 0.39038097858428955, + "style_target": 1, + "support_probability": 0.8413007259368896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011510987010947321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2716394364833832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3471965193748474, + 0.9787213802337646, + 0.18873324990272522, + 0.3289461135864258, + 0.04885760322213173, + 0.06217184290289879 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 1, + "style_probability": 0.032045476138591766, + "style_target": 0, + "support_probability": 0.3314281702041626, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8864384683777343, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01463390700519085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9943140745162964, + 0.17567501962184906, + 0.3693624436855316, + 0.25203248858451843, + 0.07668979465961456, + 0.032538577914237976, + 0.03436333313584328, + 0.12392600625753403, + 0.0423809178173542 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 1, + "style_probability": 0.7427197694778442, + "style_target": 1, + "support_probability": 0.9733127951622009, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9582182169958681, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02126323990523815, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:7276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9987934827804565, + 0.17115411162376404, + 0.3237231969833374, + 0.25581255555152893, + 0.039333753287792206, + 0.01028275303542614, + 0.017585499212145805, + 0.027214255183935165, + 0.025669924914836884 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 0, + "style_probability": 0.8777583241462708, + "style_target": 0, + "support_probability": 0.9730960726737976, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2925026284208009, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0333738774061203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21732363104820251, + 0.9640374779701233, + 0.1471697837114334, + 0.5946387648582458, + 0.038093581795692444, + 0.018615692853927612, + 0.08427190780639648, + 0.02651837281882763, + 0.03688586875796318, + 0.08307768404483795, + 0.025703610852360725 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 1, + "style_probability": 0.5082418918609619, + "style_target": 1, + "support_probability": 0.1690850555896759, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1411859603971747, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.050408292561769485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25856271386146545, + 0.9744786024093628, + 0.11420432478189468, + 0.5772782564163208, + 0.03366212919354439, + 0.09839370101690292 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 1, + "style_probability": 0.28241443634033203, + "style_target": 0, + "support_probability": 0.11804354935884476, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13918796537763856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2700681984424591, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29218533635139465, + 0.9838236570358276, + 0.13648708164691925, + 0.29218533635139465, + 0.025554589927196503 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 1, + "style_probability": 0.5226203203201294, + "style_target": 1, + "support_probability": 0.04992721229791641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09275478406295394, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11353360861539841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24269846081733704, + 0.9960792660713196, + 0.09290531277656555, + 0.24269846081733704, + 0.05521303787827492 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 1, + "style_probability": 0.15349116921424866, + "style_target": 0, + "support_probability": 0.1972278207540512, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22831426523941412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15651634335517883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7842094302177429, + 0.25811490416526794, + 0.5741763710975647, + 0.7842094302177429, + 0.04883183538913727, + 0.03065730631351471 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 1, + "style_probability": 0.6171198487281799, + "style_target": 1, + "support_probability": 0.2223319113254547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4043874496834277, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2688881754875183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8278856873512268, + 0.2752467691898346, + 0.6482184529304504, + 0.8278856873512268, + 0.06305336207151413, + 0.10658417642116547 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 0, + "style_probability": 0.9078267812728882, + "style_target": 0, + "support_probability": 0.10048811882734299, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8344291098071291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010307964868843555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37767642736434937, + 0.9774951338768005, + 0.43389588594436646, + 0.32295888662338257, + 0.08127191662788391, + 0.011720577254891396, + 0.12750747799873352, + 0.032106366008520126, + 0.01982106640934944 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 0, + "style_probability": 0.3784010112285614, + "style_target": 1, + "support_probability": 0.9637724757194519, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7788756426194168, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.008389187045395374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38625895977020264, + 0.9847570657730103, + 0.39819249510765076, + 0.358532577753067, + 0.011135220527648926 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 1, + "style_probability": 0.2511919140815735, + "style_target": 0, + "support_probability": 0.9636052846908569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5514725844478862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1045914962887764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9478392004966736, + 0.69283127784729, + 0.16381822526454926, + 0.14242681860923767, + 0.034440360963344574, + 0.0351933054625988, + 0.2686752378940582, + 0.0663689374923706, + 0.09342663735151291, + 0.10339392721652985, + 0.03127700090408325 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.5078169703483582, + "style_target": 1, + "support_probability": 0.5627692341804504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2577109415690671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13336266577243805, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9726393818855286, + 0.5915514230728149, + 0.10151124000549316, + 0.12520307302474976, + 0.02491537481546402, + 0.03520435839891434, + 0.021814530715346336, + 0.026299040764570236, + 0.03459158167243004, + 0.06084202229976654 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.14110273122787476, + "style_target": 0, + "support_probability": 0.7967284321784973, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6196638511218734, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1943243443965912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1587459146976471, + 0.21282808482646942, + 0.13542170822620392, + 0.998214840888977, + 0.07130026072263718 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.8209422826766968, + "style_target": 1, + "support_probability": 0.3377527892589569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2694358894057713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15846937894821167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15225501358509064, + 0.21721448004245758, + 0.14802810549736023, + 0.9963093400001526, + 0.10053844749927521, + 0.12266655266284943, + 0.06156972423195839, + 0.05887637659907341, + 0.05135997757315636 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.3760479986667633, + "style_target": 0, + "support_probability": 0.29016825556755066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14036212575243565, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10727319866418839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30876848101615906, + 0.192177876830101, + 0.41715705394744873, + 0.4675341546535492, + 0.007707469165325165, + 0.01927981898188591 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.9070982933044434, + "style_target": 1, + "support_probability": 0.041180163621902466, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014291740871300637, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16975834965705872, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.347604364156723, + 0.11153046786785126, + 0.3043520450592041, + 0.4880319833755493, + 0.01041082851588726, + 0.018406983464956284 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.17571870982646942, + "style_target": 0, + "support_probability": 0.14041896164417267, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8390724120523421, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008802967146039009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18229064345359802, + 0.9621237516403198, + 0.12215662747621536, + 0.12990157306194305, + 0.009534879587590694, + 0.009786500595510006, + 0.016840176656842232, + 0.014871582388877869, + 0.008679714985191822 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 1, + "style_probability": 0.6489814519882202, + "style_target": 1, + "support_probability": 0.9595831632614136, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7710836009579211, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01193656399846077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15908417105674744, + 0.9299033880233765, + 0.20149938762187958, + 0.1577177494764328, + 0.009650472551584244, + 0.012983291409909725, + 0.014239904470741749, + 0.0234468262642622, + 0.027976183220744133 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 1, + "style_probability": 0.4462455213069916, + "style_target": 0, + "support_probability": 0.9747057557106018, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.751387663935416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03535528853535652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.420479416847229, + 0.35638394951820374, + 0.6270992159843445, + 0.9939845204353333, + 0.021363897249102592, + 0.12475903332233429, + 0.1512560248374939, + 0.01552283763885498, + 0.04783453792333603, + 0.0343402624130249, + 0.04694363474845886 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.8766686320304871, + "style_target": 1, + "support_probability": 0.5717130303382874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5219670890617797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.025694631040096283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42082399129867554, + 0.31461209058761597, + 0.6016319990158081, + 0.9921730756759644, + 0.21551397442817688, + 0.05962388962507248 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.31128957867622375, + "style_target": 0, + "support_probability": 0.6966380476951599, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9414271087513894, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.004482822958379984, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:7316:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9987736344337463, + 0.6742112636566162, + 0.3622272312641144, + 0.4947773218154907, + 0.07400587201118469 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.7604638934135437, + "style_target": 1, + "support_probability": 0.9646732807159424, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.35605259730494127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.00449782470241189, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7316:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9992386102676392, + 0.549358606338501, + 0.2949799597263336, + 0.29115766286849976, + 0.0579199343919754 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.07634454220533371, + "style_target": 0, + "support_probability": 0.9539896249771118, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7851262918482632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22154808044433594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.48137912154197693, + 0.9423259496688843, + 0.2314363569021225, + 0.48137912154197693, + 0.013496491126716137, + 0.03863859921693802 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 1, + "style_probability": 0.8075920343399048, + "style_target": 1, + "support_probability": 0.6856090426445007, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5369674917107463, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3031788170337677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4853856861591339, + 0.908291220664978, + 0.24745512008666992, + 0.4853856861591339, + 0.01128179021179676, + 0.022310959175229073 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 1, + "style_probability": 0.4012663662433624, + "style_target": 0, + "support_probability": 0.6415548920631409, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2653534941716645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11265266686677933, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9910675883293152, + 0.0946800708770752, + 0.2774834632873535, + 0.22533559799194336, + 0.027012333273887634, + 0.018979504704475403, + 0.21217764914035797, + 0.0561494417488575, + 0.044822074472904205 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 1, + "style_probability": 0.8811086416244507, + "style_target": 1, + "support_probability": 0.07992439717054367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2253484547567815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07847627997398376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9898926615715027, + 0.09743433445692062, + 0.2708739936351776, + 0.21857696771621704, + 0.2408202737569809 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 0, + "style_probability": 0.5001736283302307, + "style_target": 0, + "support_probability": 0.22265686094760895, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5580301143991626, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0248566921800375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3064899444580078, + 0.1045176163315773, + 0.9970302581787109, + 0.3064899444580078, + 0.05373590812087059, + 0.01586798205971718, + 0.049074072390794754, + 0.01176436897367239, + 0.3076135516166687, + 0.09322688728570938, + 0.018361251801252365 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.6620543599128723, + "style_target": 1, + "support_probability": 0.24986781179904938, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10307808289687291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04270040988922119, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2581011950969696, + 0.1099570095539093, + 0.9976404905319214, + 0.2581011950969696, + 0.0322142094373703, + 0.015257861465215683, + 0.04052954912185669, + 0.05652778968214989, + 0.06725951284170151, + 0.07690078020095825 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.1281263530254364, + "style_target": 0, + "support_probability": 0.20122812688350677, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6994705580278104, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034912314265966415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9904932379722595, + 0.4424178898334503, + 0.13430015742778778, + 0.358967125415802, + 0.06246453896164894 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 0, + "style_probability": 0.3286875784397125, + "style_target": 1, + "support_probability": 0.8905733823776245, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6275336193561047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05545201525092125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9826589822769165, + 0.42407408356666565, + 0.15736614167690277, + 0.3766520023345947, + 0.052041977643966675 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 1, + "style_probability": 0.49349677562713623, + "style_target": 0, + "support_probability": 0.8234114050865173, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3100209782403118, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2858729064464569, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3301642835140228, + 0.9356793761253357, + 0.38624250888824463, + 0.1263679414987564, + 0.03496594354510307, + 0.11741826683282852 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.3039242625236511, + "style_target": 1, + "support_probability": 0.4234946668148041, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4155514622282516, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23056024312973022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2820514738559723, + 0.9605470299720764, + 0.44059425592422485, + 0.15369921922683716, + 0.3887679874897003, + 0.03510800749063492 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.7116715908050537, + "style_target": 0, + "support_probability": 0.4522729218006134, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2985417266526833, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18605107069015503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3293406665325165, + 0.9646313786506653, + 0.5100839734077454, + 0.5109763145446777, + 0.13350073993206024, + 0.1604905128479004, + 0.040727049112319946, + 0.039689332246780396, + 0.08814404159784317 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.5586902499198914, + "style_target": 1, + "support_probability": 0.5517187714576721, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36743079373873694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07654394209384918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37030619382858276, + 0.9837604761123657, + 0.5602293014526367, + 0.5046507716178894, + 0.07275853306055069, + 0.04400499910116196, + 0.057920828461647034, + 0.04057852551341057, + 0.08730798959732056 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.41066521406173706, + "style_target": 0, + "support_probability": 0.8797926306724548, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6648908973580435, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04685984179377556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16245488822460175, + 0.21946722269058228, + 0.9740470051765442, + 0.1377381682395935, + 0.0313928984105587, + 0.07522659748792648, + 0.04816493019461632, + 0.01775185763835907, + 0.019703295081853867, + 0.015409273095428944, + 0.025384332984685898 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 0, + "style_probability": 0.3897281289100647, + "style_target": 1, + "support_probability": 0.9533445239067078, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7670184736906431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024521969258785248, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13366663455963135, + 0.16953007876873016, + 0.962631106376648, + 0.09735498577356339, + 0.018942195922136307, + 0.014005133882164955, + 0.013785980641841888, + 0.01910327933728695, + 0.018255610018968582, + 0.05492701753973961, + 0.01426110602915287 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 0, + "style_probability": 0.5722934603691101, + "style_target": 0, + "support_probability": 0.9583459496498108, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7127430854838706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.27792614698410034, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1262548565864563, + 0.23014961183071136, + 0.218204528093338, + 0.9508115649223328, + 0.032225098460912704 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.8357649445533752, + "style_target": 1, + "support_probability": 0.661672055721283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4948320335480847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15331658720970154, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14846833050251007, + 0.2115451991558075, + 0.22169539332389832, + 0.9465695023536682, + 0.023212628439068794, + 0.03420105576515198, + 0.014710744842886925, + 0.018232814967632294, + 0.023902548477053642 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.405751496553421, + "style_target": 0, + "support_probability": 0.7628338932991028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0794658109688179, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4183051586151123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.624619722366333, + 0.2898332476615906, + 0.6083331108093262, + 0.8996430039405823, + 0.01178361102938652, + 0.03409610688686371 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.7942807078361511, + "style_target": 1, + "support_probability": 0.03883801028132439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009013402906845672, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5219714045524597, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5590277910232544, + 0.23399409651756287, + 0.5877882242202759, + 0.9394593238830566, + 0.017739340662956238, + 0.03669764846563339 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.1495816707611084, + "style_target": 0, + "support_probability": 0.063441202044487, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17442224121514652, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04945743829011917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8837969303131104, + 0.060488227754831314, + 0.22666588425636292, + 0.11313072592020035, + 0.029552962630987167, + 0.01597645878791809, + 0.02768201194703579, + 0.010998772457242012, + 0.00976244080811739 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 1, + "style_probability": 0.634447455406189, + "style_target": 1, + "support_probability": 0.2437945008277893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07651749683473841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2305203080177307, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8220429420471191, + 0.10162252932786942, + 0.2979157865047455, + 0.18590357899665833, + 0.02940937876701355, + 0.039617978036403656, + 0.0351361483335495, + 0.09443870931863785, + 0.04632527753710747 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 0, + "style_probability": 0.5100677609443665, + "style_target": 0, + "support_probability": 0.09969376027584076, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8571919188937669, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.046438492834568024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.394297331571579, + 0.4844551980495453, + 0.9931625127792358, + 0.6389577984809875, + 0.031097253784537315, + 0.024426622316241264, + 0.01951006054878235, + 0.05619027838110924, + 0.024232232943177223, + 0.017881911247968674, + 0.011828476563096046 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.8706167936325073, + "style_target": 1, + "support_probability": 0.7471866607666016, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3384943315087271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06044967472553253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3619701564311981, + 0.45094239711761475, + 0.9887397885322571, + 0.6954271793365479, + 0.01716458983719349, + 0.028398847207427025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.2839210331439972, + "style_target": 0, + "support_probability": 0.6695144176483154, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8398013600858605, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15069523453712463, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44562017917633057, + 0.391825795173645, + 0.27284467220306396, + 0.9437593817710876, + 0.03870036453008652 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.9278895854949951, + "style_target": 1, + "support_probability": 0.8604646325111389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04657219065428347, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6600697040557861, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40044063329696655, + 0.3196854293346405, + 0.17407216131687164, + 0.9432928562164307, + 0.044930171221494675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.07944341003894806, + "style_target": 0, + "support_probability": 0.37614548206329346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13862570781023542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6307761073112488, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11559852957725525, + 0.3308713138103485, + 0.2645948827266693, + 0.11559852957725525, + 0.10523538291454315, + 0.02958030439913273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 1, + "style_probability": 0.8852986097335815, + "style_target": 1, + "support_probability": 0.15739373862743378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02516700743605143, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8717891573905945, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09790068864822388, + 0.3601793348789215, + 0.27917230129241943, + 0.09790068864822388, + 0.045090679079294205, + 0.045776188373565674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 1, + "style_probability": 0.465879887342453, + "style_target": 0, + "support_probability": 0.05898686498403549, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008415699827047502, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8832826018333435, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34587690234184265, + 0.668170690536499, + 0.3320210874080658, + 0.45819002389907837, + 0.0721769854426384, + 0.05270062014460564, + 0.07938747853040695, + 0.27201047539711, + 0.1174113005399704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.6406629085540771, + "style_target": 1, + "support_probability": 0.0035059864167124033, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011809614422984858, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6828573942184448, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30371931195259094, + 0.6496303677558899, + 0.3126571476459503, + 0.39836978912353516, + 0.43454501032829285 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.20772038400173187, + "style_target": 0, + "support_probability": 0.01030680164694786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01728387925348392, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8638593554496765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3339900076389313, + 0.5891770124435425, + 0.6762385368347168, + 0.4220093786716461, + 0.0340665727853775, + 0.029826896265149117, + 0.04709412902593613, + 0.025863833725452423, + 0.04601701721549034, + 0.0918492004275322, + 0.09988581389188766 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 1, + "style_probability": 0.6833750009536743, + "style_target": 1, + "support_probability": 0.006535935215651989, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012302455898618178, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7766456007957458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32283976674079895, + 0.5358712077140808, + 0.6428147554397583, + 0.35104215145111084, + 0.06909459829330444, + 0.04080144315958023, + 0.12654945254325867, + 0.09289594739675522, + 0.05302237346768379, + 0.1446160078048706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 1, + "style_probability": 0.43227002024650574, + "style_target": 0, + "support_probability": 0.017312055453658104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8217050881800297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.021622413769364357, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06709668785333633, + 0.39757537841796875, + 0.6566738486289978, + 0.9875348806381226, + 0.08019144088029861 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.717722475528717, + "style_target": 1, + "support_probability": 0.9503095149993896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3792515878082989, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.04538589343428612, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.05686977133154869, + 0.23742905259132385, + 0.6498939990997314, + 0.9940252900123596, + 0.06898218393325806 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.23799897730350494, + "style_target": 0, + "support_probability": 0.897124171257019, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00753815427582928, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7790976166725159, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3440675735473633, + 0.5375751256942749, + 0.22179122269153595, + 0.41031673550605774, + 0.0870099663734436, + 0.08568910509347916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 1, + "style_probability": 0.5262860059738159, + "style_target": 1, + "support_probability": 0.0040665073320269585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006729789008064317, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8477702736854553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.238661989569664, + 0.4036189615726471, + 0.18406188488006592, + 0.34371793270111084, + 0.10093596577644348, + 0.06500044465065002 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 0, + "style_probability": 0.687332808971405, + "style_target": 0, + "support_probability": 0.00650253240019083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6578434472234952, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03058716654777527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17733651399612427, + 0.9730956554412842, + 0.057995252311229706, + 0.5187927484512329, + 0.025275791063904762, + 0.04001222178339958, + 0.017062654718756676, + 0.024491755291819572, + 0.04511582851409912 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 1, + "style_probability": 0.5870071649551392, + "style_target": 1, + "support_probability": 0.710345983505249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3056119299366543, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04329945892095566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17022021114826202, + 0.9704076051712036, + 0.04922908544540405, + 0.46600404381752014, + 0.06691034138202667 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 1, + "style_probability": 0.34746912121772766, + "style_target": 0, + "support_probability": 0.37876325845718384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3830667535095441, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1783466339111328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9930906295776367, + 0.3459908366203308, + 0.2656177282333374, + 0.21138088405132294, + 0.050641607493162155, + 0.062173355370759964, + 0.048567529767751694, + 0.020785938948392868, + 0.025791110470891, + 0.1021934375166893 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 0, + "style_probability": 0.40574154257774353, + "style_target": 1, + "support_probability": 0.3126080334186554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18015261775623614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.062439702451229095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.999411940574646, + 0.3563480079174042, + 0.26180967688560486, + 0.21041157841682434, + 0.02620006538927555, + 0.02689002826809883, + 0.0666976273059845, + 0.04716044291853905, + 0.04793161526322365, + 0.1274382770061493 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 1, + "style_probability": 0.17787642776966095, + "style_target": 0, + "support_probability": 0.5438426733016968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010880381143000521, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36806973814964294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16661444306373596, + 0.5895702838897705, + 0.22966933250427246, + 0.2606578767299652, + 0.033717915415763855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.7360733151435852, + "style_target": 1, + "support_probability": 0.007117690518498421, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004954339680421958, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6247344613075256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18137045204639435, + 0.5924863815307617, + 0.19483168423175812, + 0.301809161901474, + 0.024557318538427353, + 0.11877470463514328, + 0.09039190411567688, + 0.028080489486455917, + 0.0841389074921608 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.3524629771709442, + "style_target": 0, + "support_probability": 0.002904644701629877, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6294958326337131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10463973879814148, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14704890549182892, + 0.9918979406356812, + 0.23167042434215546, + 0.20354628562927246, + 0.02123783901333809, + 0.02341911569237709 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.5802485346794128, + "style_target": 1, + "support_probability": 0.7316110134124756, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22201047253197143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16287349164485931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15905886888504028, + 0.9890545606613159, + 0.23856201767921448, + 0.23891326785087585, + 0.008115019649267197, + 0.028586680069565773 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.2832375764846802, + "style_target": 0, + "support_probability": 0.6202278137207031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.055034831330934814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5643956661224365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07644771784543991, + 0.5349318981170654, + 0.7413367629051208, + 0.12565387785434723, + 0.017658891156315804, + 0.020990945398807526, + 0.02271898463368416, + 0.04512958973646164, + 0.015992959961295128 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 1, + "style_probability": 0.8475112915039062, + "style_target": 1, + "support_probability": 0.036665163934230804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08284234195463668, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5123929977416992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11840406805276871, + 0.5953323841094971, + 0.7826886177062988, + 0.18342451751232147, + 0.008709835819900036, + 0.06757868081331253, + 0.03266267478466034, + 0.03486897051334381, + 0.03801586851477623 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 0, + "style_probability": 0.6437007188796997, + "style_target": 0, + "support_probability": 0.08253931254148483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21522921128720207, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6740770936012268, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06607106328010559, + 0.06353501230478287, + 0.8180984258651733, + 0.7415286302566528, + 0.033197373151779175, + 0.01365895476192236, + 0.015311047434806824, + 0.017698366194963455, + 0.015858298167586327, + 0.02454487234354019, + 0.023090656846761703 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 1, + "style_probability": 0.8786951303482056, + "style_target": 1, + "support_probability": 0.09022988379001617, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4395859394171995, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24395249783992767, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07680367678403854, + 0.10782763361930847, + 0.8255640864372253, + 0.7224016785621643, + 0.029289521276950836, + 0.028321316465735435 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 0, + "style_probability": 0.5809235572814941, + "style_target": 0, + "support_probability": 0.5651192665100098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8368561892510947, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.017220821231603622, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27049019932746887, + 0.05252448841929436, + 0.9639456868171692, + 0.3402326703071594, + 0.06937608867883682 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.6896131634712219, + "style_target": 1, + "support_probability": 0.9199775457382202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37469520372161613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.017574207857251167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2916017174720764, + 0.04956412687897682, + 0.9657675623893738, + 0.26832863688468933, + 0.06288351118564606 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.21125555038452148, + "style_target": 0, + "support_probability": 0.8729217648506165, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20697739037031931, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7858812212944031, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7776833772659302, + 0.39071422815322876, + 0.3270779252052307, + 0.48184338212013245, + 0.07551068067550659, + 0.10976389050483704 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 1, + "style_probability": 0.9118033051490784, + "style_target": 1, + "support_probability": 0.18212786316871643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22605370507840208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.737368643283844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.768826425075531, + 0.3011198341846466, + 0.3306405246257782, + 0.35610368847846985, + 0.02858460694551468, + 0.023783819749951363 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 0, + "style_probability": 0.6248424053192139, + "style_target": 0, + "support_probability": 0.4564622640609741, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002410363187595116, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9246890544891357, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10705135762691498, + 0.42922595143318176, + 0.2392856478691101, + 0.10705135762691498, + 0.019004782661795616, + 0.05362314358353615, + 0.026957832276821136, + 0.0997890904545784, + 0.08059026300907135 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.5414975881576538, + "style_target": 1, + "support_probability": 0.006628689356148243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026083873465212928, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.933016836643219, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1415300816297531, + 0.47314193844795227, + 0.33643487095832825, + 0.1415300816297531, + 0.03888560086488724 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.19516006112098694, + "style_target": 0, + "support_probability": 0.0038355521392077208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15894969675742435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8525824546813965, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16460256278514862, + 0.12705306708812714, + 0.16460256278514862, + 0.6784493327140808, + 0.07764903455972672, + 0.028307875618338585, + 0.03611699491739273, + 0.06495846062898636, + 0.027512921020388603, + 0.14915557205677032, + 0.04364309087395668 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.8468913435935974, + "style_target": 1, + "support_probability": 0.03364117816090584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0362468411394596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5666846036911011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18249937891960144, + 0.0986839160323143, + 0.18249937891960144, + 0.7418444752693176, + 0.06972513347864151, + 0.02342378906905651, + 0.10718607902526855, + 0.021603556349873543, + 0.017088785767555237, + 0.028485126793384552 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.09686657041311264, + "style_target": 0, + "support_probability": 0.14827004075050354, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30008946798376684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08876633644104004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22892215847969055, + 0.2691236138343811, + 0.22892215847969055, + 0.9448361992835999, + 0.05903269350528717 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 1, + "style_probability": 0.755903959274292, + "style_target": 1, + "support_probability": 0.20280371606349945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2287084229119225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12272225320339203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2858788073062897, + 0.3181043267250061, + 0.2858788073062897, + 0.9382433891296387, + 0.06355945765972137 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 0, + "style_probability": 0.5366138219833374, + "style_target": 0, + "support_probability": 0.34479713439941406, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7731977000958185, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05884988233447075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9937649965286255, + 0.1619577556848526, + 0.25862568616867065, + 0.17943482100963593, + 0.011674354784190655, + 0.06511291861534119 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 1, + "style_probability": 0.8421224355697632, + "style_target": 1, + "support_probability": 0.8637173771858215, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48092020206234937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03766397386789322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9916701316833496, + 0.14850987493991852, + 0.32032129168510437, + 0.16012465953826904, + 0.019636213779449463, + 0.023155488073825836, + 0.006551866419613361, + 0.1800709217786789, + 0.026363302022218704, + 0.012236329726874828, + 0.02209106832742691 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 0, + "style_probability": 0.58409184217453, + "style_target": 0, + "support_probability": 0.8445037603378296, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08372086089876178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16163718700408936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9183357954025269, + 0.16528770327568054, + 0.19304364919662476, + 0.5316081047058105, + 0.020812466740608215, + 0.017295774072408676, + 0.01252114400267601, + 0.047948699444532394, + 0.059211649000644684 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.8305196166038513, + "style_target": 1, + "support_probability": 0.08462110906839371, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009739769029007937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1790170669555664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.921860933303833, + 0.16376830637454987, + 0.20692698657512665, + 0.5111410021781921, + 0.04770669713616371, + 0.10104606300592422, + 0.06715121120214462, + 0.01725885644555092, + 0.04091548174619675 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.23678803443908691, + "style_target": 0, + "support_probability": 0.07040715962648392, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6341883497241766, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09389539062976837, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3351721465587616, + 0.09522563219070435, + 0.9908724427223206, + 0.1896687150001526, + 0.016605639830231667, + 0.171462744474411, + 0.010016454383730888, + 0.014054584316909313, + 0.00717563834041357, + 0.021834595128893852, + 0.026044338941574097 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 1, + "style_probability": 0.8278815150260925, + "style_target": 1, + "support_probability": 0.8142284154891968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39749927361081067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1872113049030304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28044843673706055, + 0.10507678985595703, + 0.9875420928001404, + 0.15776745975017548, + 0.05509180575609207, + 0.06257406622171402, + 0.020805612206459045, + 0.10787703841924667, + 0.02228538691997528, + 0.03420951962471008, + 0.04554601386189461 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 0, + "style_probability": 0.5223646759986877, + "style_target": 0, + "support_probability": 0.7561037540435791, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07680484583776427, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5297648906707764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3900300860404968, + 0.4288966655731201, + 0.6979541778564453, + 0.4500564634799957, + 0.031640227884054184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.7587069869041443, + "style_target": 1, + "support_probability": 0.04719877988100052, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01225556435078873, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8370651006698608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36333519220352173, + 0.4007294774055481, + 0.633651614189148, + 0.4484669268131256, + 0.07245617359876633, + 0.033244237303733826, + 0.024101950228214264, + 0.011203542351722717, + 0.09445390105247498 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.3479189872741699, + "style_target": 0, + "support_probability": 0.013421478681266308, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011886092805492726, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9970082640647888, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6409655213356018, + 0.170669823884964, + 0.2232663482427597, + 0.25245144963264465, + 0.09323317557573318, + 0.044439591467380524 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.7475828528404236, + "style_target": 1, + "support_probability": 0.0012764399871230125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003876736493226494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9988701939582825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5679919123649597, + 0.1446533054113388, + 0.21416465938091278, + 0.254240483045578, + 0.07404433190822601, + 0.0931188091635704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.2285117506980896, + "style_target": 0, + "support_probability": 0.0011159569257870317, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9315085920875216, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008887268602848053, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:7553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2736291289329529, + 0.9864369034767151, + 0.12869693338871002, + 0.2736291289329529, + 0.017202261835336685, + 0.038579534739255905, + 0.014316209591925144, + 0.03038683533668518, + 0.06253333389759064 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 1, + "style_probability": 0.9311670064926147, + "style_target": 1, + "support_probability": 0.995701253414154, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8611542975570257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006881860550493002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7553:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23158656060695648, + 0.9855177402496338, + 0.11815168708562851, + 0.23158656060695648, + 0.021364077925682068, + 0.037704549729824066, + 0.02560955286026001, + 0.050429459661245346, + 0.03714726120233536 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 0, + "style_probability": 0.5592974424362183, + "style_target": 0, + "support_probability": 0.9959611296653748, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.659473604606182, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2017662525177002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20205093920230865, + 0.8321653604507446, + 0.7892664074897766, + 0.5753499269485474, + 0.01130302157253027, + 0.01205506268888712, + 0.015901198610663414, + 0.06160909682512283, + 0.01014486700296402, + 0.013839236460626125, + 0.043033432215452194 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 1, + "style_probability": 0.8885549306869507, + "style_target": 1, + "support_probability": 0.7046566009521484, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5420358413092509, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17816710472106934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2149592638015747, + 0.8229582905769348, + 0.7619400024414062, + 0.4478922486305237, + 0.028211846947669983, + 0.022055165842175484 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 0, + "style_probability": 0.562192440032959, + "style_target": 0, + "support_probability": 0.8135389089584351, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.1858666677789133, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6495766639709473, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8158737421035767, + 0.25729140639305115, + 0.21678270399570465, + 0.7391290664672852, + 0.0968821719288826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 0, + "style_probability": 0.4300244450569153, + "style_target": 1, + "support_probability": 0.27830877900123596, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24622678691222208, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16599568724632263, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.868127703666687, + 0.21859920024871826, + 0.18076029419898987, + 0.788047194480896, + 0.025349460542201996, + 0.04653589054942131, + 0.026676880195736885, + 0.034928128123283386, + 0.04975804686546326 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 1, + "style_probability": 0.21065166592597961, + "style_target": 0, + "support_probability": 0.7024761438369751, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13380426485706257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03666817024350166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25315138697624207, + 0.11416248232126236, + 0.15761221945285797, + 0.9010165333747864, + 0.028050987049937248, + 0.019773293286561966 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.5536933541297913, + "style_target": 1, + "support_probability": 0.1921592503786087, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02551987624877671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08063021302223206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3149385154247284, + 0.10869993269443512, + 0.15195026993751526, + 0.9344644546508789, + 0.01373071875423193, + 0.04558906704187393 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.2100508213043213, + "style_target": 0, + "support_probability": 0.15150344371795654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4231038120570406, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3587486147880554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3982910215854645, + 0.5363581776618958, + 0.8974684476852417, + 0.2990870773792267, + 0.009192612022161484, + 0.02512924186885357, + 0.010596267879009247, + 0.012250503525137901, + 0.014824863523244858 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 0, + "style_probability": 0.4320434331893921, + "style_target": 1, + "support_probability": 0.49098634719848633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.614097101259501, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49714475870132446, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3018064796924591, + 0.4453134536743164, + 0.9205032587051392, + 0.2725541293621063, + 0.017893321812152863, + 0.01700274646282196, + 0.007865491323173046, + 0.011402751319110394, + 0.017755616456270218 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 0, + "style_probability": 0.8309167623519897, + "style_target": 0, + "support_probability": 0.2709493041038513, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5535031260523595, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13397295773029327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16612640023231506, + 0.24738909304141998, + 0.9562363028526306, + 0.3977876901626587, + 0.02021634951233864, + 0.018886815756559372, + 0.027923395857214928, + 0.012011876329779625, + 0.04253270849585533, + 0.03156455606222153, + 0.014843923039734364 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 1, + "style_probability": 0.5575748085975647, + "style_target": 1, + "support_probability": 0.3753906190395355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44219699421701775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06750096380710602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16224610805511475, + 0.22775132954120636, + 0.9684402346611023, + 0.27447575330734253, + 0.023091115057468414, + 0.06750119477510452 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 1, + "style_probability": 0.2767241597175598, + "style_target": 0, + "support_probability": 0.6278759837150574, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3198930371706931, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.41933396458625793, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2917551100254059, + 0.08841194212436676, + 0.08252621442079544, + 0.8852101564407349, + 0.12192463129758835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 1, + "style_probability": 0.8240713477134705, + "style_target": 1, + "support_probability": 0.2291317880153656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44693535829610215, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10977435111999512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26321110129356384, + 0.15115924179553986, + 0.11975237727165222, + 0.8360180854797363, + 0.015136832371354103 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 0, + "style_probability": 0.685007631778717, + "style_target": 0, + "support_probability": 0.5987130403518677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03723132996588082, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6702225804328918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5425865054130554, + 0.8865853548049927, + 0.09916452318429947, + 0.4755811095237732, + 0.06092267483472824, + 0.07327428460121155 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.60886150598526, + "style_target": 1, + "support_probability": 0.025933973491191864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06502670112526869, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6366031765937805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5810182690620422, + 0.8655301928520203, + 0.10077698528766632, + 0.5917120575904846, + 0.026777364313602448, + 0.04005355387926102, + 0.04333081841468811, + 0.05140342935919762, + 0.05034800246357918, + 0.06417527049779892 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.2911704480648041, + "style_target": 0, + "support_probability": 0.059938397258520126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1648525844133472, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45331740379333496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6789974570274353, + 0.450366348028183, + 0.1921846568584442, + 0.48253917694091797, + 0.031370025128126144, + 0.03180678188800812, + 0.07025596499443054, + 0.04316738620400429, + 0.06467822194099426 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 1, + "style_probability": 0.932077169418335, + "style_target": 1, + "support_probability": 0.0864378809928894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23140317018337894, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3036983609199524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6685853004455566, + 0.5024088025093079, + 0.23066198825836182, + 0.4806087613105774, + 0.04296956956386566, + 0.17191635072231293, + 0.04690852016210556, + 0.03317919000983238, + 0.0491497740149498 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 0, + "style_probability": 0.7549101114273071, + "style_target": 0, + "support_probability": 0.3038467764854431, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8240408168065575, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017298247665166855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19403941929340363, + 0.22607608139514923, + 0.19403941929340363, + 0.9499950408935547, + 0.02393757924437523, + 0.014339308254420757, + 0.01911729760468006, + 0.021009517833590508, + 0.040227245539426804, + 0.02194174937903881 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 1, + "style_probability": 0.8655450344085693, + "style_target": 1, + "support_probability": 0.9546362161636353, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8116020618982017, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03700955957174301, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17045122385025024, + 0.19798699021339417, + 0.17045122385025024, + 0.9479355812072754, + 0.08172225952148438, + 0.05758736655116081, + 0.06909596920013428, + 0.05426543578505516, + 0.01814078353345394, + 0.008286443538963795 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 0, + "style_probability": 0.653412401676178, + "style_target": 0, + "support_probability": 0.9651204943656921, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41965224339055834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4360705018043518, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8938175439834595, + 0.6012754440307617, + 0.10797562450170517, + 0.2461988627910614, + 0.04401395469903946 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 1, + "style_probability": 0.5696434378623962, + "style_target": 1, + "support_probability": 0.5959859490394592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4928307909951088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6155569553375244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.904963493347168, + 0.5438255667686462, + 0.07909021526575089, + 0.17549508810043335, + 0.0810803547501564 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 0, + "style_probability": 0.8326928615570068, + "style_target": 0, + "support_probability": 0.3967330753803253, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018440649951348243, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8867347240447998, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.893703281879425, + 0.6512431502342224, + 0.26143744587898254, + 0.22767433524131775, + 0.2175922989845276, + 0.09223911166191101 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.5727297067642212, + "style_target": 1, + "support_probability": 0.007431224454194307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030094650508193294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9862759113311768, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8490518927574158, + 0.5445540547370911, + 0.273936927318573, + 0.21648061275482178, + 0.09290745854377747, + 0.0707632377743721, + 0.11285573244094849, + 0.0935274288058281, + 0.03923674300312996, + 0.08957381546497345, + 0.08363843709230423 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.4260096848011017, + "style_target": 0, + "support_probability": 0.0003109992539975792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5188024656540183, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07603543996810913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07954736053943634, + 0.4266768991947174, + 0.07954736053943634, + 0.9936352968215942, + 0.021351981908082962, + 0.022870730608701706, + 0.06978312134742737, + 0.04839951917529106, + 0.03313114494085312 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.5954543352127075, + "style_target": 1, + "support_probability": 0.721703052520752, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11309733373306396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1023840606212616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08176592737436295, + 0.5504900217056274, + 0.08176592737436295, + 0.9962527751922607, + 0.0301666222512722, + 0.04061927646398544, + 0.01637183502316475, + 0.019407371059060097, + 0.029235271736979485 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.18373265862464905, + "style_target": 0, + "support_probability": 0.6704191565513611, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6080890774717368, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03957879915833473, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.14297449588775635, + 0.2764560878276825, + 0.9884645342826843, + 0.8983921408653259, + 0.026813095435500145, + 0.04568273946642876, + 0.04669647663831711, + 0.05654604732990265, + 0.10534964501857758, + 0.04688941314816475 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 1, + "style_probability": 0.6849151253700256, + "style_target": 1, + "support_probability": 0.692954957485199, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7417200359263634, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1487722545862198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12235574424266815, + 0.27627214789390564, + 0.9886278510093689, + 0.9200950860977173, + 0.025375831872224808, + 0.020557088777422905, + 0.02316177450120449, + 0.0540262870490551, + 0.07495797425508499, + 0.042181238532066345, + 0.025248732417821884 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 0, + "style_probability": 0.9367349147796631, + "style_target": 0, + "support_probability": 0.5315451622009277, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7111210481389758, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07186147570610046, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3001335561275482, + 0.514301061630249, + 0.7070944309234619, + 0.9987953901290894, + 0.03371761739253998 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.8247612714767456, + "style_target": 1, + "support_probability": 0.6609472632408142, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26195673789543117, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07009054720401764, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.246999591588974, + 0.515984058380127, + 0.7193335294723511, + 0.9995377063751221, + 0.042091332376003265 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.2665344178676605, + "style_target": 0, + "support_probability": 0.6380056738853455, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7302841858017991, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03467772528529167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6164999604225159, + 0.1914922446012497, + 0.23180393874645233, + 0.9979264736175537, + 0.0295394454151392, + 0.0179455503821373 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 1, + "style_probability": 0.8596687316894531, + "style_target": 1, + "support_probability": 0.6639363169670105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6791143714487968, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.029097331687808037, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6170437335968018, + 0.17090296745300293, + 0.2842562794685364, + 0.9942751526832581, + 0.03842093423008919, + 0.016435259953141212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 0, + "style_probability": 0.6307706832885742, + "style_target": 0, + "support_probability": 0.8570687174797058, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08041020651303028, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8466547727584839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15643157064914703, + 0.7642744779586792, + 0.5442337989807129, + 0.8848007917404175, + 0.0353158563375473, + 0.027643263339996338, + 0.05934161692857742, + 0.0491580069065094, + 0.027239469811320305 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.7971758246421814, + "style_target": 1, + "support_probability": 0.03612786903977394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1001255633336108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7167195081710815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25479045510292053, + 0.722345232963562, + 0.5272108912467957, + 0.8403539657592773, + 0.026061587035655975 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.36241504549980164, + "style_target": 0, + "support_probability": 0.08200223743915558, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7909146063093202, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04057049751281738, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9931129813194275, + 0.452230304479599, + 0.2507054805755615, + 0.6808509230613708, + 0.014447798021137714, + 0.024750063195824623, + 0.007569406647235155, + 0.009434198960661888, + 0.011696333065629005, + 0.005999451037496328 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.8991658687591553, + "style_target": 1, + "support_probability": 0.47409817576408386, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30581189102523265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02573436312377453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9954074025154114, + 0.4272104501724243, + 0.21277128159999847, + 0.6727117300033569, + 0.014547270722687244, + 0.019587507471442223, + 0.015333415940403938, + 0.018472131341695786, + 0.013321956619620323, + 0.021711016073822975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.3144966661930084, + "style_target": 0, + "support_probability": 0.5986963510513306, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08293106840004283, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8447768688201904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8319175243377686, + 0.24664980173110962, + 0.7519751191139221, + 0.5911628603935242, + 0.04136817902326584 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 0, + "style_probability": 0.4166271686553955, + "style_target": 1, + "support_probability": 0.08974377810955048, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018982170514201927, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7126122713088989, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8398942947387695, + 0.19836698472499847, + 0.8766760230064392, + 0.685319185256958, + 0.06609747558832169 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 1, + "style_probability": 0.07342499494552612, + "style_target": 0, + "support_probability": 0.32025179266929626, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7322280125558293, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03539617359638214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5099131464958191, + 0.8708791136741638, + 0.15870767831802368, + 0.920525074005127, + 0.01056938711553812, + 0.03346073627471924 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 1, + "style_probability": 0.539106011390686, + "style_target": 1, + "support_probability": 0.9369596242904663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.85518790210395, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.025377772748470306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6552407145500183, + 0.9092808961868286, + 0.08895035088062286, + 0.950372040271759, + 0.01667991653084755, + 0.02425602823495865 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 0, + "style_probability": 0.8693165183067322, + "style_target": 0, + "support_probability": 0.9500121474266052, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4128658659256683, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09593971818685532, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24219661951065063, + 0.18375176191329956, + 0.9439118504524231, + 0.33873388171195984, + 0.023445019498467445, + 0.06806701421737671, + 0.060020409524440765, + 0.028287436813116074, + 0.04706242308020592 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.5295146107673645, + "style_target": 1, + "support_probability": 0.727192759513855, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26290911712581455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1826818585395813, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2364431917667389, + 0.1252695918083191, + 0.9603469967842102, + 0.3597736060619354, + 0.04010631889104843 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.4120262861251831, + "style_target": 0, + "support_probability": 0.6388451457023621, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3728441585689929, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0433003269135952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9700958728790283, + 0.2598262429237366, + 0.0948592871427536, + 0.3133917450904846, + 0.13133110105991364, + 0.02048433944582939, + 0.012667018920183182, + 0.05951956287026405, + 0.10930777341127396, + 0.019940750673413277, + 0.03832097351551056 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 1, + "style_probability": 0.5758323073387146, + "style_target": 1, + "support_probability": 0.39319267868995667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2316562373685985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0765293538570404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9870649576187134, + 0.2522059679031372, + 0.08607880026102066, + 0.24235008656978607, + 0.03507991135120392, + 0.05110706761479378, + 0.06350301951169968, + 0.03994601592421532, + 0.04253531992435455, + 0.17019569873809814 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 1, + "style_probability": 0.4484005868434906, + "style_target": 0, + "support_probability": 0.2285340130329132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35398721984348164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10198742896318436, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44282016158103943, + 0.08233918249607086, + 0.9665818214416504, + 0.3976711928844452, + 0.12522946298122406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 1, + "style_probability": 0.5471813678741455, + "style_target": 1, + "support_probability": 0.47883811593055725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13026945191147832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27259668707847595, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33780279755592346, + 0.06813351809978485, + 0.9752393364906311, + 0.3805983364582062, + 0.0677272379398346 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 0, + "style_probability": 0.5478527545928955, + "style_target": 0, + "support_probability": 0.20596230030059814, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8230172178269693, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0259046982973814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35456427931785583, + 0.9528805613517761, + 0.42446157336235046, + 0.28180259466171265, + 0.04819120466709137, + 0.05109590291976929 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.622903048992157, + "style_target": 1, + "support_probability": 0.9731888771057129, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7452661967446308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02954399399459362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2537117600440979, + 0.9694446921348572, + 0.4142704904079437, + 0.19816681742668152, + 0.021003732457756996, + 0.05496228486299515, + 0.034549664705991745, + 0.06815316528081894, + 0.06545962393283844, + 0.023947207257151604, + 0.027648963034152985 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.38445571064949036, + "style_target": 0, + "support_probability": 0.9822567105293274, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009841402340038326, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9309217929840088, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42300036549568176, + 0.33114612102508545, + 0.29693976044654846, + 0.751285195350647, + 0.1753530651330948, + 0.05346977710723877, + 0.2081984281539917, + 0.06378399580717087, + 0.2996803820133209 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 0, + "style_probability": 0.4024706184864044, + "style_target": 1, + "support_probability": 0.005892233457416296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026535119021904754, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9867311716079712, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43376395106315613, + 0.25622251629829407, + 0.32595252990722656, + 0.72947096824646, + 0.038672156631946564, + 0.12343814969062805, + 0.06968056410551071, + 0.05269910395145416, + 0.1535090059041977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 1, + "style_probability": 0.18144068121910095, + "style_target": 0, + "support_probability": 0.005656460765749216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0047450405141114095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9251220226287842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4187738597393036, + 0.7148373126983643, + 0.4187738597393036, + 0.08487042784690857, + 0.03073173016309738, + 0.014132554642856121, + 0.02321353740990162, + 0.025327438488602638, + 0.07680759578943253, + 0.05065669119358063, + 0.07028362154960632 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 1, + "style_probability": 0.5609614253044128, + "style_target": 1, + "support_probability": 0.0007657901733182371, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008929520943241823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8903999924659729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3495790362358093, + 0.7901579737663269, + 0.3495790362358093, + 0.10064688324928284, + 0.018257124349474907, + 0.019335079938173294, + 0.013594996184110641, + 0.033175110816955566, + 0.015755141153931618, + 0.06130427494645119 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 0, + "style_probability": 0.848325252532959, + "style_target": 0, + "support_probability": 0.0006613125442527235, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5112781994346687, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.35412293672561646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24104996025562286, + 0.30739036202430725, + 0.3909142315387726, + 0.887173056602478, + 0.06587252765893936 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.6380215287208557, + "style_target": 1, + "support_probability": 0.5450453162193298, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05275161876750975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5888293385505676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18001048266887665, + 0.2161121666431427, + 0.3604756295681, + 0.909593403339386, + 0.06995702534914017 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.22901953756809235, + "style_target": 0, + "support_probability": 0.19393734633922577, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19892271147956492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24586327373981476, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8922086358070374, + 0.5472773313522339, + 0.1322336345911026, + 0.5472773313522339, + 0.03482061251997948, + 0.03332372009754181 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 0, + "style_probability": 0.36606135964393616, + "style_target": 1, + "support_probability": 0.3540583848953247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07750253948549883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5347111225128174, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7994548678398132, + 0.5418545603752136, + 0.1508370190858841, + 0.5418545603752136, + 0.05336490646004677, + 0.036726515740156174 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 1, + "style_probability": 0.45223692059516907, + "style_target": 0, + "support_probability": 0.11553031206130981, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45852076390243773, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.062059152871370316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36775702238082886, + 0.24630452692508698, + 0.5724810361862183, + 0.9769871234893799, + 0.02555139735341072, + 0.014048435725271702, + 0.018053172156214714, + 0.04516633599996567, + 0.07740046083927155 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 1, + "style_probability": 0.6053874492645264, + "style_target": 1, + "support_probability": 0.6491008400917053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7894839453107482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09293973445892334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34158799052238464, + 0.26162341237068176, + 0.5741373896598816, + 0.9800644516944885, + 0.011677342467010021, + 0.0628301203250885, + 0.04055057093501091, + 0.0172876063734293, + 0.02577362023293972 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 0, + "style_probability": 0.9433941841125488, + "style_target": 0, + "support_probability": 0.5986511707305908, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024636664169524192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43667709827423096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6771806478500366, + 0.13882893323898315, + 0.8789905905723572, + 0.6765510439872742, + 0.1371118724346161, + 0.1524430513381958, + 0.04077167436480522, + 0.0525716096162796, + 0.02708457037806511, + 0.06292705237865448, + 0.10665414482355118 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 0, + "style_probability": 0.334629625082016, + "style_target": 1, + "support_probability": 0.008678787387907505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04083951993177736, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23524217307567596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6575186252593994, + 0.10985652357339859, + 0.9510824680328369, + 0.6895164251327515, + 0.029647892341017723, + 0.02431616373360157, + 0.05769122764468193, + 0.05542784556746483, + 0.05452728644013405, + 0.07053292542695999 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 1, + "style_probability": 0.3115905523300171, + "style_target": 0, + "support_probability": 0.046452656388282776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009265269027199494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5999273061752319, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18544930219650269, + 0.20967981219291687, + 0.210142582654953, + 0.7483287453651428, + 0.07524759322404861 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 0, + "style_probability": 0.22770647704601288, + "style_target": 1, + "support_probability": 0.012123201973736286, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0070206625905012456, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.689858078956604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16975650191307068, + 0.18412251770496368, + 0.20786114037036896, + 0.7130816578865051, + 0.06148931011557579 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 1, + "style_probability": 0.37830567359924316, + "style_target": 0, + "support_probability": 0.007545920088887215, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.35256199622085993, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0714978501200676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9859510064125061, + 0.13830675184726715, + 0.16770067811012268, + 0.2506815493106842, + 0.041344765573740005, + 0.023809293285012245 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 0, + "style_probability": 0.18301230669021606, + "style_target": 1, + "support_probability": 0.5859459042549133, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4691519078020648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11420591175556183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9866974949836731, + 0.14796335995197296, + 0.15739306807518005, + 0.24293078482151031, + 0.015928762033581734, + 0.024168798699975014 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 0, + "style_probability": 0.6092594861984253, + "style_target": 0, + "support_probability": 0.42868784070014954, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21540606293840847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21183982491493225, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14589859545230865, + 0.45002397894859314, + 0.09875663369894028, + 0.9334900975227356, + 0.0570109598338604, + 0.03829469904303551, + 0.013544535264372826, + 0.11311309039592743, + 0.033377524465322495 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 0, + "style_probability": 0.3361033797264099, + "style_target": 1, + "support_probability": 0.26247188448905945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04506705667187101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44083940982818604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1420866698026657, + 0.38327229022979736, + 0.13928157091140747, + 0.9590241312980652, + 0.03722153231501579, + 0.017986735329031944, + 0.03088649921119213, + 0.028827566653490067, + 0.053816307336091995 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 1, + "style_probability": 0.14241266250610352, + "style_target": 0, + "support_probability": 0.1764892041683197, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4327361992667589, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005999146960675716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9993383288383484, + 0.22121772170066833, + 0.26545387506484985, + 0.09156463295221329, + 0.018683232367038727, + 0.0838412418961525, + 0.061561498790979385, + 0.06379982829093933, + 0.025634117424488068, + 0.040298230946063995, + 0.02040884457528591 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.4004552364349365, + "style_target": 1, + "support_probability": 0.6501559019088745, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.507949665395125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005669367499649525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.998331606388092, + 0.23694565892219543, + 0.27429431676864624, + 0.14607523381710052, + 0.019012730568647385, + 0.04091650992631912, + 0.020018668845295906, + 0.03228553757071495, + 0.01533629558980465, + 0.02412661537528038 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.5853449702262878, + "style_target": 0, + "support_probability": 0.5299997329711914, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.046136968624622625, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9142528176307678, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16025544703006744, + 0.6524062156677246, + 0.24367481470108032, + 0.09523113071918488, + 0.024517465382814407 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.8599585890769958, + "style_target": 1, + "support_probability": 0.04149336367845535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005117248368782016, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9855284094810486, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16680096089839935, + 0.5631318688392639, + 0.2504592537879944, + 0.1008266881108284, + 0.03306954354047775, + 0.06548354774713516, + 0.03405144065618515, + 0.023399656638503075, + 0.0696214959025383 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.3014988303184509, + "style_target": 0, + "support_probability": 0.0035975605715066195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1329682355457644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7374906539916992, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23716600239276886, + 0.11836270242929459, + 0.09572131931781769, + 0.5494662523269653, + 0.07319984585046768, + 0.0390988327562809 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.954617440700531, + "style_target": 1, + "support_probability": 0.10715379565954208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02966920745504489, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3484257757663727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28314682841300964, + 0.12746164202690125, + 0.08452634513378143, + 0.667967677116394, + 0.06747660040855408, + 0.048574186861515045 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.13177363574504852, + "style_target": 0, + "support_probability": 0.4370313882827759, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031797000054712386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15315678715705872, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22390371561050415, + 0.13130700588226318, + 0.12611590325832367, + 0.9010441899299622, + 0.07631993293762207, + 0.025068866088986397, + 0.053484898060560226, + 0.024556370452046394, + 0.012219356372952461 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.7635083198547363, + "style_target": 1, + "support_probability": 0.008538229390978813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06019085099078013, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08581271022558212, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2507922053337097, + 0.12142494320869446, + 0.1299661248922348, + 0.896918535232544, + 0.2418423593044281, + 0.05633756145834923, + 0.03146309033036232, + 0.02369692176580429, + 0.06231044605374336 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.44427335262298584, + "style_target": 0, + "support_probability": 0.04486574977636337, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8438286430845333, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.013565091416239738, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9865672588348389, + 0.19919843971729279, + 0.18763728439807892, + 0.30709025263786316, + 0.06514985114336014, + 0.04591377452015877, + 0.08737462013959885, + 0.01656697690486908, + 0.022863993421196938, + 0.0452999547123909, + 0.013686618767678738 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.9401828050613403, + "style_target": 1, + "support_probability": 0.7743890285491943, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45451776800623694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.013067241758108139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9794390797615051, + 0.17852161824703217, + 0.1470971554517746, + 0.363506019115448, + 0.0238049253821373, + 0.045191891491413116 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.46768683195114136, + "style_target": 0, + "support_probability": 0.820026695728302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4603640610610096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03947602957487106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18398284912109375, + 0.8692787885665894, + 0.13002461194992065, + 0.3971315324306488, + 0.08556852489709854 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 0, + "style_probability": 0.47645723819732666, + "style_target": 1, + "support_probability": 0.6547115445137024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37230995113434195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0628906786441803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.218633770942688, + 0.8875171542167664, + 0.11448696255683899, + 0.4458787739276886, + 0.047445233911275864, + 0.14515237510204315, + 0.04273001849651337, + 0.027059759944677353, + 0.0354689359664917 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 1, + "style_probability": 0.4043281674385071, + "style_target": 0, + "support_probability": 0.7282449007034302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37380804645227655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.348080575466156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8315244913101196, + 0.15221388638019562, + 0.04967895895242691, + 0.15221388638019562, + 0.008778258226811886, + 0.03185709938406944 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 1, + "style_probability": 0.5016588568687439, + "style_target": 1, + "support_probability": 0.6741448044776917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011304697635273126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5067212581634521, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8123983144760132, + 0.17743080854415894, + 0.0576532781124115, + 0.17743080854415894, + 0.011454998515546322, + 0.025807294994592667 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 1, + "style_probability": 0.06694445013999939, + "style_target": 0, + "support_probability": 0.3353731632232666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007594318274540247, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9864891767501831, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30585554242134094, + 0.26981619000434875, + 0.4092817008495331, + 0.4212627708911896, + 0.01890735700726509, + 0.0107874209061265, + 0.018256567418575287, + 0.023189740255475044, + 0.01579095423221588 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 0, + "style_probability": 0.47044169902801514, + "style_target": 1, + "support_probability": 0.007049682084470987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02286904816715814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9411394000053406, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3112047612667084, + 0.1425439417362213, + 0.42885521054267883, + 0.47884702682495117, + 0.0223790742456913, + 0.032878126949071884, + 0.015848560258746147, + 0.034639082849025726, + 0.013076267205178738 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 0, + "style_probability": 0.7055272459983826, + "style_target": 0, + "support_probability": 0.020953385159373283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0377718367346761, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7628393173217773, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1938815861940384, + 0.06583885103464127, + 0.22961029410362244, + 0.7120361328125, + 0.019352618604898453, + 0.014261863194406033, + 0.021777255460619926, + 0.01711377687752247, + 0.012047836557030678, + 0.05602322146296501 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.5587308406829834, + "style_target": 1, + "support_probability": 0.023260729387402534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00785173312018727, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8833909630775452, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14910821616649628, + 0.06372568011283875, + 0.2545488178730011, + 0.57497638463974, + 0.0112136360257864, + 0.021589748561382294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.37971752882003784, + "style_target": 0, + "support_probability": 0.0037048778031021357, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23509495415580106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.214753657579422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33004847168922424, + 0.4969470798969269, + 0.33004847168922424, + 0.9808825850486755, + 0.08687292039394379 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.6507930755615234, + "style_target": 1, + "support_probability": 0.09704185277223587, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1534753769330408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20207853615283966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2565224766731262, + 0.5175723433494568, + 0.2565224766731262, + 0.9664608836174011, + 0.01396612823009491 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.3397284746170044, + "style_target": 0, + "support_probability": 0.14137841761112213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017357796337524586, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8172863721847534, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09092254936695099, + 0.19633623957633972, + 0.42613673210144043, + 0.15688635408878326, + 0.10036235302686691, + 0.4605010747909546 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 1, + "style_probability": 0.8824900388717651, + "style_target": 1, + "support_probability": 0.002094362396746874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019923921246443295, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8122929930686951, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.0877995491027832, + 0.18868990242481232, + 0.39065665006637573, + 0.14714232087135315, + 0.14572958648204803, + 0.05950493738055229, + 0.08451730012893677, + 0.02099195122718811, + 0.057533785700798035, + 0.04291204363107681 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 1, + "style_probability": 0.43500518798828125, + "style_target": 0, + "support_probability": 0.00017814878083299845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36673818453083895, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48475003242492676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08525869250297546, + 0.4996483325958252, + 0.5499699711799622, + 0.10646875202655792, + 0.009123171679675579, + 0.03170913830399513, + 0.07412756234407425, + 0.008516475558280945, + 0.037806436419487 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.9220325350761414, + "style_target": 1, + "support_probability": 0.36287546157836914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031141944011013112, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46590331196784973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.06857465207576752, + 0.4579939544200897, + 0.6090725660324097, + 0.07882624864578247, + 0.03259165957570076, + 0.12181685864925385, + 0.03166203573346138, + 0.01715772971510887, + 0.021236591041088104 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.09147873520851135, + "style_target": 0, + "support_probability": 0.5100150108337402, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8622131415875517, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01361070480197668, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1189078763127327, + 0.15565268695354462, + 0.3051891624927521, + 0.9892305731773376, + 0.039335981011390686, + 0.028157411143183708, + 0.015835599973797798, + 0.027079187333583832, + 0.0471368245780468, + 0.01733890362083912, + 0.08048295229673386 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 1, + "style_probability": 0.8625354170799255, + "style_target": 1, + "support_probability": 0.894540011882782, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5193055156996389, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015483906492590904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1091446727514267, + 0.16655638813972473, + 0.2188418209552765, + 0.9858428835868835, + 0.029109539464116096, + 0.06579277664422989, + 0.052355725318193436, + 0.026245713233947754, + 0.049558088183403015, + 0.006277667824178934, + 0.01543264277279377 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 1, + "style_probability": 0.3919385075569153, + "style_target": 0, + "support_probability": 0.8559552431106567, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18156760828777624, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5410267114639282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7973839044570923, + 0.2285350114107132, + 0.7272421717643738, + 0.7646269202232361, + 0.04709082096815109 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.6951199769973755, + "style_target": 1, + "support_probability": 0.08925946801900864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06091450072449844, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.813285231590271, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8241772055625916, + 0.2014966607093811, + 0.7490035891532898, + 0.7656488418579102, + 0.032486651092767715, + 0.11973880976438522, + 0.027131635695695877, + 0.04185496270656586, + 0.0778905525803566 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.29067903757095337, + "style_target": 0, + "support_probability": 0.026360277086496353, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21765453979271265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16189885139465332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9649167656898499, + 0.1385817974805832, + 0.19634848833084106, + 0.49844956398010254, + 0.042089901864528656, + 0.09971146285533905 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.6876221299171448, + "style_target": 1, + "support_probability": 0.06125452369451523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05557871921140984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3426360785961151, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9824946522712708, + 0.13206301629543304, + 0.17258265614509583, + 0.5385295152664185, + 0.039427682757377625, + 0.08942446857690811 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.2046145647764206, + "style_target": 0, + "support_probability": 0.08528392761945724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8265668444515061, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041478466242551804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9399669170379639, + 0.09898079186677933, + 0.29884669184684753, + 0.18898551166057587, + 0.23419493436813354, + 0.05059888958930969, + 0.08758743852376938, + 0.011246337555348873, + 0.024359174072742462 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 1, + "style_probability": 0.83817058801651, + "style_target": 1, + "support_probability": 0.9144022464752197, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6750435956718087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08383581787347794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9188133478164673, + 0.08640949428081512, + 0.4675353467464447, + 0.23105156421661377, + 0.02295173518359661, + 0.01903536543250084, + 0.013109846971929073, + 0.04867912083864212, + 0.02800171822309494 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 1, + "style_probability": 0.42073678970336914, + "style_target": 0, + "support_probability": 0.8416534662246704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7403673586959608, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1644493043422699, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.03773403912782669, + 0.16207759082317352, + 0.7264880537986755, + 0.9731245636940002, + 0.02672365866601467, + 0.011288579553365707, + 0.03320996090769768, + 0.024723466485738754, + 0.09257007390260696, + 0.011767590418457985, + 0.03636792302131653 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 1, + "style_probability": 0.9278680086135864, + "style_target": 1, + "support_probability": 0.41610583662986755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37195948662345274, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11327514052391052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.055760644376277924, + 0.19359053671360016, + 0.7167958617210388, + 0.9711425304412842, + 0.022187992930412292, + 0.029857147485017776 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 0, + "style_probability": 0.5220539569854736, + "style_target": 0, + "support_probability": 0.4781288504600525, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2971964319381276, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11829037219285965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40639448165893555, + 0.8903058767318726, + 0.138511061668396, + 0.36390945315361023, + 0.10831592977046967 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 0, + "style_probability": 0.30573728680610657, + "style_target": 1, + "support_probability": 0.5097050070762634, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14086406512787011, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14145399630069733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3344082832336426, + 0.9164692759513855, + 0.11905108392238617, + 0.34065258502960205, + 0.039109181612730026 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 1, + "style_probability": 0.19714762270450592, + "style_target": 0, + "support_probability": 0.5196389555931091, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01664852642929944, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8281720876693726, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2534882724285126, + 0.4554433524608612, + 0.3731018304824829, + 0.46964630484580994, + 0.106315016746521, + 0.14829058945178986 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 0, + "style_probability": 0.2993059456348419, + "style_target": 1, + "support_probability": 0.011117933318018913, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009908493270767508, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9031445384025574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23279045522212982, + 0.3339580297470093, + 0.29376062750816345, + 0.33827775716781616, + 0.20483842492103577, + 0.10558172315359116 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 0, + "style_probability": 0.5235643982887268, + "style_target": 0, + "support_probability": 0.00706252409145236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.138994053575396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15972544252872467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6480787396430969, + 0.23741336166858673, + 0.8429117202758789, + 0.7102382779121399, + 0.05808613449335098, + 0.08929470926523209, + 0.12048733979463577, + 0.021182764321565628, + 0.1787901669740677 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.47361236810684204, + "style_target": 1, + "support_probability": 0.07842621207237244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14985911844327857, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1946752369403839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6316854357719421, + 0.15602852404117584, + 0.884006679058075, + 0.6703740358352661, + 0.01766200177371502, + 0.0227795597165823, + 0.016786472871899605, + 0.017819996923208237, + 0.05009063333272934 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.8168789148330688, + "style_target": 0, + "support_probability": 0.047010719776153564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0075100654536097424, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8408830165863037, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4345364570617676, + 0.29892730712890625, + 0.37048831582069397, + 0.21804621815681458, + 0.05883893743157387, + 0.031278323382139206, + 0.07939387112855911, + 0.1353522539138794, + 0.15869131684303284, + 0.02732742764055729, + 0.40965479612350464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 0, + "style_probability": 0.1798476129770279, + "style_target": 1, + "support_probability": 0.007757522631436586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000683184497532922, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9723471403121948, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24670565128326416, + 0.15660612285137177, + 0.27798330783843994, + 0.19005176424980164, + 0.07280192524194717, + 0.0965978354215622, + 0.15865261852741241, + 0.11168401688337326, + 0.08634410798549652, + 0.17250339686870575 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 1, + "style_probability": 0.08860369026660919, + "style_target": 0, + "support_probability": 0.008539194241166115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25509476077989435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9534679055213928, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30627644062042236, + 0.39333343505859375, + 0.4545383155345917, + 0.20516613125801086, + 0.06261221319437027 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 1, + "style_probability": 0.944556713104248, + "style_target": 1, + "support_probability": 0.17213954031467438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06296992893578456, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9819400906562805, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2616213858127594, + 0.45767706632614136, + 0.5104439854621887, + 0.20767742395401, + 0.03436935320496559 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 0, + "style_probability": 0.7527582049369812, + "style_target": 0, + "support_probability": 0.05743124708533287, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7302437549915334, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.017149915918707848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2306639701128006, + 0.19380736351013184, + 0.9636611938476562, + 0.26492103934288025, + 0.013405412435531616, + 0.01358953770250082 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.6879206895828247, + "style_target": 1, + "support_probability": 0.7997963428497314, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20516863025171478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06157030165195465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26577216386795044, + 0.17815503478050232, + 0.9306327104568481, + 0.2712104022502899, + 0.02726609632372856, + 0.024682817980647087, + 0.11877565830945969, + 0.11037001013755798, + 0.02562054805457592, + 0.05303347855806351, + 0.030112374573946 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.3237510025501251, + "style_target": 0, + "support_probability": 0.4151333272457123, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8768223527351807, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.030417153611779213, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9671005606651306, + 0.01706453412771225, + 0.088966004550457, + 0.4662545621395111, + 0.01117380429059267, + 0.024902064353227615, + 0.0055102016776800156, + 0.006696500815451145, + 0.003757319413125515 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.8915741443634033, + "style_target": 1, + "support_probability": 0.9855983853340149, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2576186076578495, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04329377040266991, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9751180410385132, + 0.026286832988262177, + 0.06868061423301697, + 0.4549786150455475, + 0.0314389206469059, + 0.025118747726082802, + 0.0345296673476696, + 0.012487100437283516, + 0.03205651789903641 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.11186926066875458, + "style_target": 0, + "support_probability": 0.9732667207717896, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45208322656948186, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03550298511981964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36458277702331543, + 0.15819032490253448, + 0.9292006492614746, + 0.7589969635009766, + 0.019465742632746696, + 0.017671676352620125, + 0.012913402169942856, + 0.013427913188934326, + 0.010256576351821423, + 0.015760643407702446 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 1, + "style_probability": 0.7328802943229675, + "style_target": 1, + "support_probability": 0.1980040967464447, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08246674347276246, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07628189027309418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3759732246398926, + 0.13223861157894135, + 0.8670153021812439, + 0.7448009848594666, + 0.020966781303286552, + 0.022129710763692856, + 0.035460423678159714, + 0.07088997215032578, + 0.25201916694641113, + 0.02351979911327362, + 0.038597021251916885 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 1, + "style_probability": 0.45507121086120605, + "style_target": 0, + "support_probability": 0.03898100182414055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12132252755085178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5277370810508728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44818735122680664, + 0.1835837960243225, + 0.44818735122680664, + 0.8869388103485107, + 0.0975291058421135 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.8102543354034424, + "style_target": 1, + "support_probability": 0.07618972659111023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018342375121338697, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.752951443195343, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44299203157424927, + 0.18018236756324768, + 0.44299203157424927, + 0.8792886734008789, + 0.0758935883641243, + 0.07448998838663101, + 0.0809948593378067, + 0.030911143869161606, + 0.04317961260676384 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.31099793314933777, + "style_target": 0, + "support_probability": 0.01258775219321251, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4858221091654851, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.481307715177536, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10725372284650803, + 0.1921280473470688, + 0.9688072204589844, + 0.1873939037322998, + 0.0269890446215868, + 0.008777046576142311 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.8565598726272583, + "style_target": 1, + "support_probability": 0.4367149770259857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031311783723253196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5528762936592102, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13109266757965088, + 0.22050805389881134, + 0.9720042943954468, + 0.12729112803936005, + 0.01579689234495163, + 0.04084903001785278 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.17141881585121155, + "style_target": 0, + "support_probability": 0.3728363811969757, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7184731729443409, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1234377771615982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9656552076339722, + 0.3422125279903412, + 0.31740376353263855, + 0.17061953246593475, + 0.08920302987098694, + 0.042071178555488586, + 0.009496261365711689, + 0.03110252134501934, + 0.01552424393594265 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 1, + "style_probability": 0.8910556435585022, + "style_target": 1, + "support_probability": 0.30791276693344116, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49123529270833927, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08302590250968933, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9662132263183594, + 0.25963664054870605, + 0.2764454185962677, + 0.21042154729366302, + 0.06620261818170547, + 0.15710598230361938, + 0.039493363350629807, + 0.07522346079349518, + 0.03039543889462948 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 0, + "style_probability": 0.6214880347251892, + "style_target": 0, + "support_probability": 0.407532662153244, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009274235158733178, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9866471290588379, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3471761643886566, + 0.3592948317527771, + 0.5121726393699646, + 0.6782635450363159, + 0.03020797297358513, + 0.12131474912166595, + 0.20145563781261444, + 0.22665327787399292, + 0.12069258838891983, + 0.03382490947842598, + 0.03857407718896866 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.6870756149291992, + "style_target": 1, + "support_probability": 0.005628049839287996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0022789718176329616, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9914910197257996, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33746662735939026, + 0.3644038736820221, + 0.45627257227897644, + 0.6552444100379944, + 0.05034654214978218, + 0.029152845963835716 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.30286169052124023, + "style_target": 0, + "support_probability": 0.0009472191450186074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1916213503051944, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18163001537322998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13661517202854156, + 0.8583819270133972, + 0.3470921516418457, + 0.31573158502578735, + 0.03184087201952934 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.6192930340766907, + "style_target": 1, + "support_probability": 0.11975933611392975, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17728943295136257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18349440395832062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15513011813163757, + 0.8923692107200623, + 0.35145071148872375, + 0.30223894119262695, + 0.02829606831073761, + 0.017314603552222252, + 0.0790145993232727, + 0.10451694577932358, + 0.02850695513188839 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.3837922215461731, + "style_target": 0, + "support_probability": 0.16047993302345276, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6314018981229149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.046829044818878174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08600712567567825, + 0.13167129456996918, + 0.04178524762392044, + 0.8829699158668518, + 0.02321964129805565, + 0.011555295437574387 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 1, + "style_probability": 0.5542467832565308, + "style_target": 1, + "support_probability": 0.7480510473251343, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.034569165306512016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0978793129324913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09495693445205688, + 0.10634324699640274, + 0.06700335443019867, + 0.951328694820404, + 0.03180225193500519, + 0.045198868960142136 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 1, + "style_probability": 0.07378627359867096, + "style_target": 0, + "support_probability": 0.5878799557685852, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4937996704116893, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10221273452043533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9130749702453613, + 0.10286471992731094, + 0.3394348919391632, + 0.12701623141765594, + 0.055860601365566254, + 0.03476457670331001, + 0.12031302601099014, + 0.037379585206508636, + 0.013694880530238152 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 1, + "style_probability": 0.5666526556015015, + "style_target": 1, + "support_probability": 0.6922175288200378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7646113352551315, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06451035290956497, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9306324124336243, + 0.11270102858543396, + 0.4111212193965912, + 0.13111916184425354, + 0.027183376252651215, + 0.04150686040520668, + 0.10179565846920013, + 0.02434217557311058, + 0.029555927962064743 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 0, + "style_probability": 0.8649957180023193, + "style_target": 0, + "support_probability": 0.866741418838501, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.30389027538209473, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.361698716878891, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8134751915931702, + 0.16920609772205353, + 0.5889058113098145, + 0.29776954650878906, + 0.027803633362054825, + 0.018650781363248825, + 0.05068057030439377, + 0.019478848204016685, + 0.02125776745378971, + 0.03448834642767906, + 0.014020445756614208 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 0, + "style_probability": 0.46992501616477966, + "style_target": 1, + "support_probability": 0.4922690987586975, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2506113556616347, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12455485761165619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7717845439910889, + 0.122202567756176, + 0.5847303867340088, + 0.22370263934135437, + 0.019601810723543167, + 0.04703977331519127 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 1, + "style_probability": 0.3464025557041168, + "style_target": 0, + "support_probability": 0.520555853843689, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2190181892055364, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4703644812107086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7149429321289062, + 0.568587064743042, + 0.2269873023033142, + 0.568587064743042, + 0.06674681603908539 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 0, + "style_probability": 0.47746074199676514, + "style_target": 1, + "support_probability": 0.3196524679660797, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03267900272737898, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08168637752532959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7817197442054749, + 0.38002336025238037, + 0.21283859014511108, + 0.38002336025238037, + 0.06361138820648193 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 1, + "style_probability": 0.08644932508468628, + "style_target": 0, + "support_probability": 0.44754061102867126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21924133338635654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14214447140693665, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5419507026672363, + 0.48713767528533936, + 0.5419507026672363, + 0.8102301955223083, + 0.014574754983186722, + 0.020244771614670753 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.4869365096092224, + "style_target": 1, + "support_probability": 0.38113951683044434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4020000848201037, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08016035705804825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47067388892173767, + 0.44157159328460693, + 0.47067388892173767, + 0.8274543285369873, + 0.022377697750926018, + 0.02770020253956318 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.7906835079193115, + "style_target": 0, + "support_probability": 0.3925313353538513, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6764519557890402, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06603716313838959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5001890659332275, + 0.5353701114654541, + 0.7875853180885315, + 0.8163555264472961, + 0.008964435197412968, + 0.011072599329054356, + 0.009386819787323475, + 0.010216876864433289, + 0.01281878910958767 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 0, + "style_probability": 0.49536141753196716, + "style_target": 1, + "support_probability": 0.9769974946975708, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6256997290026672, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.02726862207055092, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4578503668308258, + 0.4589247405529022, + 0.861086368560791, + 0.7792496085166931, + 0.012873669154942036 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 1, + "style_probability": 0.293538361787796, + "style_target": 0, + "support_probability": 0.9800381064414978, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.405399560368969, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0773535668849945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1350826621055603, + 0.125854954123497, + 0.9874795079231262, + 0.13073961436748505, + 0.1503472775220871, + 0.021634355187416077, + 0.028779756277799606, + 0.031053993850946426, + 0.02510400488972664, + 0.05631037428975105 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.5096263289451599, + "style_target": 1, + "support_probability": 0.6266200542449951, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24944684773409523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03440498560667038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09290288388729095, + 0.07843522727489471, + 0.9982399940490723, + 0.06606771796941757, + 0.047304630279541016, + 0.027346452698111534, + 0.15736936032772064, + 0.025760959833860397, + 0.03957449272274971, + 0.0570366345345974 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.21642176806926727, + "style_target": 0, + "support_probability": 0.7489520907402039, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004489885067794008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9878847002983093, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2087099254131317, + 0.49465110898017883, + 0.15981826186180115, + 0.39664578437805176, + 0.023362107574939728 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.8029349446296692, + "style_target": 1, + "support_probability": 0.004564018454402685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008866558613955048, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9683389663696289, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21978993713855743, + 0.3988654911518097, + 0.1550789326429367, + 0.37062880396842957, + 0.03129734843969345, + 0.12413274496793747, + 0.06796415895223618, + 0.06338164210319519, + 0.055010728538036346 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.336832195520401, + "style_target": 0, + "support_probability": 0.011252022348344326, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8837142209277309, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021214138716459274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1641998142004013, + 0.9671520590782166, + 0.2187914103269577, + 0.14198321104049683, + 0.009680029936134815, + 0.005685445852577686 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.9198991060256958, + "style_target": 1, + "support_probability": 0.9655511379241943, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11483308170198804, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028080500662326813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16682890057563782, + 0.9860413670539856, + 0.1640896201133728, + 0.17748886346817017, + 0.011848821304738522, + 0.028337664902210236 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.10167037695646286, + "style_target": 0, + "support_probability": 0.8554497957229614, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8906084394041655, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03866061195731163, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9735944867134094, + 0.04861276596784592, + 0.10444283485412598, + 0.31433844566345215, + 0.007313917391002178, + 0.014835209585726261, + 0.011935839429497719, + 0.006694264709949493, + 0.008951045572757721 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 1, + "style_probability": 0.9454964399337769, + "style_target": 1, + "support_probability": 0.9195659756660461, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8467395664869664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02731059305369854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9687673449516296, + 0.06849174201488495, + 0.09799842536449432, + 0.24157820641994476, + 0.027295714244246483, + 0.012205775827169418, + 0.02014697901904583, + 0.035209864377975464, + 0.015697194263339043 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 0, + "style_probability": 0.7495310306549072, + "style_target": 0, + "support_probability": 0.9646319150924683, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7871761704214989, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021897973492741585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17234523594379425, + 0.4754992723464966, + 0.2844372093677521, + 0.8818774819374084, + 0.011415217071771622, + 0.011154853738844395, + 0.009334211237728596, + 0.0391516275703907, + 0.005114964209496975, + 0.00696531543508172, + 0.00691733043640852 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.9274255633354187, + "style_target": 1, + "support_probability": 0.9803820848464966, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6458417021153943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03514326363801956, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14171625673770905, + 0.425546795129776, + 0.2871089279651642, + 0.8968987464904785, + 0.055654365569353104, + 0.07866518199443817 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.4797571897506714, + "style_target": 0, + "support_probability": 0.9253601431846619, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19144580037275022, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3315175175666809, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1440226435661316, + 0.7466831207275391, + 0.07814370095729828, + 0.22381830215454102, + 0.04587910696864128 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.5745822787284851, + "style_target": 1, + "support_probability": 0.12744905054569244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029772550632491967, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.39354097843170166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1250484138727188, + 0.7590651512145996, + 0.07331434637308121, + 0.2350786328315735, + 0.023587122559547424 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.14155703783035278, + "style_target": 0, + "support_probability": 0.15017226338386536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8580783300374357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03307697921991348, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17188885807991028, + 0.1576574742794037, + 0.9984084963798523, + 0.36153003573417664, + 0.07257937639951706, + 0.03028036653995514 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 1, + "style_probability": 0.8474365472793579, + "style_target": 1, + "support_probability": 0.7400233149528503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7701769692588647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.028709672391414642, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18434377014636993, + 0.23054929077625275, + 0.9940106272697449, + 0.5039147734642029, + 0.12283775210380554, + 0.029634885489940643 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 0, + "style_probability": 0.7818815112113953, + "style_target": 0, + "support_probability": 0.7752132415771484, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008722066168937793, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7391058802604675, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.349002480506897, + 0.28129199147224426, + 0.21733613312244415, + 0.28129199147224426, + 0.06750974804162979, + 0.1170424148440361, + 0.0859249085187912, + 0.13564741611480713, + 0.12547437846660614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.9288918972015381, + "style_target": 1, + "support_probability": 0.009817597456276417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006270780177386871, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7266660928726196, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5064616203308105, + 0.2710047960281372, + 0.2613648772239685, + 0.2710047960281372, + 0.1134805679321289 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.17434050142765045, + "style_target": 0, + "support_probability": 0.011012069880962372, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7685871626851399, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09549426287412643, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5255727171897888, + 0.7442895770072937, + 0.8690621852874756, + 0.6913304328918457, + 0.06614504009485245, + 0.025679485872387886, + 0.01782579906284809, + 0.034635383635759354, + 0.0700688436627388, + 0.07214542478322983, + 0.04604170098900795 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.9493846297264099, + "style_target": 1, + "support_probability": 0.9497401118278503, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2209308769521142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09912145137786865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.43033716082572937, + 0.6433032751083374, + 0.9032620191574097, + 0.6170558929443359, + 0.036991558969020844, + 0.012748314999043941, + 0.03383665159344673, + 0.02210938185453415, + 0.03014625608921051, + 0.07458055019378662 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.08764082938432693, + "style_target": 0, + "support_probability": 0.8987432718276978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4642330456478617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6571935415267944, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3270156979560852, + 0.9672797918319702, + 0.5552949905395508, + 0.054176170378923416, + 0.02081363834440708 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 1, + "style_probability": 0.6325585246086121, + "style_target": 1, + "support_probability": 0.3785189986228943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4702965214455882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37695327401161194, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40106168389320374, + 0.9462341666221619, + 0.6305446028709412, + 0.08430061489343643, + 0.15683580935001373 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 1, + "style_probability": 0.350516140460968, + "style_target": 0, + "support_probability": 0.5728457570075989, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8769788054346432, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04880775883793831, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42665740847587585, + 0.3612150549888611, + 0.279757022857666, + 0.996715784072876, + 0.027967333793640137, + 0.0188693068921566 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 1, + "style_probability": 0.8710470199584961, + "style_target": 1, + "support_probability": 0.8635660409927368, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5087719430363118, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060672253370285034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4099107086658478, + 0.30065831542015076, + 0.2596293091773987, + 0.9929672479629517, + 0.019726401194930077, + 0.032352689653635025, + 0.04369266703724861, + 0.03215214982628822, + 0.03722749650478363, + 0.038028307259082794 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 1, + "style_probability": 0.49012506008148193, + "style_target": 0, + "support_probability": 0.8291978240013123, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6677190841791258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0483926497399807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12170816957950592, + 0.05047812685370445, + 0.15297135710716248, + 0.9712309241294861, + 0.037220969796180725, + 0.0243108868598938, + 0.04427706450223923, + 0.04114212840795517, + 0.018127169460058212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.8949135541915894, + "style_target": 1, + "support_probability": 0.441874623298645, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05275104670815557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07820437848567963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10105103999376297, + 0.03583810478448868, + 0.19035395979881287, + 0.978870153427124, + 0.04732050374150276, + 0.0923992320895195, + 0.0420859195291996, + 0.01753130927681923, + 0.056818630546331406 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.16340775787830353, + "style_target": 0, + "support_probability": 0.4682902991771698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05781163474581857, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8498060703277588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11797500401735306, + 0.790826678276062, + 0.051735639572143555, + 0.5440449118614197, + 0.03469880670309067, + 0.026709679514169693, + 0.036683667451143265, + 0.13905847072601318, + 0.02626722678542137, + 0.12229887396097183, + 0.02028898149728775 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.8755965232849121, + "style_target": 1, + "support_probability": 0.010399105027318, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01672850816663214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7939110398292542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1226167157292366, + 0.7177317142486572, + 0.07599744200706482, + 0.4375198185443878, + 0.020255541428923607, + 0.00885752122849226, + 0.20751158893108368, + 0.034952469170093536, + 0.060210078954696655, + 0.040559589862823486, + 0.062449779361486435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.4401516914367676, + "style_target": 0, + "support_probability": 0.008594281040132046, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7386853439398781, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.137917622923851, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9077097773551941, + 0.05014796555042267, + 0.27676841616630554, + 0.5425068140029907, + 0.008923301473259926 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.8441472053527832, + "style_target": 1, + "support_probability": 0.86238032579422, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5485361238855151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08024284243583679, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.912282407283783, + 0.0634128674864769, + 0.24958060681819916, + 0.6225562691688538, + 0.015963397920131683, + 0.026854094117879868, + 0.04455898702144623, + 0.025175483897328377, + 0.025391239672899246 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.34545671939849854, + "style_target": 0, + "support_probability": 0.9296199679374695, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6188259256241554, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04730106517672539, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41201871633529663, + 0.07828918844461441, + 0.898081362247467, + 0.15347307920455933, + 0.038669403642416, + 0.015378577634692192 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.8691242933273315, + "style_target": 1, + "support_probability": 0.4198785722255707, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.031162412256457488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05416285619139671, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35740357637405396, + 0.0960167944431305, + 0.9315743446350098, + 0.13095465302467346, + 0.027189107611775398, + 0.04222961142659187 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.12001573294401169, + "style_target": 0, + "support_probability": 0.44818830490112305, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9391226394020833, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006258904468268156, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8213:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17670366168022156, + 0.9978179931640625, + 0.3087831735610962, + 0.08966117352247238, + 0.044216934591531754, + 0.038182616233825684, + 0.02200048230588436, + 0.0637924075126648, + 0.030869001522660255 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 1, + "style_probability": 0.8597944974899292, + "style_target": 1, + "support_probability": 0.9766581654548645, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8491935423488137, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012866836041212082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8213:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18956497311592102, + 0.9975742697715759, + 0.25996050238609314, + 0.11890912801027298, + 0.02594796195626259, + 0.05127762630581856, + 0.11088410764932632, + 0.17274866998195648, + 0.04620050638914108 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 1, + "style_probability": 0.3758479058742523, + "style_target": 0, + "support_probability": 0.9684731960296631, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012875336084361916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8341434597969055, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19097375869750977, + 0.40005889534950256, + 0.25904250144958496, + 0.4390234053134918, + 0.024593662470579147, + 0.0454016774892807, + 0.12759704887866974, + 0.01776786893606186, + 0.012997610494494438, + 0.27162495255470276, + 0.013180188834667206 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.6086777448654175, + "style_target": 1, + "support_probability": 0.009591220878064632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012003605817359455, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9535844326019287, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2117164433002472, + 0.47806882858276367, + 0.25754332542419434, + 0.48045408725738525, + 0.07554443925619125, + 0.06547072529792786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.19452811777591705, + "style_target": 0, + "support_probability": 0.01167555432766676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11662554492794526, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.39359432458877563, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16171817481517792, + 0.7485805153846741, + 0.14139224588871002, + 0.375830739736557, + 0.01840502768754959 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.7391666173934937, + "style_target": 1, + "support_probability": 0.07555614411830902, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004753043762549487, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7076427936553955, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17275312542915344, + 0.6521676182746887, + 0.13185890018939972, + 0.21069620549678802, + 0.04038875550031662 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.09688936918973923, + "style_target": 0, + "support_probability": 0.06634984165430069, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9588082379954767, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.008779424242675304, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8231:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4025232493877411, + 0.23784899711608887, + 0.15033937990665436, + 0.9995821118354797, + 0.0739312469959259, + 0.05426262319087982 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 1, + "style_probability": 0.8262983560562134, + "style_target": 1, + "support_probability": 0.9777120351791382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8623618999294385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.008856329135596752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8231:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3928336501121521, + 0.22913260757923126, + 0.15171095728874207, + 0.9987695813179016, + 0.035055167973041534, + 0.02814377099275589 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 1, + "style_probability": 0.4376767575740814, + "style_target": 0, + "support_probability": 0.9839929938316345, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.23228158498412527, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46445325016975403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9789344668388367, + 0.345000296831131, + 0.1524040251970291, + 0.21742816269397736, + 0.07760622352361679, + 0.022708531469106674, + 0.15805980563163757, + 0.04892390966415405, + 0.16364839673042297 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.811833381652832, + "style_target": 1, + "support_probability": 0.13305863738059998, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15524780128472795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40102872252464294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9734551906585693, + 0.36443352699279785, + 0.16465307772159576, + 0.22331218421459198, + 0.042489685118198395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.45037412643432617, + "style_target": 0, + "support_probability": 0.1500953882932663, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8314957539906541, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2971755266189575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2656917870044708, + 0.2253531962633133, + 0.9567562341690063, + 0.14531466364860535, + 0.010738824494183064, + 0.008722162805497646, + 0.028657089918851852, + 0.08783421665430069, + 0.04805156588554382, + 0.017409760504961014 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.8726440668106079, + "style_target": 1, + "support_probability": 0.8203202486038208, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42409010823453414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09774990379810333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20327609777450562, + 0.15784263610839844, + 0.9845907688140869, + 0.12926676869392395, + 0.030871301889419556, + 0.021910464391112328, + 0.024341687560081482, + 0.018812134861946106, + 0.013048689812421799, + 0.032984524965286255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.14310255646705627, + "style_target": 0, + "support_probability": 0.9583871960639954, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21631108242599573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26073017716407776, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09647604823112488, + 0.2204846441745758, + 0.7850189208984375, + 0.5122440457344055, + 0.05880206823348999 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 1, + "style_probability": 0.8629090189933777, + "style_target": 1, + "support_probability": 0.05910969898104668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.208929491516927, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15874260663986206, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08357017487287521, + 0.22233380377292633, + 0.8310827016830444, + 0.4596247673034668, + 0.05304533988237381 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 1, + "style_probability": 0.38808852434158325, + "style_target": 0, + "support_probability": 0.13917703926563263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08180465969236787, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22773925960063934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41600242257118225, + 0.15063129365444183, + 0.43249446153640747, + 0.2461584359407425, + 0.030925629660487175, + 0.0343671552836895 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.8443019986152649, + "style_target": 1, + "support_probability": 0.027913931757211685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030361283596935993, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23856081068515778, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3707963526248932, + 0.172464981675148, + 0.44191774725914, + 0.25615379214286804, + 0.02957153134047985, + 0.018325168639421463, + 0.02229570969939232, + 0.07000456750392914, + 0.031006604433059692, + 0.03180346265435219, + 0.01560419425368309 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.2138492465019226, + "style_target": 0, + "support_probability": 0.03284845128655434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002181862492912007, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9965949654579163, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23005616664886475, + 0.6430293917655945, + 0.16667549312114716, + 0.33233320713043213, + 0.04095594584941864, + 0.06145578250288963, + 0.029901234433054924, + 0.03421126306056976, + 0.06508949398994446 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.7396939396858215, + "style_target": 1, + "support_probability": 0.0008167162886820734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013898215399006768, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9852972626686096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24117906391620636, + 0.7313951253890991, + 0.11061133444309235, + 0.25652000308036804, + 0.03569011017680168, + 0.10545850545167923, + 0.08518215268850327, + 0.021198756992816925, + 0.05175330489873886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.0927584245800972, + "style_target": 0, + "support_probability": 0.006926580332219601, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019107262897453577, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6093348264694214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22037595510482788, + 0.3821627199649811, + 0.25134411454200745, + 0.40264955163002014, + 0.15565268695354462, + 0.05217485874891281, + 0.0543966181576252, + 0.06962549686431885, + 0.07292148470878601, + 0.038710661232471466, + 0.04926328733563423 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.5339323282241821, + "style_target": 1, + "support_probability": 0.0004908606642857194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014556456715881319, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6911305785179138, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24920526146888733, + 0.3973524570465088, + 0.3052329421043396, + 0.43207764625549316, + 0.08501727133989334, + 0.038282305002212524, + 0.03099815919995308, + 0.07346516102552414, + 0.09867562353610992, + 0.09365496784448624, + 0.07689783722162247 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.4010162055492401, + "style_target": 0, + "support_probability": 0.00022653801715932786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30965777320680493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39454716444015503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8464275002479553, + 0.05677087604999542, + 0.030955296009778976, + 0.0766286700963974, + 0.028627697378396988 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.7413082122802734, + "style_target": 1, + "support_probability": 0.22207437455654144, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22329882223156677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32805436849594116, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8705977201461792, + 0.07949156314134598, + 0.029321812093257904, + 0.07928471267223358, + 0.086861252784729, + 0.14783072471618652, + 0.06594932824373245, + 0.11921139061450958, + 0.0659908801317215 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.34157487750053406, + "style_target": 0, + "support_probability": 0.3064010739326477, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007024850971320987, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9706062078475952, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15293268859386444, + 0.5065186023712158, + 0.16024158895015717, + 0.15293268859386444, + 0.03561033308506012, + 0.014211758971214294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.7772152423858643, + "style_target": 1, + "support_probability": 0.0018700965447351336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015669437526563725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9669085144996643, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10138466954231262, + 0.5459023118019104, + 0.11552150547504425, + 0.10138466954231262, + 0.10922878235578537, + 0.24957431852817535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.14865657687187195, + "style_target": 0, + "support_probability": 0.012193271890282631, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8580282969782834, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06275384873151779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.984973669052124, + 0.18142089247703552, + 0.4306075870990753, + 0.18142089247703552, + 0.015201643109321594, + 0.007017229218035936, + 0.012767905369400978, + 0.0279147457331419, + 0.007886240258812904 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 1, + "style_probability": 0.6389970779418945, + "style_target": 1, + "support_probability": 0.9384352564811707, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7623382240783966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11568838357925415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9739637970924377, + 0.26380404829978943, + 0.49953100085258484, + 0.26380404829978943, + 0.08067608624696732, + 0.026415729895234108, + 0.01695389859378338, + 0.026892157271504402, + 0.026591874659061432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 0, + "style_probability": 0.5249871611595154, + "style_target": 0, + "support_probability": 0.9089974761009216, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013308972035971624, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9690143465995789, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19811025261878967, + 0.12126987427473068, + 0.5091807842254639, + 0.19811025261878967, + 0.04534941166639328, + 0.025540297850966454, + 0.023221192881464958, + 0.02535093203186989, + 0.023512620478868484, + 0.028843626379966736, + 0.016461865976452827 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.913766622543335, + "style_target": 1, + "support_probability": 0.005280639976263046, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030338946587222873, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9801363945007324, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17858123779296875, + 0.09659034758806229, + 0.5052049160003662, + 0.17858123779296875, + 0.013411852531135082, + 0.02803986892104149 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.2577314078807831, + "style_target": 0, + "support_probability": 0.001519945333711803, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.757234004394632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05911984294652939, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24854174256324768, + 0.29567545652389526, + 0.07695043832063675, + 0.9506388306617737, + 0.029996465891599655 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.6666868925094604, + "style_target": 1, + "support_probability": 0.8524066209793091, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0342342650932023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2392587512731552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2066996991634369, + 0.25119850039482117, + 0.052532557398080826, + 0.9274550080299377, + 0.04217996820807457 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.04410495236515999, + "style_target": 0, + "support_probability": 0.5306492447853088, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44024357941710957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09869523346424103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11508507281541824, + 0.14201706647872925, + 0.2586246728897095, + 0.9472130537033081, + 0.0759723037481308, + 0.1113213524222374 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 1, + "style_probability": 0.9011563658714294, + "style_target": 1, + "support_probability": 0.41967734694480896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24137287962237802, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12953172624111176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12195923924446106, + 0.18509243428707123, + 0.3256922960281372, + 0.9215086102485657, + 0.12968403100967407, + 0.09059863537549973 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 0, + "style_probability": 0.6724957227706909, + "style_target": 0, + "support_probability": 0.22606922686100006, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0738038852136667, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9177833795547485, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1712133288383484, + 0.13697220385074615, + 0.1927393674850464, + 0.26894402503967285, + 0.08154885470867157, + 0.03918967396020889, + 0.024250738322734833, + 0.18656644225120544, + 0.06014597415924072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.8453416228294373, + "style_target": 1, + "support_probability": 0.061530400067567825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027369512021896022, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.902163565158844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13068503141403198, + 0.1904667168855667, + 0.22515720129013062, + 0.2649978995323181, + 0.11851752549409866 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.16120555996894836, + "style_target": 0, + "support_probability": 0.15694861114025116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07535431021173691, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5876815319061279, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1786271035671234, + 0.1649860441684723, + 0.2937890887260437, + 0.7506932020187378, + 0.07492491602897644, + 0.04223434627056122, + 0.2998610734939575, + 0.04860755056142807, + 0.031519390642642975, + 0.17672991752624512, + 0.1145775094628334 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.8101570010185242, + "style_target": 1, + "support_probability": 0.06357045471668243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01738989407212558, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6051803231239319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12244071066379547, + 0.15336254239082336, + 0.40814724564552307, + 0.8183557391166687, + 0.11049272865056992, + 0.052795182913541794, + 0.09224573522806168, + 0.08417113870382309, + 0.029991500079631805, + 0.0963943600654602 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.15423130989074707, + "style_target": 0, + "support_probability": 0.07160098105669022, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04704439151524065, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7453155517578125, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6914610862731934, + 0.22212667763233185, + 0.23995879292488098, + 0.20075790584087372, + 0.060870587825775146 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 1, + "style_probability": 0.7632485032081604, + "style_target": 1, + "support_probability": 0.010111303068697453, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05881009502451265, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48414310812950134, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6503304839134216, + 0.22691364586353302, + 0.20078834891319275, + 0.22326703369617462, + 0.0310786385089159 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 1, + "style_probability": 0.471897691488266, + "style_target": 0, + "support_probability": 0.04485978186130524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3135048895602332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5280285477638245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.294528603553772, + 0.43891051411628723, + 0.1849260777235031, + 0.8246437907218933, + 0.03270372003316879, + 0.08704162389039993 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.8148843050003052, + "style_target": 1, + "support_probability": 0.16165964305400848, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11037130323655783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5875017046928406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3393042981624603, + 0.5253469347953796, + 0.1833837926387787, + 0.7903294563293457, + 0.030359799042344093, + 0.02797703817486763, + 0.029146350920200348, + 0.012694593518972397, + 0.02397075854241848, + 0.0566822849214077, + 0.14303827285766602 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.4079055190086365, + "style_target": 0, + "support_probability": 0.09060665220022202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0035488292676232555, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9683765172958374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17245250940322876, + 0.31658580899238586, + 0.2815590500831604, + 0.09985846281051636, + 0.019647760316729546, + 0.03050735592842102, + 0.04203268513083458, + 0.032613009214401245, + 0.03783676028251648 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.6216353178024292, + "style_target": 1, + "support_probability": 0.002317039994522929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014885802889295409, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9001700282096863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13992451131343842, + 0.3578694760799408, + 0.231175035238266, + 0.07176677137613297, + 0.030973153188824654, + 0.186997190117836, + 0.04191112518310547, + 0.021705619990825653, + 0.06982152163982391 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.15447579324245453, + "style_target": 0, + "support_probability": 0.005318245384842157, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.628743061191365, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1361418217420578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5481101870536804, + 0.7013208866119385, + 0.4583631157875061, + 0.4250551462173462, + 0.06646785140037537, + 0.036287322640419006, + 0.021864082664251328, + 0.03247666358947754, + 0.018076689913868904, + 0.017069470137357712, + 0.039358530193567276 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 1, + "style_probability": 0.8565933108329773, + "style_target": 1, + "support_probability": 0.9688252210617065, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5089916353745967, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5266914367675781, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5251762270927429, + 0.6352108716964722, + 0.63033127784729, + 0.5417998433113098, + 0.01480902824550867, + 0.016492413356900215, + 0.018943658098578453, + 0.03617219626903534, + 0.008272512815892696, + 0.010465312749147415, + 0.017596401274204254 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 1, + "style_probability": 0.4583486318588257, + "style_target": 0, + "support_probability": 0.9546696543693542, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5374777375419258, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03462095186114311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9193952083587646, + 0.19283275306224823, + 0.5010924339294434, + 0.240256205201149, + 0.03672400489449501 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 1, + "style_probability": 0.5082371830940247, + "style_target": 1, + "support_probability": 0.8666107058525085, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7475686870829155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03508884832262993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9583193063735962, + 0.1741628497838974, + 0.4959012567996979, + 0.19814960658550262, + 0.014530252665281296 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 0, + "style_probability": 0.7729091048240662, + "style_target": 0, + "support_probability": 0.8542805910110474, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.021391465886145454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9313068985939026, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7636301517486572, + 0.3844601809978485, + 0.26767370104789734, + 0.4605180025100708, + 0.1304536759853363, + 0.04291496053338051 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 0, + "style_probability": 0.4928854703903198, + "style_target": 1, + "support_probability": 0.027148008346557617, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0037105015181583675, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9940645098686218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7331840991973877, + 0.33816054463386536, + 0.22645454108715057, + 0.4681597948074341, + 0.045314472168684006, + 0.14084991812705994, + 0.11099683493375778, + 0.1503870189189911, + 0.05249262601137161, + 0.05158799886703491, + 0.05443195998668671 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 1, + "style_probability": 0.26865652203559875, + "style_target": 0, + "support_probability": 0.002455055480822921, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.900554042121212, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019841128960251808, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8384:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9790481328964233, + 0.4673551321029663, + 0.1922970712184906, + 0.3705122470855713, + 0.04342097416520119, + 0.043521251529455185, + 0.016872521489858627, + 0.017489364370703697, + 0.012669583782553673 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.5402407646179199, + "style_target": 1, + "support_probability": 0.9871929883956909, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4439049653875755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03565184399485588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8384:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9915652275085449, + 0.5309820175170898, + 0.20354075729846954, + 0.2425907552242279, + 0.02924102358520031, + 0.03464525565505028, + 0.04019058868288994, + 0.014534547924995422, + 0.0336725078523159 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.0750049501657486, + "style_target": 0, + "support_probability": 0.9557684659957886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007280298671540853, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9492008686065674, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41484516859054565, + 0.3833366334438324, + 0.16110989451408386, + 0.485545814037323, + 0.02310623601078987, + 0.027311258018016815, + 0.09582000970840454, + 0.05820414051413536, + 0.03497724235057831, + 0.11697332561016083, + 0.03788772597908974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 1, + "style_probability": 0.5133742690086365, + "style_target": 1, + "support_probability": 0.004455317743122578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025856310945670457, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.972331702709198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4180186688899994, + 0.3031871020793915, + 0.12632884085178375, + 0.49876660108566284, + 0.06651698052883148, + 0.08050571382045746, + 0.023863535374403, + 0.049522656947374344, + 0.032553404569625854, + 0.04693114385008812, + 0.07167158275842667 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 0, + "style_probability": 0.6876354217529297, + "style_target": 0, + "support_probability": 0.0008379087666980922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000998900859040098, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9969844222068787, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3873119056224823, + 0.2819730341434479, + 0.2768164873123169, + 0.5423557758331299, + 0.07307171821594238 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.5770429372787476, + "style_target": 1, + "support_probability": 0.0004562961112242192, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00014517113061610106, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.997022807598114, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4362400472164154, + 0.17880640923976898, + 0.2873724699020386, + 0.5258411169052124, + 0.09731774032115936 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.03795451670885086, + "style_target": 0, + "support_probability": 0.00045482051791623235, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04571633164348297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.35662713646888733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2564575672149658, + 0.36176151037216187, + 0.3328101634979248, + 0.8823349475860596, + 0.028548508882522583, + 0.07105392962694168 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 1, + "style_probability": 0.5954819917678833, + "style_target": 1, + "support_probability": 0.04434901848435402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12676168235205582, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1789667159318924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3386695384979248, + 0.4075762629508972, + 0.375724732875824, + 0.8852250576019287, + 0.016628088429570198, + 0.050192683935165405 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 0, + "style_probability": 0.5028512477874756, + "style_target": 0, + "support_probability": 0.10723349452018738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5931713639578026, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45356878638267517, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14337287843227386, + 0.8215014934539795, + 0.5438930988311768, + 0.26270633935928345, + 0.09730542451143265, + 0.018274087458848953, + 0.0069713774137198925, + 0.010297744534909725, + 0.05311531573534012 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 1, + "style_probability": 0.8482975959777832, + "style_target": 1, + "support_probability": 0.4680542051792145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4836718767520024, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21428324282169342, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14312826097011566, + 0.8388593792915344, + 0.3841954469680786, + 0.32268470525741577, + 0.016390858218073845 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 1, + "style_probability": 0.46819445490837097, + "style_target": 0, + "support_probability": 0.6432961225509644, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4872359487209018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05679359287023544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15379399061203003, + 0.43692314624786377, + 0.6259034872055054, + 0.9714727401733398, + 0.06701505929231644, + 0.05204661190509796, + 0.04164986312389374, + 0.02804509922862053, + 0.032744765281677246, + 0.09443441033363342 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.5670671463012695, + "style_target": 1, + "support_probability": 0.6051574349403381, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08650816585147691, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06699495017528534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.14991723001003265, + 0.37363022565841675, + 0.6553496718406677, + 0.9735225439071655, + 0.06921131908893585, + 0.022868428379297256, + 0.059020645916461945, + 0.07123105227947235, + 0.045861758291721344, + 0.068802610039711 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.12628352642059326, + "style_target": 0, + "support_probability": 0.5489843487739563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06751752009289531, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6624780297279358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36184605956077576, + 0.22920991480350494, + 0.36019039154052734, + 0.28295260667800903, + 0.02969149313867092 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 0, + "style_probability": 0.4893432557582855, + "style_target": 1, + "support_probability": 0.10682711750268936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00796088334943566, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9159870147705078, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28903666138648987, + 0.20722466707229614, + 0.47424861788749695, + 0.1648227721452713, + 0.029747022315859795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 1, + "style_probability": 0.21169480681419373, + "style_target": 0, + "support_probability": 0.04709438234567642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07516710373915052, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1262865662574768, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34395211935043335, + 0.9201732873916626, + 0.8116849660873413, + 0.12488764524459839, + 0.025400716811418533, + 0.12262672930955887 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 1, + "style_probability": 0.6015533804893494, + "style_target": 1, + "support_probability": 0.027960211038589478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11959551385059983, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19828280806541443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31763941049575806, + 0.9359081983566284, + 0.8010912537574768, + 0.12836192548274994, + 0.02135663665831089, + 0.01211309153586626 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 0, + "style_probability": 0.8324357867240906, + "style_target": 0, + "support_probability": 0.020823366940021515, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20578064193821544, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2961454391479492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7497767210006714, + 0.41914916038513184, + 0.07660574465990067, + 0.3028995394706726, + 0.013741073198616505, + 0.00714581273496151, + 0.019170720130205154, + 0.014059826731681824, + 0.013988438062369823 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 1, + "style_probability": 0.5401198267936707, + "style_target": 1, + "support_probability": 0.14072461426258087, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3560098527562751, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11917922645807266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.802152693271637, + 0.4599708914756775, + 0.057828959077596664, + 0.3825208842754364, + 0.029931219294667244 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 1, + "style_probability": 0.2897217869758606, + "style_target": 0, + "support_probability": 0.36601439118385315, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005528993986237355, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8686429262161255, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7100570201873779, + 0.49676862359046936, + 0.6981611251831055, + 0.26053765416145325, + 0.02606523595750332, + 0.02027874067425728, + 0.01876397244632244, + 0.01902574673295021, + 0.09602338075637817, + 0.054321520030498505, + 0.02562079392373562 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.5591917037963867, + "style_target": 1, + "support_probability": 0.0035486507695168257, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011041013135374627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4642234146595001, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8355138897895813, + 0.5516384840011597, + 0.6976770758628845, + 0.15907269716262817, + 0.05074308067560196, + 0.03701446205377579, + 0.11160482466220856, + 0.02724084071815014, + 0.020019110292196274, + 0.03862876072525978 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.24806609749794006, + "style_target": 0, + "support_probability": 0.08206040412187576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.062222353204029225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.809560239315033, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32998567819595337, + 0.2844001352787018, + 0.03378484398126602, + 0.8045291900634766, + 0.0795762836933136 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.7585174441337585, + "style_target": 1, + "support_probability": 0.09029439091682434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01308004901063665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7684261202812195, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34291863441467285, + 0.22344693541526794, + 0.03154381737112999, + 0.7827854156494141, + 0.1423346996307373, + 0.07615435123443604, + 0.07905521243810654, + 0.06917047500610352, + 0.21764381229877472 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.20538945496082306, + "style_target": 0, + "support_probability": 0.025141213089227676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04904232278852663, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6912274956703186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.485433429479599, + 0.646213948726654, + 0.8819392919540405, + 0.1580342948436737, + 0.16351084411144257, + 0.026790855452418327 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.7262281179428101, + "style_target": 1, + "support_probability": 0.02704887092113495, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033372640600322794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7191661596298218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.330141544342041, + 0.6851839423179626, + 0.8738430142402649, + 0.10537386685609818, + 0.03655698895454407, + 0.062214773148298264 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.1406715363264084, + "style_target": 0, + "support_probability": 0.018147524446249008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023792128425977138, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3404623568058014, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6594135165214539, + 0.3544946610927582, + 0.2824647128582001, + 0.3987663686275482, + 0.05014675110578537, + 0.1249670460820198, + 0.08193601667881012, + 0.033144667744636536, + 0.17331084609031677 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.6947064399719238, + "style_target": 1, + "support_probability": 0.011501273140311241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009843633733392684, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3780936598777771, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5546496510505676, + 0.33612117171287537, + 0.25704482197761536, + 0.40684494376182556, + 0.09900224953889847, + 0.017861900851130486, + 0.05639127269387245, + 0.04568973183631897, + 0.0641031414270401 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.26219668984413147, + "style_target": 0, + "support_probability": 0.008266631513834, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5977922694361433, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07082701474428177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9665665626525879, + 0.06070122495293617, + 0.10595127195119858, + 0.0639946460723877, + 0.011595082469284534, + 0.005254039075225592, + 0.04424716904759407, + 0.02360045723617077, + 0.055279504507780075, + 0.038359932601451874, + 0.0985809788107872 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.7319011092185974, + "style_target": 1, + "support_probability": 0.5523173809051514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3003611144027892, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04695310443639755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9563835263252258, + 0.07927367836236954, + 0.14322900772094727, + 0.06266692280769348, + 0.01764967106282711, + 0.014650529250502586 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.2085096389055252, + "style_target": 0, + "support_probability": 0.718466579914093, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0959274992857123, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8814441561698914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5650622844696045, + 0.4154646694660187, + 0.3718179762363434, + 0.5650622844696045, + 0.018034152686595917 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.8114412426948547, + "style_target": 1, + "support_probability": 0.059473615139722824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00843643738909095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9000403881072998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7158436179161072, + 0.3274668753147125, + 0.44176897406578064, + 0.7158436179161072, + 0.053278811275959015 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.12673024833202362, + "style_target": 0, + "support_probability": 0.06333769857883453, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001296048322269419, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.985798716545105, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23984043300151825, + 0.21229545772075653, + 0.294980525970459, + 0.2687844932079315, + 0.028935382142663002, + 0.033926256000995636 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 1, + "style_probability": 0.5992116332054138, + "style_target": 1, + "support_probability": 0.0011837437050417066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016880342045710393, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9081971049308777, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3053309917449951, + 0.23434622585773468, + 0.45587337017059326, + 0.26980382204055786, + 0.11276231706142426, + 0.03151092305779457 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 1, + "style_probability": 0.3612995445728302, + "style_target": 0, + "support_probability": 0.015124711208045483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7270503433645873, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20922181010246277, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3796130120754242, + 0.3046809136867523, + 0.3796130120754242, + 0.8521011471748352, + 0.007238768041133881, + 0.029958883300423622, + 0.011853030882775784, + 0.009931284002959728, + 0.02111605368554592 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.845410168170929, + "style_target": 1, + "support_probability": 0.8464581966400146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47382268008225836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.231634721159935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35259371995925903, + 0.28695493936538696, + 0.35259371995925903, + 0.8265893459320068, + 0.05226727947592735 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.3494829833507538, + "style_target": 0, + "support_probability": 0.8266289830207825, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6793840882315297, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8178908228874207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12412147969007492, + 0.9624889492988586, + 0.12412147969007492, + 0.25332432985305786, + 0.02471577376127243, + 0.030381763353943825, + 0.0344507098197937, + 0.011941278353333473, + 0.036356329917907715, + 0.04329640045762062, + 0.03486845642328262 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.8172925114631653, + "style_target": 1, + "support_probability": 0.6268580555915833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23651663222338667, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.725831925868988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.10057269036769867, + 0.9830005168914795, + 0.10057269036769867, + 0.35112783312797546, + 0.1230258196592331, + 0.028547868132591248, + 0.24934890866279602, + 0.039574459195137024, + 0.02188967354595661, + 0.07407420128583908 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.2625666856765747, + "style_target": 0, + "support_probability": 0.7469329237937927, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028480485253485045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8897256851196289, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3107381761074066, + 0.548841118812561, + 0.8339086174964905, + 0.3107381761074066, + 0.044244781136512756 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.7045491933822632, + "style_target": 1, + "support_probability": 0.010645090602338314, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02232998910009044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.779578685760498, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2958478629589081, + 0.44366905093193054, + 0.9200407862663269, + 0.2958478629589081, + 0.04117271676659584 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.406841516494751, + "style_target": 0, + "support_probability": 0.04740457236766815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10856113206964843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.48288941383361816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.414325088262558, + 0.2233375459909439, + 0.13950689136981964, + 0.9911508560180664, + 0.043007880449295044, + 0.10007575154304504 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 1, + "style_probability": 0.5274471640586853, + "style_target": 1, + "support_probability": 0.09386543184518814, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19916720349413422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4351692199707031, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3619331121444702, + 0.18780353665351868, + 0.13757994771003723, + 0.9975447058677673, + 0.014428101480007172, + 0.04495828598737717 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 0, + "style_probability": 0.6960180401802063, + "style_target": 0, + "support_probability": 0.1174505427479744, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20826477320415684, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0792846605181694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.207245871424675, + 0.03965115547180176, + 0.4080125093460083, + 0.9641703367233276, + 0.025278644636273384, + 0.07262454926967621, + 0.042145680636167526, + 0.02523965761065483, + 0.03775932639837265 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 0, + "style_probability": 0.3317171335220337, + "style_target": 1, + "support_probability": 0.18189175426959991, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10227794185210497, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11047525703907013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21207208931446075, + 0.054294686764478683, + 0.4437146484851837, + 0.967118501663208, + 0.1263536512851715 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 1, + "style_probability": 0.23781655728816986, + "style_target": 0, + "support_probability": 0.1095200926065445, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03881118120147916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8169653415679932, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6941847205162048, + 0.3470022678375244, + 0.1544615924358368, + 0.3470022678375244, + 0.026249151676893234, + 0.03186110407114029, + 0.09571543335914612, + 0.05933219566941261, + 0.022990116849541664, + 0.059578873217105865 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 0, + "style_probability": 0.4153097867965698, + "style_target": 1, + "support_probability": 0.02765544131398201, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019025144016592437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4088921844959259, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8897187113761902, + 0.22871358692646027, + 0.23114013671875, + 0.22871358692646027, + 0.04413958638906479, + 0.058506738394498825, + 0.11068601906299591, + 0.04099778085947037, + 0.03703960031270981, + 0.08480745553970337 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 1, + "style_probability": 0.16879524290561676, + "style_target": 0, + "support_probability": 0.07470577210187912, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38829146496295536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7643296122550964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38785311579704285, + 0.9691568613052368, + 0.21954865753650665, + 0.2560984492301941, + 0.08227314800024033 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.8872207403182983, + "style_target": 1, + "support_probability": 0.20477263629436493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1805338817590811, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7480387687683105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4103422164916992, + 0.9660842418670654, + 0.23125416040420532, + 0.22617870569229126, + 0.05784028023481369, + 0.04462112858891487, + 0.12897801399230957, + 0.04737980291247368, + 0.10832767188549042 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.43618297576904297, + "style_target": 0, + "support_probability": 0.15980802476406097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6698773301578669, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34689369797706604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9694737195968628, + 0.09817040711641312, + 0.37067362666130066, + 0.09817040711641312, + 0.027522893622517586, + 0.0423123762011528 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.7957426309585571, + "style_target": 1, + "support_probability": 0.5805252194404602, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0724348424952129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4938598871231079, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.979253888130188, + 0.11027891933917999, + 0.3725501000881195, + 0.11027891933917999, + 0.011410736478865147, + 0.040268439799547195 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.15881775319576263, + "style_target": 0, + "support_probability": 0.49684396386146545, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8402741991526395, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10218913853168488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9568966031074524, + 0.41938477754592896, + 0.4671473503112793, + 0.5034294128417969, + 0.0084322365000844, + 0.010441060177981853, + 0.04051891341805458, + 0.023905469104647636, + 0.06027631834149361 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 1, + "style_probability": 0.7866714596748352, + "style_target": 1, + "support_probability": 0.9227787852287292, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5578505780263514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29100674390792847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9178194403648376, + 0.24949966371059418, + 0.3092248737812042, + 0.344468355178833, + 0.04156904295086861, + 0.06827632337808609, + 0.022112000733613968, + 0.03208375349640846, + 0.0677051693201065 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 0, + "style_probability": 0.5223620533943176, + "style_target": 0, + "support_probability": 0.7563701868057251, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8183017505149612, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05463551729917526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9610716104507446, + 0.07110275328159332, + 0.07110275328159332, + 0.1279606819152832, + 0.008467056788504124, + 0.014799799770116806, + 0.012614402920007706, + 0.013452363200485706, + 0.045802708715200424, + 0.02753305435180664, + 0.04558413848280907 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 1, + "style_probability": 0.9102709293365479, + "style_target": 1, + "support_probability": 0.7233661413192749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5790960736746735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03720477223396301, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9634227752685547, + 0.0726502314209938, + 0.0726502314209938, + 0.10599587857723236, + 0.046619873493909836, + 0.02034124545753002 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 1, + "style_probability": 0.42139866948127747, + "style_target": 0, + "support_probability": 0.821386456489563, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8626263036928918, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.023018373176455498, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13901813328266144, + 0.9906044602394104, + 0.10861147195100784, + 0.4192718267440796, + 0.02519005723297596 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.7923908233642578, + "style_target": 1, + "support_probability": 0.9139374494552612, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6130553534885311, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.013711688108742237, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13896675407886505, + 0.9942923784255981, + 0.12407036870718002, + 0.4344325661659241, + 0.05600346252322197 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.25320160388946533, + "style_target": 0, + "support_probability": 0.9443582892417908, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005269920686088171, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.999123752117157, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44696980714797974, + 0.32626938819885254, + 0.4768839478492737, + 0.32626938819885254, + 0.03754911944270134, + 0.039036307483911514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 1, + "style_probability": 0.7287808060646057, + "style_target": 1, + "support_probability": 0.0001083004244719632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0029299833035314983, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9931604862213135, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5872015953063965, + 0.33031922578811646, + 0.5170196890830994, + 0.33031922578811646, + 0.14037084579467773, + 0.056903447955846786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 0, + "style_probability": 0.5382525324821472, + "style_target": 0, + "support_probability": 0.0013380508171394467, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40188342228588764, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23367023468017578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4835764169692993, + 0.5240287780761719, + 0.43463727831840515, + 0.5344560742378235, + 0.009315529838204384, + 0.011155164800584316, + 0.014813892543315887, + 0.03638727217912674, + 0.031142212450504303 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.9158800840377808, + "style_target": 1, + "support_probability": 0.443206250667572, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15925568801809, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1805916726589203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45307621359825134, + 0.46373990178108215, + 0.40058210492134094, + 0.4755861759185791, + 0.0869075357913971 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.4504687786102295, + "style_target": 0, + "support_probability": 0.43607157468795776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45716650598337166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03690579533576965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05025264993309975, + 0.9698136448860168, + 0.3471721410751343, + 0.13090841472148895, + 0.005849300418049097, + 0.0855906754732132, + 0.012500036507844925, + 0.020395226776599884, + 0.017045557498931885, + 0.03664396330714226, + 0.014319050125777721 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.5860275626182556, + "style_target": 1, + "support_probability": 0.5483912825584412, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13071625258578565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031737908720970154, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.06425968557596207, + 0.979395866394043, + 0.33228546380996704, + 0.12018211930990219, + 0.010998876765370369, + 0.024296684190630913, + 0.02436254359781742, + 0.01998046413064003, + 0.007869447581470013, + 0.018789326772093773 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.10543104261159897, + "style_target": 0, + "support_probability": 0.7046425342559814, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7943427811465276, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0904960110783577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21657392382621765, + 0.22230345010757446, + 0.9413678646087646, + 0.3288716971874237, + 0.01999867521226406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 1, + "style_probability": 0.696532666683197, + "style_target": 1, + "support_probability": 0.9281526207923889, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5902067393752656, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23959751427173615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2209751158952713, + 0.19502489268779755, + 0.8942106366157532, + 0.30575260519981384, + 0.048484936356544495 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 1, + "style_probability": 0.48991408944129944, + "style_target": 0, + "support_probability": 0.7949832081794739, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5053265179850291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30855992436408997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.930665135383606, + 0.2639960050582886, + 0.3289560377597809, + 0.12697741389274597, + 0.01812640205025673, + 0.08956944197416306 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.9058631658554077, + "style_target": 1, + "support_probability": 0.24689961969852448, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2003265454124652, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17228640615940094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9393054246902466, + 0.23888985812664032, + 0.3159772753715515, + 0.10388073325157166, + 0.018796492367982864, + 0.20310555398464203, + 0.020272361114621162, + 0.016738714650273323, + 0.022342631593346596, + 0.054153088480234146, + 0.06394921988248825 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.4020228087902069, + "style_target": 0, + "support_probability": 0.2697801887989044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5233533034027339, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09158839285373688, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9852594137191772, + 0.1312214434146881, + 0.09497267007827759, + 0.38039106130599976, + 0.029719874262809753, + 0.026409391313791275, + 0.03123888000845909, + 0.02794121764600277, + 0.028568975627422333 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.6925824284553528, + "style_target": 1, + "support_probability": 0.5771715044975281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10813202957345272, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.042917195707559586, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9901841282844543, + 0.11405722796916962, + 0.0984296202659607, + 0.4481622576713562, + 0.026418335735797882, + 0.015034082345664501, + 0.0605681911110878, + 0.057842057198286057, + 0.14960169792175293 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.1608847826719284, + "style_target": 0, + "support_probability": 0.6081435680389404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18114142723194135, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15914765000343323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12731879949569702, + 0.9644308686256409, + 0.03375517949461937, + 0.2530859410762787, + 0.0926964208483696, + 0.028824269771575928, + 0.034418486058712006, + 0.05167897045612335, + 0.2201089859008789, + 0.05135120078921318 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 1, + "style_probability": 0.8359121084213257, + "style_target": 1, + "support_probability": 0.11108305305242538, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11901527503234116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08992993086576462, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12930165231227875, + 0.9668821692466736, + 0.03932618722319603, + 0.2569039762020111, + 0.02975611574947834, + 0.05272235721349716, + 0.0877622663974762, + 0.02787046879529953, + 0.044923122972249985, + 0.2542267143726349 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 1, + "style_probability": 0.47843098640441895, + "style_target": 0, + "support_probability": 0.21574190258979797, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8342524422426724, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06259574741125107, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14253970980644226, + 0.09507682919502258, + 0.983223021030426, + 0.2641478478908539, + 0.022516503930091858 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 1, + "style_probability": 0.941001832485199, + "style_target": 1, + "support_probability": 0.8763071298599243, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.481575548766995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0959215834736824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14943619072437286, + 0.0737498328089714, + 0.9718155860900879, + 0.22434702515602112, + 0.04251383617520332, + 0.040777623653411865, + 0.06256518512964249, + 0.09673741459846497, + 0.03966711089015007 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 0, + "style_probability": 0.5405662059783936, + "style_target": 0, + "support_probability": 0.7923158407211304, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8221213466531342, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.009415924549102783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2094874233007431, + 0.0338931530714035, + 0.9996961355209351, + 0.1968296319246292, + 0.007097987458109856, + 0.008103321306407452 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.5894058346748352, + "style_target": 1, + "support_probability": 0.9169984459877014, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4844096206559953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.009232668206095695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18397784233093262, + 0.02778681181371212, + 0.9998641014099121, + 0.227992981672287, + 0.009080660529434681, + 0.03090299665927887 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.2377600371837616, + "style_target": 0, + "support_probability": 0.9231650829315186, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6973018047192768, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01014127116650343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18808968365192413, + 0.2094498723745346, + 0.15691465139389038, + 0.9996281862258911, + 0.00646370742470026, + 0.030327869579195976, + 0.008753233589231968, + 0.0071999686770141125, + 0.008292127400636673 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 1, + "style_probability": 0.5732063055038452, + "style_target": 1, + "support_probability": 0.8287006616592407, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5723780989109295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012952159158885479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16540652513504028, + 0.24124056100845337, + 0.18972396850585938, + 0.9986327290534973, + 0.021294616162776947, + 0.01905735582113266, + 0.017878830432891846, + 0.005728462245315313, + 0.0707065761089325 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 1, + "style_probability": 0.4956061542034149, + "style_target": 0, + "support_probability": 0.7224879264831543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014142454603284239, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8970498442649841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31630340218544006, + 0.27807971835136414, + 0.35573872923851013, + 0.2542318105697632, + 0.016121266409754753, + 0.11872587352991104, + 0.06447785347700119, + 0.06403491646051407, + 0.04208638146519661, + 0.03838252276182175, + 0.04471975192427635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.7275571823120117, + "style_target": 1, + "support_probability": 0.0190663430839777, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019573049737833617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9571369886398315, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28873738646507263, + 0.22745564579963684, + 0.3404731750488281, + 0.21972011029720306, + 0.13263770937919617, + 0.1051679402589798 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.24370554089546204, + "style_target": 0, + "support_probability": 0.002115133684128523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20765187463133375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3699093759059906, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40338221192359924, + 0.28235235810279846, + 0.3130911886692047, + 0.9622290730476379, + 0.061651088297367096 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.8288280963897705, + "style_target": 1, + "support_probability": 0.08774729818105698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0021394593800819614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5511986017227173, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3160741627216339, + 0.26462578773498535, + 0.22474996745586395, + 0.9481683969497681, + 0.09405766427516937 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.052094873040914536, + "style_target": 0, + "support_probability": 0.014733105897903442, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11231497489973157, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5599294900894165, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40229111909866333, + 0.4900898337364197, + 0.13690884411334991, + 0.36819374561309814, + 0.05420846864581108, + 0.07720354944467545 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.8219628930091858, + "style_target": 1, + "support_probability": 0.09447702765464783, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028015033836014377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5407816171646118, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4760071337223053, + 0.3557755947113037, + 0.14059799909591675, + 0.3105562627315521, + 0.10243473201990128, + 0.05576726421713829 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.3140524923801422, + "style_target": 0, + "support_probability": 0.06925954669713974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07527059643375215, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6902223825454712, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2238292396068573, + 0.9161229729652405, + 0.19023118913173676, + 0.20493870973587036, + 0.06988443434238434, + 0.1466119885444641, + 0.05471270531415939, + 0.03468090668320656, + 0.199386328458786 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 1, + "style_probability": 0.844578742980957, + "style_target": 1, + "support_probability": 0.01716332696378231, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06848971661499408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5102404356002808, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28150010108947754, + 0.8996007442474365, + 0.237538143992424, + 0.20822270214557648, + 0.08017667382955551 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 1, + "style_probability": 0.4900183379650116, + "style_target": 0, + "support_probability": 0.0362795852124691, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5740835546263081, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4851085841655731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3265119194984436, + 0.2198692411184311, + 0.3265119194984436, + 0.6426127552986145, + 0.025104884058237076, + 0.011487489566206932, + 0.018492987379431725, + 0.00738211115822196, + 0.009615289978682995, + 0.012186341919004917 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.8646708726882935, + "style_target": 1, + "support_probability": 0.8366368412971497, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08155052573665401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5174471735954285, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17477141320705414, + 0.1335337907075882, + 0.17477141320705414, + 0.7513325214385986, + 0.03256755322217941, + 0.03951411321759224, + 0.047646041959524155, + 0.015224953182041645, + 0.02525985985994339, + 0.04688825085759163 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.07880499213933945, + "style_target": 0, + "support_probability": 0.772820234298706, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008816036481459158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14749372005462646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9590631723403931, + 0.5850386023521423, + 0.22917385399341583, + 0.23368267714977264, + 0.029326528310775757 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.609494149684906, + "style_target": 1, + "support_probability": 0.004134358838200569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02673378324927197, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07615683227777481, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9545058608055115, + 0.5903907418251038, + 0.2818912863731384, + 0.2957898676395416, + 0.02237883396446705 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.49502673745155334, + "style_target": 0, + "support_probability": 0.020863374695181847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00538818646353272, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9457325339317322, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22655346989631653, + 0.7390298247337341, + 0.8746999502182007, + 0.21765729784965515, + 0.09296990931034088, + 0.04985218122601509 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.7682707905769348, + "style_target": 1, + "support_probability": 0.0013230823678895831, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002983264237111365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9835451245307922, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28780511021614075, + 0.7432762384414673, + 0.853586733341217, + 0.2260400801897049, + 0.08281269669532776, + 0.04549269750714302, + 0.043971139937639236, + 0.0458197295665741, + 0.11113511025905609, + 0.07367124408483505, + 0.07133302837610245 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.42472967505455017, + "style_target": 0, + "support_probability": 0.00042156866402365267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.435614326515771, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31075987219810486, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6123836636543274, + 0.24189139902591705, + 0.4204773008823395, + 0.6486884355545044, + 0.03250911086797714, + 0.01912464201450348, + 0.028164945542812347, + 0.027260513976216316, + 0.030300473794341087 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.9049432873725891, + "style_target": 1, + "support_probability": 0.2917267382144928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013108943415482802, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1920730173587799, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6534223556518555, + 0.3491673171520233, + 0.3196926712989807, + 0.6739780306816101, + 0.09334821254014969, + 0.10447000712156296, + 0.046601925045251846, + 0.03182169795036316, + 0.06315142661333084 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.11048848927021027, + "style_target": 0, + "support_probability": 0.15920338034629822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5802975671054452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07228801399469376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39015278220176697, + 0.9829180836677551, + 0.5772756934165955, + 0.21459056437015533, + 0.040477972477674484, + 0.013140815310180187, + 0.0123817203566432, + 0.005862780846655369, + 0.018769782036542892, + 0.02536645531654358, + 0.017268847674131393 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 1, + "style_probability": 0.8500158190727234, + "style_target": 1, + "support_probability": 0.3742833137512207, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25855555894866744, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15384849905967712, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3906421661376953, + 0.9689642190933228, + 0.6877827644348145, + 0.19864517450332642, + 0.01881847344338894, + 0.02763931080698967, + 0.008598300628364086, + 0.05687117949128151, + 0.0681840032339096, + 0.034920834004879, + 0.013666692189872265 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 0, + "style_probability": 0.758285641670227, + "style_target": 0, + "support_probability": 0.08413449674844742, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.25869698505544164, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6524450778961182, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5687358975410461, + 0.5912712812423706, + 0.5190269947052002, + 0.21186897158622742, + 0.11424006521701813 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 0, + "style_probability": 0.38953909277915955, + "style_target": 1, + "support_probability": 0.6419543623924255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6033995140055871, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27444660663604736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.607765793800354, + 0.6753284931182861, + 0.5830830931663513, + 0.2645857036113739, + 0.022838681936264038 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 0, + "style_probability": 0.9210168719291687, + "style_target": 0, + "support_probability": 0.8940250873565674, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026855349710259413, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9921603798866272, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26954013109207153, + 0.21270672976970673, + 0.20297595858573914, + 0.2609569728374481, + 0.03571505472064018, + 0.06856169551610947 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 0, + "style_probability": 0.47604960203170776, + "style_target": 1, + "support_probability": 0.0020669016521424055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003284284256213439, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9775430560112, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.23643964529037476, + 0.22713667154312134, + 0.23086626827716827, + 0.2148430347442627, + 0.056353937834501266, + 0.03802923113107681, + 0.0658634752035141, + 0.019640179350972176, + 0.0716240257024765, + 0.12563814222812653, + 0.06325235962867737, + 0.06089356541633606 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 1, + "style_probability": 0.34129470586776733, + "style_target": 0, + "support_probability": 0.0024228952825069427, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5685898063508037, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08608522266149521, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32411208748817444, + 0.3826078772544861, + 0.15279871225357056, + 0.9911034107208252, + 0.04994853213429451, + 0.17134273052215576, + 0.08475968986749649, + 0.03526568412780762, + 0.1206769198179245 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 1, + "style_probability": 0.6452920436859131, + "style_target": 1, + "support_probability": 0.5848413109779358, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09432846417887382, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0882544070482254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1900293529033661, + 0.3144823908805847, + 0.16834944486618042, + 0.9972731471061707, + 0.03705810010433197, + 0.1050221249461174, + 0.08714710921049118, + 0.055103883147239685, + 0.23816300928592682 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 1, + "style_probability": 0.1594037413597107, + "style_target": 0, + "support_probability": 0.5348914861679077, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6229586889202748, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05423000082373619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9874798655509949, + 0.1371668428182602, + 0.1371668428182602, + 0.3094750940799713, + 0.0252117570489645, + 0.02673656865954399, + 0.04000590741634369, + 0.009154320694506168, + 0.014953524805605412, + 0.10350720584392548 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 1, + "style_probability": 0.5072029232978821, + "style_target": 1, + "support_probability": 0.8304648995399475, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8551198966734075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04390399530529976, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9910997748374939, + 0.1505858600139618, + 0.1505858600139618, + 0.3708491325378418, + 0.051197201013565063, + 0.04166810214519501, + 0.17170777916908264, + 0.2797107994556427, + 0.051286015659570694, + 0.014829169027507305 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 0, + "style_probability": 0.8836942315101624, + "style_target": 0, + "support_probability": 0.8725650906562805, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07776541912014519, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7690639495849609, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30628761649131775, + 0.6450123190879822, + 0.2764848470687866, + 0.33524009585380554, + 0.1821393519639969 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.8181363344192505, + "style_target": 1, + "support_probability": 0.054699674248695374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006064108522709659, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9472610950469971, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26620468497276306, + 0.4401922821998596, + 0.38661953806877136, + 0.3267486095428467, + 0.1249941885471344 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.11144640296697617, + "style_target": 0, + "support_probability": 0.008185455575585365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6071720102885124, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06940235197544098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.99090975522995, + 0.7527701258659363, + 0.16004852950572968, + 0.4370652437210083, + 0.05864270403981209, + 0.1018977239727974 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 0, + "style_probability": 0.4951278865337372, + "style_target": 1, + "support_probability": 0.6832476258277893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.516830856210182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06480800360441208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9808233976364136, + 0.6919333338737488, + 0.21163329482078552, + 0.40930482745170593, + 0.02583787776529789, + 0.023689990863204002 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 0, + "style_probability": 0.5246815085411072, + "style_target": 0, + "support_probability": 0.48847678303718567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22304956144808752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6795451045036316, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24367496371269226, + 0.06868250668048859, + 0.4164727032184601, + 0.9772162437438965, + 0.08203121274709702, + 0.09319821745157242, + 0.03331432864069939, + 0.021991178393363953, + 0.05130615457892418 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 1, + "style_probability": 0.7376118898391724, + "style_target": 1, + "support_probability": 0.21371415257453918, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0756628871354678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7546004056930542, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.233779639005661, + 0.08571228384971619, + 0.4481222331523895, + 0.9712492823600769, + 0.08729249238967896 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 1, + "style_probability": 0.33381038904190063, + "style_target": 0, + "support_probability": 0.142311230301857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6135019448098353, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13630960881710052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22680407762527466, + 0.9913530945777893, + 0.4486309885978699, + 0.5562188625335693, + 0.0534370131790638, + 0.08023083955049515, + 0.08928825706243515, + 0.03251027315855026, + 0.07775474339723587, + 0.024882283061742783, + 0.06294418126344681 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 1, + "style_probability": 0.8110095858573914, + "style_target": 1, + "support_probability": 0.3137151300907135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08247207160624398, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1830148994922638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19517530500888824, + 0.9919446110725403, + 0.3034595251083374, + 0.4763888716697693, + 0.03644595295190811, + 0.01321408897638321, + 0.06623053550720215, + 0.04745924472808838, + 0.02569723315536976, + 0.14697560667991638 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 1, + "style_probability": 0.3048308193683624, + "style_target": 0, + "support_probability": 0.17017796635627747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02416729099923081, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8364771008491516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1506681889295578, + 0.8978207111358643, + 0.17024490237236023, + 0.09231013804674149, + 0.070516437292099 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 0, + "style_probability": 0.48790663480758667, + "style_target": 1, + "support_probability": 0.02430814318358898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007123117807113041, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9486798644065857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15932701528072357, + 0.8861401677131653, + 0.15476034581661224, + 0.0905575081706047, + 0.10734861344099045 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 1, + "style_probability": 0.11620907485485077, + "style_target": 0, + "support_probability": 0.004132853355258703, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7331880427500898, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26015111804008484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3375537693500519, + 0.3991326689720154, + 0.3395925462245941, + 0.8809152245521545, + 0.04024985805153847, + 0.020481139421463013 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 1, + "style_probability": 0.6403350234031677, + "style_target": 1, + "support_probability": 0.9799632430076599, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.850597185603434, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0811053141951561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32412463426589966, + 0.3596959412097931, + 0.23319558799266815, + 0.9364895820617676, + 0.016525184735655785, + 0.02132541872560978 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 0, + "style_probability": 0.9333429336547852, + "style_target": 0, + "support_probability": 0.980510413646698, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1399290330404126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8637782335281372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8073514699935913, + 0.32555997371673584, + 0.3991146683692932, + 0.11401095986366272, + 0.05177284777164459, + 0.02599945478141308, + 0.01632574014365673, + 0.026802146807312965, + 0.02108161337673664 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.5403052568435669, + "style_target": 1, + "support_probability": 0.09461609274148941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07560854797052702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9232578277587891, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7912790775299072, + 0.2642434239387512, + 0.4272621273994446, + 0.0879150852560997, + 0.06328456103801727 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.3383745849132538, + "style_target": 0, + "support_probability": 0.046448662877082825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02261876765577169, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9597880244255066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20743359625339508, + 0.9508928060531616, + 0.2813050448894501, + 0.21386787295341492, + 0.06756643950939178, + 0.09153583645820618, + 0.0920395702123642, + 0.14018221199512482, + 0.025099022313952446, + 0.031449951231479645, + 0.0415959507226944 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.5400604605674744, + "style_target": 1, + "support_probability": 0.011269627138972282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02942588620837945, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5844790935516357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18940509855747223, + 0.9745955467224121, + 0.1614595502614975, + 0.17885877192020416, + 0.08610167354345322, + 0.07249072939157486, + 0.26023024320602417, + 0.041189733892679214, + 0.04408799856901169, + 0.10069315880537033 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.2073669731616974, + "style_target": 0, + "support_probability": 0.17473001778125763, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7391618384266244, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.036939263343811035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4853328764438629, + 0.8668487668037415, + 0.26285362243652344, + 0.5058140158653259, + 0.024820473045110703 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.8826315402984619, + "style_target": 1, + "support_probability": 0.8059417009353638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2140663972704665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0980941578745842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3275874853134155, + 0.7934677600860596, + 0.24683573842048645, + 0.4641736149787903, + 0.019123822450637817, + 0.040767643600702286, + 0.02927447482943535, + 0.03181036561727524, + 0.025574440136551857 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.1524547040462494, + "style_target": 0, + "support_probability": 0.5069814920425415, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7714142062371327, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027324307709932327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9770137667655945, + 0.11777880787849426, + 0.5833736062049866, + 0.3041561543941498, + 0.09452254325151443, + 0.08449256420135498 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.8430562019348145, + "style_target": 1, + "support_probability": 0.7511810064315796, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23166257794951645, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008590049110352993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9964794516563416, + 0.1214287206530571, + 0.7806132435798645, + 0.4145984947681427, + 0.11210884153842926, + 0.1614382117986679 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.10632507503032684, + "style_target": 0, + "support_probability": 0.8032575249671936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.003224346932552248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9413550496101379, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7571645379066467, + 0.30474933981895447, + 0.3283578157424927, + 0.2575954794883728, + 0.16368496417999268, + 0.08154475688934326, + 0.19002394378185272, + 0.04206741973757744, + 0.11266091465950012 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 1, + "style_probability": 0.7413434386253357, + "style_target": 1, + "support_probability": 0.0014368054689839482, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0022494589318513987, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8600978851318359, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5551864504814148, + 0.34357473254203796, + 0.3982023596763611, + 0.27726516127586365, + 0.07465076446533203, + 0.1418536752462387, + 0.07869654148817062, + 0.028451140969991684, + 0.04625770449638367 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 0, + "style_probability": 0.6070668697357178, + "style_target": 0, + "support_probability": 0.0009871479123830795, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20852378673474986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5094301700592041, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.48145267367362976, + 0.6692078709602356, + 0.3784908354282379, + 0.48145267367362976, + 0.08229546248912811, + 0.02797243371605873, + 0.024295147508382797, + 0.13754913210868835, + 0.04308704286813736, + 0.03611554950475693 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 1, + "style_probability": 0.8820427656173706, + "style_target": 1, + "support_probability": 0.17375968396663666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2353786309529171, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5103856325149536, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5121164917945862, + 0.6479884386062622, + 0.3575814962387085, + 0.5121164917945862, + 0.03304288908839226, + 0.05206511914730072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 0, + "style_probability": 0.6619523167610168, + "style_target": 0, + "support_probability": 0.31843340396881104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21330242267672914, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8239765763282776, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45653432607650757, + 0.16259098052978516, + 0.1910366415977478, + 0.16259098052978516, + 0.019335368648171425 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.6999287009239197, + "style_target": 1, + "support_probability": 0.20542658865451813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00872549450903068, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8355320692062378, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3856145441532135, + 0.26486387848854065, + 0.26027753949165344, + 0.26486387848854065, + 0.06690600514411926 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.04105955362319946, + "style_target": 0, + "support_probability": 0.17662304639816284, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28208565372037003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17301861941814423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19265785813331604, + 0.24264875054359436, + 0.2062065154314041, + 0.7740004658699036, + 0.015729887410998344, + 0.02901717834174633 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 1, + "style_probability": 0.8605954647064209, + "style_target": 1, + "support_probability": 0.072660431265831, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2074569881709165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17936594784259796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21140870451927185, + 0.2642098367214203, + 0.2815495431423187, + 0.7613292932510376, + 0.007100982125848532, + 0.03757339343428612 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 0, + "style_probability": 0.5696738958358765, + "style_target": 0, + "support_probability": 0.12111160159111023, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8209195473194132, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16642749309539795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5962013006210327, + 0.13935379683971405, + 0.9791494011878967, + 0.23693963885307312, + 0.03428123518824577, + 0.008662516251206398, + 0.02760366164147854, + 0.02190799079835415, + 0.04525674879550934 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.8712981939315796, + "style_target": 1, + "support_probability": 0.7642064690589905, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5878800039397725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21214266121387482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5219283699989319, + 0.1312270611524582, + 0.972426176071167, + 0.19589412212371826, + 0.05363881587982178 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.47492414712905884, + "style_target": 0, + "support_probability": 0.7920315861701965, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9295468109349576, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023263724520802498, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8822:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.04254244640469551, + 0.3996303081512451, + 0.9956911206245422, + 0.28729498386383057, + 0.037333518266677856, + 0.02259982004761696, + 0.0314282663166523, + 0.02501283586025238, + 0.14318567514419556, + 0.06298688799142838, + 0.04079675301909447 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.8932023644447327, + "style_target": 1, + "support_probability": 0.9524431824684143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32694469872582865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.046347733587026596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8822:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.0696120634675026, + 0.4181246757507324, + 0.9994200468063354, + 0.24974723160266876, + 0.10151604562997818, + 0.1071251854300499, + 0.048405420035123825, + 0.16107800602912903, + 0.17845146358013153, + 0.20112892985343933 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.17071665823459625, + "style_target": 0, + "support_probability": 0.9014310240745544, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06611418552988145, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.288078248500824, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36736682057380676, + 0.18648017942905426, + 0.5085033774375916, + 0.154196098446846, + 0.01849413849413395 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 1, + "style_probability": 0.6755399703979492, + "style_target": 1, + "support_probability": 0.0916518121957779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016448713910919545, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47066405415534973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41866162419319153, + 0.1447085440158844, + 0.40510985255241394, + 0.15778885781764984, + 0.046564165502786636 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 1, + "style_probability": 0.32390910387039185, + "style_target": 0, + "support_probability": 0.03473830968141556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39563315170242447, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22527015209197998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08467563986778259, + 0.6743482947349548, + 0.09363982826471329, + 0.08467563986778259, + 0.052516061812639236, + 0.02547590620815754 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.7967441082000732, + "style_target": 1, + "support_probability": 0.23303279280662537, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08166289825430718, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3272148668766022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09651021659374237, + 0.6931383013725281, + 0.08920504152774811, + 0.09651021659374237, + 0.0368332713842392, + 0.11251945793628693, + 0.06280123442411423, + 0.030910780653357506, + 0.19180509448051453, + 0.0709899365901947, + 0.05437460169196129 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.32817384600639343, + "style_target": 0, + "support_probability": 0.07983236759901047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8086641055285594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03339828923344612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22518882155418396, + 0.9723507761955261, + 0.07051659375429153, + 0.21416179835796356, + 0.03014325723052025, + 0.021554915234446526, + 0.03802678734064102, + 0.02226504124701023, + 0.042939841747283936 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.9198072552680969, + "style_target": 1, + "support_probability": 0.8510224223136902, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31431303138814526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022903313860297203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2552972137928009, + 0.9905215501785278, + 0.09039090573787689, + 0.20082104206085205, + 0.12328536063432693, + 0.04693065211176872, + 0.017255354672670364, + 0.031476642936468124, + 0.06849425286054611 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.21707643568515778, + "style_target": 0, + "support_probability": 0.9303008913993835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9292829633997954, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.001943312818184495, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8835:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2034142017364502, + 0.14261293411254883, + 0.09094980359077454, + 0.9809017777442932, + 0.015642056241631508, + 0.010900286957621574, + 0.018857698887586594, + 0.013181337155401707, + 0.0325106717646122, + 0.011198789812624454, + 0.014519182033836842 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 1, + "style_probability": 0.919362485408783, + "style_target": 1, + "support_probability": 0.9993502497673035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8811139332925825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0023916412610560656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8835:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2897753417491913, + 0.25292080640792847, + 0.1173999011516571, + 0.9728862047195435, + 0.02633100189268589, + 0.023139137774705887, + 0.03745246306061745, + 0.14436258375644684, + 0.029267119243741035, + 0.033487509936094284 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 0, + "style_probability": 0.6080062985420227, + "style_target": 0, + "support_probability": 0.9989373087882996, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3065938695636335, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6164472699165344, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3168603479862213, + 0.41638997197151184, + 0.6519238352775574, + 0.2583865821361542, + 0.014977972954511642 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.8176261782646179, + "style_target": 1, + "support_probability": 0.19452625513076782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018836519311983404, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8254561424255371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2605088949203491, + 0.4608071446418762, + 0.6155207753181458, + 0.24422980844974518, + 0.024421311914920807, + 0.025469643995165825, + 0.020352208986878395, + 0.022022567689418793, + 0.04016399011015892 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.36966997385025024, + "style_target": 0, + "support_probability": 0.023792563006281853, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22334804825169643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2373507022857666, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09388650208711624, + 0.29202061891555786, + 0.19499564170837402, + 0.7799610495567322, + 0.1011580377817154, + 0.005813270341604948 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.5894179344177246, + "style_target": 1, + "support_probability": 0.3139837682247162, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007120766424196632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3290875554084778, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07387056201696396, + 0.2871670424938202, + 0.20994752645492554, + 0.8135609030723572, + 0.01783612184226513, + 0.04102553427219391 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.06514682620763779, + "style_target": 0, + "support_probability": 0.15899257361888885, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6892849358496815, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0485382117331028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3487411439418793, + 0.8714885711669922, + 0.1621955931186676, + 0.10229389369487762, + 0.010247276164591312, + 0.03187856078147888, + 0.010329210199415684, + 0.005692642647773027, + 0.004896847531199455 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.7454860210418701, + "style_target": 1, + "support_probability": 0.8767322301864624, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5418821623337713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09668667614459991, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39241623878479004, + 0.8223230838775635, + 0.2349158376455307, + 0.0981905534863472, + 0.019073670729994774, + 0.052180685102939606, + 0.023520953953266144, + 0.01053625252097845, + 0.052333880215883255 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.4706357717514038, + "style_target": 0, + "support_probability": 0.8875234723091125, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8827659705075201, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044735778123140335, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2122184932231903, + 0.1386221945285797, + 0.18265917897224426, + 0.9858507513999939, + 0.045241937041282654, + 0.05337046459317207, + 0.02453121729195118, + 0.05560402199625969, + 0.030963702127337456, + 0.03289797157049179, + 0.016104528680443764 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.848602831363678, + "style_target": 1, + "support_probability": 0.9493350386619568, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.517831385037331, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11252393573522568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2011004537343979, + 0.12759776413440704, + 0.1860925704240799, + 0.9785378575325012, + 0.0674605518579483, + 0.04075576737523079 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.3272964656352997, + "style_target": 0, + "support_probability": 0.8691808581352234, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41456846496260796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5620542764663696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2863568663597107, + 0.9481447339057922, + 0.13938045501708984, + 0.16739527881145477, + 0.0627596527338028 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.9283331036567688, + "style_target": 1, + "support_probability": 0.18427222967147827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0090075960965522, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7475765943527222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2599528133869171, + 0.933307409286499, + 0.1373068243265152, + 0.08494894206523895, + 0.16330155730247498 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.07599440962076187, + "style_target": 0, + "support_probability": 0.10900186002254486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40694863934873166, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.34905344247817993, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3993126451969147, + 0.0680774375796318, + 0.32214149832725525, + 0.6315313577651978, + 0.01915483921766281, + 0.024568762630224228 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.8930278420448303, + "style_target": 1, + "support_probability": 0.211081400513649, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10011947423699608, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.47385305166244507, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3445538878440857, + 0.06122003495693207, + 0.3610106408596039, + 0.5348674058914185, + 0.010922069661319256, + 0.028478166088461876 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.31835922598838806, + "style_target": 0, + "support_probability": 0.1873120665550232, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33959070409298836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05862035974860191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1224256306886673, + 0.4290015995502472, + 0.9459035992622375, + 0.46164968609809875, + 0.02687445469200611, + 0.0292621199041605, + 0.01318121887743473, + 0.02931157872080803, + 0.03787769749760628 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.7223019003868103, + "style_target": 1, + "support_probability": 0.2512343227863312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31788056868268105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.022657101973891258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17982670664787292, + 0.4332655072212219, + 0.946359395980835, + 0.44774895906448364, + 0.036555904895067215 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.3875051736831665, + "style_target": 0, + "support_probability": 0.48626431822776794, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8960195825749072, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02697867900133133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9813971519470215, + 0.228371262550354, + 0.26534727215766907, + 0.228371262550354, + 0.03077756054699421, + 0.01862022839486599, + 0.02096273936331272, + 0.05057162791490555, + 0.022397253662347794, + 0.07942067831754684, + 0.04911090061068535 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.8238739371299744, + "style_target": 1, + "support_probability": 0.979889988899231, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14828269337592512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03513025864958763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9878144264221191, + 0.14932475984096527, + 0.17447587847709656, + 0.14932475984096527, + 0.03490385785698891, + 0.04639536514878273, + 0.05675482749938965, + 0.07508094608783722, + 0.042822107672691345, + 0.06249053776264191 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.06409753859043121, + "style_target": 0, + "support_probability": 0.9461120367050171, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5036542106422942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.43425220251083374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15720263123512268, + 0.5014784932136536, + 0.987725555896759, + 0.16194677352905273, + 0.05769634246826172 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 1, + "style_probability": 0.8164108991622925, + "style_target": 1, + "support_probability": 0.3553505837917328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5942623563273304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.18972304463386536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12465468049049377, + 0.6407831311225891, + 0.9735555052757263, + 0.25566643476486206, + 0.05021054297685623 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 0, + "style_probability": 0.5608832240104675, + "style_target": 0, + "support_probability": 0.7911466360092163, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8791340740356506, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.013504385948181152, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3289047181606293, + 0.09502234309911728, + 0.9841361045837402, + 0.1317787766456604, + 0.018539998680353165, + 0.032900385558605194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.8817939162254333, + "style_target": 1, + "support_probability": 0.8764441013336182, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.502383654628966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.013610991649329662, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36295124888420105, + 0.10133969038724899, + 0.9727602601051331, + 0.14833258092403412, + 0.037822872400283813, + 0.0155215198174119, + 0.08616326749324799, + 0.024740492925047874, + 0.010293745435774326, + 0.0379633903503418, + 0.03575972467660904 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.33487963676452637, + "style_target": 0, + "support_probability": 0.7502950429916382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8889494545881133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.035235557705163956, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41119691729545593, + 0.349863201379776, + 0.9788113236427307, + 0.5098890066146851, + 0.042220037430524826, + 0.054947465658187866, + 0.02210245467722416, + 0.03229191526770592, + 0.032502707093954086 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.8761208057403564, + "style_target": 1, + "support_probability": 0.8598974943161011, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5050401593780016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017294825986027718, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4177449643611908, + 0.31150439381599426, + 0.9928869605064392, + 0.48588892817497253, + 0.03161770850419998, + 0.049427330493927, + 0.02228451333940029, + 0.027094641700387, + 0.02726234868168831 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.23395739495754242, + "style_target": 0, + "support_probability": 0.8607914447784424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5282668380777574, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27568894624710083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3060366213321686, + 0.2569379508495331, + 0.27716064453125, + 0.7401002049446106, + 0.02684921957552433, + 0.011644172482192516, + 0.031525880098342896, + 0.06951960921287537, + 0.013988549821078777, + 0.028544336557388306, + 0.06431637704372406 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.8827238082885742, + "style_target": 1, + "support_probability": 0.43922632932662964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18987776372741746, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4514869749546051, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27898234128952026, + 0.20703697204589844, + 0.2692904472351074, + 0.7092384099960327, + 0.019174855202436447, + 0.017872728407382965, + 0.05925687402486801, + 0.026052214205265045, + 0.030411705374717712, + 0.047801364213228226 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.43359294533729553, + "style_target": 0, + "support_probability": 0.3396097719669342, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.75517465271544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16038264334201813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9060255885124207, + 0.8884110450744629, + 0.17528608441352844, + 0.9060255885124207, + 0.08410991728305817 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 1, + "style_probability": 0.8887289762496948, + "style_target": 1, + "support_probability": 0.5119271278381348, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38080892145728085, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2221028357744217, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.886715292930603, + 0.8750903010368347, + 0.13763560354709625, + 0.886715292930603, + 0.042552705854177475, + 0.16714802384376526, + 0.03282385319471359, + 0.08431436121463776, + 0.04570258408784866 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 1, + "style_probability": 0.44431355595588684, + "style_target": 0, + "support_probability": 0.3929136097431183, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25202743027323127, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10590619593858719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7054577469825745, + 0.08858176320791245, + 0.41668805480003357, + 0.21180139482021332, + 0.032708946615457535, + 0.12048451602458954 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.7207874059677124, + "style_target": 1, + "support_probability": 0.15181797742843628, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02753313086939895, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19083766639232635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7291722893714905, + 0.10987541079521179, + 0.3737073242664337, + 0.21208547055721283, + 0.025432046502828598, + 0.048856738954782486 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.26757970452308655, + "style_target": 0, + "support_probability": 0.14653894305229187, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8809758419310516, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010439745150506496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9713594913482666, + 0.6592453718185425, + 0.20453909039497375, + 0.39763498306274414, + 0.01438848115503788, + 0.010616843588650227, + 0.01988772489130497, + 0.01947180926799774, + 0.03137846663594246 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.7917625904083252, + "style_target": 1, + "support_probability": 0.9865212440490723, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6300095924793182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017337985336780548, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9496622681617737, + 0.6040229797363281, + 0.20885241031646729, + 0.40738198161125183, + 0.059808745980262756, + 0.02894544042646885, + 0.03542407229542732, + 0.03020605817437172, + 0.05079587548971176 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.40487974882125854, + "style_target": 0, + "support_probability": 0.9619934558868408, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5058687292054236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03549138084053993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33700665831565857, + 0.9284137487411499, + 0.594757080078125, + 0.17533493041992188, + 0.016812622547149658, + 0.014158781617879868, + 0.014834269881248474, + 0.01555612962692976, + 0.03156786039471626, + 0.023810651153326035, + 0.032387930899858475 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.7926878333091736, + "style_target": 1, + "support_probability": 0.5942237377166748, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19779406872623803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0499253012239933, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3082617521286011, + 0.9277737140655518, + 0.641788899898529, + 0.20340318977832794, + 0.05204087123274803, + 0.15036992728710175 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.37575477361679077, + "style_target": 0, + "support_probability": 0.5577605366706848, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14859819384392906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39657801389694214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8452432155609131, + 0.264195054769516, + 0.4079442322254181, + 0.264195054769516, + 0.1502072662115097 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 1, + "style_probability": 0.8790794610977173, + "style_target": 1, + "support_probability": 0.014841030351817608, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05551527717851008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.390889972448349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8653904795646667, + 0.15374253690242767, + 0.409837007522583, + 0.15374253690242767, + 0.07044466584920883 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 0, + "style_probability": 0.5851091742515564, + "style_target": 0, + "support_probability": 0.03663082793354988, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8974260065088164, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.032292235642671585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1244247779250145, + 0.9941142201423645, + 0.04718458652496338, + 0.13185884058475494, + 0.013520130887627602, + 0.014280766248703003 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 1, + "style_probability": 0.8911142945289612, + "style_target": 1, + "support_probability": 0.9723197817802429, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6969893017465084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11027003824710846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13149406015872955, + 0.9886335134506226, + 0.06960832327604294, + 0.13604451715946198, + 0.03218400850892067, + 0.060260605067014694 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 0, + "style_probability": 0.5577212572097778, + "style_target": 0, + "support_probability": 0.9500095248222351, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8720407590597872, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0577811598777771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9634025692939758, + 0.15332792699337006, + 0.03211382031440735, + 0.15332792699337006, + 0.09678364545106888, + 0.04983844980597496, + 0.011328249238431454, + 0.022573629394173622, + 0.01706768572330475 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.9223260283470154, + "style_target": 1, + "support_probability": 0.9169302582740784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.567773397113959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10619508475065231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9437183737754822, + 0.1552199125289917, + 0.03168882057070732, + 0.1552199125289917, + 0.059465061873197556 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.42439427971839905, + "style_target": 0, + "support_probability": 0.8030960559844971, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004481242141063271, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7793824672698975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14070719480514526, + 0.21200279891490936, + 0.3988739550113678, + 0.560637891292572, + 0.057427968829870224, + 0.08996614813804626, + 0.046529874205589294, + 0.0368477962911129, + 0.09690754115581512, + 0.08552621304988861, + 0.0585748553276062 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 1, + "style_probability": 0.871016263961792, + "style_target": 1, + "support_probability": 0.0006474275724031031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015392509181539672, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.832831621170044, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12228511273860931, + 0.2171820104122162, + 0.4091169834136963, + 0.5208373069763184, + 0.042054433375597, + 0.008739115670323372, + 0.03756774589419365, + 0.1320754587650299, + 0.13890327513217926, + 0.15634220838546753 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 0, + "style_probability": 0.5399172306060791, + "style_target": 0, + "support_probability": 0.0007645130972377956, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3036164143403784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21942970156669617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6627922654151917, + 0.1156100258231163, + 0.8373020887374878, + 0.6627922654151917, + 0.04610564559698105 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 0, + "style_probability": 0.23387901484966278, + "style_target": 1, + "support_probability": 0.6780677437782288, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.667446914859592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11404502391815186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7147220969200134, + 0.11894995719194412, + 0.8621147871017456, + 0.7147220969200134, + 0.009134774096310139 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 1, + "style_probability": 0.48793286085128784, + "style_target": 0, + "support_probability": 0.9207937717437744, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.42145219736295697, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3228263258934021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3998658359050751, + 0.4663234055042267, + 0.4115893244743347, + 0.8419236540794373, + 0.057734712958335876, + 0.07094085216522217 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.4000183939933777, + "style_target": 1, + "support_probability": 0.7615681886672974, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.680644771149167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2176327258348465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4390368163585663, + 0.37747156620025635, + 0.4389049708843231, + 0.7720363736152649, + 0.027667120099067688, + 0.024124139919877052 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.9624602198600769, + "style_target": 0, + "support_probability": 0.849433422088623, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.06563072107196471, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6716142296791077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7085754871368408, + 0.2581859230995178, + 0.07024263590574265, + 0.25446563959121704, + 0.16112978756427765, + 0.0891210064291954, + 0.03469915688037872, + 0.04173418879508972, + 0.07404352724552155 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 0, + "style_probability": 0.31323185563087463, + "style_target": 1, + "support_probability": 0.1226111575961113, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08340842756306223, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2733558416366577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.85895836353302, + 0.18858756124973297, + 0.051103219389915466, + 0.1688268929719925, + 0.06735194474458694, + 0.038955286145210266, + 0.016177678480744362, + 0.018374130129814148, + 0.02885464020073414 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 1, + "style_probability": 0.05228964239358902, + "style_target": 0, + "support_probability": 0.7369235754013062, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4281667287962384, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19002434611320496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2028345912694931, + 0.20565856993198395, + 0.11687497794628143, + 0.7929349541664124, + 0.019585996866226196, + 0.01742914319038391, + 0.018853534013032913, + 0.013407002203166485, + 0.032772693783044815, + 0.011790208518505096, + 0.00928102619946003 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 0, + "style_probability": 0.2160072922706604, + "style_target": 1, + "support_probability": 0.8806039690971375, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5965415542790757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17013506591320038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.206470787525177, + 0.18498747050762177, + 0.1307811290025711, + 0.8406620025634766, + 0.020845752209424973, + 0.0063568176701664925, + 0.011864363215863705, + 0.0557582825422287, + 0.006336180493235588, + 0.006369214504957199, + 0.007990042679011822 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 1, + "style_probability": 0.3212502896785736, + "style_target": 0, + "support_probability": 0.9428757429122925, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.050156115435610875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32633113861083984, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8403126001358032, + 0.12042844295501709, + 0.05960439518094063, + 0.48591458797454834, + 0.028792111203074455 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 1, + "style_probability": 0.7567821145057678, + "style_target": 1, + "support_probability": 0.015166888944804668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033691492223571484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2725033164024353, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7988243699073792, + 0.12412156164646149, + 0.06009851023554802, + 0.46249231696128845, + 0.021147606894373894, + 0.032716054469347, + 0.03499644622206688, + 0.039252229034900665, + 0.03817202150821686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 1, + "style_probability": 0.4428452253341675, + "style_target": 0, + "support_probability": 0.017537765204906464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014833261965198649, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6815187335014343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4726698398590088, + 0.49522966146469116, + 0.18596917390823364, + 0.9519105553627014, + 0.0327274315059185, + 0.0672253966331482 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.7780672907829285, + "style_target": 1, + "support_probability": 0.006335582584142685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005842993470183493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7350549697875977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.557262122631073, + 0.44750136137008667, + 0.21298807859420776, + 0.9625375270843506, + 0.053534697741270065, + 0.07890111953020096 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.4758627116680145, + "style_target": 0, + "support_probability": 0.0066819265484809875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040133923418550356, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3996163010597229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5785706043243408, + 0.32605835795402527, + 0.34530091285705566, + 0.27462249994277954, + 0.034685391932725906, + 0.019510991871356964, + 0.030575357377529144, + 0.03444661945104599, + 0.02965526096522808 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 1, + "style_probability": 0.8512373566627502, + "style_target": 1, + "support_probability": 0.009382227435708046, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01796779789086672, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.407825231552124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45787885785102844, + 0.3108918070793152, + 0.27480608224868774, + 0.30819833278656006, + 0.015749944373965263, + 0.030794912949204445, + 0.006700726691633463, + 0.021271679550409317, + 0.02248436026275158 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 1, + "style_probability": 0.41208556294441223, + "style_target": 0, + "support_probability": 0.018178191035985947, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8598205388428795, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021003728732466698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9801470637321472, + 0.13728304207324982, + 0.640149712562561, + 0.3420391082763672, + 0.01935833878815174, + 0.032403960824012756, + 0.02654423378407955, + 0.010750344954431057, + 0.021703574806451797, + 0.009028224274516106 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.6950730085372925, + "style_target": 1, + "support_probability": 0.9775961637496948, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5695172266893708, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03548261150717735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9749772548675537, + 0.11951512843370438, + 0.6554790735244751, + 0.33756619691848755, + 0.020312540233135223, + 0.03477609530091286 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.2557298243045807, + "style_target": 0, + "support_probability": 0.9643503427505493, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30452883995696467, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.026208369061350822, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13623732328414917, + 0.47563451528549194, + 0.13623732328414917, + 0.9933168888092041, + 0.101852647960186 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 1, + "style_probability": 0.5527986288070679, + "style_target": 1, + "support_probability": 0.31510084867477417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25413658236358544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012846678495407104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08090043067932129, + 0.35485386848449707, + 0.08090043067932129, + 0.996709942817688, + 0.01942182332277298, + 0.09042543172836304, + 0.022734075784683228, + 0.023744158446788788, + 0.038806650787591934 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 1, + "style_probability": 0.31573471426963806, + "style_target": 0, + "support_probability": 0.4285185635089874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19752524768189372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25710010528564453, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8396387100219727, + 0.20488522946834564, + 0.3770556151866913, + 0.20357073843479156, + 0.0374891459941864, + 0.15295350551605225 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.5504931807518005, + "style_target": 1, + "support_probability": 0.14492398500442505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015725065939398188, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7107785940170288, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7561152577400208, + 0.14456655085086823, + 0.25477662682533264, + 0.15384648740291595, + 0.022219538688659668, + 0.036448147147893906 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.058642324060201645, + "style_target": 0, + "support_probability": 0.01253251638263464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001470279968829618, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7395997643470764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1812622994184494, + 0.22997644543647766, + 0.1341695785522461, + 0.3621745705604553, + 0.05682234838604927, + 0.014863942749798298, + 0.058141253888607025, + 0.05261549726128578, + 0.11598922312259674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.41659092903137207, + "style_target": 1, + "support_probability": 0.0011242373147979379, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002290312825577913, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9339776635169983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16489814221858978, + 0.17126978933811188, + 0.12717919051647186, + 0.3973357379436493, + 0.04523344337940216, + 0.09453974664211273, + 0.04059690237045288, + 0.07296865433454514, + 0.027117468416690826 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.7195888757705688, + "style_target": 0, + "support_probability": 0.001065390883013606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007209824916811147, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6026076078414917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.07886119186878204, + 0.6840095520019531, + 0.8344770669937134, + 0.09062987565994263, + 0.019104458391666412, + 0.021253567188978195, + 0.024571139365434647, + 0.013575826771557331, + 0.026774320751428604, + 0.045927494764328, + 0.10950332134962082 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 1, + "style_probability": 0.5629516839981079, + "style_target": 1, + "support_probability": 0.0015383705031126738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025225977237538985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36362993717193604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06795311719179153, + 0.7251196503639221, + 0.842912495136261, + 0.07908318191766739, + 0.049657005816698074, + 0.10389841347932816 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 1, + "style_probability": 0.32614582777023315, + "style_target": 0, + "support_probability": 0.008088135160505772, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.850353211669613, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07478919625282288, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34437334537506104, + 0.9883279204368591, + 0.30662843585014343, + 0.5854049324989319, + 0.017331812530755997 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 0, + "style_probability": 0.45635131001472473, + "style_target": 1, + "support_probability": 0.9826335310935974, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42343420403824555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12364658713340759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2989281415939331, + 0.9968034029006958, + 0.2249356508255005, + 0.5252110958099365, + 0.06632616370916367 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 1, + "style_probability": 0.14449262619018555, + "style_target": 0, + "support_probability": 0.9313834309577942, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9118425776586586, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.038719527423381805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8994:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5848777890205383, + 0.3388647139072418, + 0.991925060749054, + 0.11018459498882294, + 0.02416781336069107, + 0.016510428860783577 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 0, + "style_probability": 0.4936361610889435, + "style_target": 1, + "support_probability": 0.996688187122345, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9630376649111482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04412730410695076, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8994:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5094258189201355, + 0.2623942494392395, + 0.9966050386428833, + 0.11725235730409622, + 0.012758239172399044, + 0.01075066439807415 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 0, + "style_probability": 0.8575012683868408, + "style_target": 0, + "support_probability": 0.9869379997253418, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06369092122007738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7263243794441223, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3225604295730591, + 0.7072140574455261, + 0.21431274712085724, + 0.7580437660217285, + 0.030403515323996544, + 0.05827169120311737, + 0.013584941625595093, + 0.0308738574385643, + 0.023267805576324463 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.5521202683448792, + "style_target": 1, + "support_probability": 0.09250422567129135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5299638088471879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10027877986431122, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27862751483917236, + 0.7313240170478821, + 0.32332345843315125, + 0.7797505855560303, + 0.05641750618815422 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.45909616351127625, + "style_target": 0, + "support_probability": 0.8768890500068665, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7797940011371796, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16955111920833588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1310964822769165, + 0.20942121744155884, + 0.2650723159313202, + 0.9419305920600891, + 0.010518703609704971, + 0.016293616965413094, + 0.025538180023431778, + 0.00790458358824253, + 0.015405149199068546, + 0.013085518963634968 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 1, + "style_probability": 0.5381214022636414, + "style_target": 1, + "support_probability": 0.9543130993843079, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08554491961847537, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49007341265678406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.14279627799987793, + 0.2904413640499115, + 0.2787020206451416, + 0.9300445318222046, + 0.01089454535394907, + 0.024378275498747826, + 0.021945515647530556, + 0.029240205883979797, + 0.007375883404165506, + 0.03316381201148033 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 1, + "style_probability": 0.09396471828222275, + "style_target": 0, + "support_probability": 0.7399252653121948, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07984450656505641, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4364844262599945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31724610924720764, + 0.3337540328502655, + 0.14137963950634003, + 0.6184874773025513, + 0.0262067262083292 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.8241757154464722, + "style_target": 1, + "support_probability": 0.024008342996239662, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011604989386329617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20600946247577667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2066570669412613, + 0.2926572561264038, + 0.11008141189813614, + 0.6500732898712158, + 0.09037448465824127, + 0.051209308207035065, + 0.028434734791517258, + 0.01738991029560566, + 0.07595177739858627 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.11706972867250443, + "style_target": 0, + "support_probability": 0.018519993871450424, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.06609977264846459, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24434146285057068, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21338343620300293, + 0.06947580724954605, + 0.15437999367713928, + 0.9587480425834656, + 0.04077219218015671, + 0.13629050552845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8250738382339478, + "style_target": 1, + "support_probability": 0.018905680626630783, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.017568476341383407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2942555844783783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22636663913726807, + 0.06601603329181671, + 0.12461773306131363, + 0.9620044827461243, + 0.011440012603998184, + 0.02816263772547245 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.3235158324241638, + "style_target": 0, + "support_probability": 0.025535089895129204, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36509332250242754, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16247916221618652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.02016252465546131, + 0.3014494776725769, + 0.8903657793998718, + 0.08114033937454224, + 0.01871488429605961, + 0.05943572521209717, + 0.018974628299474716, + 0.02373724989593029, + 0.040584199130535126 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 1, + "style_probability": 0.8015108108520508, + "style_target": 1, + "support_probability": 0.30762261152267456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31911258995674435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11359596997499466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.022664230316877365, + 0.2813052535057068, + 0.8974403142929077, + 0.11768561601638794, + 0.030038274824619293, + 0.029911760240793228, + 0.05313945561647415, + 0.08168936520814896, + 0.011225004680454731 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 1, + "style_probability": 0.28476688265800476, + "style_target": 0, + "support_probability": 0.6319639086723328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03293476173309496, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9830600619316101, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12109137326478958, + 0.33070409297943115, + 0.4342084228992462, + 0.4910484850406647, + 0.0201970674097538, + 0.01699335314333439, + 0.03934485465288162, + 0.020177194848656654, + 0.024540642276406288, + 0.022794965654611588 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.8516657948493958, + "style_target": 1, + "support_probability": 0.025172995403409004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006393120570240183, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9926910996437073, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14306499063968658, + 0.3696577548980713, + 0.3613273799419403, + 0.41049090027809143, + 0.04312095046043396, + 0.048844706267118454 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.3428857624530792, + "style_target": 0, + "support_probability": 0.005404948722571135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34276629802269276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08793500810861588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9967222809791565, + 0.3858599066734314, + 0.12942172586917877, + 0.053601525723934174, + 0.023359036073088646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.6947479844093323, + "style_target": 1, + "support_probability": 0.4969854950904846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.060801148938424504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12655238807201385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9986048340797424, + 0.3838842511177063, + 0.08835315704345703, + 0.05569450184702873, + 0.11639649420976639 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.15706495940685272, + "style_target": 0, + "support_probability": 0.48696789145469666, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.896043014206109, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.01740366965532303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.04355889931321144, + 0.6927092671394348, + 0.9868571162223816, + 0.17297090590000153, + 0.22604963183403015, + 0.0216971505433321 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 1, + "style_probability": 0.6705588698387146, + "style_target": 1, + "support_probability": 0.9903978109359741, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8654622037124132, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.021027106791734695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.05419633165001869, + 0.675739586353302, + 0.9737796187400818, + 0.19588454067707062, + 0.014558177441358566, + 0.02374497428536415 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 0, + "style_probability": 0.6067133545875549, + "style_target": 0, + "support_probability": 0.993613064289093, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001654475978735936, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9970340728759766, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6036814451217651, + 0.2955794334411621, + 0.26426705718040466, + 0.2955794334411621, + 0.08269625902175903, + 0.03745157644152641, + 0.04494640976190567, + 0.17500364780426025, + 0.07784479111433029 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.7680469751358032, + "style_target": 1, + "support_probability": 0.0012601304333657026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002337843921246831, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9913365840911865, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6298719644546509, + 0.3229807913303375, + 0.3044467270374298, + 0.3229807913303375, + 0.15704981982707977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.2986050844192505, + "style_target": 0, + "support_probability": 0.006731693632900715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.629086080421273, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1317146271467209, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2798541486263275, + 0.39829790592193604, + 0.4157038927078247, + 0.9544928073883057, + 0.09302802383899689, + 0.1284623146057129, + 0.037764664739370346, + 0.05272667855024338, + 0.025899477303028107, + 0.19308827817440033, + 0.055519040673971176 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.858406126499176, + "style_target": 1, + "support_probability": 0.666813313961029, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04470612289289544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1171799898147583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23705552518367767, + 0.353198766708374, + 0.3395272195339203, + 0.964809775352478, + 0.044258080422878265, + 0.0667891874909401, + 0.03379695862531662, + 0.03236984834074974, + 0.06220705062150955, + 0.08043654263019562 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.1022610068321228, + "style_target": 0, + "support_probability": 0.4445009231567383, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8075632165493777, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026061516255140305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9677467942237854, + 0.09896618127822876, + 0.09896618127822876, + 0.1018536388874054, + 0.017404654994606972 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 1, + "style_probability": 0.8011115193367004, + "style_target": 1, + "support_probability": 0.6764040589332581, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6817340783258344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0333356112241745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9484302997589111, + 0.18735197186470032, + 0.18735197186470032, + 0.1603424996137619, + 0.02514740079641342 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 0, + "style_probability": 0.6890409588813782, + "style_target": 0, + "support_probability": 0.6429072022438049, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.695025875305092, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10688027739524841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07519601285457611, + 0.1133967861533165, + 0.9629467129707336, + 0.6428625583648682, + 0.014370572753250599, + 0.018222946673631668 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 1, + "style_probability": 0.79300457239151, + "style_target": 1, + "support_probability": 0.42617079615592957, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3728323708186251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060263846069574356, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11995020508766174, + 0.17048119008541107, + 0.9627317190170288, + 0.6073382496833801, + 0.023849884048104286, + 0.03328746557235718, + 0.06762200593948364, + 0.01816604658961296, + 0.006964395754039288, + 0.018176276236772537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 0, + "style_probability": 0.5431040525436401, + "style_target": 0, + "support_probability": 0.3216138482093811, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014441821932706506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9719223976135254, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42990732192993164, + 0.45402348041534424, + 0.27909067273139954, + 0.7588440179824829, + 0.04567131772637367, + 0.055944204330444336, + 0.04360916092991829, + 0.027755584567785263, + 0.058510921895504 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.9075068831443787, + "style_target": 1, + "support_probability": 0.0038117861840873957, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0021094539825969125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9508323669433594, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38800427317619324, + 0.4912855327129364, + 0.19177304208278656, + 0.7966845631599426, + 0.059073444455862045, + 0.14915725588798523, + 0.13088960945606232, + 0.06852596998214722, + 0.24850663542747498 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.18616466224193573, + "style_target": 0, + "support_probability": 0.012049163691699505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9154212671978758, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02458775043487549, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9080:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9941427111625671, + 0.36316004395484924, + 0.3078511655330658, + 0.1450691968202591, + 0.06381651014089584, + 0.2738025188446045, + 0.015427564270794392, + 0.01618131436407566, + 0.05056987702846527, + 0.014314251951873302, + 0.03593359887599945 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 1, + "style_probability": 0.9068357348442078, + "style_target": 1, + "support_probability": 0.9445723295211792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6395629635277912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041432321071624756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9080:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9886705875396729, + 0.4123804569244385, + 0.30321332812309265, + 0.18958325684070587, + 0.030452223494648933, + 0.029748758301138878, + 0.025245804339647293, + 0.10076810419559479, + 0.09309571981430054, + 0.08702675253152847, + 0.02567989192903042 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 0, + "style_probability": 0.5400001406669617, + "style_target": 0, + "support_probability": 0.8563223481178284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5366630767961891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19735173881053925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4427179992198944, + 0.979771077632904, + 0.5642166137695312, + 0.25637730956077576, + 0.1386459916830063 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.21179229021072388, + "style_target": 1, + "support_probability": 0.8920903205871582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8796795445454021, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1203894093632698, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5005574822425842, + 0.9833902716636658, + 0.6043179631233215, + 0.2774248421192169, + 0.02520967647433281 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.8517573475837708, + "style_target": 0, + "support_probability": 0.9282304644584656, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20348654274035802, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5123946070671082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10605723410844803, + 0.3254798650741577, + 0.5567375421524048, + 0.836206316947937, + 0.044075511395931244, + 0.054344069212675095 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 0, + "style_probability": 0.2563796043395996, + "style_target": 1, + "support_probability": 0.5050047039985657, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028620783126268163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.761542797088623, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09925819933414459, + 0.2644643485546112, + 0.6312947273254395, + 0.9229639768600464, + 0.020402057096362114, + 0.03773362562060356 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 1, + "style_probability": 0.07633281499147415, + "style_target": 0, + "support_probability": 0.4440891742706299, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1091155318538247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04078715294599533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8347238898277283, + 0.49591708183288574, + 0.1831095814704895, + 0.11245334148406982, + 0.01782456412911415, + 0.013395179994404316, + 0.033840108662843704, + 0.061047740280628204, + 0.014806942082941532 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 0, + "style_probability": 0.26038599014282227, + "style_target": 1, + "support_probability": 0.13583363592624664, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17277210884313718, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01956239715218544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8596523404121399, + 0.4743165671825409, + 0.1733151376247406, + 0.1587914079427719, + 0.012782098725438118, + 0.015121962875127792, + 0.03325920179486275, + 0.01745631732046604, + 0.028386233374476433 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 1, + "style_probability": 0.3213847577571869, + "style_target": 0, + "support_probability": 0.23391588032245636, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1915267609313944, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4156248867511749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35285696387290955, + 0.30967241525650024, + 0.2951146960258484, + 0.9655543565750122, + 0.101922906935215, + 0.04227963835000992, + 0.035814814269542694, + 0.06911369413137436, + 0.1925555169582367, + 0.11167261004447937 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 0, + "style_probability": 0.2738431990146637, + "style_target": 1, + "support_probability": 0.20887120068073273, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6328451884436677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4286496639251709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3769594430923462, + 0.3693838119506836, + 0.2783930003643036, + 0.9733445048332214, + 0.12037010490894318, + 0.013483134098351002, + 0.06738366931676865, + 0.011961864307522774, + 0.07023034244775772, + 0.017367370426654816 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 0, + "style_probability": 0.755237340927124, + "style_target": 0, + "support_probability": 0.42887043952941895, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010142223662885544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7299397587776184, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2165057361125946, + 0.17808347940444946, + 0.552789568901062, + 0.36925017833709717, + 0.14237453043460846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.9138827919960022, + "style_target": 1, + "support_probability": 0.0017589854542165995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028021811748230176, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6757200360298157, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19205650687217712, + 0.16209325194358826, + 0.7076865434646606, + 0.25807443261146545, + 0.058958906680345535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.13739590346813202, + "style_target": 0, + "support_probability": 0.010796829126775265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5062085066105797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06993070989847183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42304283380508423, + 0.39646559953689575, + 0.42304283380508423, + 0.9987686276435852, + 0.03541170805692673, + 0.05573581904172897 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 1, + "style_probability": 0.7554295659065247, + "style_target": 1, + "support_probability": 0.16145847737789154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4170898808652991, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07517403364181519, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.491931289434433, + 0.43339428305625916, + 0.491931289434433, + 0.9955610036849976, + 0.039559029042720795, + 0.020719178020954132 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 0, + "style_probability": 0.6164098978042603, + "style_target": 0, + "support_probability": 0.19953519105911255, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8818775499417819, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05645386502146721, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08583944290876389, + 0.9940120577812195, + 0.4261398911476135, + 0.3124944269657135, + 0.01048567146062851, + 0.05681600794196129, + 0.02930259145796299, + 0.08624203503131866, + 0.02004065178334713 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 1, + "style_probability": 0.6965136528015137, + "style_target": 1, + "support_probability": 0.9640457034111023, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.528467179904648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07500670850276947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07422415912151337, + 0.9924667477607727, + 0.5056135654449463, + 0.3679467737674713, + 0.020771313458681107 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 1, + "style_probability": 0.18512903153896332, + "style_target": 0, + "support_probability": 0.9533259868621826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015554629022882132, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9296640753746033, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.225722536444664, + 0.8731226325035095, + 0.38645118474960327, + 0.42541518807411194, + 0.07750885933637619, + 0.10448620468378067, + 0.11414408683776855, + 0.07873030006885529, + 0.1066293716430664, + 0.08902192860841751, + 0.046627260744571686 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.8026848435401917, + "style_target": 1, + "support_probability": 0.007648605853319168, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012939565348248337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9760639667510986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.13858576118946075, + 0.849502682685852, + 0.32882410287857056, + 0.29608309268951416, + 0.07155634462833405, + 0.12360070645809174, + 0.08945488929748535, + 0.08994565159082413, + 0.1432734876871109, + 0.1007363349199295 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.042522214353084564, + "style_target": 0, + "support_probability": 0.01417703740298748, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4720962939676249, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02072095312178135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4053056538105011, + 0.9487792253494263, + 0.46097707748413086, + 0.345074862241745, + 0.14717645943164825 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.5085248351097107, + "style_target": 1, + "support_probability": 0.7571080923080444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08842147243563836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.040997207164764404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3764764666557312, + 0.9757994413375854, + 0.3551937937736511, + 0.19194424152374268, + 0.11977885663509369 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.08797624707221985, + "style_target": 0, + "support_probability": 0.6793234348297119, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6910776087401228, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05962827801704407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8543872237205505, + 0.9725454449653625, + 0.21297681331634521, + 0.20870281755924225, + 0.01954292505979538, + 0.14272964000701904 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 1, + "style_probability": 0.5758172869682312, + "style_target": 1, + "support_probability": 0.9020368456840515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.867747836457381, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.055158667266368866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.790442705154419, + 0.9905233979225159, + 0.19771556556224823, + 0.18112048506736755, + 0.0335860475897789, + 0.20936234295368195 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 0, + "style_probability": 0.8667912483215332, + "style_target": 0, + "support_probability": 0.8955826163291931, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5403185767689835, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39578792452812195, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9090665578842163, + 0.07821626216173172, + 0.11746542155742645, + 0.3757631182670593, + 0.054469313472509384, + 0.056241296231746674, + 0.06452789157629013, + 0.06289102882146835, + 0.13360661268234253 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 0, + "style_probability": 0.465672105550766, + "style_target": 1, + "support_probability": 0.7474385499954224, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3222162203031232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4973757565021515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9274839162826538, + 0.11262322217226028, + 0.10908973962068558, + 0.2965281903743744, + 0.12471736967563629 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 1, + "style_probability": 0.19948065280914307, + "style_target": 0, + "support_probability": 0.5934896469116211, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6901355244349062, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016348347067832947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4675136208534241, + 0.11997480690479279, + 0.9875984787940979, + 0.4675136208534241, + 0.021293392404913902, + 0.06771939247846603, + 0.07776088267564774, + 0.04292408749461174, + 0.05534898489713669, + 0.13343432545661926 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 1, + "style_probability": 0.5058519840240479, + "style_target": 1, + "support_probability": 0.9536953568458557, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1858311905953917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02176324650645256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6342969536781311, + 0.11857151240110397, + 0.9968034029006958, + 0.6342969536781311, + 0.021250586956739426, + 0.012328213080763817, + 0.0109111238270998, + 0.048490073531866074, + 0.056758392602205276, + 0.0747702568769455 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 1, + "style_probability": 0.13055220246315002, + "style_target": 0, + "support_probability": 0.9599436521530151, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04778260892969561, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20062461495399475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09314563870429993, + 0.837550938129425, + 0.2038324773311615, + 0.143666610121727, + 0.039341870695352554 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.42791464924812317, + "style_target": 1, + "support_probability": 0.02340884879231453, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.349452350094512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10491283982992172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08827172964811325, + 0.8363285064697266, + 0.18593713641166687, + 0.11342836171388626, + 0.04238151013851166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.9256929159164429, + "style_target": 0, + "support_probability": 0.14848576486110687, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5606441699664559, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0648626759648323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10903006047010422, + 0.13964968919754028, + 0.968380331993103, + 0.3019641935825348, + 0.014791007153689861, + 0.017814356833696365 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 0, + "style_probability": 0.4815598130226135, + "style_target": 1, + "support_probability": 0.9094377160072327, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05587826062131868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10439176112413406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06329601258039474, + 0.18285857141017914, + 0.9743948578834534, + 0.36587458848953247, + 0.028847521170973778, + 0.06288723647594452 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 1, + "style_probability": 0.07190259546041489, + "style_target": 0, + "support_probability": 0.6358461976051331, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01120485717213885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9612268209457397, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1659279614686966, + 0.3335583508014679, + 0.53717440366745, + 0.7550115585327148, + 0.044890400022268295, + 0.08034977316856384, + 0.07635181397199631, + 0.0431193970143795, + 0.06475213170051575 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 0, + "style_probability": 0.4171627163887024, + "style_target": 1, + "support_probability": 0.004715304356068373, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03859163667667707, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8081610202789307, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15902937948703766, + 0.43628624081611633, + 0.655572772026062, + 0.7947359681129456, + 0.030074909329414368, + 0.030983973294496536, + 0.04781104624271393, + 0.05276716873049736, + 0.0623030923306942 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 0, + "style_probability": 0.5911680459976196, + "style_target": 0, + "support_probability": 0.020124217495322227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10741899261674161, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5812847018241882, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06802912801504135, + 0.7437243461608887, + 0.2936081290245056, + 0.5187253952026367, + 0.050827935338020325, + 0.05513579770922661, + 0.02145528420805931, + 0.05241139978170395, + 0.04156298190355301, + 0.06310545653104782, + 0.04905291646718979 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.4020327925682068, + "style_target": 1, + "support_probability": 0.09545837342739105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12883506920195126, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5624810457229614, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05357243865728378, + 0.6544790267944336, + 0.27228203415870667, + 0.5169990658760071, + 0.018838923424482346, + 0.03123690001666546, + 0.0334710069000721, + 0.029501505196094513, + 0.1310391128063202, + 0.04925057291984558, + 0.05833401158452034 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.8710953593254089, + "style_target": 0, + "support_probability": 0.07058766484260559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5215917954782742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1712457686662674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9161021113395691, + 0.5141128301620483, + 0.15432903170585632, + 0.20152980089187622, + 0.04500187188386917 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.8066816926002502, + "style_target": 1, + "support_probability": 0.4905249774456024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02533401379629774, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3727569580078125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8848530650138855, + 0.4816674292087555, + 0.2043001502752304, + 0.12892919778823853, + 0.11564090847969055 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.10000038146972656, + "style_target": 0, + "support_probability": 0.2550722360610962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00426803745511424, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9790247082710266, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11661326885223389, + 0.698178768157959, + 0.3653954863548279, + 0.49858933687210083, + 0.08463867008686066, + 0.061345331370830536 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 1, + "style_probability": 0.7510484457015991, + "style_target": 1, + "support_probability": 0.0035146933514624834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045062548567015914, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8901457190513611, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16085833311080933, + 0.7431391477584839, + 0.33491504192352295, + 0.39669305086135864, + 0.03868836537003517, + 0.02861018478870392 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 0, + "style_probability": 0.5468634963035583, + "style_target": 0, + "support_probability": 0.023664675652980804, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.48107525017865727, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3179856240749359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2742406725883484, + 0.187682643532753, + 0.9352636337280273, + 0.15881651639938354, + 0.009039506316184998, + 0.012074995785951614, + 0.017630411311984062, + 0.17337244749069214, + 0.015142470598220825 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.868660032749176, + "style_target": 1, + "support_probability": 0.24033746123313904, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22850600935478807, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2534780204296112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2700006365776062, + 0.20888783037662506, + 0.9150143265724182, + 0.15554934740066528, + 0.023655444383621216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.4417598247528076, + "style_target": 0, + "support_probability": 0.2754799425601959, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8916341505879565, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023101748898625374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6166315674781799, + 0.6693845987319946, + 0.45450204610824585, + 0.9949398040771484, + 0.012367017567157745, + 0.013557588681578636, + 0.03257513418793678, + 0.042520128190517426, + 0.12651948630809784, + 0.039888765662908554 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.7748480439186096, + "style_target": 1, + "support_probability": 0.9814093112945557, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3710951314018447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02358916588127613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5920097827911377, + 0.5872304439544678, + 0.4039407968521118, + 0.9944630265235901, + 0.02696279250085354, + 0.036760929971933365, + 0.07569963485002518, + 0.05931863561272621, + 0.020500333979725838, + 0.05052937939763069 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.13720494508743286, + "style_target": 0, + "support_probability": 0.9783110022544861, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8387476091527866, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06901387125253677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1198970302939415, + 0.9872151613235474, + 0.09743615984916687, + 0.35909104347229004, + 0.013783024623990059 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 1, + "style_probability": 0.8425242900848389, + "style_target": 1, + "support_probability": 0.8722997307777405, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6715411915576145, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.050032321363687515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12057886272668839, + 0.9745927453041077, + 0.13039514422416687, + 0.30566641688346863, + 0.028089482337236404 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 0, + "style_probability": 0.5119773149490356, + "style_target": 0, + "support_probability": 0.8995447158813477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08384676420840156, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9491296410560608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5370018482208252, + 0.6435447335243225, + 0.8537414073944092, + 0.5370018482208252, + 0.10639894753694534, + 0.07936174422502518 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.8166851997375488, + "style_target": 1, + "support_probability": 0.06648117303848267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03409412494755652, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9753469228744507, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.49738436937332153, + 0.6161075234413147, + 0.8314705491065979, + 0.49738436937332153, + 0.07437290996313095, + 0.0489312969148159, + 0.08076946437358856, + 0.07121630758047104, + 0.054420046508312225, + 0.0615776926279068, + 0.05377949774265289 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.3617519736289978, + "style_target": 0, + "support_probability": 0.027154389768838882, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.805422991274843, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025035960599780083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.04493024945259094, + 0.16078539192676544, + 0.9815667867660522, + 0.11437085270881653, + 0.014834004454314709, + 0.028872592374682426, + 0.005358757451176643, + 0.011871576309204102, + 0.011137236841022968 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 0, + "style_probability": 0.4031914472579956, + "style_target": 1, + "support_probability": 0.9905097484588623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19726831937576536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.036081112921237946, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.050309740006923676, + 0.22349143028259277, + 0.9833827614784241, + 0.1692982316017151, + 0.015305678360164165, + 0.01035973522812128, + 0.03026210144162178, + 0.013641871511936188, + 0.031216096132993698 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 1, + "style_probability": 0.039644110947847366, + "style_target": 0, + "support_probability": 0.9900912046432495, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11117376489481165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6314743161201477, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7450628876686096, + 0.20629410445690155, + 0.1592024713754654, + 0.20629410445690155, + 0.028940850868821144, + 0.046433743089437485, + 0.03007219359278679, + 0.027460819110274315, + 0.027563923969864845, + 0.055207569152116776, + 0.027999643236398697 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 1, + "style_probability": 0.8368796706199646, + "style_target": 1, + "support_probability": 0.08080320805311203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16378514483944107, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4117598831653595, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7507927417755127, + 0.25382933020591736, + 0.14922365546226501, + 0.25382933020591736, + 0.021120769903063774, + 0.028731077909469604, + 0.019104721024632454, + 0.06132400780916214, + 0.029796628281474113, + 0.03132684901356697 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 0, + "style_probability": 0.5026469826698303, + "style_target": 0, + "support_probability": 0.22705429792404175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21367467351202607, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8627864122390747, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.987555205821991, + 0.30964431166648865, + 0.335448294878006, + 0.10306873917579651, + 0.03829236701130867 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 1, + "style_probability": 0.5112699866294861, + "style_target": 1, + "support_probability": 0.19112099707126617, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25054907387022685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5440109372138977, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9890483021736145, + 0.29560136795043945, + 0.2856145203113556, + 0.0883307084441185, + 0.02216426469385624, + 0.0261244997382164, + 0.030720900744199753, + 0.04131888970732689, + 0.05429819971323013 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 1, + "style_probability": 0.1924479752779007, + "style_target": 0, + "support_probability": 0.45313698053359985, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009416512525685474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9970480799674988, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2559003531932831, + 0.1196221187710762, + 0.29084765911102295, + 0.1207626461982727, + 0.033603910356760025, + 0.04820721596479416 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 1, + "style_probability": 0.8176014423370361, + "style_target": 1, + "support_probability": 0.00037119409535080194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00017991571917858734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9995672106742859, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2757226228713989, + 0.15180866420269012, + 0.2535446584224701, + 0.1314121037721634, + 0.060929056257009506, + 0.06921713799238205 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 0, + "style_probability": 0.5027202367782593, + "style_target": 0, + "support_probability": 0.0001794730342226103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10124620035607934, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37727057933807373, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24436937272548676, + 0.49695897102355957, + 0.26530417799949646, + 0.5390558242797852, + 0.05455218255519867, + 0.03897705301642418, + 0.22492088377475739, + 0.029902679845690727, + 0.17437532544136047 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 1, + "style_probability": 0.8795117735862732, + "style_target": 1, + "support_probability": 0.07435843348503113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058053437113128235, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4774561822414398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36047685146331787, + 0.48657840490341187, + 0.2278793454170227, + 0.5107126235961914, + 0.12791889905929565, + 0.1519845724105835, + 0.0446293018758297, + 0.022030692547559738, + 0.04332873597741127 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 0, + "style_probability": 0.5044943690299988, + "style_target": 0, + "support_probability": 0.06686453521251678, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5282826951375043, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20771190524101257, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33949458599090576, + 0.24003085494041443, + 0.8653904795646667, + 0.5124773383140564, + 0.02763574756681919, + 0.008133039809763432, + 0.011702794581651688, + 0.2799912691116333, + 0.024123433977365494, + 0.01159700658172369, + 0.0186763908714056 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.7585426568984985, + "style_target": 1, + "support_probability": 0.6329864859580994, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2813943552942225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2125205546617508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39889806509017944, + 0.32367396354675293, + 0.8153485655784607, + 0.5001409649848938, + 0.10425466299057007, + 0.01629897579550743 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.3087354302406311, + "style_target": 0, + "support_probability": 0.6321464776992798, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9032962423554032, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02891193889081478, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.982638955116272, + 0.3118995726108551, + 0.07150682806968689, + 0.14572183787822723, + 0.16370786726474762 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.9682156443595886, + "style_target": 1, + "support_probability": 0.9343972206115723, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5157795090686684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0026158876717090607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9991205334663391, + 0.28229859471321106, + 0.08887232840061188, + 0.05622800812125206, + 0.1354687511920929 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.16072897613048553, + "style_target": 0, + "support_probability": 0.9890079498291016, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7784183650121702, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04793853685259819, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6697915196418762, + 0.9828422665596008, + 0.19509418308734894, + 0.21012932062149048, + 0.029079943895339966, + 0.07848821580410004 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 1, + "style_probability": 0.592279314994812, + "style_target": 1, + "support_probability": 0.9520564079284668, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7959396941523048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.048381298780441284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6087496280670166, + 0.9800490140914917, + 0.33096471428871155, + 0.2201140969991684, + 0.07070133835077286, + 0.05943065881729126 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 1, + "style_probability": 0.4698353707790375, + "style_target": 0, + "support_probability": 0.9659063816070557, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8734309947695881, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04104449599981308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9965823292732239, + 0.5367931723594666, + 0.4785022735595703, + 0.7203899621963501, + 0.01476939208805561, + 0.013986846432089806, + 0.01662316918373108, + 0.031245261430740356, + 0.03658768907189369 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.8498325347900391, + "style_target": 1, + "support_probability": 0.9363371729850769, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4609281895674249, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.049619607627391815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9948379397392273, + 0.5416906476020813, + 0.4500383734703064, + 0.687502920627594, + 0.2830826938152313 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.3490598201751709, + "style_target": 0, + "support_probability": 0.8987210392951965, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11770898209205528, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4245098829269409, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17588956654071808, + 0.8327896595001221, + 0.7964769005775452, + 0.37619802355766296, + 0.02695148065686226, + 0.008353570476174355, + 0.029055722057819366, + 0.02306247688829899, + 0.08264816552400589, + 0.05418027937412262 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7762746214866638, + "style_target": 1, + "support_probability": 0.09944606572389603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008339418447954694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42431914806365967, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19502809643745422, + 0.8263550996780396, + 0.7408822774887085, + 0.40425288677215576, + 0.02542831189930439, + 0.041294779628515244, + 0.049852024763822556, + 0.05203493312001228, + 0.023650627583265305, + 0.06640629470348358 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.12634888291358948, + "style_target": 0, + "support_probability": 0.07749541103839874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23701712538158942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.18669725954532623, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09653616696596146, + 0.6710333824157715, + 0.3928636908531189, + 0.7518329620361328, + 0.0562005490064621 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 0, + "style_probability": 0.2670554518699646, + "style_target": 1, + "support_probability": 0.5113608837127686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26936753323517565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1990336924791336, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07987991720438004, + 0.5970664024353027, + 0.26711806654930115, + 0.7529689073562622, + 0.03945954889059067 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 1, + "style_probability": 0.33553099632263184, + "style_target": 0, + "support_probability": 0.5822067856788635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02284743614612057, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8198223114013672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6522724032402039, + 0.11560484021902084, + 0.2481657862663269, + 0.20384547114372253, + 0.057080890983343124, + 0.04450366273522377 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.33002179861068726, + "style_target": 1, + "support_probability": 0.006441113539040089, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02490349397635815, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8648374080657959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6443259716033936, + 0.10384179651737213, + 0.254631906747818, + 0.16326187551021576, + 0.017882121726870537, + 0.06148757413029671 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.8048551678657532, + "style_target": 0, + "support_probability": 0.004801492672413588, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2651658771844966, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1347137689590454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39218270778656006, + 0.7307062149047852, + 0.3358778953552246, + 0.1385440081357956, + 0.06027046963572502, + 0.048126328736543655, + 0.12024077028036118, + 0.01973092555999756, + 0.06884832680225372 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 0, + "style_probability": 0.42111334204673767, + "style_target": 1, + "support_probability": 0.3372008800506592, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11442198869843458, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10710148513317108, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3332185447216034, + 0.7710781097412109, + 0.39263641834259033, + 0.09466776996850967, + 0.028043227270245552, + 0.03531531244516373, + 0.022912802174687386, + 0.01826414093375206, + 0.058108847588300705 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 1, + "style_probability": 0.17756758630275726, + "style_target": 0, + "support_probability": 0.4366092383861542, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21322629734286538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5950006246566772, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.414838045835495, + 0.26858222484588623, + 0.23482249677181244, + 0.983349084854126, + 0.13787208497524261, + 0.40737709403038025, + 0.11331161856651306, + 0.1917070746421814, + 0.29767945408821106, + 0.07628700137138367, + 0.1535196304321289 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 0, + "style_probability": 0.3003127872943878, + "style_target": 1, + "support_probability": 0.27637168765068054, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3762012305771562, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4823457598686218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40292173624038696, + 0.252944678068161, + 0.21491262316703796, + 0.9805636405944824, + 0.09186914563179016, + 0.05457288771867752, + 0.10706537216901779, + 0.05804327875375748, + 0.0612553171813488, + 0.12054723501205444 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 0, + "style_probability": 0.5291863679885864, + "style_target": 0, + "support_probability": 0.4096282422542572, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04260401816785609, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7063673138618469, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.050462041050195694, + 0.07506293803453445, + 0.27181750535964966, + 0.24754667282104492, + 0.1133330687880516 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.3048609793186188, + "style_target": 1, + "support_probability": 0.1408994495868683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01321078890295783, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8069977760314941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0506783090531826, + 0.07391639053821564, + 0.23364956676959991, + 0.190252885222435, + 0.023720165714621544 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.8752885460853577, + "style_target": 0, + "support_probability": 0.0200225617736578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03221328149056867, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8306241631507874, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3362793028354645, + 0.19180093705654144, + 0.34856000542640686, + 0.25136855244636536, + 0.05360490083694458, + 0.11648590117692947 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 0, + "style_probability": 0.37989166378974915, + "style_target": 1, + "support_probability": 0.020525889471173286, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01972937909143281, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5296956896781921, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3675157129764557, + 0.20872409641742706, + 0.6546092629432678, + 0.3451146185398102, + 0.04573305323719978, + 0.07575356215238571 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 1, + "style_probability": 0.17174026370048523, + "style_target": 0, + "support_probability": 0.11885541677474976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004019190154123686, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9891952276229858, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.338748574256897, + 0.6071884036064148, + 0.48074400424957275, + 0.4187915623188019, + 0.056425515562295914, + 0.07494831830263138, + 0.19669713079929352, + 0.03986809775233269, + 0.10800713300704956 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 0, + "style_probability": 0.26586222648620605, + "style_target": 1, + "support_probability": 0.002038789913058281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006466647623092752, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9768090844154358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27569580078125, + 0.7194082140922546, + 0.5287904143333435, + 0.3300059735774994, + 0.08679284155368805, + 0.08398883789777756, + 0.06019367650151253, + 0.06309973448514938, + 0.1908111274242401 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 1, + "style_probability": 0.4772639572620392, + "style_target": 0, + "support_probability": 0.0069432430900633335, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.26217245328457195, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17481330037117004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12708692252635956, + 0.1496018022298813, + 0.19061841070652008, + 0.975283682346344, + 0.045542553067207336, + 0.041889797896146774, + 0.0666271224617958, + 0.06047073006629944, + 0.04042629525065422, + 0.06399624794721603 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.33915218710899353, + "style_target": 1, + "support_probability": 0.6898640394210815, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6164344834815202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20742514729499817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16325995326042175, + 0.13290774822235107, + 0.20286144316196442, + 0.9793410897254944, + 0.029862916097044945, + 0.06522734463214874, + 0.035342730581760406, + 0.08736908435821533, + 0.03956974670290947, + 0.0432775504887104, + 0.030918866395950317 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.8762801289558411, + "style_target": 0, + "support_probability": 0.6827263236045837, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.22301579032395036, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3189474642276764, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.265352725982666, + 0.1916433721780777, + 0.31030645966529846, + 0.9391409754753113, + 0.010941614396870136 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.759281575679779, + "style_target": 1, + "support_probability": 0.04058851674199104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02121586272366868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5096478462219238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20973935723304749, + 0.15468695759773254, + 0.29286131262779236, + 0.9306607842445374, + 0.05667119473218918 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.2708260118961334, + "style_target": 0, + "support_probability": 0.015833891928195953, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05134858098485863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5440980792045593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5627948641777039, + 0.7958942651748657, + 0.12667880952358246, + 0.4610823690891266, + 0.08071880042552948, + 0.04587172344326973 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 1, + "style_probability": 0.5712876319885254, + "style_target": 1, + "support_probability": 0.02504388615489006, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13526545195138998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33650413155555725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.633368194103241, + 0.813683032989502, + 0.19970838725566864, + 0.504889965057373, + 0.028292180970311165, + 0.047624193131923676 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 1, + "style_probability": 0.4978881776332855, + "style_target": 0, + "support_probability": 0.08661612868309021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06913687332863379, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48814886808395386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.332834929227829, + 0.5778661370277405, + 0.9732531309127808, + 0.6147655248641968, + 0.026887010782957077, + 0.04999954625964165, + 0.03275994583964348, + 0.12217632681131363, + 0.04701829329133034 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.6523703336715698, + "style_target": 1, + "support_probability": 0.02107025496661663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06732604160330573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2831111550331116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.375300794839859, + 0.5935617685317993, + 0.9679232835769653, + 0.6080198287963867, + 0.049077484756708145 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.2515055239200592, + "style_target": 0, + "support_probability": 0.035849250853061676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05140110627617698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8796055316925049, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7846691012382507, + 0.11592164635658264, + 0.18524602055549622, + 0.0936676561832428, + 0.015151483938097954, + 0.05597350001335144, + 0.029584240168333054, + 0.03034706972539425, + 0.05352000892162323, + 0.215178981423378, + 0.023809608072042465 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.896662175655365, + "style_target": 1, + "support_probability": 0.00945234950631857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00470099058743971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.747230589389801, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8262344598770142, + 0.11357729136943817, + 0.19908705353736877, + 0.08797430992126465, + 0.04026176035404205, + 0.03768569231033325, + 0.11346963047981262, + 0.02672443352639675, + 0.018761908635497093, + 0.04510379955172539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.1824771910905838, + "style_target": 0, + "support_probability": 0.01949118822813034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0043659428062329635, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.774192214012146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3093913197517395, + 0.2069934904575348, + 0.3093913197517395, + 0.32570475339889526, + 0.03435284644365311 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 1, + "style_probability": 0.839044451713562, + "style_target": 1, + "support_probability": 0.0006385399610735476, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006901639789039878, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7264310717582703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32193681597709656, + 0.28375986218452454, + 0.32193681597709656, + 0.3320922553539276, + 0.1265021711587906 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 0, + "style_probability": 0.7503905296325684, + "style_target": 0, + "support_probability": 0.0023854635655879974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8433258897566953, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05867068096995354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7804774045944214, + 0.9517619609832764, + 0.26449456810951233, + 0.09833166003227234, + 0.021088428795337677, + 0.018833743408322334 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 1, + "style_probability": 0.9336686730384827, + "style_target": 1, + "support_probability": 0.9324325323104858, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5573821614080239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13756342232227325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7473475933074951, + 0.9410794377326965, + 0.42448151111602783, + 0.11890885978937149, + 0.03016471676528454, + 0.035616811364889145, + 0.03888064622879028, + 0.11212145537137985, + 0.08734872937202454, + 0.0306113101541996 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 0, + "style_probability": 0.5283342599868774, + "style_target": 0, + "support_probability": 0.8767009377479553, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5061497761470239, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4494803249835968, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26709142327308655, + 0.4582764804363251, + 0.25561222434043884, + 0.7123300433158875, + 0.017819488421082497, + 0.011761980131268501, + 0.016568630933761597, + 0.01577695459127426, + 0.016509754583239555 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.8825263381004333, + "style_target": 1, + "support_probability": 0.48093000054359436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007565044470086994, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6814081072807312, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23378834128379822, + 0.37668582797050476, + 0.20603103935718536, + 0.5568118095397949, + 0.06054307147860527, + 0.04933784902095795, + 0.052613917738199234, + 0.042218539863824844, + 0.11044815182685852 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.05481766536831856, + "style_target": 0, + "support_probability": 0.27724432945251465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9267034092080864, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031995195895433426, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6197232007980347, + 0.12954668700695038, + 0.34223175048828125, + 0.9939035773277283, + 0.029789330437779427, + 0.018932752311229706, + 0.02047411911189556, + 0.01582619547843933, + 0.022184066474437714, + 0.022585580125451088, + 0.027805302292108536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 1, + "style_probability": 0.9320898056030273, + "style_target": 1, + "support_probability": 0.9374754428863525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8088220161117884, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1050294041633606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9347:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5856765508651733, + 0.10429608076810837, + 0.21744851768016815, + 0.9849462509155273, + 0.03336950019001961, + 0.026197662577033043, + 0.0791480764746666, + 0.030047733336687088, + 0.05191170051693916, + 0.044172026216983795, + 0.23976220190525055 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 0, + "style_probability": 0.7282785773277283, + "style_target": 0, + "support_probability": 0.9036650657653809, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9421870114601696, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04616489261388779, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9349:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2553926706314087, + 0.9957677125930786, + 0.5082951188087463, + 0.1654718965291977, + 0.17327547073364258 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.7803735136985779, + "style_target": 1, + "support_probability": 0.9899085760116577, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6766252251512948, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03410421311855316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18876969814300537, + 0.9947579503059387, + 0.45632997155189514, + 0.22451649606227875, + 0.2621607482433319, + 0.06945717334747314, + 0.07195591181516647, + 0.038949333131313324, + 0.11806964129209518 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.1895061433315277, + "style_target": 0, + "support_probability": 0.989011287689209, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0053902599038992615, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9234324097633362, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17204535007476807, + 0.8624217510223389, + 0.11246314644813538, + 0.037176020443439484, + 0.07284756749868393, + 0.05428920313715935 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.7852188348770142, + "style_target": 1, + "support_probability": 0.004323269240558147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0036327889088644066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.546855092048645, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19564978778362274, + 0.9048841595649719, + 0.08498051762580872, + 0.035489972680807114, + 0.0329253114759922, + 0.06886789202690125 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.24084687232971191, + "style_target": 0, + "support_probability": 0.021784666925668716, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4095445744941084, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26611196994781494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25204595923423767, + 0.9674851298332214, + 0.13433746993541718, + 0.5702133774757385, + 0.02688160166144371, + 0.045790188014507294, + 0.05294397100806236, + 0.02067592740058899, + 0.025808675214648247 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 1, + "style_probability": 0.7583407759666443, + "style_target": 1, + "support_probability": 0.42092084884643555, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08945900317846531, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6475083827972412, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2816234529018402, + 0.9609180688858032, + 0.15885835886001587, + 0.554766058921814, + 0.037167880684137344, + 0.10606089234352112, + 0.05451406538486481, + 0.11594214290380478, + 0.07304036617279053 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 0, + "style_probability": 0.5377963781356812, + "style_target": 0, + "support_probability": 0.07467079907655716, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07477265720771323, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8487452864646912, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08769776672124863, + 0.13863232731819153, + 0.3295954465866089, + 0.08769776672124863, + 0.056122202426195145, + 0.026099901646375656, + 0.013403880409896374, + 0.041377775371074677, + 0.055685676634311676, + 0.05162806808948517, + 0.03424614667892456 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.9031389951705933, + "style_target": 1, + "support_probability": 0.052401795983314514, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01645008206257059, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.935275673866272, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10608936101198196, + 0.14249078929424286, + 0.3291080594062805, + 0.10608936101198196, + 0.020711764693260193, + 0.043066542595624924 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.48511552810668945, + "style_target": 0, + "support_probability": 0.0255728829652071, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.901787058380858, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0497518926858902, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.034570418298244476, + 0.18047410249710083, + 0.0886310562491417, + 0.9991621971130371, + 0.24548542499542236 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.8224525451660156, + "style_target": 1, + "support_probability": 0.8539111614227295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3171114811435706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.09051155298948288, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.025790564715862274, + 0.1712169349193573, + 0.05308331549167633, + 0.9993786811828613, + 0.041543345898389816 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.2069305181503296, + "style_target": 0, + "support_probability": 0.7405796051025391, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5174714754151587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2091224193572998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9766502380371094, + 0.2593565881252289, + 0.4829087555408478, + 0.13013960421085358, + 0.034285079687833786, + 0.10514723509550095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.7595028281211853, + "style_target": 1, + "support_probability": 0.514589250087738, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34135656724875363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12619739770889282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9699769616127014, + 0.2882135510444641, + 0.4461511969566345, + 0.10344994813203812, + 0.04116091504693031, + 0.1297488957643509 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.39539507031440735, + "style_target": 0, + "support_probability": 0.6477044224739075, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7280462693275334, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08301693946123123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2080390900373459, + 0.23792333900928497, + 0.9621847867965698, + 0.19196996092796326, + 0.01816542074084282, + 0.027088040485978127, + 0.012473578564822674, + 0.025238798931241035, + 0.06161877140402794 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.7652935981750488, + "style_target": 1, + "support_probability": 0.8441644310951233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48667452162846586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10018777847290039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17798690497875214, + 0.30385836958885193, + 0.962191104888916, + 0.19945941865444183, + 0.13811378180980682 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.3528338670730591, + "style_target": 0, + "support_probability": 0.8559594750404358, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05515403667301588, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6662057042121887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.254842609167099, + 0.6635541915893555, + 0.3005838394165039, + 0.3046787679195404, + 0.02291199564933777, + 0.08814689517021179, + 0.07813142240047455, + 0.027659162878990173, + 0.01708153821527958, + 0.08343612402677536, + 0.2315511256456375 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.9039654731750488, + "style_target": 1, + "support_probability": 0.04591921344399452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01596432494182145, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2523929178714752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.10813450068235397, + 0.7447291612625122, + 0.19984310865402222, + 0.13156351447105408, + 0.04356956109404564, + 0.03675458952784538, + 0.18500420451164246, + 0.02630826085805893, + 0.030996548011898994, + 0.17627090215682983 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.1182822436094284, + "style_target": 0, + "support_probability": 0.1720428317785263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5558554948285287, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.30858081579208374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3024429380893707, + 0.8584899306297302, + 0.1425238400697708, + 0.08415956050157547, + 0.027342798188328743 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 1, + "style_probability": 0.830612301826477, + "style_target": 1, + "support_probability": 0.47994425892829895, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3856102264823255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27702009677886963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3406825065612793, + 0.8038085699081421, + 0.11135353147983551, + 0.10736680775880814, + 0.014939405955374241 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 0, + "style_probability": 0.5801480412483215, + "style_target": 0, + "support_probability": 0.5418408513069153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9466716719669193, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007814401760697365, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9413:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.022363821044564247, + 0.9973287582397461, + 0.8831355571746826, + 0.022363821044564247, + 0.008777448907494545, + 0.007967216894030571 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.8027809262275696, + "style_target": 1, + "support_probability": 0.9874189496040344, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7435395023998003, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006695181131362915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9413:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.02623288705945015, + 0.9946585297584534, + 0.8675358891487122, + 0.02623288705945015, + 0.020419439300894737, + 0.102955661714077, + 0.02647794596850872, + 0.009952451102435589, + 0.04196205735206604, + 0.007696943357586861, + 0.006514714099466801 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.21538282930850983, + "style_target": 0, + "support_probability": 0.9854355454444885, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37378938343959267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07560188323259354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39990243315696716, + 0.645128071308136, + 0.60871821641922, + 0.645128071308136, + 0.007313879206776619, + 0.026692435145378113, + 0.018068645149469376, + 0.024824338033795357, + 0.01694329082965851 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.8790735602378845, + "style_target": 1, + "support_probability": 0.24581871926784515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015612728711579088, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20962537825107574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34529876708984375, + 0.474850058555603, + 0.4936147630214691, + 0.474850058555603, + 0.017635606229305267, + 0.018343208357691765, + 0.019949661567807198, + 0.0370975099503994, + 0.053206589072942734 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.08059227466583252, + "style_target": 0, + "support_probability": 0.34877657890319824, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6137658467088991, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04532788321375847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13495613634586334, + 0.7409147620201111, + 0.08615074306726456, + 0.20759117603302002, + 0.025516871362924576, + 0.02289675734937191, + 0.007743294350802898, + 0.06952511519193649, + 0.014648271724581718, + 0.012215171940624714, + 0.009804797358810902 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 1, + "style_probability": 0.6627288460731506, + "style_target": 1, + "support_probability": 0.8133703470230103, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47956723942470314, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07842778414487839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22514298558235168, + 0.6991974711418152, + 0.1645197868347168, + 0.26932400465011597, + 0.0501406267285347, + 0.013396749272942543, + 0.027733830735087395, + 0.009307634085416794, + 0.007123020477592945, + 0.011951150372624397 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 1, + "style_probability": 0.34407395124435425, + "style_target": 0, + "support_probability": 0.8190553188323975, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8980438178748983, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015190411359071732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9976025223731995, + 0.1644042581319809, + 0.11738713830709457, + 0.20105789601802826, + 0.010822215117514133 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 1, + "style_probability": 0.6858112812042236, + "style_target": 1, + "support_probability": 0.9953718781471252, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8595398151264094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04334546625614166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9992005228996277, + 0.127018541097641, + 0.14520449936389923, + 0.22087755799293518, + 0.018462378531694412 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 0, + "style_probability": 0.6692301034927368, + "style_target": 0, + "support_probability": 0.9750799536705017, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07179983033521173, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.58680260181427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.07033185660839081, + 0.28133729100227356, + 0.3204808831214905, + 0.8023672699928284, + 0.02695625275373459, + 0.04426975175738335 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.6340144872665405, + "style_target": 1, + "support_probability": 0.013826792128384113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040184261271585114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49840015172958374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.07355928421020508, + 0.30279356241226196, + 0.351139634847641, + 0.8211445212364197, + 0.08087775856256485, + 0.03199166804552078, + 0.0556972473859787, + 0.030824314802885056, + 0.03191564604640007, + 0.0263507142663002 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.40331947803497314, + "style_target": 0, + "support_probability": 0.010789075866341591, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06625642466615614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.364864706993103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21238793432712555, + 0.17747506499290466, + 0.25434550642967224, + 0.8216570019721985, + 0.012067108415067196, + 0.04761553183197975, + 0.02357158251106739, + 0.01686839945614338, + 0.013322664424777031 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 0, + "style_probability": 0.3875104784965515, + "style_target": 1, + "support_probability": 0.08158013224601746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01369459284908823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24708934128284454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1666989028453827, + 0.24488942325115204, + 0.13535498082637787, + 0.9548283219337463, + 0.08912071585655212, + 0.0972566157579422, + 0.07683174312114716, + 0.085703544318676, + 0.1293572336435318 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 1, + "style_probability": 0.06137920171022415, + "style_target": 0, + "support_probability": 0.22707194089889526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08497933699185647, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18268802762031555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8656421303749084, + 0.06444799154996872, + 0.2669244408607483, + 0.6147708296775818, + 0.06952325999736786, + 0.06357374042272568, + 0.128508523106575, + 0.0496700219810009, + 0.05750654637813568, + 0.016182338818907738 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 0, + "style_probability": 0.4686693549156189, + "style_target": 1, + "support_probability": 0.05375496298074722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18339761773922844, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07981063425540924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8786684274673462, + 0.055405907332897186, + 0.3050547242164612, + 0.6903440952301025, + 0.012123778462409973, + 0.038678016513586044, + 0.027071243152022362, + 0.02664773166179657, + 0.01905856840312481, + 0.02432291768491268, + 0.040927644819021225 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 0, + "style_probability": 0.6653130650520325, + "style_target": 0, + "support_probability": 0.09000059217214584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006797091039169866, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9767338037490845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5906054377555847, + 0.14996758103370667, + 0.5769655108451843, + 0.25055623054504395, + 0.03332265838980675 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.5293272137641907, + "style_target": 1, + "support_probability": 0.007378153502941132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005023246932483242, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9935300946235657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5802315473556519, + 0.14824554324150085, + 0.5789667963981628, + 0.22983238101005554, + 0.018171096220612526, + 0.03663168475031853, + 0.04469894990324974, + 0.050908878445625305, + 0.026477400213479996 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.36568090319633484, + "style_target": 0, + "support_probability": 0.0012827073223888874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06939740522760998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48245370388031006, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8128027319908142, + 0.18721811473369598, + 0.5511160492897034, + 0.18576285243034363, + 0.031509317457675934, + 0.055016618221998215 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.5423564910888672, + "style_target": 1, + "support_probability": 0.028277665376663208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015403662386020193, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6728358268737793, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8441778421401978, + 0.1560509204864502, + 0.8489513397216797, + 0.14649322628974915, + 0.08100582659244537, + 0.09355229139328003 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.28252193331718445, + "style_target": 0, + "support_probability": 0.024094395339488983, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.46610262267523495, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04074113816022873, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1304531991481781, + 0.9519320726394653, + 0.1300245076417923, + 0.25636762380599976, + 0.013245774433016777, + 0.050825126469135284, + 0.08756881207227707, + 0.05404532328248024, + 0.0511816069483757 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 1, + "style_probability": 0.6203296184539795, + "style_target": 1, + "support_probability": 0.4648089110851288, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5455564917575917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0983332097530365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07928571105003357, + 0.9666265845298767, + 0.1352054923772812, + 0.27924081683158875, + 0.025257054716348648, + 0.016615843400359154, + 0.12673424184322357, + 0.023784706369042397, + 0.07442088425159454 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 0, + "style_probability": 0.7598164677619934, + "style_target": 0, + "support_probability": 0.3821229636669159, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5087047875794756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05303541198372841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1678093820810318, + 0.9918681979179382, + 0.7042068839073181, + 0.1678093820810318, + 0.0053060008212924, + 0.014277933165431023, + 0.004472401458770037, + 0.024793829768896103, + 0.032685257494449615, + 0.055350642651319504, + 0.016823306679725647 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 1, + "style_probability": 0.6721301674842834, + "style_target": 1, + "support_probability": 0.6747359037399292, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38557779784538826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.042021818459033966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1653241515159607, + 0.9937478303909302, + 0.7480130195617676, + 0.1653241515159607, + 0.0185471773147583, + 0.022307470440864563 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 1, + "style_probability": 0.43239036202430725, + "style_target": 0, + "support_probability": 0.7329845428466797, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4302410698620275, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23445545136928558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.662899911403656, + 0.7182480692863464, + 0.10710828006267548, + 0.10346170514822006, + 0.0109255937859416 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.7186242341995239, + "style_target": 1, + "support_probability": 0.6080536842346191, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4067671463488057, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30068355798721313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.530098557472229, + 0.6467708349227905, + 0.15276342630386353, + 0.144616037607193, + 0.12041043490171432 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.4387668967247009, + "style_target": 0, + "support_probability": 0.7267844080924988, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004647959706020991, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9940932393074036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.0806117132306099, + 0.1829315423965454, + 0.25537732243537903, + 0.0806117132306099, + 0.07653447240591049, + 0.08829940110445023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 1, + "style_probability": 0.5035585165023804, + "style_target": 1, + "support_probability": 0.00022965554671827704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006328791506483392, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.839088499546051, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11267136037349701, + 0.21359655261039734, + 0.3284003436565399, + 0.11267136037349701, + 0.044693004339933395, + 0.13722297549247742, + 0.13109958171844482, + 0.08466053009033203, + 0.034911926835775375, + 0.3799295127391815, + 0.12173962593078613 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 1, + "style_probability": 0.3345254361629486, + "style_target": 0, + "support_probability": 0.0023834500461816788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46815165776154827, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48386791348457336, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.676769495010376, + 0.46783211827278137, + 0.5013890266418457, + 0.2684963345527649, + 0.026637066155672073, + 0.032985903322696686, + 0.07172553986310959, + 0.0292454082518816, + 0.02262173965573311 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.9637860655784607, + "style_target": 1, + "support_probability": 0.46998289227485657, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015048495274493114, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5825709700584412, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7062578201293945, + 0.41201916337013245, + 0.36137986183166504, + 0.1031484305858612, + 0.03029453381896019, + 0.02265244908630848, + 0.025458084419369698, + 0.010872985236346722, + 0.03239123523235321 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.07370918989181519, + "style_target": 0, + "support_probability": 0.5492820143699646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9291670676288811, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029004471376538277, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9541:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28248336911201477, + 0.9984397292137146, + 0.15173476934432983, + 0.5629829168319702, + 0.017851881682872772, + 0.1159953847527504, + 0.02498641237616539, + 0.05133780837059021, + 0.06356073170900345, + 0.04889044165611267, + 0.03643960878252983 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 1, + "style_probability": 0.8677041530609131, + "style_target": 1, + "support_probability": 0.9427900314331055, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7367851279943796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04327252507209778, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9541:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25630396604537964, + 0.9951529502868652, + 0.23602943122386932, + 0.6416932940483093, + 0.06472212076187134, + 0.18515071272850037, + 0.019068872556090355, + 0.07785822451114655, + 0.10403057932853699, + 0.10415738075971603, + 0.1432342678308487 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 0, + "style_probability": 0.6428067684173584, + "style_target": 0, + "support_probability": 0.9092656970024109, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007532645463300797, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9852824211120605, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5238447785377502, + 0.2664603888988495, + 0.4662915766239166, + 0.2645511031150818, + 0.035694368183612823 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.9074671268463135, + "style_target": 1, + "support_probability": 0.004591899458318949, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006788910396369861, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9676581025123596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44501444697380066, + 0.2785702347755432, + 0.4937952756881714, + 0.3265092968940735, + 0.09421796351671219, + 0.08608704060316086, + 0.04063869267702103, + 0.10875976830720901, + 0.09942932426929474 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.28767508268356323, + "style_target": 0, + "support_probability": 0.009943349286913872, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9270009650396751, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09097529947757721, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1305202841758728, + 0.9929574728012085, + 0.4154053330421448, + 0.1388424038887024, + 0.05975411832332611, + 0.011247280985116959 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.7038297057151794, + "style_target": 1, + "support_probability": 0.9751592874526978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3387765954877278, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10440777242183685, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10603196918964386, + 0.9928776621818542, + 0.33660539984703064, + 0.12251448631286621, + 0.018063722178339958, + 0.0202945563942194 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.07428637146949768, + "style_target": 0, + "support_probability": 0.9574689865112305, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.051147400202074436, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8091690540313721, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7772202491760254, + 0.3671930730342865, + 0.33033788204193115, + 0.6155037879943848, + 0.09396442025899887, + 0.03753865510225296, + 0.06124487146735191, + 0.049230072647333145, + 0.021881859749555588 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 1, + "style_probability": 0.9488881826400757, + "style_target": 1, + "support_probability": 0.01467828918248415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018131702781471602, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.81089186668396, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7641584873199463, + 0.2900398075580597, + 0.26745155453681946, + 0.6091138124465942, + 0.04187026247382164, + 0.025775887072086334, + 0.03858295828104019, + 0.08243028074502945, + 0.04657147079706192 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 1, + "style_probability": 0.3599967956542969, + "style_target": 0, + "support_probability": 0.02440267615020275, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005693751339639105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9787554740905762, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8206462264060974, + 0.36818423867225647, + 0.3149164021015167, + 0.30482929944992065, + 0.06607022136449814, + 0.2022433578968048, + 0.07059350609779358, + 0.13724450767040253, + 0.11888932436704636, + 0.03150731697678566 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.7265301942825317, + "style_target": 1, + "support_probability": 0.004855101462453604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009911718275465583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.966021716594696, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7892324924468994, + 0.3724314868450165, + 0.31625595688819885, + 0.29586198925971985, + 0.046061206609010696, + 0.07396076619625092 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.3018149435520172, + "style_target": 0, + "support_probability": 0.008624113164842129, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7107390523378191, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.024503447115421295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5342889428138733, + 0.8248220086097717, + 0.11502282321453094, + 0.7338107824325562, + 0.006034113932400942 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.632339358329773, + "style_target": 1, + "support_probability": 0.983162522315979, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1276967891386897, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04799026995897293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43476951122283936, + 0.8658294677734375, + 0.09437668323516846, + 0.773809015750885, + 0.024346694350242615 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.03726552426815033, + "style_target": 0, + "support_probability": 0.9711133241653442, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8297169371663431, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10894300788640976, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9992103576660156, + 0.2945452332496643, + 0.1546684354543686, + 0.5472896695137024, + 0.010817806236445904, + 0.011922990903258324 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 1, + "style_probability": 0.7280169725418091, + "style_target": 1, + "support_probability": 0.9323721528053284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7769930823944355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12269140779972076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9961254000663757, + 0.39984411001205444, + 0.21016432344913483, + 0.6578169465065002, + 0.02679823711514473, + 0.024799294769763947 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 0, + "style_probability": 0.5509399771690369, + "style_target": 0, + "support_probability": 0.9567016959190369, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5141989769297849, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2137756496667862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6188766360282898, + 0.6283289790153503, + 0.9984933137893677, + 0.5358953475952148, + 0.04557275027036667, + 0.026631304994225502, + 0.09040208160877228, + 0.032625406980514526, + 0.03222218528389931 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 1, + "style_probability": 0.7387677431106567, + "style_target": 1, + "support_probability": 0.3629402220249176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20109351484594448, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.25884461402893066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6499902009963989, + 0.6586582064628601, + 0.9970396161079407, + 0.5740837454795837, + 0.04932073876261711 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 1, + "style_probability": 0.2711907923221588, + "style_target": 0, + "support_probability": 0.2781945466995239, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8298044012536181, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09917933493852615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.47929468750953674, + 0.4549781084060669, + 0.5185196995735168, + 0.9926722049713135, + 0.039198752492666245, + 0.026228316128253937, + 0.08272776752710342, + 0.09079178422689438, + 0.07982834428548813, + 0.11243574321269989 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.9129736423492432, + "style_target": 1, + "support_probability": 0.7907727956771851, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13973466098695256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04196476936340332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6095659732818604, + 0.331262469291687, + 0.4473264813423157, + 0.9983919262886047, + 0.04538065567612648, + 0.04164229333400726, + 0.11213929951190948, + 0.0716719999909401, + 0.025608841329813004, + 0.09193681925535202 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.10310231149196625, + "style_target": 0, + "support_probability": 0.8292136192321777, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48958495310983335, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19436071813106537, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.957547128200531, + 0.2951395511627197, + 0.10695565491914749, + 0.14569425582885742, + 0.18328656256198883 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 1, + "style_probability": 0.8271255493164062, + "style_target": 1, + "support_probability": 0.13646526634693146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30340821700553916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16468705236911774, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9335511922836304, + 0.25920775532722473, + 0.10697893053293228, + 0.14002954959869385, + 0.0855424702167511 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 0, + "style_probability": 0.5313109755516052, + "style_target": 0, + "support_probability": 0.24005188047885895, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9224877088488588, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012165390886366367, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9617:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3578189015388489, + 0.9831619262695312, + 0.27919864654541016, + 0.3578189015388489, + 0.05326026678085327, + 0.03007381409406662 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.734546959400177, + "style_target": 1, + "support_probability": 0.9959697723388672, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7694072566287087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023319894447922707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9617:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39846161007881165, + 0.9750717878341675, + 0.23592601716518402, + 0.39846161007881165, + 0.018428225070238113, + 0.026156723499298096, + 0.041065748780965805, + 0.02423388510942459, + 0.020947344601154327, + 0.03052002377808094, + 0.06905080378055573 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.3716159760951996, + "style_target": 0, + "support_probability": 0.9898408055305481, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9440841747696197, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011908365413546562, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9620:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9938840270042419, + 0.06280624121427536, + 0.6417394876480103, + 0.051899030804634094, + 0.01544488500803709, + 0.022024543955922127, + 0.018049875274300575, + 0.030282488092780113, + 0.07228399813175201 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.9024060964584351, + "style_target": 1, + "support_probability": 0.9962159991264343, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4204703622799895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012920229695737362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9620:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9981470108032227, + 0.06283004581928253, + 0.7048746943473816, + 0.08432001620531082, + 0.02837529219686985, + 0.019226957112550735, + 0.03043757751584053, + 0.07864269614219666, + 0.08178392052650452 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.11130581051111221, + "style_target": 0, + "support_probability": 0.9967592358589172, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07238271538607549, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31622418761253357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43686652183532715, + 0.1267038732767105, + 0.04177187383174896, + 0.11628291010856628, + 0.008875615894794464, + 0.013998900540173054, + 0.01128113642334938, + 0.017369870096445084, + 0.016154319047927856, + 0.01583852805197239, + 0.055320728570222855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.7150000333786011, + "style_target": 1, + "support_probability": 0.060214731842279434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05321827813177993, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42829176783561707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4064849615097046, + 0.13834911584854126, + 0.037801023572683334, + 0.07462088018655777, + 0.008173571899533272, + 0.008569265715777874, + 0.014877816662192345, + 0.020892299711704254, + 0.003962588496506214, + 0.006180023308843374, + 0.07527665048837662 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.4017775058746338, + "style_target": 0, + "support_probability": 0.0934491753578186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9050069319917178, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.001562869525514543, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9643:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3085244596004486, + 0.21549668908119202, + 0.3085244596004486, + 0.9858188033103943, + 0.05587863549590111 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 1, + "style_probability": 0.6638349890708923, + "style_target": 1, + "support_probability": 0.9899840354919434, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9528983137025567, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0017989880871027708, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9643:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3264494240283966, + 0.20840932428836823, + 0.3264494240283966, + 0.9907989501953125, + 0.022728007286787033 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 0, + "style_probability": 0.9363882541656494, + "style_target": 0, + "support_probability": 0.9946187734603882, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028030214253209392, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8409948348999023, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10456174612045288, + 0.6154177188873291, + 0.6016547083854675, + 0.5192503333091736, + 0.09498773515224457, + 0.16620631515979767 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 0, + "style_probability": 0.479324609041214, + "style_target": 1, + "support_probability": 0.021465042605996132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011544872051041032, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8896417021751404, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11689507961273193, + 0.6249003410339355, + 0.5507878065109253, + 0.6589094400405884, + 0.07663644850254059, + 0.05269252508878708, + 0.024497907608747482, + 0.03309553489089012, + 0.03700457885861397, + 0.08566616475582123, + 0.07142770290374756 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 1, + "style_probability": 0.3117118775844574, + "style_target": 0, + "support_probability": 0.009415742009878159, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5607388080984613, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02072344534099102, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5250548720359802, + 0.9463671445846558, + 0.5803452134132385, + 0.2881234288215637, + 0.004616858903318644, + 0.020951462909579277, + 0.008036007173359394, + 0.0064279381185770035, + 0.02097776159644127 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 0, + "style_probability": 0.3553437292575836, + "style_target": 1, + "support_probability": 0.802501916885376, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05891485630557271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017355648800730705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4431643784046173, + 0.9553302526473999, + 0.43860453367233276, + 0.1987600177526474, + 0.023381013423204422, + 0.04084022343158722, + 0.012213981710374355, + 0.010238218121230602, + 0.014762337319552898 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 1, + "style_probability": 0.04049069061875343, + "style_target": 0, + "support_probability": 0.7450807690620422, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009161744548798203, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9427820444107056, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1651390641927719, + 0.37431490421295166, + 0.5077238082885742, + 0.3081839680671692, + 0.07550699263811111, + 0.04553675651550293, + 0.0478106327354908, + 0.030629837885499, + 0.03348098322749138, + 0.11212074011564255, + 0.09974014014005661 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 0, + "style_probability": 0.46712228655815125, + "style_target": 1, + "support_probability": 0.0057968869805336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04796502816633911, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6490518450737, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20427964627742767, + 0.42316800355911255, + 0.5137593150138855, + 0.31236863136291504, + 0.06579400599002838, + 0.1081906259059906, + 0.0484776571393013, + 0.05605103075504303, + 0.3703269064426422, + 0.1605379581451416, + 0.06166035309433937 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 0, + "style_probability": 0.8869113326072693, + "style_target": 0, + "support_probability": 0.025686640292406082, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027716669081195338, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8792474269866943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1251291036605835, + 0.20976522564888, + 0.8551977872848511, + 0.1251291036605835, + 0.065333791077137 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.8610034584999084, + "style_target": 1, + "support_probability": 0.0035754626151174307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007765239188875266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.852481484413147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12631450593471527, + 0.19958320260047913, + 0.932200014591217, + 0.12631450593471527, + 0.08163312822580338 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.3093699514865875, + "style_target": 0, + "support_probability": 0.009602289646863937, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.753026558229962, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09111344814300537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1101510301232338, + 0.876182496547699, + 0.4178677499294281, + 0.35516759753227234, + 0.012299645692110062, + 0.09953178465366364 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 1, + "style_probability": 0.7623541355133057, + "style_target": 1, + "support_probability": 0.9478960633277893, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6287969461856777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19029343128204346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10818351060152054, + 0.8517137765884399, + 0.3660908341407776, + 0.3866979777812958, + 0.07574138045310974, + 0.017168283462524414 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 1, + "style_probability": 0.3745909631252289, + "style_target": 0, + "support_probability": 0.9416738748550415, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8447553700281509, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.012950830161571503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08387541770935059, + 0.9364453554153442, + 0.4044870436191559, + 0.0660533532500267, + 0.003906672820448875, + 0.008848847821354866, + 0.014504799619317055, + 0.020076168701052666, + 0.029089896008372307 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.664238691329956, + "style_target": 1, + "support_probability": 0.9930236339569092, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6979152871052298, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.013913397677242756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09828787297010422, + 0.9296550750732422, + 0.4888908267021179, + 0.08309683203697205, + 0.012790771201252937 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.22523507475852966, + "style_target": 0, + "support_probability": 0.9922881126403809, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14320320466416056, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3214893341064453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21982349455356598, + 0.0702061578631401, + 0.22360652685165405, + 0.8848130106925964, + 0.009068319573998451, + 0.015141718089580536, + 0.00788269005715847, + 0.011383915320038795, + 0.007929331623017788, + 0.024464458227157593, + 0.03578868508338928 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 1, + "style_probability": 0.9211208820343018, + "style_target": 1, + "support_probability": 0.019381415098905563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10782939675343597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22662121057510376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1982683688402176, + 0.05904979258775711, + 0.2251589447259903, + 0.9182933568954468, + 0.022213619202375412, + 0.011061164550483227, + 0.03097265399992466, + 0.02452848292887211, + 0.013561817817389965, + 0.017759066075086594 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 0, + "style_probability": 0.6413567662239075, + "style_target": 0, + "support_probability": 0.07475144416093826, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7847149867593153, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05383450910449028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9539706110954285, + 0.1639396995306015, + 0.3416399359703064, + 0.12895838916301727, + 0.025505805388092995 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 1, + "style_probability": 0.5978902578353882, + "style_target": 1, + "support_probability": 0.8326323628425598, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5306206573957509, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07842888683080673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9176592826843262, + 0.1765417903661728, + 0.3536733090877533, + 0.15326958894729614, + 0.00965226348489523 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 1, + "style_probability": 0.3864877223968506, + "style_target": 0, + "support_probability": 0.5895044207572937, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3759716947532177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6270997524261475, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3389221429824829, + 0.40382808446884155, + 0.16505195200443268, + 0.898491621017456, + 0.23021475970745087, + 0.05918024107813835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.8894035816192627, + "style_target": 1, + "support_probability": 0.16371381282806396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04153756045252555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.854398787021637, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38102325797080994, + 0.35743075609207153, + 0.14403080940246582, + 0.87699955701828, + 0.048824507743120193, + 0.04292043671011925, + 0.044402286410331726, + 0.05534537881612778, + 0.029971757903695107, + 0.05861538276076317, + 0.0633319690823555 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.29519814252853394, + "style_target": 0, + "support_probability": 0.01613602042198181, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9069438399094436, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029166171327233315, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9868041276931763, + 0.37712422013282776, + 0.515298068523407, + 0.35806897282600403, + 0.034578319638967514, + 0.06391360610723495, + 0.02934074029326439, + 0.0786544606089592, + 0.05927693471312523 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.8850152492523193, + "style_target": 1, + "support_probability": 0.9853543043136597, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38842171089676997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06677675247192383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9939063191413879, + 0.27535480260849, + 0.5329564809799194, + 0.3347577154636383, + 0.06478552520275116, + 0.16675937175750732, + 0.034378018230199814, + 0.021379755809903145, + 0.05742985010147095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.1287519782781601, + "style_target": 0, + "support_probability": 0.9664677977561951, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03364418338851629, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.693671703338623, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2655920088291168, + 0.38529524207115173, + 0.2655920088291168, + 0.8317761421203613, + 0.014956784434616566, + 0.028070900589227676, + 0.06432866305112839, + 0.03406911343336105, + 0.022630970925092697, + 0.041228633373975754, + 0.04613836109638214 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 1, + "style_probability": 0.714234471321106, + "style_target": 1, + "support_probability": 0.027083221822977066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006902502881325545, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.935482919216156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24222442507743835, + 0.3455982506275177, + 0.24222442507743835, + 0.7373130917549133, + 0.02813843823969364, + 0.06413580477237701, + 0.054801635444164276, + 0.03084663301706314, + 0.052767064422369, + 0.03166911005973816, + 0.017155326902866364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 0, + "style_probability": 0.5517253875732422, + "style_target": 0, + "support_probability": 0.002454601228237152, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6345492678624766, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.040746502578258514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2314440757036209, + 0.07737670838832855, + 0.38710057735443115, + 0.9901182055473328, + 0.027372488752007484 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.376586377620697, + "style_target": 1, + "support_probability": 0.9826734662055969, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8676917433576322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0409841425716877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24669265747070312, + 0.0540156215429306, + 0.33797311782836914, + 0.9930647015571594, + 0.016556747257709503 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.7032472491264343, + "style_target": 0, + "support_probability": 0.9793809056282043, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04694434498687019, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21793122589588165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.322258323431015, + 0.664717435836792, + 0.2844181954860687, + 0.31792813539505005, + 0.16742578148841858, + 0.05720214918255806 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 0, + "style_probability": 0.42946818470954895, + "style_target": 1, + "support_probability": 0.06438248604536057, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12054358464505757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09824992716312408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2982845604419708, + 0.7653183937072754, + 0.45048707723617554, + 0.25853803753852844, + 0.06420022994279861, + 0.07903077453374863 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 1, + "style_probability": 0.2770322263240814, + "style_target": 0, + "support_probability": 0.5747995376586914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017656701801423622, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9826377034187317, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16626058518886566, + 0.7216473817825317, + 0.1927119940519333, + 0.3349049985408783, + 0.15645542740821838, + 0.2029106318950653, + 0.06318852305412292, + 0.1033255085349083, + 0.08621666580438614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 0, + "style_probability": 0.1327967494726181, + "style_target": 1, + "support_probability": 0.0008360779611393809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0029382756766892374, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9600244164466858, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17564763128757477, + 0.6888092160224915, + 0.2264777272939682, + 0.43666085600852966, + 0.10004954785108566, + 0.13850945234298706, + 0.07657354325056076, + 0.1069328784942627, + 0.04665323346853256 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 1, + "style_probability": 0.41596370935440063, + "style_target": 0, + "support_probability": 0.000842922949232161, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.05721032408141546, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5216215252876282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33738964796066284, + 0.4295544922351837, + 0.7961399555206299, + 0.15101151168346405, + 0.07715919613838196, + 0.031200487166643143, + 0.04777142032980919, + 0.023238396272063255, + 0.06354748457670212, + 0.11266307532787323, + 0.049982622265815735 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 1, + "style_probability": 0.5956308245658875, + "style_target": 1, + "support_probability": 0.03060459904372692, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10109697564359997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5278382301330566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3877563774585724, + 0.42606163024902344, + 0.8369651436805725, + 0.1340543031692505, + 0.026208585128188133, + 0.02004041150212288, + 0.033212896436452866, + 0.018772244453430176, + 0.026257215067744255, + 0.05346604809165001, + 0.016416292637586594 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 0, + "style_probability": 0.9107272028923035, + "style_target": 0, + "support_probability": 0.024820396676659584, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.023629362645490863, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9044479131698608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2874475419521332, + 0.24908168613910675, + 0.29692456126213074, + 0.44245433807373047, + 0.1560957431793213 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 0, + "style_probability": 0.3184795677661896, + "style_target": 1, + "support_probability": 0.07939624041318893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015130122614513082, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9581097364425659, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2081781029701233, + 0.16374953091144562, + 0.24664878845214844, + 0.46002623438835144, + 0.03361545875668526, + 0.08136985450983047 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 1, + "style_probability": 0.21555158495903015, + "style_target": 0, + "support_probability": 0.0463142953813076, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2773615559348668, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.047747667878866196, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08811116218566895, + 0.138472318649292, + 0.06138559803366661, + 0.952487051486969, + 0.0685594454407692, + 0.10377275198698044 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 0, + "style_probability": 0.34369489550590515, + "style_target": 1, + "support_probability": 0.6660771369934082, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013104625893532743, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1691070795059204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.0776989758014679, + 0.12263040244579315, + 0.06544887274503708, + 0.9786853790283203, + 0.025323418900370598, + 0.04205038398504257 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 1, + "style_probability": 0.05720963329076767, + "style_target": 0, + "support_probability": 0.36043262481689453, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0502820941739359, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5630024075508118, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15627294778823853, + 0.15627294778823853, + 0.762125551700592, + 0.3366178274154663, + 0.024938659742474556, + 0.059974558651447296, + 0.07324692606925964, + 0.0796717181801796, + 0.05767254903912544 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 1, + "style_probability": 0.5497183799743652, + "style_target": 1, + "support_probability": 0.014345665462315083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.039283294590057756, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7805188298225403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12460237741470337, + 0.12460237741470337, + 0.731239378452301, + 0.28448909521102905, + 0.07922226935625076, + 0.09863438457250595, + 0.04296381399035454, + 0.03170573338866234, + 0.04024123027920723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 0, + "style_probability": 0.6325178742408752, + "style_target": 0, + "support_probability": 0.01676173135638237, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8123674268774835, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02320018783211708, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9862300157546997, + 0.3138233423233032, + 0.11147230118513107, + 0.10765232890844345, + 0.010359417647123337, + 0.07238754630088806, + 0.015686500817537308, + 0.1706431359052658, + 0.07253378629684448, + 0.07824209332466125, + 0.15154509246349335 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 1, + "style_probability": 0.6759089231491089, + "style_target": 1, + "support_probability": 0.9542149901390076, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5977114177092764, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025568468496203423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9889345765113831, + 0.21149761974811554, + 0.08900275081396103, + 0.08066699653863907, + 0.007832331582903862 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 1, + "style_probability": 0.4717211127281189, + "style_target": 0, + "support_probability": 0.9362437129020691, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002525991419376773, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9917303919792175, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5295162200927734, + 0.6381494998931885, + 0.6705989241600037, + 0.25986629724502563, + 0.068582683801651 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 0, + "style_probability": 0.31737786531448364, + "style_target": 1, + "support_probability": 0.0024863346479833126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00027916933640686944, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9964039325714111, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.362123042345047, + 0.4795299768447876, + 0.7206209897994995, + 0.22689783573150635, + 0.25760889053344727 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 1, + "style_probability": 0.08488933742046356, + "style_target": 0, + "support_probability": 0.0018260638462379575, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02187505906278512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7021741271018982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2691284120082855, + 0.7756653428077698, + 0.2557676434516907, + 0.22516857087612152, + 0.2253754585981369, + 0.06196320056915283 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 0, + "style_probability": 0.3491188883781433, + "style_target": 1, + "support_probability": 0.013815048150718212, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.052970379406588344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5806869268417358, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4374237358570099, + 0.8616579174995422, + 0.36144769191741943, + 0.24076154828071594, + 0.05026684328913689, + 0.02901475317776203 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 0, + "style_probability": 0.8482822179794312, + "style_target": 0, + "support_probability": 0.019163168966770172, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3408386209467881, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04422753304243088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2397240251302719, + 0.23378828167915344, + 0.9023417830467224, + 0.34585800766944885, + 0.025984322652220726, + 0.03364511951804161, + 0.026902778074145317, + 0.07040798664093018, + 0.02468256838619709 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 1, + "style_probability": 0.5203518271446228, + "style_target": 1, + "support_probability": 0.6836516261100769, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3205637979597471, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.037178341299295425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3829793334007263, + 0.23360513150691986, + 0.9264219403266907, + 0.3735755681991577, + 0.023198939859867096, + 0.016933199018239975, + 0.04199952632188797, + 0.0567365400493145, + 0.03983680531382561, + 0.024751219898462296, + 0.0242563858628273 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 0, + "style_probability": 0.5172328948974609, + "style_target": 0, + "support_probability": 0.62702476978302, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8113104956678137, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007057867478579283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31003043055534363, + 0.9889028668403625, + 0.43896880745887756, + 0.6017319560050964, + 0.053201813250780106, + 0.0686725601553917, + 0.04200105369091034, + 0.02362571656703949, + 0.06078197434544563, + 0.12521907687187195, + 0.04293084889650345 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.5002061128616333, + "style_target": 1, + "support_probability": 0.9855052828788757, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7128207350768297, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0016672250349074602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5732215046882629, + 0.9995418787002563, + 0.6274569630622864, + 0.8011268973350525, + 0.08223307132720947, + 0.018430670723319054, + 0.05898985266685486, + 0.031525176018476486, + 0.014914712868630886, + 0.04949381202459335 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.2056557834148407, + "style_target": 0, + "support_probability": 0.9961296319961548, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1857112849877835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.061587922275066376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6269786357879639, + 0.4140236973762512, + 0.06794470548629761, + 0.8591572046279907, + 0.03180430829524994 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 1, + "style_probability": 0.7166165113449097, + "style_target": 1, + "support_probability": 0.135267972946167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20681228249689002, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13566701114177704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6078289151191711, + 0.4761342704296112, + 0.05106054246425629, + 0.9020766019821167, + 0.04400092363357544 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 0, + "style_probability": 0.8791409134864807, + "style_target": 0, + "support_probability": 0.05316315218806267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.586498569277987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0159074068069458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4169868528842926, + 0.4221324324607849, + 0.9949296712875366, + 0.4169868528842926, + 0.029914183542132378, + 0.02126355655491352 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 0, + "style_probability": 0.2555939555168152, + "style_target": 1, + "support_probability": 0.8669528365135193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3543286704557199, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019480591639876366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5073854327201843, + 0.49705496430397034, + 0.9931204319000244, + 0.5073854327201843, + 0.1352265626192093, + 0.023939723148941994, + 0.02477046102285385, + 0.06371330469846725, + 0.018241984769701958 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 1, + "style_probability": 0.1597573459148407, + "style_target": 0, + "support_probability": 0.7590295672416687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6376436286194862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07981645315885544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9430955052375793, + 0.4884304404258728, + 0.3660000264644623, + 0.23549386858940125, + 0.02052966319024563, + 0.03621770814061165, + 0.019659345969557762, + 0.016185157001018524, + 0.020678596571087837 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.7839381694793701, + "style_target": 1, + "support_probability": 0.5802918076515198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4439352755417053, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044492755085229874, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.97734135389328, + 0.3308717608451843, + 0.2347106635570526, + 0.17197230458259583, + 0.015998536720871925, + 0.019768409430980682, + 0.019155843183398247, + 0.01526097021996975, + 0.08044451475143433 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.34342578053474426, + "style_target": 0, + "support_probability": 0.7238016128540039, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02832147301307487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5041437745094299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5548421144485474, + 0.10292768478393555, + 0.5548421144485474, + 0.7865661382675171, + 0.0174234788864851, + 0.024212628602981567, + 0.020674392580986023, + 0.0443316213786602, + 0.05345747247338295, + 0.040318410843610764, + 0.020918672904372215 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 1, + "style_probability": 0.5350356101989746, + "style_target": 1, + "support_probability": 0.014674123376607895, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.426295791838415, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12620986998081207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8002349138259888, + 0.06530517339706421, + 0.8002349138259888, + 0.9187036752700806, + 0.007283872924745083, + 0.018864840269088745, + 0.03457785025238991, + 0.014085985720157623, + 0.020031625404953957, + 0.061704475432634354, + 0.03577664867043495 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 0, + "style_probability": 0.8531941175460815, + "style_target": 0, + "support_probability": 0.20194070041179657, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9229718694413996, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005284756887704134, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9830641150474548, + 0.31387850642204285, + 0.3260011672973633, + 0.23867355287075043, + 0.03279054909944534 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.8779358863830566, + "style_target": 1, + "support_probability": 0.9916087985038757, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37789689854540187, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010913331061601639, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9870033860206604, + 0.29128313064575195, + 0.2711417078971863, + 0.2936207354068756, + 0.17602021992206573 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.11607187241315842, + "style_target": 0, + "support_probability": 0.9816515445709229, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6017092540015029, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04601280018687248, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7809903621673584, + 0.6789095401763916, + 0.17505498230457306, + 0.40457794070243835, + 0.0054397727362811565, + 0.019498147070407867 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 1, + "style_probability": 0.7129298448562622, + "style_target": 1, + "support_probability": 0.8762426376342773, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3902487068906917, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03647878021001816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6649014949798584, + 0.6898284554481506, + 0.15801475942134857, + 0.4101890027523041, + 0.012074170634150505, + 0.012206638231873512 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 1, + "style_probability": 0.4092327356338501, + "style_target": 0, + "support_probability": 0.8138152956962585, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.12667524854581824, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26527082920074463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48596081137657166, + 0.5744367837905884, + 0.7594410181045532, + 0.9457401037216187, + 0.01513670478016138, + 0.014288447797298431, + 0.024212302640080452, + 0.014729506336152554, + 0.008402450941503048 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.7507830858230591, + "style_target": 1, + "support_probability": 0.04608617722988129, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05748975695841805, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25557616353034973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5209623575210571, + 0.556066632270813, + 0.7167548537254333, + 0.9145275354385376, + 0.019965633749961853, + 0.032712552696466446, + 0.03987640142440796, + 0.01623450592160225, + 0.028599614277482033, + 0.059911083430051804, + 0.015705933794379234 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.347059965133667, + "style_target": 0, + "support_probability": 0.04599108546972275, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2743114179633088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1571822464466095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1597386598587036, + 0.8879066109657288, + 0.14451202750205994, + 0.26145443320274353, + 0.043787870556116104, + 0.023119661957025528, + 0.019852163270115852, + 0.04851262643933296, + 0.03259606659412384, + 0.005081871058791876, + 0.07058299332857132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.6876200437545776, + "style_target": 1, + "support_probability": 0.1905638575553894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009806883476424937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.200771301984787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.111491858959198, + 0.9051876068115234, + 0.1441488415002823, + 0.2905946969985962, + 0.09929253906011581, + 0.043813612312078476, + 0.04968712106347084, + 0.07527859508991241, + 0.052750084549188614, + 0.06403020024299622 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.055737387388944626, + "style_target": 0, + "support_probability": 0.10105687379837036, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7421281494625106, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1391896903514862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9585404396057129, + 0.10941610485315323, + 0.988174557685852, + 0.1897258758544922, + 0.026275966316461563 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 1, + "style_probability": 0.9464471340179443, + "style_target": 1, + "support_probability": 0.4199444055557251, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6969276211978427, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05197592452168465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9235553741455078, + 0.12618571519851685, + 0.9856686592102051, + 0.232598215341568, + 0.03143699839711189 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 0, + "style_probability": 0.6366137266159058, + "style_target": 0, + "support_probability": 0.7207571864128113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17426898322036377, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7300394177436829, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9776051640510559, + 0.17929567396640778, + 0.8548010587692261, + 0.4904226064682007, + 0.04686588793992996, + 0.022747989743947983 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 1, + "style_probability": 0.7863606810569763, + "style_target": 1, + "support_probability": 0.12760402262210846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19055464477799886, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41439810395240784, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9729966521263123, + 0.2038377821445465, + 0.8430715203285217, + 0.4403851628303528, + 0.04447866976261139, + 0.07380516082048416, + 0.06449898332357407, + 0.042279571294784546, + 0.05122488737106323 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 1, + "style_probability": 0.4462797939777374, + "style_target": 0, + "support_probability": 0.2418307662010193, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6461746632480279, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07694332301616669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5615746974945068, + 0.07320357859134674, + 0.9901151657104492, + 0.5908323526382446, + 0.01638580858707428, + 0.01336091011762619, + 0.026058344170451164, + 0.010552250780165195, + 0.039490848779678345 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.5583853125572205, + "style_target": 1, + "support_probability": 0.8071023225784302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0688339186062628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11829742044210434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4213109314441681, + 0.0653100535273552, + 0.9932783246040344, + 0.5638787746429443, + 0.11213812232017517, + 0.05129576846957207, + 0.04707115888595581, + 0.03828723728656769, + 0.05337490886449814 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.040532615035772324, + "style_target": 0, + "support_probability": 0.7943689227104187, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8173414037667782, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07763490080833435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.992950439453125, + 0.34967121481895447, + 0.1549248993396759, + 0.8507012724876404, + 0.0642504096031189, + 0.07292146980762482, + 0.15947391092777252, + 0.02902737632393837, + 0.045315153896808624, + 0.06858626008033752, + 0.013993043452501297 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 1, + "style_probability": 0.7679744362831116, + "style_target": 1, + "support_probability": 0.8419231176376343, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7052192076534578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1976369470357895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9863733053207397, + 0.41433024406433105, + 0.20529253780841827, + 0.714143693447113, + 0.07772345095872879, + 0.0353599488735199, + 0.04859061911702156, + 0.046923521906137466, + 0.03579244390130043, + 0.059952981770038605 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 0, + "style_probability": 0.7302293181419373, + "style_target": 0, + "support_probability": 0.7878811359405518, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07827586969025566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6001282930374146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5107633471488953, + 0.903602659702301, + 0.25402042269706726, + 0.3257597088813782, + 0.11410125344991684 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.8981135487556458, + "style_target": 1, + "support_probability": 0.03519556671380997, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006538529767029127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.884466826915741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4964793920516968, + 0.8843762874603271, + 0.23133033514022827, + 0.3083265721797943, + 0.223736971616745, + 0.13114489614963531 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.34867891669273376, + "style_target": 0, + "support_probability": 0.0019154532346874475, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013216666626521708, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9352681636810303, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37501248717308044, + 0.20647738873958588, + 0.6813079118728638, + 0.4606126844882965, + 0.03106549382209778, + 0.14477628469467163 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.7474427819252014, + "style_target": 1, + "support_probability": 0.018726205453276634, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014970458510148057, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.878525972366333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35337692499160767, + 0.22191648185253143, + 0.6894747018814087, + 0.4660549461841583, + 0.07250401377677917, + 0.11642079055309296 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.12152054905891418, + "style_target": 0, + "support_probability": 0.029084043577313423, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.45863586153086905, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.356061190366745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8629411458969116, + 0.47679126262664795, + 0.3154586851596832, + 0.36511778831481934, + 0.038315549492836, + 0.052894070744514465, + 0.03654566779732704, + 0.03442037105560303, + 0.04787039756774902 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8643758893013, + "style_target": 1, + "support_probability": 0.23522837460041046, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39283747267308244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24182459712028503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7720313668251038, + 0.3849998712539673, + 0.28225597739219666, + 0.30800801515579224, + 0.024737773463129997, + 0.03625744208693504, + 0.03394010290503502, + 0.0374458022415638, + 0.016805915161967278 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 1, + "style_probability": 0.3884860873222351, + "style_target": 0, + "support_probability": 0.6464070081710815, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.019482170550761856, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.609970211982727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22415341436862946, + 0.41271665692329407, + 0.44492945075035095, + 0.4898309111595154, + 0.04297676682472229, + 0.02104342170059681, + 0.035980768501758575, + 0.024676388129591942, + 0.026156553998589516, + 0.06020219251513481 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.8387988805770874, + "style_target": 1, + "support_probability": 0.009066352620720863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010015199145284889, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6916757225990295, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2443908154964447, + 0.4122283458709717, + 0.3220897316932678, + 0.4570712149143219, + 0.05345090851187706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.36747410893440247, + "style_target": 0, + "support_probability": 0.009614704176783562, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30216748048189074, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9068776965141296, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7607116103172302, + 0.07112342864274979, + 0.12867337465286255, + 0.07112342864274979, + 0.02874659188091755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 0, + "style_probability": 0.4668539762496948, + "style_target": 1, + "support_probability": 0.4547520875930786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14544493080577883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9507042169570923, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.785966157913208, + 0.0708284080028534, + 0.1200125440955162, + 0.0708284080028534, + 0.012146552093327045, + 0.03434401750564575 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 1, + "style_probability": 0.3438957929611206, + "style_target": 0, + "support_probability": 0.18264266848564148, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8346790578101314, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007579471450299025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33262911438941956, + 0.9891001582145691, + 0.10429941862821579, + 0.33262911438941956, + 0.01106174848973751, + 0.025235280394554138 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 1, + "style_probability": 0.6578199863433838, + "style_target": 1, + "support_probability": 0.964242160320282, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.513452076494398, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.007704588584601879, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2491728663444519, + 0.9957140684127808, + 0.09967733174562454, + 0.2491728663444519, + 0.03125825896859169, + 0.027503998950123787 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 1, + "style_probability": 0.20209404826164246, + "style_target": 0, + "support_probability": 0.9370429515838623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5470018509265225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06819596141576767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29164519906044006, + 0.19035816192626953, + 0.3769996762275696, + 0.9808380007743835, + 0.007791665382683277, + 0.03969505801796913, + 0.008806408382952213, + 0.016367964446544647, + 0.027419855818152428 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 0, + "style_probability": 0.4018436670303345, + "style_target": 1, + "support_probability": 0.7619199156761169, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6388791555879566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05700146406888962, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2843214273452759, + 0.15457367897033691, + 0.2878066301345825, + 0.9809354543685913, + 0.02287169173359871, + 0.014298356138169765, + 0.031661901623010635, + 0.017901450395584106, + 0.06001017242670059 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 0, + "style_probability": 0.670199990272522, + "style_target": 0, + "support_probability": 0.6668181419372559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012682474246768272, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9373040795326233, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32259491086006165, + 0.7912771701812744, + 0.6957671642303467, + 0.8106449842453003, + 0.05891816318035126, + 0.0406184121966362, + 0.03860938176512718, + 0.1196131631731987, + 0.04154691472649574, + 0.02131681889295578, + 0.06456484645605087 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 0, + "style_probability": 0.49881473183631897, + "style_target": 1, + "support_probability": 0.0072362362407147884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06662373408239425, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5247577428817749, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27910205721855164, + 0.7824884653091431, + 0.664592444896698, + 0.8245656490325928, + 0.0679301992058754 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 1, + "style_probability": 0.2594955861568451, + "style_target": 0, + "support_probability": 0.08155141025781631, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46548834395589095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19241301715373993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24119043350219727, + 0.9082968235015869, + 0.3116724193096161, + 0.22029778361320496, + 0.02166430465877056 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 1, + "style_probability": 0.8936338424682617, + "style_target": 1, + "support_probability": 0.20118148624897003, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21664329837644725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4326673448085785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2650606632232666, + 0.844991147518158, + 0.3067125678062439, + 0.2340621054172516, + 0.09655837714672089 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 0, + "style_probability": 0.7055398225784302, + "style_target": 0, + "support_probability": 0.20605625212192535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04957749268217491, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6521421074867249, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34750303626060486, + 0.14566397666931152, + 0.2821245789527893, + 0.3624323010444641, + 0.02800826169550419, + 0.008466132916510105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 1, + "style_probability": 0.793751060962677, + "style_target": 1, + "support_probability": 0.03258860856294632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016379739694947517, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.849341869354248, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37604013085365295, + 0.16571186482906342, + 0.3700922727584839, + 0.48237791657447815, + 0.08774678409099579, + 0.02957824431359768, + 0.009943589568138123, + 0.016875186935067177, + 0.040147751569747925 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 1, + "style_probability": 0.4429772198200226, + "style_target": 0, + "support_probability": 0.005023530684411526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08286478766968836, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6636645793914795, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07256122678518295, + 0.32940033078193665, + 0.1736268550157547, + 0.2885746955871582, + 0.08944689482450485, + 0.04430132731795311, + 0.03399517387151718, + 0.019491737708449364, + 0.008097164332866669 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.659553587436676, + "style_target": 1, + "support_probability": 0.10395989567041397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010907917535687879, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5123888850212097, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.05668210610747337, + 0.256733238697052, + 0.11772691458463669, + 0.33417341113090515, + 0.03546217083930969, + 0.036466289311647415, + 0.038610056042671204, + 0.01907121017575264, + 0.07604609429836273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.10795015841722488, + "style_target": 0, + "support_probability": 0.317142516374588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014683870283012285, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5829272866249084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.05507415905594826, + 0.40351417660713196, + 0.607799232006073, + 0.22207865118980408, + 0.09355775266885757, + 0.028249746188521385, + 0.026024293154478073, + 0.07970361411571503, + 0.07115266472101212, + 0.05885547772049904, + 0.03620712459087372 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 1, + "style_probability": 0.820408046245575, + "style_target": 1, + "support_probability": 0.01248579565435648, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011681202772621297, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6800063252449036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14144781231880188, + 0.4570048749446869, + 0.5497068762779236, + 0.34183236956596375, + 0.032798293977975845, + 0.04567808285355568, + 0.07168640196323395, + 0.0729610025882721, + 0.06850190460681915, + 0.10248866677284241, + 0.29664984345436096 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 0, + "style_probability": 0.5563347339630127, + "style_target": 0, + "support_probability": 0.01569323055446148, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.130210626037208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.473941445350647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29008856415748596, + 0.22179298102855682, + 0.11052875220775604, + 0.8921908140182495, + 0.026572637259960175 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.8920268416404724, + "style_target": 1, + "support_probability": 0.018315529450774193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06539346459933792, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2716018557548523, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28126779198646545, + 0.22576437890529633, + 0.11679846793413162, + 0.898391842842102, + 0.07062976807355881, + 0.06267049163579941 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.4034806489944458, + "style_target": 0, + "support_probability": 0.03458049148321152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2949020575115567, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12311579287052155, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9803771376609802, + 0.11992749571800232, + 0.11965937912464142, + 0.4504943788051605, + 0.11078039556741714, + 0.02091333083808422 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.6410487294197083, + "style_target": 1, + "support_probability": 0.17693643271923065, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013038905944395919, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17157402634620667, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9730374217033386, + 0.09840533882379532, + 0.12857867777347565, + 0.4822438061237335, + 0.03574748709797859, + 0.04550442844629288 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.05785178020596504, + "style_target": 0, + "support_probability": 0.08113139122724533, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8865976976220928, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06727518886327744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14713256061077118, + 0.19996993243694305, + 0.9979562759399414, + 0.09982338547706604, + 0.009105588309466839, + 0.05170533433556557, + 0.022036954760551453, + 0.0349438339471817, + 0.010991555638611317 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.7763621807098389, + "style_target": 1, + "support_probability": 0.9313082098960876, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7266449410471585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03734033927321434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13880124688148499, + 0.251706600189209, + 0.9967939257621765, + 0.10461077094078064, + 0.01931779831647873, + 0.022218067198991776, + 0.006060569081455469, + 0.01371406763792038, + 0.07629464566707611 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.3863210380077362, + "style_target": 0, + "support_probability": 0.9344691634178162, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6525957564743526, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26193705201148987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9736925959587097, + 0.21305526793003082, + 0.42637044191360474, + 0.3686795234680176, + 0.023385455831885338, + 0.039316046983003616, + 0.014126962050795555, + 0.022484924644231796, + 0.019690904766321182, + 0.052879322320222855, + 0.01784258708357811 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 1, + "style_probability": 0.9197377562522888, + "style_target": 1, + "support_probability": 0.2342187911272049, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.579564533268086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22096042335033417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9691736698150635, + 0.23699820041656494, + 0.4632950723171234, + 0.37007132172584534, + 0.016265837475657463 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 0, + "style_probability": 0.689460813999176, + "style_target": 0, + "support_probability": 0.40354859828948975, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17076074701086105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8064084053039551, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4294103682041168, + 0.3380960524082184, + 0.9117183089256287, + 0.19041576981544495, + 0.02387819066643715 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 1, + "style_probability": 0.9241684079170227, + "style_target": 1, + "support_probability": 0.015596101991832256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03663078323885111, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8979173898696899, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44826740026474, + 0.35671117901802063, + 0.9158614873886108, + 0.16722284257411957, + 0.04744743928313255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 0, + "style_probability": 0.8160430192947388, + "style_target": 0, + "support_probability": 0.005310756620019674, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0542053454583223, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7530564665794373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4424658715724945, + 0.20317015051841736, + 0.9151431918144226, + 0.18845579028129578, + 0.02208280749619007, + 0.11614838242530823 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 1, + "style_probability": 0.898351788520813, + "style_target": 1, + "support_probability": 0.00607699528336525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05166678805342263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6593864560127258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3731626570224762, + 0.16584722697734833, + 0.9221485257148743, + 0.14501912891864777, + 0.01993815042078495, + 0.04631157964468002 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 0, + "style_probability": 0.6391298770904541, + "style_target": 0, + "support_probability": 0.02029556781053543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.525117851524278, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08775783330202103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08281601220369339, + 0.1738312989473343, + 0.3466970920562744, + 0.7145830988883972, + 0.04077056422829628, + 0.016288811340928078, + 0.04243653267621994, + 0.023722734302282333, + 0.08127950876951218 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.890867292881012, + "style_target": 1, + "support_probability": 0.24013890326023102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.155029061706178, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1259722113609314, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08476760238409042, + 0.1639866977930069, + 0.324391633272171, + 0.6991702318191528, + 0.02681620605289936, + 0.020126305520534515, + 0.2808893024921417, + 0.04253730922937393, + 0.051815249025821686, + 0.04287426173686981, + 0.03250684589147568 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.3545284867286682, + "style_target": 0, + "support_probability": 0.09994758665561676, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8261411086118639, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022921307012438774, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7585672736167908, + 0.9209910035133362, + 0.430944561958313, + 0.20262984931468964, + 0.017961574718356133, + 0.01840832829475403, + 0.06690371036529541, + 0.011684867553412914, + 0.010704856365919113, + 0.015478712506592274, + 0.05254841968417168 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.8310302495956421, + "style_target": 1, + "support_probability": 0.9886782765388489, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10583360734026515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09260011464357376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6372122764587402, + 0.910747766494751, + 0.32062819600105286, + 0.21722330152988434, + 0.027271877974271774, + 0.03513312339782715, + 0.05358729138970375, + 0.015870582312345505, + 0.018474344164133072, + 0.03342585638165474 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.0407925620675087, + "style_target": 0, + "support_probability": 0.9303789138793945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08298232164558307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.25014016032218933, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48037388920783997, + 0.4010440409183502, + 0.9075708985328674, + 0.28368011116981506, + 0.18938098847866058 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 0, + "style_probability": 0.3047254681587219, + "style_target": 1, + "support_probability": 0.03405016288161278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13609167448377513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23654848337173462, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5170660614967346, + 0.44204068183898926, + 0.884670078754425, + 0.3260570168495178, + 0.05356433615088463 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 0, + "style_probability": 0.5632666349411011, + "style_target": 0, + "support_probability": 0.055551864206790924, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6925517102394217, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.02901938185095787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40949201583862305, + 0.32170915603637695, + 0.992515504360199, + 0.40949201583862305, + 0.020768489688634872, + 0.041696082800626755 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 1, + "style_probability": 0.6122933626174927, + "style_target": 1, + "support_probability": 0.8264586925506592, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8979055857426638, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.017890045419335365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.403790146112442, + 0.2706489562988281, + 0.9949550032615662, + 0.403790146112442, + 0.0415470264852047, + 0.019626857712864876 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 0, + "style_probability": 0.8714280724525452, + "style_target": 0, + "support_probability": 0.8579603433609009, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2459505007400001, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09716710448265076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9710254073143005, + 0.2529900074005127, + 0.13222181797027588, + 0.05736445635557175, + 0.006282146088778973, + 0.025439152494072914, + 0.007506771478801966, + 0.03324107825756073, + 0.050956785678863525 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 0, + "style_probability": 0.12136004120111465, + "style_target": 1, + "support_probability": 0.8217740058898926, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06969718685354803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1334150731563568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9818315505981445, + 0.20418179035186768, + 0.06966108828783035, + 0.05619838461279869, + 0.032751668244600296, + 0.011652527377009392, + 0.022010114043951035, + 0.018682243302464485, + 0.03321116417646408 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 1, + "style_probability": 0.04842684790492058, + "style_target": 0, + "support_probability": 0.8480638861656189, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.43470218941716326, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09699559211730957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23337878286838531, + 0.377816766500473, + 0.9744724631309509, + 0.23337878286838531, + 0.033047161996364594, + 0.1076231300830841, + 0.018190404400229454, + 0.0166928730905056, + 0.031052974984049797, + 0.07020262628793716, + 0.07227341830730438 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.385377436876297, + "style_target": 1, + "support_probability": 0.8050374984741211, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7404323460147452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07633672654628754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22851908206939697, + 0.405521422624588, + 0.9627305269241333, + 0.22851908206939697, + 0.04185711219906807, + 0.022387845441699028, + 0.029816314578056335, + 0.06869672238826752, + 0.11551695317029953, + 0.025992341339588165, + 0.0516030490398407 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.5537794828414917, + "style_target": 0, + "support_probability": 0.9233005046844482, + "support_target": 1 + } + ], + "schema_version": "personavoice-language-evaluation-v1", + "status": "measured_evaluation_run", + "summary": { + "accept_accuracy": { + "lower": 0.8296666666666667, + "mean": 0.8426666666666667, + "speaker_count": 1500.0, + "upper": 0.8556666666666667 + }, + "accept_brier": 0.13919691154633118, + "accept_ece": 0.15832027749572913, + "area_under_risk_coverage": 0.31461074988167886, + "budget_breakdown": { + "10": { + "acceptance_rate": 0.01444043321299639, + "count": 277, + "f1_macro": 0.7154478947104858, + "false_abstention": 0.9393939393939394, + "false_acceptance": 0.5, + "precision_macro": 0.7021660649819494, + "recall_macro": 0.7292418772563177 + }, + "11": { + "acceptance_rate": 0.019438444924406047, + "count": 463, + "f1_macro": 0.6876998348766321, + "false_abstention": 0.9263157894736842, + "false_acceptance": 0.2222222222222222, + "precision_macro": 0.6760259179265659, + "recall_macro": 0.6997840172786177 + }, + "12": { + "acceptance_rate": 0.0, + "count": 4, + "f1_macro": 0.16666666666666666, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.125, + "recall_macro": 0.25 + }, + "14": { + "acceptance_rate": 0.0, + "count": 2, + "f1_macro": 0.5, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.5, + "recall_macro": 0.5 + }, + "5": { + "acceptance_rate": 0.0214190093708166, + "count": 747, + "f1_macro": 0.757851124882163, + "false_abstention": 0.8991596638655462, + "false_acceptance": 0.25, + "precision_macro": 0.7463186077643909, + "recall_macro": 0.7697456492637216 + }, + "6": { + "acceptance_rate": 0.018300653594771243, + "count": 765, + "f1_macro": 0.7274254470535407, + "false_abstention": 0.9291338582677166, + "false_acceptance": 0.35714285714285715, + "precision_macro": 0.711764705882353, + "recall_macro": 0.7437908496732026 + }, + "9": { + "acceptance_rate": 0.02021563342318059, + "count": 742, + "f1_macro": 0.7256355004292145, + "false_abstention": 0.8968253968253969, + "false_acceptance": 0.13333333333333333, + "precision_macro": 0.7169811320754716, + "recall_macro": 0.7345013477088949 + } + }, + "citation_macro": { + "f1_macro": 0.7264573394495414, + "precision_macro": 0.7143333333333334, + "recall_macro": 0.739 + }, + "citation_precision_mean": 0.7143333333333334, + "citation_precision_micro": 0.9067484662576687, + "citation_recall_mean": 0.739, + "citation_recall_micro": 0.739, + "conflict": { + "accuracy": 0.791, + "brier": 0.1477817351687508, + "count": 3000, + "ece": 0.04846426085747467, + "f1": 0.6786263454638647, + "precision": 0.6961093585699264, + "recall": 0.662 + }, + "false_abstention": 0.914, + "false_acceptance": 0.25862068965517243, + "positive_coverage": 0.086, + "risk_coverage": [ + { + "coverage": 0.0003333333333333333, + "risk": 0.0, + "threshold": 0.980871499962252 + }, + { + "coverage": 0.0006666666666666666, + "risk": 0.5, + "threshold": 0.9630376649111482 + }, + { + "coverage": 0.001, + "risk": 0.3333333333333333, + "threshold": 0.962119850031616 + }, + { + "coverage": 0.0013333333333333333, + "risk": 0.25, + "threshold": 0.9617793654845771 + }, + { + "coverage": 0.0016666666666666668, + "risk": 0.2, + "threshold": 0.9595500645424129 + }, + { + "coverage": 0.002, + "risk": 0.16666666666666666, + "threshold": 0.9588082379954767 + }, + { + "coverage": 0.0023333333333333335, + "risk": 0.2857142857142857, + "threshold": 0.9582182169958681 + }, + { + "coverage": 0.0026666666666666666, + "risk": 0.25, + "threshold": 0.9574707596767181 + }, + { + "coverage": 0.003, + "risk": 0.3333333333333333, + "threshold": 0.9528983137025567 + }, + { + "coverage": 0.0033333333333333335, + "risk": 0.3, + "threshold": 0.9466716719669193 + }, + { + "coverage": 0.0036666666666666666, + "risk": 0.36363636363636365, + "threshold": 0.9462730523439724 + }, + { + "coverage": 0.004, + "risk": 0.3333333333333333, + "threshold": 0.9451255502738194 + }, + { + "coverage": 0.004333333333333333, + "risk": 0.3076923076923077, + "threshold": 0.9440841747696197 + }, + { + "coverage": 0.004666666666666667, + "risk": 0.2857142857142857, + "threshold": 0.9438458438392375 + }, + { + "coverage": 0.005, + "risk": 0.26666666666666666, + "threshold": 0.9421870114601696 + }, + { + "coverage": 0.005333333333333333, + "risk": 0.3125, + "threshold": 0.9421533215282208 + }, + { + "coverage": 0.005666666666666667, + "risk": 0.29411764705882354, + "threshold": 0.9414271087513894 + }, + { + "coverage": 0.006, + "risk": 0.2777777777777778, + "threshold": 0.9391226394020833 + }, + { + "coverage": 0.006333333333333333, + "risk": 0.2631578947368421, + "threshold": 0.9367006737587574 + }, + { + "coverage": 0.006666666666666667, + "risk": 0.25, + "threshold": 0.9365836946525576 + }, + { + "coverage": 0.007, + "risk": 0.2857142857142857, + "threshold": 0.936413455317297 + }, + { + "coverage": 0.007333333333333333, + "risk": 0.2727272727272727, + "threshold": 0.9364125548352636 + }, + { + "coverage": 0.007666666666666666, + "risk": 0.2608695652173913, + "threshold": 0.9342969295761835 + }, + { + "coverage": 0.008, + "risk": 0.2916666666666667, + "threshold": 0.9319170314157114 + }, + { + "coverage": 0.008333333333333333, + "risk": 0.28, + "threshold": 0.9315085920875216 + }, + { + "coverage": 0.008666666666666666, + "risk": 0.2692307692307692, + "threshold": 0.9295468109349576 + }, + { + "coverage": 0.009, + "risk": 0.25925925925925924, + "threshold": 0.9292829633997954 + }, + { + "coverage": 0.009333333333333334, + "risk": 0.25, + "threshold": 0.9291670676288811 + }, + { + "coverage": 0.009666666666666667, + "risk": 0.2413793103448276, + "threshold": 0.9288992406211072 + }, + { + "coverage": 0.01, + "risk": 0.23333333333333334, + "threshold": 0.9288408926736319 + }, + { + "coverage": 0.010333333333333333, + "risk": 0.22580645161290322, + "threshold": 0.9270009650396751 + }, + { + "coverage": 0.010666666666666666, + "risk": 0.21875, + "threshold": 0.9267034092080864 + }, + { + "coverage": 0.011, + "risk": 0.21212121212121213, + "threshold": 0.9262966745996276 + }, + { + "coverage": 0.011333333333333334, + "risk": 0.23529411764705882, + "threshold": 0.9239681621158239 + }, + { + "coverage": 0.011666666666666667, + "risk": 0.22857142857142856, + "threshold": 0.9229718694413996 + }, + { + "coverage": 0.012, + "risk": 0.2222222222222222, + "threshold": 0.9224877088488588 + }, + { + "coverage": 0.012333333333333333, + "risk": 0.21621621621621623, + "threshold": 0.9196584305640401 + }, + { + "coverage": 0.012666666666666666, + "risk": 0.21052631578947367, + "threshold": 0.9184064922912694 + }, + { + "coverage": 0.013, + "risk": 0.23076923076923078, + "threshold": 0.918162043639045 + }, + { + "coverage": 0.013333333333333334, + "risk": 0.25, + "threshold": 0.9162442140311244 + }, + { + "coverage": 0.013666666666666667, + "risk": 0.24390243902439024, + "threshold": 0.9154212671978758 + }, + { + "coverage": 0.014, + "risk": 0.2619047619047619, + "threshold": 0.9149169981817589 + }, + { + "coverage": 0.014333333333333333, + "risk": 0.2558139534883721, + "threshold": 0.9142887096158769 + }, + { + "coverage": 0.014666666666666666, + "risk": 0.25, + "threshold": 0.9136590627695077 + }, + { + "coverage": 0.015, + "risk": 0.26666666666666666, + "threshold": 0.912429530880306 + }, + { + "coverage": 0.015333333333333332, + "risk": 0.2608695652173913, + "threshold": 0.9123938185497315 + }, + { + "coverage": 0.015666666666666666, + "risk": 0.2765957446808511, + "threshold": 0.9118425776586586 + }, + { + "coverage": 0.016, + "risk": 0.2916666666666667, + "threshold": 0.9101888330826928 + }, + { + "coverage": 0.01633333333333333, + "risk": 0.2857142857142857, + "threshold": 0.9069438399094436 + }, + { + "coverage": 0.016666666666666666, + "risk": 0.28, + "threshold": 0.9068250425076 + }, + { + "coverage": 0.017, + "risk": 0.27450980392156865, + "threshold": 0.9068149070798308 + }, + { + "coverage": 0.017333333333333333, + "risk": 0.2692307692307692, + "threshold": 0.905347854499053 + }, + { + "coverage": 0.017666666666666667, + "risk": 0.2641509433962264, + "threshold": 0.9050069319917178 + }, + { + "coverage": 0.018, + "risk": 0.25925925925925924, + "threshold": 0.9046746190094649 + }, + { + "coverage": 0.018333333333333333, + "risk": 0.2545454545454545, + "threshold": 0.9040448641149557 + }, + { + "coverage": 0.018666666666666668, + "risk": 0.25, + "threshold": 0.9032962423554032 + }, + { + "coverage": 0.019, + "risk": 0.2631578947368421, + "threshold": 0.9029113077896147 + }, + { + "coverage": 0.019333333333333334, + "risk": 0.27586206896551724, + "threshold": 0.901787058380858 + }, + { + "coverage": 0.019666666666666666, + "risk": 0.2711864406779661, + "threshold": 0.900554042121212 + }, + { + "coverage": 0.02, + "risk": 0.2833333333333333, + "threshold": 0.8997519047061358 + }, + { + "coverage": 0.02033333333333333, + "risk": 0.29508196721311475, + "threshold": 0.8980438178748983 + }, + { + "coverage": 0.020666666666666667, + "risk": 0.2903225806451613, + "threshold": 0.8979055857426638 + }, + { + "coverage": 0.021, + "risk": 0.30158730158730157, + "threshold": 0.8974260065088164 + }, + { + "coverage": 0.021333333333333333, + "risk": 0.3125, + "threshold": 0.897404037137591 + }, + { + "coverage": 0.021666666666666667, + "risk": 0.3230769230769231, + "threshold": 0.896043014206109 + }, + { + "coverage": 0.022, + "risk": 0.3333333333333333, + "threshold": 0.8960195825749072 + }, + { + "coverage": 0.022333333333333334, + "risk": 0.34328358208955223, + "threshold": 0.8956068753179532 + }, + { + "coverage": 0.02266666666666667, + "risk": 0.3382352941176471, + "threshold": 0.8953974298061895 + }, + { + "coverage": 0.023, + "risk": 0.34782608695652173, + "threshold": 0.8953237998892383 + }, + { + "coverage": 0.023333333333333334, + "risk": 0.34285714285714286, + "threshold": 0.895183840250472 + }, + { + "coverage": 0.023666666666666666, + "risk": 0.352112676056338, + "threshold": 0.8950512200444983 + }, + { + "coverage": 0.024, + "risk": 0.3611111111111111, + "threshold": 0.894425392688926 + }, + { + "coverage": 0.024333333333333332, + "risk": 0.3698630136986301, + "threshold": 0.8923030513036917 + }, + { + "coverage": 0.024666666666666667, + "risk": 0.36486486486486486, + "threshold": 0.892197478964409 + }, + { + "coverage": 0.025, + "risk": 0.37333333333333335, + "threshold": 0.8916341505879565 + }, + { + "coverage": 0.025333333333333333, + "risk": 0.3815789473684211, + "threshold": 0.89145895466845 + }, + { + "coverage": 0.025666666666666667, + "risk": 0.37662337662337664, + "threshold": 0.8911089913174554 + }, + { + "coverage": 0.026, + "risk": 0.38461538461538464, + "threshold": 0.8907391364005326 + }, + { + "coverage": 0.026333333333333334, + "risk": 0.3924050632911392, + "threshold": 0.8906084394041655 + }, + { + "coverage": 0.02666666666666667, + "risk": 0.4, + "threshold": 0.8904933143005249 + }, + { + "coverage": 0.027, + "risk": 0.4074074074074074, + "threshold": 0.8904290602786376 + }, + { + "coverage": 0.027333333333333334, + "risk": 0.4024390243902439, + "threshold": 0.890126173723752 + }, + { + "coverage": 0.027666666666666666, + "risk": 0.40963855421686746, + "threshold": 0.8897720995967054 + }, + { + "coverage": 0.028, + "risk": 0.4166666666666667, + "threshold": 0.88974789286737 + }, + { + "coverage": 0.028333333333333332, + "risk": 0.4235294117647059, + "threshold": 0.8895435678192452 + }, + { + "coverage": 0.028666666666666667, + "risk": 0.43023255813953487, + "threshold": 0.8894184488423635 + }, + { + "coverage": 0.029, + "risk": 0.42528735632183906, + "threshold": 0.8889494545881133 + }, + { + "coverage": 0.029333333333333333, + "risk": 0.42045454545454547, + "threshold": 0.8878403740794578 + }, + { + "coverage": 0.029666666666666668, + "risk": 0.4157303370786517, + "threshold": 0.8877808528150216 + }, + { + "coverage": 0.03, + "risk": 0.4222222222222222, + "threshold": 0.887704618437303 + }, + { + "coverage": 0.030333333333333334, + "risk": 0.42857142857142855, + "threshold": 0.887155668808581 + }, + { + "coverage": 0.030666666666666665, + "risk": 0.43478260869565216, + "threshold": 0.8867557922599271 + }, + { + "coverage": 0.031, + "risk": 0.44086021505376344, + "threshold": 0.8865976976220928 + }, + { + "coverage": 0.03133333333333333, + "risk": 0.44680851063829785, + "threshold": 0.8864384683777343 + }, + { + "coverage": 0.03166666666666667, + "risk": 0.45263157894736844, + "threshold": 0.8864170610259139 + }, + { + "coverage": 0.032, + "risk": 0.4583333333333333, + "threshold": 0.8863561180691739 + }, + { + "coverage": 0.03233333333333333, + "risk": 0.4536082474226804, + "threshold": 0.8858792743428452 + }, + { + "coverage": 0.03266666666666666, + "risk": 0.45918367346938777, + "threshold": 0.8846556997694722 + }, + { + "coverage": 0.033, + "risk": 0.45454545454545453, + "threshold": 0.8841273993181573 + }, + { + "coverage": 0.03333333333333333, + "risk": 0.46, + "threshold": 0.8837142209277309 + }, + { + "coverage": 0.033666666666666664, + "risk": 0.45544554455445546, + "threshold": 0.8835405466881369 + }, + { + "coverage": 0.034, + "risk": 0.46078431372549017, + "threshold": 0.8827659705075201 + }, + { + "coverage": 0.034333333333333334, + "risk": 0.46601941747572817, + "threshold": 0.8818775499417819 + }, + { + "coverage": 0.034666666666666665, + "risk": 0.46153846153846156, + "threshold": 0.8811139332925825 + }, + { + "coverage": 0.035, + "risk": 0.4666666666666667, + "threshold": 0.8809758419310516 + }, + { + "coverage": 0.035333333333333335, + "risk": 0.4716981132075472, + "threshold": 0.8807615770510469 + }, + { + "coverage": 0.035666666666666666, + "risk": 0.4672897196261682, + "threshold": 0.8807074457516316 + }, + { + "coverage": 0.036, + "risk": 0.46296296296296297, + "threshold": 0.880283961615177 + }, + { + "coverage": 0.036333333333333336, + "risk": 0.45871559633027525, + "threshold": 0.880149522278614 + }, + { + "coverage": 0.03666666666666667, + "risk": 0.4636363636363636, + "threshold": 0.8800919458702177 + }, + { + "coverage": 0.037, + "risk": 0.4594594594594595, + "threshold": 0.8796795445454021 + }, + { + "coverage": 0.037333333333333336, + "risk": 0.4642857142857143, + "threshold": 0.8791340740356506 + }, + { + "coverage": 0.03766666666666667, + "risk": 0.4690265486725664, + "threshold": 0.8769788054346432 + }, + { + "coverage": 0.038, + "risk": 0.47368421052631576, + "threshold": 0.8768223527351807 + }, + { + "coverage": 0.03833333333333333, + "risk": 0.46956521739130436, + "threshold": 0.8762497067623969 + }, + { + "coverage": 0.03866666666666667, + "risk": 0.47413793103448276, + "threshold": 0.875300379208311 + }, + { + "coverage": 0.039, + "risk": 0.47863247863247865, + "threshold": 0.8737887854508183 + }, + { + "coverage": 0.03933333333333333, + "risk": 0.4830508474576271, + "threshold": 0.8737491779501148 + }, + { + "coverage": 0.03966666666666667, + "risk": 0.48739495798319327, + "threshold": 0.8734309947695881 + }, + { + "coverage": 0.04, + "risk": 0.49166666666666664, + "threshold": 0.872560422886696 + }, + { + "coverage": 0.04033333333333333, + "risk": 0.49586776859504134, + "threshold": 0.872457754723782 + }, + { + "coverage": 0.04066666666666666, + "risk": 0.5, + "threshold": 0.8722924617825871 + }, + { + "coverage": 0.041, + "risk": 0.5040650406504065, + "threshold": 0.8722184639890577 + }, + { + "coverage": 0.04133333333333333, + "risk": 0.5, + "threshold": 0.8720407590597872 + }, + { + "coverage": 0.041666666666666664, + "risk": 0.504, + "threshold": 0.871662345994065 + }, + { + "coverage": 0.042, + "risk": 0.5, + "threshold": 0.8710674633944143 + }, + { + "coverage": 0.042333333333333334, + "risk": 0.5039370078740157, + "threshold": 0.8679086441255456 + }, + { + "coverage": 0.042666666666666665, + "risk": 0.5, + "threshold": 0.8677732991403999 + }, + { + "coverage": 0.043, + "risk": 0.49612403100775193, + "threshold": 0.867747836457381 + }, + { + "coverage": 0.043333333333333335, + "risk": 0.49230769230769234, + "threshold": 0.8676917433576322 + }, + { + "coverage": 0.043666666666666666, + "risk": 0.48854961832061067, + "threshold": 0.866788168403709 + }, + { + "coverage": 0.044, + "risk": 0.48484848484848486, + "threshold": 0.8660250598840239 + }, + { + "coverage": 0.044333333333333336, + "risk": 0.48872180451127817, + "threshold": 0.8658723095610156 + }, + { + "coverage": 0.04466666666666667, + "risk": 0.4925373134328358, + "threshold": 0.8654901210651327 + }, + { + "coverage": 0.045, + "risk": 0.4888888888888889, + "threshold": 0.8654622037124132 + }, + { + "coverage": 0.04533333333333334, + "risk": 0.4852941176470588, + "threshold": 0.865276105551573 + }, + { + "coverage": 0.04566666666666667, + "risk": 0.48905109489051096, + "threshold": 0.8647167962260112 + }, + { + "coverage": 0.046, + "risk": 0.4927536231884058, + "threshold": 0.8643403326248076 + }, + { + "coverage": 0.04633333333333333, + "risk": 0.4892086330935252, + "threshold": 0.8633123369441275 + }, + { + "coverage": 0.04666666666666667, + "risk": 0.4928571428571429, + "threshold": 0.8629458422943301 + }, + { + "coverage": 0.047, + "risk": 0.48936170212765956, + "threshold": 0.8626263036928918 + }, + { + "coverage": 0.04733333333333333, + "risk": 0.4859154929577465, + "threshold": 0.8624631180650614 + }, + { + "coverage": 0.04766666666666667, + "risk": 0.4825174825174825, + "threshold": 0.8623618999294385 + }, + { + "coverage": 0.048, + "risk": 0.4861111111111111, + "threshold": 0.8623234083458715 + }, + { + "coverage": 0.04833333333333333, + "risk": 0.4896551724137931, + "threshold": 0.8622131415875517 + }, + { + "coverage": 0.048666666666666664, + "risk": 0.4931506849315068, + "threshold": 0.8621750006157817 + }, + { + "coverage": 0.049, + "risk": 0.4965986394557823, + "threshold": 0.8618229819630717 + }, + { + "coverage": 0.04933333333333333, + "risk": 0.5, + "threshold": 0.8613947261572372 + }, + { + "coverage": 0.049666666666666665, + "risk": 0.5033557046979866, + "threshold": 0.8612573530412618 + }, + { + "coverage": 0.05, + "risk": 0.5, + "threshold": 0.8611542975570257 + }, + { + "coverage": 0.050333333333333334, + "risk": 0.4966887417218543, + "threshold": 0.8604972645163184 + }, + { + "coverage": 0.050666666666666665, + "risk": 0.5, + "threshold": 0.8598205388428795 + }, + { + "coverage": 0.051, + "risk": 0.49673202614379086, + "threshold": 0.8595398151264094 + }, + { + "coverage": 0.051333333333333335, + "risk": 0.5, + "threshold": 0.8594700093149209 + }, + { + "coverage": 0.051666666666666666, + "risk": 0.5032258064516129, + "threshold": 0.8590924434343793 + }, + { + "coverage": 0.052, + "risk": 0.5064102564102564, + "threshold": 0.8584860336223858 + }, + { + "coverage": 0.052333333333333336, + "risk": 0.5031847133757962, + "threshold": 0.8582370897641352 + }, + { + "coverage": 0.05266666666666667, + "risk": 0.5063291139240507, + "threshold": 0.8581459373356062 + }, + { + "coverage": 0.053, + "risk": 0.5031446540880503, + "threshold": 0.8580783300374357 + }, + { + "coverage": 0.05333333333333334, + "risk": 0.50625, + "threshold": 0.8580282969782834 + }, + { + "coverage": 0.05366666666666667, + "risk": 0.5093167701863354, + "threshold": 0.8571919188937669 + }, + { + "coverage": 0.054, + "risk": 0.5123456790123457, + "threshold": 0.8565231031781181 + }, + { + "coverage": 0.05433333333333333, + "risk": 0.5153374233128835, + "threshold": 0.8565128702429045 + }, + { + "coverage": 0.05466666666666667, + "risk": 0.5121951219512195, + "threshold": 0.8565001672686833 + }, + { + "coverage": 0.055, + "risk": 0.5151515151515151, + "threshold": 0.8560713868640584 + }, + { + "coverage": 0.05533333333333333, + "risk": 0.5180722891566265, + "threshold": 0.8560632084168572 + }, + { + "coverage": 0.05566666666666667, + "risk": 0.5149700598802395, + "threshold": 0.8551939206319408 + }, + { + "coverage": 0.056, + "risk": 0.5119047619047619, + "threshold": 0.85518790210395 + }, + { + "coverage": 0.05633333333333333, + "risk": 0.5088757396449705, + "threshold": 0.8551198966734075 + }, + { + "coverage": 0.056666666666666664, + "risk": 0.5117647058823529, + "threshold": 0.8529992957880467 + }, + { + "coverage": 0.057, + "risk": 0.5087719298245614, + "threshold": 0.850597185603434 + }, + { + "coverage": 0.05733333333333333, + "risk": 0.5116279069767442, + "threshold": 0.850353211669613 + }, + { + "coverage": 0.057666666666666665, + "risk": 0.5144508670520231, + "threshold": 0.849902394154927 + }, + { + "coverage": 0.058, + "risk": 0.5114942528735632, + "threshold": 0.8496824966024196 + }, + { + "coverage": 0.058333333333333334, + "risk": 0.5085714285714286, + "threshold": 0.8495011485462497 + }, + { + "coverage": 0.058666666666666666, + "risk": 0.5113636363636364, + "threshold": 0.8493195516170345 + }, + { + "coverage": 0.059, + "risk": 0.5084745762711864, + "threshold": 0.8491935423488137 + }, + { + "coverage": 0.059333333333333335, + "risk": 0.5112359550561798, + "threshold": 0.8489545756926555 + }, + { + "coverage": 0.059666666666666666, + "risk": 0.5139664804469274, + "threshold": 0.8478121465971588 + }, + { + "coverage": 0.06, + "risk": 0.5166666666666667, + "threshold": 0.8476149338916044 + }, + { + "coverage": 0.060333333333333336, + "risk": 0.5193370165745856, + "threshold": 0.8471982185110246 + }, + { + "coverage": 0.06066666666666667, + "risk": 0.5164835164835165, + "threshold": 0.8467395664869664 + }, + { + "coverage": 0.061, + "risk": 0.5191256830601093, + "threshold": 0.8463531899144451 + }, + { + "coverage": 0.06133333333333333, + "risk": 0.5217391304347826, + "threshold": 0.845853202254809 + }, + { + "coverage": 0.06166666666666667, + "risk": 0.5243243243243243, + "threshold": 0.8457870761291614 + }, + { + "coverage": 0.062, + "risk": 0.5268817204301075, + "threshold": 0.8457299824277843 + }, + { + "coverage": 0.06233333333333333, + "risk": 0.5294117647058824, + "threshold": 0.8454831019567317 + }, + { + "coverage": 0.06266666666666666, + "risk": 0.5319148936170213, + "threshold": 0.8447553700281509 + }, + { + "coverage": 0.063, + "risk": 0.5291005291005291, + "threshold": 0.8443539430995066 + }, + { + "coverage": 0.06333333333333334, + "risk": 0.531578947368421, + "threshold": 0.8439413442674834 + }, + { + "coverage": 0.06366666666666666, + "risk": 0.5340314136125655, + "threshold": 0.8438286430845333 + }, + { + "coverage": 0.064, + "risk": 0.53125, + "threshold": 0.8437976940572413 + }, + { + "coverage": 0.06433333333333334, + "risk": 0.533678756476684, + "threshold": 0.8435666253595765 + }, + { + "coverage": 0.06466666666666666, + "risk": 0.5309278350515464, + "threshold": 0.8433258897566953 + }, + { + "coverage": 0.065, + "risk": 0.5282051282051282, + "threshold": 0.8431831588492164 + }, + { + "coverage": 0.06533333333333333, + "risk": 0.5306122448979592, + "threshold": 0.8415699596506908 + }, + { + "coverage": 0.06566666666666666, + "risk": 0.5279187817258884, + "threshold": 0.8414657239540437 + }, + { + "coverage": 0.066, + "risk": 0.5303030303030303, + "threshold": 0.8402741991526395 + }, + { + "coverage": 0.06633333333333333, + "risk": 0.5276381909547738, + "threshold": 0.8398013600858605 + }, + { + "coverage": 0.06666666666666667, + "risk": 0.53, + "threshold": 0.8394514041322806 + }, + { + "coverage": 0.067, + "risk": 0.5323383084577115, + "threshold": 0.8390724120523421 + }, + { + "coverage": 0.06733333333333333, + "risk": 0.5297029702970297, + "threshold": 0.83879125477437 + }, + { + "coverage": 0.06766666666666667, + "risk": 0.5320197044334976, + "threshold": 0.8387476091527866 + }, + { + "coverage": 0.068, + "risk": 0.5294117647058824, + "threshold": 0.8384967600537471 + }, + { + "coverage": 0.06833333333333333, + "risk": 0.5317073170731708, + "threshold": 0.8379646984782819 + }, + { + "coverage": 0.06866666666666667, + "risk": 0.5339805825242718, + "threshold": 0.8379291800050774 + }, + { + "coverage": 0.069, + "risk": 0.5362318840579711, + "threshold": 0.8373373342451247 + }, + { + "coverage": 0.06933333333333333, + "risk": 0.5384615384615384, + "threshold": 0.8371351313962082 + }, + { + "coverage": 0.06966666666666667, + "risk": 0.5406698564593302, + "threshold": 0.8370760183552424 + }, + { + "coverage": 0.07, + "risk": 0.5380952380952381, + "threshold": 0.8368561892510947 + }, + { + "coverage": 0.07033333333333333, + "risk": 0.5402843601895735, + "threshold": 0.836575954978763 + }, + { + "coverage": 0.07066666666666667, + "risk": 0.5377358490566038, + "threshold": 0.8349715514488936 + }, + { + "coverage": 0.071, + "risk": 0.539906103286385, + "threshold": 0.8346790578101314 + }, + { + "coverage": 0.07133333333333333, + "risk": 0.5373831775700935, + "threshold": 0.8344291098071291 + }, + { + "coverage": 0.07166666666666667, + "risk": 0.5395348837209303, + "threshold": 0.8342524422426724 + }, + { + "coverage": 0.072, + "risk": 0.5370370370370371, + "threshold": 0.834229324118019 + }, + { + "coverage": 0.07233333333333333, + "risk": 0.5345622119815668, + "threshold": 0.8326418366186921 + }, + { + "coverage": 0.07266666666666667, + "risk": 0.536697247706422, + "threshold": 0.8324515149255629 + }, + { + "coverage": 0.073, + "risk": 0.5342465753424658, + "threshold": 0.8324361381902157 + }, + { + "coverage": 0.07333333333333333, + "risk": 0.5363636363636364, + "threshold": 0.8324184346350472 + }, + { + "coverage": 0.07366666666666667, + "risk": 0.5339366515837104, + "threshold": 0.8319989399511366 + }, + { + "coverage": 0.074, + "risk": 0.536036036036036, + "threshold": 0.8314957539906541 + }, + { + "coverage": 0.07433333333333333, + "risk": 0.5336322869955157, + "threshold": 0.8306565098634647 + }, + { + "coverage": 0.07466666666666667, + "risk": 0.5357142857142857, + "threshold": 0.8298044012536181 + }, + { + "coverage": 0.075, + "risk": 0.5377777777777778, + "threshold": 0.8297169371663431 + }, + { + "coverage": 0.07533333333333334, + "risk": 0.5398230088495575, + "threshold": 0.8295925549571436 + }, + { + "coverage": 0.07566666666666666, + "risk": 0.5418502202643172, + "threshold": 0.829574533226223 + }, + { + "coverage": 0.076, + "risk": 0.5394736842105263, + "threshold": 0.8295551139866136 + }, + { + "coverage": 0.07633333333333334, + "risk": 0.537117903930131, + "threshold": 0.8290949160263322 + }, + { + "coverage": 0.07666666666666666, + "risk": 0.5391304347826087, + "threshold": 0.8288731159269414 + }, + { + "coverage": 0.077, + "risk": 0.5411255411255411, + "threshold": 0.8285392960397078 + }, + { + "coverage": 0.07733333333333334, + "risk": 0.5387931034482759, + "threshold": 0.8281553747678676 + }, + { + "coverage": 0.07766666666666666, + "risk": 0.5407725321888412, + "threshold": 0.8279945953207898 + }, + { + "coverage": 0.078, + "risk": 0.5384615384615384, + "threshold": 0.8273883254552723 + }, + { + "coverage": 0.07833333333333334, + "risk": 0.5404255319148936, + "threshold": 0.8273434330796126 + }, + { + "coverage": 0.07866666666666666, + "risk": 0.5423728813559322, + "threshold": 0.8270759039327835 + }, + { + "coverage": 0.079, + "risk": 0.540084388185654, + "threshold": 0.8270155695607126 + }, + { + "coverage": 0.07933333333333334, + "risk": 0.542016806722689, + "threshold": 0.8265820099543408 + }, + { + "coverage": 0.07966666666666666, + "risk": 0.5397489539748954, + "threshold": 0.8265668444515061 + }, + { + "coverage": 0.08, + "risk": 0.5416666666666666, + "threshold": 0.8264996968559432 + }, + { + "coverage": 0.08033333333333334, + "risk": 0.5435684647302904, + "threshold": 0.8261411086118639 + }, + { + "coverage": 0.08066666666666666, + "risk": 0.5454545454545454, + "threshold": 0.8253447415353532 + }, + { + "coverage": 0.081, + "risk": 0.5473251028806584, + "threshold": 0.8248865043573647 + }, + { + "coverage": 0.08133333333333333, + "risk": 0.5450819672131147, + "threshold": 0.8243133626141752 + }, + { + "coverage": 0.08166666666666667, + "risk": 0.5469387755102041, + "threshold": 0.8240408168065575 + }, + { + "coverage": 0.082, + "risk": 0.5447154471544715, + "threshold": 0.8230478614003354 + }, + { + "coverage": 0.08233333333333333, + "risk": 0.5465587044534413, + "threshold": 0.8230172178269693 + }, + { + "coverage": 0.08266666666666667, + "risk": 0.5483870967741935, + "threshold": 0.8221213466531342 + }, + { + "coverage": 0.083, + "risk": 0.5461847389558233, + "threshold": 0.8217050881800297 + }, + { + "coverage": 0.08333333333333333, + "risk": 0.548, + "threshold": 0.8211921061961547 + }, + { + "coverage": 0.08366666666666667, + "risk": 0.545816733067729, + "threshold": 0.8211813727896784 + }, + { + "coverage": 0.084, + "risk": 0.5476190476190477, + "threshold": 0.8211106120127714 + }, + { + "coverage": 0.08433333333333333, + "risk": 0.549407114624506, + "threshold": 0.8209195473194132 + }, + { + "coverage": 0.08466666666666667, + "risk": 0.5511811023622047, + "threshold": 0.8202919564878228 + }, + { + "coverage": 0.085, + "risk": 0.5529411764705883, + "threshold": 0.8202396419140549 + }, + { + "coverage": 0.08533333333333333, + "risk": 0.5546875, + "threshold": 0.8187850220417608 + }, + { + "coverage": 0.08566666666666667, + "risk": 0.556420233463035, + "threshold": 0.8183017505149612 + }, + { + "coverage": 0.086, + "risk": 0.5581395348837209, + "threshold": 0.818231285008926 + }, + { + "coverage": 0.08633333333333333, + "risk": 0.555984555984556, + "threshold": 0.8178514580602007 + }, + { + "coverage": 0.08666666666666667, + "risk": 0.5576923076923077, + "threshold": 0.8175727347382485 + }, + { + "coverage": 0.087, + "risk": 0.5593869731800766, + "threshold": 0.8173414037667782 + }, + { + "coverage": 0.08733333333333333, + "risk": 0.5610687022900763, + "threshold": 0.817290341687996 + }, + { + "coverage": 0.08766666666666667, + "risk": 0.55893536121673, + "threshold": 0.8162326047718135 + }, + { + "coverage": 0.088, + "risk": 0.5568181818181818, + "threshold": 0.8154878772678295 + }, + { + "coverage": 0.08833333333333333, + "risk": 0.5547169811320755, + "threshold": 0.8146631807507192 + }, + { + "coverage": 0.08866666666666667, + "risk": 0.5526315789473685, + "threshold": 0.8140026985441295 + }, + { + "coverage": 0.089, + "risk": 0.5543071161048689, + "threshold": 0.8138303841902712 + }, + { + "coverage": 0.08933333333333333, + "risk": 0.5522388059701493, + "threshold": 0.8136520479887979 + }, + { + "coverage": 0.08966666666666667, + "risk": 0.550185873605948, + "threshold": 0.8132739985421087 + }, + { + "coverage": 0.09, + "risk": 0.5481481481481482, + "threshold": 0.8130799552685516 + }, + { + "coverage": 0.09033333333333333, + "risk": 0.5498154981549815, + "threshold": 0.8123674268774835 + }, + { + "coverage": 0.09066666666666667, + "risk": 0.5514705882352942, + "threshold": 0.8121013624740968 + }, + { + "coverage": 0.091, + "risk": 0.5494505494505495, + "threshold": 0.8116020618982017 + }, + { + "coverage": 0.09133333333333334, + "risk": 0.551094890510949, + "threshold": 0.8113104956678137 + }, + { + "coverage": 0.09166666666666666, + "risk": 0.5527272727272727, + "threshold": 0.808914425799644 + }, + { + "coverage": 0.092, + "risk": 0.5507246376811594, + "threshold": 0.8088220161117884 + }, + { + "coverage": 0.09233333333333334, + "risk": 0.5487364620938628, + "threshold": 0.8086641055285594 + }, + { + "coverage": 0.09266666666666666, + "risk": 0.5503597122302158, + "threshold": 0.807585261427428 + }, + { + "coverage": 0.093, + "risk": 0.5519713261648745, + "threshold": 0.8075632165493777 + }, + { + "coverage": 0.09333333333333334, + "risk": 0.55, + "threshold": 0.8072713259277032 + }, + { + "coverage": 0.09366666666666666, + "risk": 0.5516014234875445, + "threshold": 0.8069734833742439 + }, + { + "coverage": 0.094, + "risk": 0.5531914893617021, + "threshold": 0.8067586502125963 + }, + { + "coverage": 0.09433333333333334, + "risk": 0.5512367491166078, + "threshold": 0.8067532867755915 + }, + { + "coverage": 0.09466666666666666, + "risk": 0.5528169014084507, + "threshold": 0.805422991274843 + }, + { + "coverage": 0.095, + "risk": 0.5543859649122806, + "threshold": 0.8052207829216372 + }, + { + "coverage": 0.09533333333333334, + "risk": 0.5559440559440559, + "threshold": 0.8046760550674178 + }, + { + "coverage": 0.09566666666666666, + "risk": 0.554006968641115, + "threshold": 0.8046114251785861 + }, + { + "coverage": 0.096, + "risk": 0.5520833333333334, + "threshold": 0.804116807500454 + }, + { + "coverage": 0.09633333333333334, + "risk": 0.5536332179930796, + "threshold": 0.8037060667550192 + }, + { + "coverage": 0.09666666666666666, + "risk": 0.5517241379310345, + "threshold": 0.8022400002793312 + }, + { + "coverage": 0.097, + "risk": 0.5498281786941581, + "threshold": 0.8011490951284512 + }, + { + "coverage": 0.09733333333333333, + "risk": 0.547945205479452, + "threshold": 0.8010655828558949 + }, + { + "coverage": 0.09766666666666667, + "risk": 0.5460750853242321, + "threshold": 0.7982768083458234 + }, + { + "coverage": 0.098, + "risk": 0.5476190476190477, + "threshold": 0.7975606998083482 + }, + { + "coverage": 0.09833333333333333, + "risk": 0.5491525423728814, + "threshold": 0.7965457415556668 + }, + { + "coverage": 0.09866666666666667, + "risk": 0.5472972972972973, + "threshold": 0.7959396941523048 + }, + { + "coverage": 0.099, + "risk": 0.5488215488215489, + "threshold": 0.7958036614975738 + }, + { + "coverage": 0.09933333333333333, + "risk": 0.5503355704697986, + "threshold": 0.7943427811465276 + }, + { + "coverage": 0.09966666666666667, + "risk": 0.5484949832775919, + "threshold": 0.7938875725457918 + }, + { + "coverage": 0.1, + "risk": 0.55, + "threshold": 0.7930531703731363 + }, + { + "coverage": 0.10033333333333333, + "risk": 0.5514950166112956, + "threshold": 0.7929853689352545 + }, + { + "coverage": 0.10066666666666667, + "risk": 0.5496688741721855, + "threshold": 0.7924180893995018 + }, + { + "coverage": 0.101, + "risk": 0.5478547854785478, + "threshold": 0.7918369517483576 + }, + { + "coverage": 0.10133333333333333, + "risk": 0.5493421052631579, + "threshold": 0.7913979362946997 + }, + { + "coverage": 0.10166666666666667, + "risk": 0.5475409836065573, + "threshold": 0.7913182496156088 + }, + { + "coverage": 0.102, + "risk": 0.5490196078431373, + "threshold": 0.7912179674309412 + }, + { + "coverage": 0.10233333333333333, + "risk": 0.5504885993485342, + "threshold": 0.7909146063093202 + }, + { + "coverage": 0.10266666666666667, + "risk": 0.5487012987012987, + "threshold": 0.7904754667149659 + }, + { + "coverage": 0.103, + "risk": 0.5469255663430421, + "threshold": 0.7900667956079064 + }, + { + "coverage": 0.10333333333333333, + "risk": 0.5483870967741935, + "threshold": 0.7900621706651283 + }, + { + "coverage": 0.10366666666666667, + "risk": 0.5466237942122186, + "threshold": 0.7897372924307051 + }, + { + "coverage": 0.104, + "risk": 0.5448717948717948, + "threshold": 0.7894839453107482 + }, + { + "coverage": 0.10433333333333333, + "risk": 0.5431309904153354, + "threshold": 0.7892863696462203 + }, + { + "coverage": 0.10466666666666667, + "risk": 0.5445859872611465, + "threshold": 0.7877073428706611 + }, + { + "coverage": 0.105, + "risk": 0.546031746031746, + "threshold": 0.7871761704214989 + }, + { + "coverage": 0.10533333333333333, + "risk": 0.5443037974683544, + "threshold": 0.7869441676406866 + }, + { + "coverage": 0.10566666666666667, + "risk": 0.5457413249211357, + "threshold": 0.7859272569862706 + }, + { + "coverage": 0.106, + "risk": 0.5440251572327044, + "threshold": 0.7857370484365923 + }, + { + "coverage": 0.10633333333333334, + "risk": 0.542319749216301, + "threshold": 0.7852020038919093 + }, + { + "coverage": 0.10666666666666667, + "risk": 0.54375, + "threshold": 0.7851262918482632 + }, + { + "coverage": 0.107, + "risk": 0.5451713395638629, + "threshold": 0.7847149867593153 + }, + { + "coverage": 0.10733333333333334, + "risk": 0.5434782608695652, + "threshold": 0.7846551569961808 + }, + { + "coverage": 0.10766666666666666, + "risk": 0.541795665634675, + "threshold": 0.7841914053167688 + }, + { + "coverage": 0.108, + "risk": 0.5401234567901234, + "threshold": 0.7834566090663704 + }, + { + "coverage": 0.10833333333333334, + "risk": 0.5415384615384615, + "threshold": 0.7833611668509979 + }, + { + "coverage": 0.10866666666666666, + "risk": 0.5429447852760736, + "threshold": 0.7830529442137819 + }, + { + "coverage": 0.109, + "risk": 0.5412844036697247, + "threshold": 0.7829354333055615 + }, + { + "coverage": 0.10933333333333334, + "risk": 0.5426829268292683, + "threshold": 0.7823763570474392 + }, + { + "coverage": 0.10966666666666666, + "risk": 0.5440729483282675, + "threshold": 0.7821257810138112 + }, + { + "coverage": 0.11, + "risk": 0.5424242424242425, + "threshold": 0.7820799991825892 + }, + { + "coverage": 0.11033333333333334, + "risk": 0.540785498489426, + "threshold": 0.7816837753298096 + }, + { + "coverage": 0.11066666666666666, + "risk": 0.5391566265060241, + "threshold": 0.7802862321629219 + }, + { + "coverage": 0.111, + "risk": 0.5405405405405406, + "threshold": 0.7797940011371796 + }, + { + "coverage": 0.11133333333333334, + "risk": 0.5419161676646707, + "threshold": 0.7790768866118487 + }, + { + "coverage": 0.11166666666666666, + "risk": 0.5402985074626866, + "threshold": 0.7788756426194168 + }, + { + "coverage": 0.112, + "risk": 0.5386904761904762, + "threshold": 0.7787776790503143 + }, + { + "coverage": 0.11233333333333333, + "risk": 0.5400593471810089, + "threshold": 0.7787329464478275 + }, + { + "coverage": 0.11266666666666666, + "risk": 0.5384615384615384, + "threshold": 0.7785796800320365 + }, + { + "coverage": 0.113, + "risk": 0.5398230088495575, + "threshold": 0.7784183650121702 + }, + { + "coverage": 0.11333333333333333, + "risk": 0.538235294117647, + "threshold": 0.7772692427787826 + }, + { + "coverage": 0.11366666666666667, + "risk": 0.5366568914956011, + "threshold": 0.7771016093582835 + }, + { + "coverage": 0.114, + "risk": 0.5350877192982456, + "threshold": 0.7769930823944355 + }, + { + "coverage": 0.11433333333333333, + "risk": 0.5364431486880467, + "threshold": 0.7765977725029071 + }, + { + "coverage": 0.11466666666666667, + "risk": 0.5348837209302325, + "threshold": 0.7763804135216771 + }, + { + "coverage": 0.115, + "risk": 0.5362318840579711, + "threshold": 0.7757807982334435 + }, + { + "coverage": 0.11533333333333333, + "risk": 0.5346820809248555, + "threshold": 0.775310263731737 + }, + { + "coverage": 0.11566666666666667, + "risk": 0.5360230547550432, + "threshold": 0.774933976504579 + }, + { + "coverage": 0.116, + "risk": 0.5344827586206896, + "threshold": 0.774823565549405 + }, + { + "coverage": 0.11633333333333333, + "risk": 0.5329512893982808, + "threshold": 0.7742513843491634 + }, + { + "coverage": 0.11666666666666667, + "risk": 0.5342857142857143, + "threshold": 0.7731977000958185 + }, + { + "coverage": 0.117, + "risk": 0.5327635327635327, + "threshold": 0.7730997573191731 + }, + { + "coverage": 0.11733333333333333, + "risk": 0.53125, + "threshold": 0.7730726129402943 + }, + { + "coverage": 0.11766666666666667, + "risk": 0.5297450424929179, + "threshold": 0.7729586335104425 + }, + { + "coverage": 0.118, + "risk": 0.5282485875706214, + "threshold": 0.772941945729427 + }, + { + "coverage": 0.11833333333333333, + "risk": 0.5267605633802817, + "threshold": 0.7729069006796113 + }, + { + "coverage": 0.11866666666666667, + "risk": 0.5252808988764045, + "threshold": 0.7726191120125351 + }, + { + "coverage": 0.119, + "risk": 0.5238095238095238, + "threshold": 0.7725472704438089 + }, + { + "coverage": 0.11933333333333333, + "risk": 0.5223463687150838, + "threshold": 0.7724593073539776 + }, + { + "coverage": 0.11966666666666667, + "risk": 0.520891364902507, + "threshold": 0.7722371174041956 + }, + { + "coverage": 0.12, + "risk": 0.5222222222222223, + "threshold": 0.7719126975481869 + }, + { + "coverage": 0.12033333333333333, + "risk": 0.5235457063711911, + "threshold": 0.7714142062371327 + }, + { + "coverage": 0.12066666666666667, + "risk": 0.5220994475138122, + "threshold": 0.7712456881432891 + }, + { + "coverage": 0.121, + "risk": 0.5206611570247934, + "threshold": 0.7710836009579211 + }, + { + "coverage": 0.12133333333333333, + "risk": 0.521978021978022, + "threshold": 0.7710101658588393 + }, + { + "coverage": 0.12166666666666667, + "risk": 0.5205479452054794, + "threshold": 0.7709389634394114 + }, + { + "coverage": 0.122, + "risk": 0.5218579234972678, + "threshold": 0.7707313797432791 + }, + { + "coverage": 0.12233333333333334, + "risk": 0.5204359673024523, + "threshold": 0.7704327151690206 + }, + { + "coverage": 0.12266666666666666, + "risk": 0.5190217391304348, + "threshold": 0.7701769692588647 + }, + { + "coverage": 0.123, + "risk": 0.5176151761517616, + "threshold": 0.7694072566287087 + }, + { + "coverage": 0.12333333333333334, + "risk": 0.518918918918919, + "threshold": 0.7686488806124951 + }, + { + "coverage": 0.12366666666666666, + "risk": 0.5202156334231806, + "threshold": 0.7685871626851399 + }, + { + "coverage": 0.124, + "risk": 0.5188172043010753, + "threshold": 0.7678650658246864 + }, + { + "coverage": 0.12433333333333334, + "risk": 0.517426273458445, + "threshold": 0.7670184736906431 + }, + { + "coverage": 0.12466666666666666, + "risk": 0.516042780748663, + "threshold": 0.7668720942892164 + }, + { + "coverage": 0.125, + "risk": 0.5173333333333333, + "threshold": 0.7656202940141595 + }, + { + "coverage": 0.12533333333333332, + "risk": 0.5159574468085106, + "threshold": 0.7646113352551315 + }, + { + "coverage": 0.12566666666666668, + "risk": 0.5172413793103449, + "threshold": 0.7632260879153296 + }, + { + "coverage": 0.126, + "risk": 0.5158730158730159, + "threshold": 0.7623382240783966 + }, + { + "coverage": 0.12633333333333333, + "risk": 0.5171503957783641, + "threshold": 0.7617608947028107 + }, + { + "coverage": 0.12666666666666668, + "risk": 0.5184210526315789, + "threshold": 0.7604820989988669 + }, + { + "coverage": 0.127, + "risk": 0.5196850393700787, + "threshold": 0.7604767895435496 + }, + { + "coverage": 0.12733333333333333, + "risk": 0.5209424083769634, + "threshold": 0.7601507632708409 + }, + { + "coverage": 0.12766666666666668, + "risk": 0.5195822454308094, + "threshold": 0.7592239868226188 + }, + { + "coverage": 0.128, + "risk": 0.5208333333333334, + "threshold": 0.7592118936887983 + }, + { + "coverage": 0.12833333333333333, + "risk": 0.5194805194805194, + "threshold": 0.7590894613997288 + }, + { + "coverage": 0.12866666666666668, + "risk": 0.5181347150259067, + "threshold": 0.7589368579562571 + }, + { + "coverage": 0.129, + "risk": 0.5193798449612403, + "threshold": 0.7585929356387616 + }, + { + "coverage": 0.12933333333333333, + "risk": 0.5180412371134021, + "threshold": 0.7580546962262309 + }, + { + "coverage": 0.12966666666666668, + "risk": 0.5167095115681234, + "threshold": 0.757770760404671 + }, + { + "coverage": 0.13, + "risk": 0.517948717948718, + "threshold": 0.757234004394632 + }, + { + "coverage": 0.13033333333333333, + "risk": 0.5191815856777494, + "threshold": 0.7557935071379124 + }, + { + "coverage": 0.13066666666666665, + "risk": 0.5178571428571429, + "threshold": 0.75517465271544 + }, + { + "coverage": 0.131, + "risk": 0.5190839694656488, + "threshold": 0.7547863975124626 + }, + { + "coverage": 0.13133333333333333, + "risk": 0.5177664974619289, + "threshold": 0.7546383830770612 + }, + { + "coverage": 0.13166666666666665, + "risk": 0.5164556962025316, + "threshold": 0.7543650006746364 + }, + { + "coverage": 0.132, + "risk": 0.5151515151515151, + "threshold": 0.7542042831212227 + }, + { + "coverage": 0.13233333333333333, + "risk": 0.5163727959697733, + "threshold": 0.7538302565210315 + }, + { + "coverage": 0.13266666666666665, + "risk": 0.5175879396984925, + "threshold": 0.753026558229962 + }, + { + "coverage": 0.133, + "risk": 0.5162907268170426, + "threshold": 0.751387663935416 + }, + { + "coverage": 0.13333333333333333, + "risk": 0.5175, + "threshold": 0.7505729343643577 + }, + { + "coverage": 0.13366666666666666, + "risk": 0.516209476309227, + "threshold": 0.7503195894896735 + }, + { + "coverage": 0.134, + "risk": 0.5174129353233831, + "threshold": 0.7496956731408856 + }, + { + "coverage": 0.13433333333333333, + "risk": 0.5161290322580645, + "threshold": 0.7491390671700513 + }, + { + "coverage": 0.13466666666666666, + "risk": 0.5148514851485149, + "threshold": 0.7482678370530186 + }, + { + "coverage": 0.135, + "risk": 0.5135802469135803, + "threshold": 0.7475686870829155 + }, + { + "coverage": 0.13533333333333333, + "risk": 0.5123152709359606, + "threshold": 0.7473503257180053 + }, + { + "coverage": 0.13566666666666666, + "risk": 0.5110565110565111, + "threshold": 0.7466567804822176 + }, + { + "coverage": 0.136, + "risk": 0.5098039215686274, + "threshold": 0.745547649675832 + }, + { + "coverage": 0.13633333333333333, + "risk": 0.508557457212714, + "threshold": 0.7452661967446308 + }, + { + "coverage": 0.13666666666666666, + "risk": 0.5097560975609756, + "threshold": 0.744705544108379 + }, + { + "coverage": 0.137, + "risk": 0.5085158150851582, + "threshold": 0.7444813124713153 + }, + { + "coverage": 0.13733333333333334, + "risk": 0.5097087378640777, + "threshold": 0.7438778728963753 + }, + { + "coverage": 0.13766666666666666, + "risk": 0.5084745762711864, + "threshold": 0.7435395023998003 + }, + { + "coverage": 0.138, + "risk": 0.5096618357487923, + "threshold": 0.7429686635232573 + }, + { + "coverage": 0.13833333333333334, + "risk": 0.5084337349397591, + "threshold": 0.7429393861948603 + }, + { + "coverage": 0.13866666666666666, + "risk": 0.5096153846153846, + "threshold": 0.7428258855610492 + }, + { + "coverage": 0.139, + "risk": 0.5107913669064749, + "threshold": 0.7427163899990853 + }, + { + "coverage": 0.13933333333333334, + "risk": 0.5095693779904307, + "threshold": 0.742527041013988 + }, + { + "coverage": 0.13966666666666666, + "risk": 0.5107398568019093, + "threshold": 0.7424268902634423 + }, + { + "coverage": 0.14, + "risk": 0.5095238095238095, + "threshold": 0.7421281494625106 + }, + { + "coverage": 0.14033333333333334, + "risk": 0.5106888361045131, + "threshold": 0.7419364513093214 + }, + { + "coverage": 0.14066666666666666, + "risk": 0.509478672985782, + "threshold": 0.7417200359263634 + }, + { + "coverage": 0.141, + "risk": 0.5106382978723404, + "threshold": 0.7410176842025317 + }, + { + "coverage": 0.14133333333333334, + "risk": 0.5094339622641509, + "threshold": 0.7404323460147452 + }, + { + "coverage": 0.14166666666666666, + "risk": 0.508235294117647, + "threshold": 0.7403673586959608 + }, + { + "coverage": 0.142, + "risk": 0.5070422535211268, + "threshold": 0.7397283267878244 + }, + { + "coverage": 0.14233333333333334, + "risk": 0.5058548009367682, + "threshold": 0.7396069530568343 + }, + { + "coverage": 0.14266666666666666, + "risk": 0.5070093457943925, + "threshold": 0.7391618384266244 + }, + { + "coverage": 0.143, + "risk": 0.5081585081585082, + "threshold": 0.7386853439398781 + }, + { + "coverage": 0.14333333333333334, + "risk": 0.5093023255813953, + "threshold": 0.738458242502226 + }, + { + "coverage": 0.14366666666666666, + "risk": 0.5104408352668214, + "threshold": 0.7374985155661811 + }, + { + "coverage": 0.144, + "risk": 0.5092592592592593, + "threshold": 0.7367851279943796 + }, + { + "coverage": 0.14433333333333334, + "risk": 0.5080831408775982, + "threshold": 0.7366687904428098 + }, + { + "coverage": 0.14466666666666667, + "risk": 0.5092165898617511, + "threshold": 0.7366535457347853 + }, + { + "coverage": 0.145, + "risk": 0.5080459770114942, + "threshold": 0.7352715219057246 + }, + { + "coverage": 0.14533333333333334, + "risk": 0.5068807339449541, + "threshold": 0.7341475463725202 + }, + { + "coverage": 0.14566666666666667, + "risk": 0.5080091533180778, + "threshold": 0.7338000192939249 + }, + { + "coverage": 0.146, + "risk": 0.5091324200913242, + "threshold": 0.7331880427500898 + }, + { + "coverage": 0.14633333333333334, + "risk": 0.5079726651480638, + "threshold": 0.7324791272881876 + }, + { + "coverage": 0.14666666666666667, + "risk": 0.5068181818181818, + "threshold": 0.7322331185529536 + }, + { + "coverage": 0.147, + "risk": 0.5056689342403629, + "threshold": 0.7322280125558293 + }, + { + "coverage": 0.14733333333333334, + "risk": 0.504524886877828, + "threshold": 0.7322221055744379 + }, + { + "coverage": 0.14766666666666667, + "risk": 0.5056433408577878, + "threshold": 0.7317098371468695 + }, + { + "coverage": 0.148, + "risk": 0.5067567567567568, + "threshold": 0.731043133223254 + }, + { + "coverage": 0.14833333333333334, + "risk": 0.5056179775280899, + "threshold": 0.730919603533359 + }, + { + "coverage": 0.14866666666666667, + "risk": 0.5044843049327354, + "threshold": 0.7305293810740701 + }, + { + "coverage": 0.149, + "risk": 0.5033557046979866, + "threshold": 0.7302841858017991 + }, + { + "coverage": 0.14933333333333335, + "risk": 0.5044642857142857, + "threshold": 0.7302437549915334 + }, + { + "coverage": 0.14966666666666667, + "risk": 0.5033407572383074, + "threshold": 0.7300026890565817 + }, + { + "coverage": 0.15, + "risk": 0.5022222222222222, + "threshold": 0.7291263730867835 + }, + { + "coverage": 0.15033333333333335, + "risk": 0.5011086474501109, + "threshold": 0.7291199995626091 + }, + { + "coverage": 0.15066666666666667, + "risk": 0.5, + "threshold": 0.7288077746870958 + }, + { + "coverage": 0.151, + "risk": 0.5011037527593819, + "threshold": 0.7284884859168164 + }, + { + "coverage": 0.15133333333333332, + "risk": 0.5022026431718062, + "threshold": 0.7280462693275334 + }, + { + "coverage": 0.15166666666666667, + "risk": 0.5010989010989011, + "threshold": 0.7272605413760024 + }, + { + "coverage": 0.152, + "risk": 0.5021929824561403, + "threshold": 0.7271561471588831 + }, + { + "coverage": 0.15233333333333332, + "risk": 0.5010940919037199, + "threshold": 0.7270503433645873 + }, + { + "coverage": 0.15266666666666667, + "risk": 0.5021834061135371, + "threshold": 0.7269589078065128 + }, + { + "coverage": 0.153, + "risk": 0.5010893246187363, + "threshold": 0.7266449410471585 + }, + { + "coverage": 0.15333333333333332, + "risk": 0.5, + "threshold": 0.7264024666241511 + }, + { + "coverage": 0.15366666666666667, + "risk": 0.49891540130151846, + "threshold": 0.7259466873215654 + }, + { + "coverage": 0.154, + "risk": 0.5, + "threshold": 0.7250011036979762 + }, + { + "coverage": 0.15433333333333332, + "risk": 0.49892008639308855, + "threshold": 0.723786426943784 + }, + { + "coverage": 0.15466666666666667, + "risk": 0.5, + "threshold": 0.7216621511210753 + }, + { + "coverage": 0.155, + "risk": 0.4989247311827957, + "threshold": 0.7214099458321163 + }, + { + "coverage": 0.15533333333333332, + "risk": 0.4978540772532189, + "threshold": 0.7204289285301114 + }, + { + "coverage": 0.15566666666666668, + "risk": 0.4989293361884368, + "threshold": 0.7184731729443409 + }, + { + "coverage": 0.156, + "risk": 0.5, + "threshold": 0.7178347354301827 + }, + { + "coverage": 0.15633333333333332, + "risk": 0.5010660980810234, + "threshold": 0.7175156857918048 + }, + { + "coverage": 0.15666666666666668, + "risk": 0.502127659574468, + "threshold": 0.7173214135289923 + }, + { + "coverage": 0.157, + "risk": 0.5010615711252654, + "threshold": 0.7162272132448667 + }, + { + "coverage": 0.15733333333333333, + "risk": 0.5021186440677966, + "threshold": 0.7146017327179601 + }, + { + "coverage": 0.15766666666666668, + "risk": 0.5031712473572939, + "threshold": 0.7131542477568401 + }, + { + "coverage": 0.158, + "risk": 0.5021097046413502, + "threshold": 0.7128207350768297 + }, + { + "coverage": 0.15833333333333333, + "risk": 0.5010526315789474, + "threshold": 0.7127430854838706 + }, + { + "coverage": 0.15866666666666668, + "risk": 0.5, + "threshold": 0.7126082869882785 + }, + { + "coverage": 0.159, + "risk": 0.5010482180293501, + "threshold": 0.7126065292452219 + }, + { + "coverage": 0.15933333333333333, + "risk": 0.5, + "threshold": 0.7119796561561028 + }, + { + "coverage": 0.15966666666666668, + "risk": 0.5010438413361169, + "threshold": 0.7115301863637158 + }, + { + "coverage": 0.16, + "risk": 0.5020833333333333, + "threshold": 0.7111210481389758 + }, + { + "coverage": 0.16033333333333333, + "risk": 0.5031185031185031, + "threshold": 0.7107390523378191 + }, + { + "coverage": 0.16066666666666668, + "risk": 0.5020746887966805, + "threshold": 0.7098632692513008 + }, + { + "coverage": 0.161, + "risk": 0.5031055900621118, + "threshold": 0.709230119420325 + }, + { + "coverage": 0.16133333333333333, + "risk": 0.5020661157024794, + "threshold": 0.7092147043076507 + }, + { + "coverage": 0.16166666666666665, + "risk": 0.5010309278350515, + "threshold": 0.7083577646444438 + }, + { + "coverage": 0.162, + "risk": 0.5, + "threshold": 0.7083538871957543 + }, + { + "coverage": 0.16233333333333333, + "risk": 0.5010266940451745, + "threshold": 0.7082406624470365 + }, + { + "coverage": 0.16266666666666665, + "risk": 0.5, + "threshold": 0.7080283850457619 + }, + { + "coverage": 0.163, + "risk": 0.5010224948875256, + "threshold": 0.7065516464393646 + }, + { + "coverage": 0.16333333333333333, + "risk": 0.5, + "threshold": 0.7063388423195728 + }, + { + "coverage": 0.16366666666666665, + "risk": 0.4989816700610998, + "threshold": 0.7061078097524699 + }, + { + "coverage": 0.164, + "risk": 0.49796747967479676, + "threshold": 0.7056622965335466 + }, + { + "coverage": 0.16433333333333333, + "risk": 0.4969574036511156, + "threshold": 0.7052192076534578 + }, + { + "coverage": 0.16466666666666666, + "risk": 0.4959514170040486, + "threshold": 0.7046056778014129 + }, + { + "coverage": 0.165, + "risk": 0.49696969696969695, + "threshold": 0.7044662730939366 + }, + { + "coverage": 0.16533333333333333, + "risk": 0.4959677419354839, + "threshold": 0.7042020137638534 + }, + { + "coverage": 0.16566666666666666, + "risk": 0.4949698189134809, + "threshold": 0.7040378748192865 + }, + { + "coverage": 0.166, + "risk": 0.4939759036144578, + "threshold": 0.7037267237098538 + }, + { + "coverage": 0.16633333333333333, + "risk": 0.49298597194388777, + "threshold": 0.7028145302348149 + }, + { + "coverage": 0.16666666666666666, + "risk": 0.492, + "threshold": 0.7027838440984702 + }, + { + "coverage": 0.167, + "risk": 0.49101796407185627, + "threshold": 0.7021458007754828 + }, + { + "coverage": 0.16733333333333333, + "risk": 0.4900398406374502, + "threshold": 0.7020791690703349 + }, + { + "coverage": 0.16766666666666666, + "risk": 0.49105367793240556, + "threshold": 0.7018353616382864 + }, + { + "coverage": 0.168, + "risk": 0.49007936507936506, + "threshold": 0.7008076461803787 + }, + { + "coverage": 0.16833333333333333, + "risk": 0.4891089108910891, + "threshold": 0.699912540565826 + }, + { + "coverage": 0.16866666666666666, + "risk": 0.4881422924901186, + "threshold": 0.6997844691875946 + }, + { + "coverage": 0.169, + "risk": 0.48717948717948717, + "threshold": 0.6995268562777368 + }, + { + "coverage": 0.16933333333333334, + "risk": 0.4881889763779528, + "threshold": 0.6994705580278104 + }, + { + "coverage": 0.16966666666666666, + "risk": 0.48722986247544203, + "threshold": 0.6993716705466886 + }, + { + "coverage": 0.17, + "risk": 0.48627450980392156, + "threshold": 0.6989291476037421 + }, + { + "coverage": 0.17033333333333334, + "risk": 0.48532289628180036, + "threshold": 0.6984490417347837 + }, + { + "coverage": 0.17066666666666666, + "risk": 0.484375, + "threshold": 0.698267151495874 + }, + { + "coverage": 0.171, + "risk": 0.4834307992202729, + "threshold": 0.6982355840311005 + }, + { + "coverage": 0.17133333333333334, + "risk": 0.48249027237354086, + "threshold": 0.6979152871052298 + }, + { + "coverage": 0.17166666666666666, + "risk": 0.48349514563106794, + "threshold": 0.6973018047192768 + }, + { + "coverage": 0.172, + "risk": 0.48255813953488375, + "threshold": 0.6969893017465084 + }, + { + "coverage": 0.17233333333333334, + "risk": 0.4816247582205029, + "threshold": 0.6969276211978427 + }, + { + "coverage": 0.17266666666666666, + "risk": 0.4826254826254826, + "threshold": 0.6966839627267115 + }, + { + "coverage": 0.173, + "risk": 0.4816955684007707, + "threshold": 0.696518441162548 + }, + { + "coverage": 0.17333333333333334, + "risk": 0.4807692307692308, + "threshold": 0.695025875305092 + }, + { + "coverage": 0.17366666666666666, + "risk": 0.4798464491362764, + "threshold": 0.6941400473915234 + }, + { + "coverage": 0.174, + "risk": 0.4789272030651341, + "threshold": 0.6934663241019361 + }, + { + "coverage": 0.17433333333333334, + "risk": 0.47992351816443596, + "threshold": 0.6927935568262793 + }, + { + "coverage": 0.17466666666666666, + "risk": 0.48091603053435117, + "threshold": 0.6925517102394217 + }, + { + "coverage": 0.175, + "risk": 0.48, + "threshold": 0.6921122415025196 + }, + { + "coverage": 0.17533333333333334, + "risk": 0.48098859315589354, + "threshold": 0.6915848814991006 + }, + { + "coverage": 0.17566666666666667, + "risk": 0.48007590132827327, + "threshold": 0.6914216065306404 + }, + { + "coverage": 0.176, + "risk": 0.4791666666666667, + "threshold": 0.6914032151066377 + }, + { + "coverage": 0.17633333333333334, + "risk": 0.4782608695652174, + "threshold": 0.6910776087401228 + }, + { + "coverage": 0.17666666666666667, + "risk": 0.47924528301886793, + "threshold": 0.6910513282922477 + }, + { + "coverage": 0.177, + "risk": 0.480225988700565, + "threshold": 0.6907204405428058 + }, + { + "coverage": 0.17733333333333334, + "risk": 0.48120300751879697, + "threshold": 0.6901355244349062 + }, + { + "coverage": 0.17766666666666667, + "risk": 0.4803001876172608, + "threshold": 0.6901115743371289 + }, + { + "coverage": 0.178, + "risk": 0.4812734082397004, + "threshold": 0.6895383124581027 + }, + { + "coverage": 0.17833333333333334, + "risk": 0.4822429906542056, + "threshold": 0.6892849358496815 + }, + { + "coverage": 0.17866666666666667, + "risk": 0.4832089552238806, + "threshold": 0.6890766983592331 + }, + { + "coverage": 0.179, + "risk": 0.4823091247672253, + "threshold": 0.6885967493256028 + }, + { + "coverage": 0.17933333333333334, + "risk": 0.483271375464684, + "threshold": 0.6885055109577181 + }, + { + "coverage": 0.17966666666666667, + "risk": 0.4842300556586271, + "threshold": 0.6883253284832915 + }, + { + "coverage": 0.18, + "risk": 0.48333333333333334, + "threshold": 0.6870923227590459 + }, + { + "coverage": 0.18033333333333335, + "risk": 0.4824399260628466, + "threshold": 0.68493536540864 + }, + { + "coverage": 0.18066666666666667, + "risk": 0.48154981549815495, + "threshold": 0.684924137746975 + }, + { + "coverage": 0.181, + "risk": 0.48066298342541436, + "threshold": 0.684321976713811 + }, + { + "coverage": 0.18133333333333335, + "risk": 0.4797794117647059, + "threshold": 0.6841576267609224 + }, + { + "coverage": 0.18166666666666667, + "risk": 0.47889908256880737, + "threshold": 0.6822595529052944 + }, + { + "coverage": 0.182, + "risk": 0.47802197802197804, + "threshold": 0.6817340783258344 + }, + { + "coverage": 0.18233333333333332, + "risk": 0.4789762340036563, + "threshold": 0.680681563993053 + }, + { + "coverage": 0.18266666666666667, + "risk": 0.4781021897810219, + "threshold": 0.680644771149167 + }, + { + "coverage": 0.183, + "risk": 0.4790528233151184, + "threshold": 0.6805778930100798 + }, + { + "coverage": 0.18333333333333332, + "risk": 0.48, + "threshold": 0.6793840882315297 + }, + { + "coverage": 0.18366666666666667, + "risk": 0.4791288566243194, + "threshold": 0.6791143714487968 + }, + { + "coverage": 0.184, + "risk": 0.4782608695652174, + "threshold": 0.6774930263007413 + }, + { + "coverage": 0.18433333333333332, + "risk": 0.47739602169981915, + "threshold": 0.6772711474880823 + }, + { + "coverage": 0.18466666666666667, + "risk": 0.47653429602888087, + "threshold": 0.6770067557267188 + }, + { + "coverage": 0.185, + "risk": 0.4756756756756757, + "threshold": 0.6767236114465782 + }, + { + "coverage": 0.18533333333333332, + "risk": 0.4748201438848921, + "threshold": 0.6766252251512948 + }, + { + "coverage": 0.18566666666666667, + "risk": 0.4757630161579892, + "threshold": 0.6764519557890402 + }, + { + "coverage": 0.186, + "risk": 0.47491039426523296, + "threshold": 0.6750435956718087 + }, + { + "coverage": 0.18633333333333332, + "risk": 0.4740608228980322, + "threshold": 0.6739663975163549 + }, + { + "coverage": 0.18666666666666668, + "risk": 0.4732142857142857, + "threshold": 0.6725735321479944 + }, + { + "coverage": 0.187, + "risk": 0.47237076648841353, + "threshold": 0.6715411915576145 + }, + { + "coverage": 0.18733333333333332, + "risk": 0.47153024911032027, + "threshold": 0.6707017568477474 + }, + { + "coverage": 0.18766666666666668, + "risk": 0.4706927175843694, + "threshold": 0.6706789385187321 + }, + { + "coverage": 0.188, + "risk": 0.4698581560283688, + "threshold": 0.6704571996317119 + }, + { + "coverage": 0.18833333333333332, + "risk": 0.4690265486725664, + "threshold": 0.670306831393465 + }, + { + "coverage": 0.18866666666666668, + "risk": 0.46819787985865724, + "threshold": 0.6699548293199784 + }, + { + "coverage": 0.189, + "risk": 0.4673721340388007, + "threshold": 0.6698773301578669 + }, + { + "coverage": 0.18933333333333333, + "risk": 0.4665492957746479, + "threshold": 0.6696270375375883 + }, + { + "coverage": 0.18966666666666668, + "risk": 0.46572934973637964, + "threshold": 0.6694831027879552 + }, + { + "coverage": 0.19, + "risk": 0.4649122807017544, + "threshold": 0.6692718381370122 + }, + { + "coverage": 0.19033333333333333, + "risk": 0.46409807355516636, + "threshold": 0.6688523617281888 + }, + { + "coverage": 0.19066666666666668, + "risk": 0.4632867132867133, + "threshold": 0.6677190841791258 + }, + { + "coverage": 0.191, + "risk": 0.462478184991274, + "threshold": 0.6675851370276655 + }, + { + "coverage": 0.19133333333333333, + "risk": 0.4616724738675958, + "threshold": 0.667446914859592 + }, + { + "coverage": 0.19166666666666668, + "risk": 0.4608695652173913, + "threshold": 0.6670904338164486 + }, + { + "coverage": 0.192, + "risk": 0.4600694444444444, + "threshold": 0.6668675933090136 + }, + { + "coverage": 0.19233333333333333, + "risk": 0.4610051993067591, + "threshold": 0.6658179528493126 + }, + { + "coverage": 0.19266666666666668, + "risk": 0.4619377162629758, + "threshold": 0.6648908973580435 + }, + { + "coverage": 0.193, + "risk": 0.46113989637305697, + "threshold": 0.6639340381531813 + }, + { + "coverage": 0.19333333333333333, + "risk": 0.4603448275862069, + "threshold": 0.6627117893712737 + }, + { + "coverage": 0.19366666666666665, + "risk": 0.459552495697074, + "threshold": 0.6619125288328037 + }, + { + "coverage": 0.194, + "risk": 0.4587628865979381, + "threshold": 0.6614492951550187 + }, + { + "coverage": 0.19433333333333333, + "risk": 0.45797598627787306, + "threshold": 0.6614024629898472 + }, + { + "coverage": 0.19466666666666665, + "risk": 0.4589041095890411, + "threshold": 0.6613999455552684 + }, + { + "coverage": 0.195, + "risk": 0.4598290598290598, + "threshold": 0.6613529221795031 + }, + { + "coverage": 0.19533333333333333, + "risk": 0.46075085324232085, + "threshold": 0.6602414226383914 + }, + { + "coverage": 0.19566666666666666, + "risk": 0.4599659284497445, + "threshold": 0.659631348771643 + }, + { + "coverage": 0.196, + "risk": 0.4608843537414966, + "threshold": 0.659473604606182 + }, + { + "coverage": 0.19633333333333333, + "risk": 0.4601018675721562, + "threshold": 0.6592567010242902 + }, + { + "coverage": 0.19666666666666666, + "risk": 0.45932203389830506, + "threshold": 0.6592159091670549 + }, + { + "coverage": 0.197, + "risk": 0.45854483925549916, + "threshold": 0.6578939491821801 + }, + { + "coverage": 0.19733333333333333, + "risk": 0.4577702702702703, + "threshold": 0.6578434472234952 + }, + { + "coverage": 0.19766666666666666, + "risk": 0.45868465430016864, + "threshold": 0.6576352827599766 + }, + { + "coverage": 0.198, + "risk": 0.45791245791245794, + "threshold": 0.6574768475742222 + }, + { + "coverage": 0.19833333333333333, + "risk": 0.45714285714285713, + "threshold": 0.6567382900683457 + }, + { + "coverage": 0.19866666666666666, + "risk": 0.45805369127516776, + "threshold": 0.6565516629490338 + }, + { + "coverage": 0.199, + "risk": 0.45896147403685095, + "threshold": 0.6539380151941964 + }, + { + "coverage": 0.19933333333333333, + "risk": 0.45819397993311034, + "threshold": 0.6532788165392014 + }, + { + "coverage": 0.19966666666666666, + "risk": 0.4590984974958264, + "threshold": 0.6525957564743526 + }, + { + "coverage": 0.2, + "risk": 0.4583333333333333, + "threshold": 0.6520711505183874 + }, + { + "coverage": 0.20033333333333334, + "risk": 0.45757071547420963, + "threshold": 0.6507726127725177 + }, + { + "coverage": 0.20066666666666666, + "risk": 0.4568106312292359, + "threshold": 0.6502664341775061 + }, + { + "coverage": 0.201, + "risk": 0.4560530679933665, + "threshold": 0.6486173425573251 + }, + { + "coverage": 0.20133333333333334, + "risk": 0.4552980132450331, + "threshold": 0.6482598313673801 + }, + { + "coverage": 0.20166666666666666, + "risk": 0.45454545454545453, + "threshold": 0.6467853016782025 + }, + { + "coverage": 0.202, + "risk": 0.4537953795379538, + "threshold": 0.64670251197164 + }, + { + "coverage": 0.20233333333333334, + "risk": 0.4546952224052718, + "threshold": 0.6461746632480279 + }, + { + "coverage": 0.20266666666666666, + "risk": 0.45394736842105265, + "threshold": 0.6458417021153943 + }, + { + "coverage": 0.203, + "risk": 0.4548440065681445, + "threshold": 0.6450265868543476 + }, + { + "coverage": 0.20333333333333334, + "risk": 0.4540983606557377, + "threshold": 0.6445442025022253 + }, + { + "coverage": 0.20366666666666666, + "risk": 0.45499181669394434, + "threshold": 0.6445318178413283 + }, + { + "coverage": 0.204, + "risk": 0.4542483660130719, + "threshold": 0.6444465267804081 + }, + { + "coverage": 0.20433333333333334, + "risk": 0.4535073409461664, + "threshold": 0.644319727233022 + }, + { + "coverage": 0.20466666666666666, + "risk": 0.4527687296416938, + "threshold": 0.644222770529468 + }, + { + "coverage": 0.205, + "risk": 0.45365853658536587, + "threshold": 0.6440506806334865 + }, + { + "coverage": 0.20533333333333334, + "risk": 0.45292207792207795, + "threshold": 0.6437121299718084 + }, + { + "coverage": 0.20566666666666666, + "risk": 0.4538087520259319, + "threshold": 0.6435617069487056 + }, + { + "coverage": 0.206, + "risk": 0.45307443365695793, + "threshold": 0.6432806754986254 + }, + { + "coverage": 0.20633333333333334, + "risk": 0.45234248788368336, + "threshold": 0.6432698484676369 + }, + { + "coverage": 0.20666666666666667, + "risk": 0.4532258064516129, + "threshold": 0.6429372983575233 + }, + { + "coverage": 0.207, + "risk": 0.45410628019323673, + "threshold": 0.6428995599676497 + }, + { + "coverage": 0.20733333333333334, + "risk": 0.4533762057877814, + "threshold": 0.6428959500366401 + }, + { + "coverage": 0.20766666666666667, + "risk": 0.45425361155698235, + "threshold": 0.6424470465703095 + }, + { + "coverage": 0.208, + "risk": 0.453525641025641, + "threshold": 0.6424137584809556 + }, + { + "coverage": 0.20833333333333334, + "risk": 0.4528, + "threshold": 0.64192102979114 + }, + { + "coverage": 0.20866666666666667, + "risk": 0.4520766773162939, + "threshold": 0.641580437075492 + }, + { + "coverage": 0.209, + "risk": 0.4529505582137161, + "threshold": 0.6412685039618834 + }, + { + "coverage": 0.20933333333333334, + "risk": 0.45222929936305734, + "threshold": 0.6407065871165748 + }, + { + "coverage": 0.20966666666666667, + "risk": 0.45151033386327505, + "threshold": 0.6398574118200835 + }, + { + "coverage": 0.21, + "risk": 0.4507936507936508, + "threshold": 0.6396679070698102 + }, + { + "coverage": 0.21033333333333334, + "risk": 0.4500792393026941, + "threshold": 0.6395629635277912 + }, + { + "coverage": 0.21066666666666667, + "risk": 0.4509493670886076, + "threshold": 0.6391382817044113 + }, + { + "coverage": 0.211, + "risk": 0.45023696682464454, + "threshold": 0.6388791555879566 + }, + { + "coverage": 0.21133333333333335, + "risk": 0.44952681388012616, + "threshold": 0.6385120170000222 + }, + { + "coverage": 0.21166666666666667, + "risk": 0.4503937007874016, + "threshold": 0.6384111157693816 + }, + { + "coverage": 0.212, + "risk": 0.449685534591195, + "threshold": 0.6376436286194862 + }, + { + "coverage": 0.21233333333333335, + "risk": 0.4489795918367347, + "threshold": 0.6368518752020638 + }, + { + "coverage": 0.21266666666666667, + "risk": 0.4482758620689655, + "threshold": 0.6360813809788027 + }, + { + "coverage": 0.213, + "risk": 0.4475743348982786, + "threshold": 0.6347940761551013 + }, + { + "coverage": 0.21333333333333335, + "risk": 0.4484375, + "threshold": 0.6345492678624766 + }, + { + "coverage": 0.21366666666666667, + "risk": 0.44929797191887677, + "threshold": 0.6341883497241766 + }, + { + "coverage": 0.214, + "risk": 0.4501557632398754, + "threshold": 0.6338752190347525 + }, + { + "coverage": 0.21433333333333332, + "risk": 0.4494556765163297, + "threshold": 0.6328451884436677 + }, + { + "coverage": 0.21466666666666667, + "risk": 0.44875776397515527, + "threshold": 0.6323960278942518 + }, + { + "coverage": 0.215, + "risk": 0.448062015503876, + "threshold": 0.6321218287376387 + }, + { + "coverage": 0.21533333333333332, + "risk": 0.44891640866873067, + "threshold": 0.6314018981229149 + }, + { + "coverage": 0.21566666666666667, + "risk": 0.4482225656877898, + "threshold": 0.631240013839934 + }, + { + "coverage": 0.216, + "risk": 0.44907407407407407, + "threshold": 0.6312082392886539 + }, + { + "coverage": 0.21633333333333332, + "risk": 0.44838212634822805, + "threshold": 0.6308431529218487 + }, + { + "coverage": 0.21666666666666667, + "risk": 0.44769230769230767, + "threshold": 0.6308132364962528 + }, + { + "coverage": 0.217, + "risk": 0.4485407066052227, + "threshold": 0.6302318784580125 + }, + { + "coverage": 0.21733333333333332, + "risk": 0.44785276073619634, + "threshold": 0.6300095924793182 + }, + { + "coverage": 0.21766666666666667, + "risk": 0.44716692189892804, + "threshold": 0.6294958326337131 + }, + { + "coverage": 0.218, + "risk": 0.44648318042813456, + "threshold": 0.629086080421273 + }, + { + "coverage": 0.21833333333333332, + "risk": 0.44580152671755724, + "threshold": 0.6287969461856777 + }, + { + "coverage": 0.21866666666666668, + "risk": 0.44664634146341464, + "threshold": 0.628743061191365 + }, + { + "coverage": 0.219, + "risk": 0.4474885844748858, + "threshold": 0.6287190164556008 + }, + { + "coverage": 0.21933333333333332, + "risk": 0.44832826747720367, + "threshold": 0.6283818365383087 + }, + { + "coverage": 0.21966666666666668, + "risk": 0.44764795144157815, + "threshold": 0.6282777033159118 + }, + { + "coverage": 0.22, + "risk": 0.44696969696969696, + "threshold": 0.6281327266899427 + }, + { + "coverage": 0.22033333333333333, + "risk": 0.4462934947049924, + "threshold": 0.6277902256915283 + }, + { + "coverage": 0.22066666666666668, + "risk": 0.44561933534743203, + "threshold": 0.6275336193561047 + }, + { + "coverage": 0.221, + "risk": 0.4464555052790347, + "threshold": 0.6274335401839917 + }, + { + "coverage": 0.22133333333333333, + "risk": 0.4457831325301205, + "threshold": 0.6273410151675557 + }, + { + "coverage": 0.22166666666666668, + "risk": 0.44511278195488724, + "threshold": 0.6270570399838981 + }, + { + "coverage": 0.222, + "risk": 0.4444444444444444, + "threshold": 0.6266342699309423 + }, + { + "coverage": 0.22233333333333333, + "risk": 0.4452773613193403, + "threshold": 0.6264740557266587 + }, + { + "coverage": 0.22266666666666668, + "risk": 0.44461077844311375, + "threshold": 0.6256997290026672 + }, + { + "coverage": 0.223, + "risk": 0.445440956651719, + "threshold": 0.6254189866674659 + }, + { + "coverage": 0.22333333333333333, + "risk": 0.44477611940298506, + "threshold": 0.6246081539318464 + }, + { + "coverage": 0.22366666666666668, + "risk": 0.4456035767511177, + "threshold": 0.6237098661133939 + }, + { + "coverage": 0.224, + "risk": 0.44642857142857145, + "threshold": 0.6229586889202748 + }, + { + "coverage": 0.22433333333333333, + "risk": 0.4472511144130758, + "threshold": 0.6224369932242789 + }, + { + "coverage": 0.22466666666666665, + "risk": 0.44807121661721067, + "threshold": 0.6222675284695149 + }, + { + "coverage": 0.225, + "risk": 0.4488888888888889, + "threshold": 0.6219902664725088 + }, + { + "coverage": 0.22533333333333333, + "risk": 0.44970414201183434, + "threshold": 0.6217681776946318 + }, + { + "coverage": 0.22566666666666665, + "risk": 0.44903988183161003, + "threshold": 0.6209188129796661 + }, + { + "coverage": 0.226, + "risk": 0.44837758112094395, + "threshold": 0.6205335142783815 + }, + { + "coverage": 0.22633333333333333, + "risk": 0.4477172312223859, + "threshold": 0.6200415518652654 + }, + { + "coverage": 0.22666666666666666, + "risk": 0.4485294117647059, + "threshold": 0.6199221569669167 + }, + { + "coverage": 0.227, + "risk": 0.447870778267254, + "threshold": 0.6196638511218734 + }, + { + "coverage": 0.22733333333333333, + "risk": 0.4472140762463343, + "threshold": 0.6192852509931256 + }, + { + "coverage": 0.22766666666666666, + "risk": 0.4465592972181552, + "threshold": 0.6189080699142865 + }, + { + "coverage": 0.228, + "risk": 0.4473684210526316, + "threshold": 0.6188259256241554 + }, + { + "coverage": 0.22833333333333333, + "risk": 0.4467153284671533, + "threshold": 0.6178026152251881 + }, + { + "coverage": 0.22866666666666666, + "risk": 0.446064139941691, + "threshold": 0.6164344834815202 + }, + { + "coverage": 0.229, + "risk": 0.4468704512372635, + "threshold": 0.614656671705443 + }, + { + "coverage": 0.22933333333333333, + "risk": 0.44622093023255816, + "threshold": 0.6142581618933354 + }, + { + "coverage": 0.22966666666666666, + "risk": 0.4455732946298984, + "threshold": 0.614097101259501 + }, + { + "coverage": 0.23, + "risk": 0.4463768115942029, + "threshold": 0.6140364076693968 + }, + { + "coverage": 0.23033333333333333, + "risk": 0.44573082489146165, + "threshold": 0.6137729792996263 + }, + { + "coverage": 0.23066666666666666, + "risk": 0.44653179190751446, + "threshold": 0.6137658467088991 + }, + { + "coverage": 0.231, + "risk": 0.4458874458874459, + "threshold": 0.6137038809313378 + }, + { + "coverage": 0.23133333333333334, + "risk": 0.4452449567723343, + "threshold": 0.6135019448098353 + }, + { + "coverage": 0.23166666666666666, + "risk": 0.4460431654676259, + "threshold": 0.6132454323220523 + }, + { + "coverage": 0.232, + "risk": 0.4454022988505747, + "threshold": 0.6130553534885311 + }, + { + "coverage": 0.23233333333333334, + "risk": 0.44619799139167865, + "threshold": 0.6123177814395186 + }, + { + "coverage": 0.23266666666666666, + "risk": 0.4455587392550143, + "threshold": 0.6119232005244416 + }, + { + "coverage": 0.233, + "risk": 0.44635193133047213, + "threshold": 0.6118075148116526 + }, + { + "coverage": 0.23333333333333334, + "risk": 0.44571428571428573, + "threshold": 0.6116698096116934 + }, + { + "coverage": 0.23366666666666666, + "risk": 0.4450784593437946, + "threshold": 0.6111739596974131 + }, + { + "coverage": 0.234, + "risk": 0.4458689458689459, + "threshold": 0.6098774173455574 + }, + { + "coverage": 0.23433333333333334, + "risk": 0.44523470839260315, + "threshold": 0.6091817579730036 + }, + { + "coverage": 0.23466666666666666, + "risk": 0.4446022727272727, + "threshold": 0.6091595905170768 + }, + { + "coverage": 0.235, + "risk": 0.4453900709219858, + "threshold": 0.6090673862079348 + }, + { + "coverage": 0.23533333333333334, + "risk": 0.4447592067988669, + "threshold": 0.6089826040289025 + }, + { + "coverage": 0.23566666666666666, + "risk": 0.44413012729844414, + "threshold": 0.608796614257991 + }, + { + "coverage": 0.236, + "risk": 0.4435028248587571, + "threshold": 0.6080890774717368 + }, + { + "coverage": 0.23633333333333334, + "risk": 0.44287729196050774, + "threshold": 0.6073196954047735 + }, + { + "coverage": 0.23666666666666666, + "risk": 0.4422535211267606, + "threshold": 0.6071720102885124 + }, + { + "coverage": 0.237, + "risk": 0.44163150492264414, + "threshold": 0.6051481524673209 + }, + { + "coverage": 0.23733333333333334, + "risk": 0.44241573033707865, + "threshold": 0.6046594224178641 + }, + { + "coverage": 0.23766666666666666, + "risk": 0.4417952314165498, + "threshold": 0.6033995140055871 + }, + { + "coverage": 0.238, + "risk": 0.4411764705882353, + "threshold": 0.6021583228272078 + }, + { + "coverage": 0.23833333333333334, + "risk": 0.44195804195804195, + "threshold": 0.6017092540015029 + }, + { + "coverage": 0.23866666666666667, + "risk": 0.441340782122905, + "threshold": 0.6004903406869836 + }, + { + "coverage": 0.239, + "risk": 0.4407252440725244, + "threshold": 0.6002350627913694 + }, + { + "coverage": 0.23933333333333334, + "risk": 0.4415041782729805, + "threshold": 0.5994746693322359 + }, + { + "coverage": 0.23966666666666667, + "risk": 0.4422809457579972, + "threshold": 0.5983132672971933 + }, + { + "coverage": 0.24, + "risk": 0.44166666666666665, + "threshold": 0.5978420473010927 + }, + { + "coverage": 0.24033333333333334, + "risk": 0.44244105409153955, + "threshold": 0.5977922694361433 + }, + { + "coverage": 0.24066666666666667, + "risk": 0.44182825484764543, + "threshold": 0.5977114177092764 + }, + { + "coverage": 0.241, + "risk": 0.4412171507607192, + "threshold": 0.5967439408895612 + }, + { + "coverage": 0.24133333333333334, + "risk": 0.44060773480662985, + "threshold": 0.5965415542790757 + }, + { + "coverage": 0.24166666666666667, + "risk": 0.44, + "threshold": 0.5958376100525077 + }, + { + "coverage": 0.242, + "risk": 0.4393939393939394, + "threshold": 0.5946316206304826 + }, + { + "coverage": 0.24233333333333335, + "risk": 0.43878954607977994, + "threshold": 0.5942623563273304 + }, + { + "coverage": 0.24266666666666667, + "risk": 0.4381868131868132, + "threshold": 0.5934502165014321 + }, + { + "coverage": 0.243, + "risk": 0.4375857338820302, + "threshold": 0.5931713639578026 + }, + { + "coverage": 0.24333333333333335, + "risk": 0.4383561643835616, + "threshold": 0.5921856631591842 + }, + { + "coverage": 0.24366666666666667, + "risk": 0.43912448700410395, + "threshold": 0.5908227911156926 + }, + { + "coverage": 0.244, + "risk": 0.4385245901639344, + "threshold": 0.5902067393752656 + }, + { + "coverage": 0.24433333333333335, + "risk": 0.4392905866302865, + "threshold": 0.5896199888968885 + }, + { + "coverage": 0.24466666666666667, + "risk": 0.43869209809264303, + "threshold": 0.5888456482476171 + }, + { + "coverage": 0.245, + "risk": 0.4380952380952381, + "threshold": 0.5886907720161955 + }, + { + "coverage": 0.24533333333333332, + "risk": 0.4388586956521739, + "threshold": 0.5886483806135467 + }, + { + "coverage": 0.24566666666666667, + "risk": 0.43826322930800543, + "threshold": 0.5884030807107259 + }, + { + "coverage": 0.246, + "risk": 0.43766937669376693, + "threshold": 0.5880239423037925 + }, + { + "coverage": 0.24633333333333332, + "risk": 0.4370771312584574, + "threshold": 0.5878800039397725 + }, + { + "coverage": 0.24666666666666667, + "risk": 0.43648648648648647, + "threshold": 0.5876477201569836 + }, + { + "coverage": 0.247, + "risk": 0.4358974358974359, + "threshold": 0.586498569277987 + }, + { + "coverage": 0.24733333333333332, + "risk": 0.4366576819407008, + "threshold": 0.5849251823041323 + }, + { + "coverage": 0.24766666666666667, + "risk": 0.43741588156123823, + "threshold": 0.5844289326809085 + }, + { + "coverage": 0.248, + "risk": 0.4368279569892473, + "threshold": 0.5843520731669961 + }, + { + "coverage": 0.24833333333333332, + "risk": 0.4375838926174497, + "threshold": 0.5843460341348685 + }, + { + "coverage": 0.24866666666666667, + "risk": 0.43699731903485256, + "threshold": 0.5842341657637689 + }, + { + "coverage": 0.249, + "risk": 0.43775100401606426, + "threshold": 0.5837104009113224 + }, + { + "coverage": 0.24933333333333332, + "risk": 0.43716577540106955, + "threshold": 0.5836085950801236 + }, + { + "coverage": 0.24966666666666668, + "risk": 0.43658210947930576, + "threshold": 0.5833360178178788 + }, + { + "coverage": 0.25, + "risk": 0.436, + "threshold": 0.5824582150264103 + }, + { + "coverage": 0.25033333333333335, + "risk": 0.4367509986684421, + "threshold": 0.5819560304850149 + }, + { + "coverage": 0.25066666666666665, + "risk": 0.43617021276595747, + "threshold": 0.5802975671054452 + }, + { + "coverage": 0.251, + "risk": 0.4355909694555113, + "threshold": 0.5802641031178185 + }, + { + "coverage": 0.25133333333333335, + "risk": 0.4350132625994695, + "threshold": 0.5801754995351218 + }, + { + "coverage": 0.25166666666666665, + "risk": 0.43443708609271525, + "threshold": 0.579564533268086 + }, + { + "coverage": 0.252, + "risk": 0.4351851851851852, + "threshold": 0.5794091965501592 + }, + { + "coverage": 0.25233333333333335, + "risk": 0.4346103038309115, + "threshold": 0.5790960736746735 + }, + { + "coverage": 0.25266666666666665, + "risk": 0.4340369393139842, + "threshold": 0.579060532004099 + }, + { + "coverage": 0.253, + "risk": 0.43346508563899866, + "threshold": 0.5787668422081635 + }, + { + "coverage": 0.25333333333333335, + "risk": 0.4328947368421053, + "threshold": 0.5782027961874764 + }, + { + "coverage": 0.25366666666666665, + "risk": 0.4323258869908016, + "threshold": 0.5781657494847262 + }, + { + "coverage": 0.254, + "risk": 0.431758530183727, + "threshold": 0.5779010067045726 + }, + { + "coverage": 0.25433333333333336, + "risk": 0.43119266055045874, + "threshold": 0.5771999369626146 + }, + { + "coverage": 0.25466666666666665, + "risk": 0.4319371727748691, + "threshold": 0.5771758724093632 + }, + { + "coverage": 0.255, + "risk": 0.43137254901960786, + "threshold": 0.575411991951718 + }, + { + "coverage": 0.25533333333333336, + "risk": 0.4308093994778068, + "threshold": 0.5752380682337823 + }, + { + "coverage": 0.25566666666666665, + "risk": 0.4315514993481095, + "threshold": 0.5749131898603368 + }, + { + "coverage": 0.256, + "risk": 0.4322916666666667, + "threshold": 0.5745026584693629 + }, + { + "coverage": 0.25633333333333336, + "risk": 0.43302990897269183, + "threshold": 0.5740835546263081 + }, + { + "coverage": 0.25666666666666665, + "risk": 0.43376623376623374, + "threshold": 0.5735181111260277 + }, + { + "coverage": 0.257, + "risk": 0.43320363164721143, + "threshold": 0.5728801177650105 + }, + { + "coverage": 0.25733333333333336, + "risk": 0.4326424870466321, + "threshold": 0.5727892559079595 + }, + { + "coverage": 0.25766666666666665, + "risk": 0.4333764553686934, + "threshold": 0.5724820225964999 + }, + { + "coverage": 0.258, + "risk": 0.43281653746770027, + "threshold": 0.5724153162258613 + }, + { + "coverage": 0.25833333333333336, + "risk": 0.432258064516129, + "threshold": 0.5723780989109295 + }, + { + "coverage": 0.25866666666666666, + "risk": 0.4329896907216495, + "threshold": 0.5722874269498903 + }, + { + "coverage": 0.259, + "risk": 0.43243243243243246, + "threshold": 0.5711589784809225 + }, + { + "coverage": 0.25933333333333336, + "risk": 0.4318766066838046, + "threshold": 0.5703461952241954 + }, + { + "coverage": 0.25966666666666666, + "risk": 0.4313222079589217, + "threshold": 0.5697801880259907 + }, + { + "coverage": 0.26, + "risk": 0.4307692307692308, + "threshold": 0.5697036858228214 + }, + { + "coverage": 0.26033333333333336, + "risk": 0.43021766965428937, + "threshold": 0.5695172266893708 + }, + { + "coverage": 0.26066666666666666, + "risk": 0.4296675191815857, + "threshold": 0.5694548351157412 + }, + { + "coverage": 0.261, + "risk": 0.43039591315453385, + "threshold": 0.5687782052371064 + }, + { + "coverage": 0.2613333333333333, + "risk": 0.43112244897959184, + "threshold": 0.5685898063508037 + }, + { + "coverage": 0.26166666666666666, + "risk": 0.4305732484076433, + "threshold": 0.5684565705959242 + }, + { + "coverage": 0.262, + "risk": 0.4300254452926209, + "threshold": 0.567773397113959 + }, + { + "coverage": 0.2623333333333333, + "risk": 0.4307496823379924, + "threshold": 0.5676461241698018 + }, + { + "coverage": 0.26266666666666666, + "risk": 0.4302030456852792, + "threshold": 0.5670241088804 + }, + { + "coverage": 0.263, + "risk": 0.4296577946768061, + "threshold": 0.5665916256454586 + }, + { + "coverage": 0.2633333333333333, + "risk": 0.4291139240506329, + "threshold": 0.566390604188026 + }, + { + "coverage": 0.26366666666666666, + "risk": 0.42857142857142855, + "threshold": 0.5661757390264391 + }, + { + "coverage": 0.264, + "risk": 0.42803030303030304, + "threshold": 0.5660703090868802 + }, + { + "coverage": 0.2643333333333333, + "risk": 0.4274905422446406, + "threshold": 0.565957034703422 + }, + { + "coverage": 0.26466666666666666, + "risk": 0.4269521410579345, + "threshold": 0.562815808357 + }, + { + "coverage": 0.265, + "risk": 0.42641509433962266, + "threshold": 0.5627870911358708 + }, + { + "coverage": 0.2653333333333333, + "risk": 0.4258793969849246, + "threshold": 0.5625416425852103 + }, + { + "coverage": 0.26566666666666666, + "risk": 0.42534504391468003, + "threshold": 0.5616358677981738 + }, + { + "coverage": 0.266, + "risk": 0.424812030075188, + "threshold": 0.5609314169100242 + }, + { + "coverage": 0.2663333333333333, + "risk": 0.425531914893617, + "threshold": 0.5607388080984613 + }, + { + "coverage": 0.26666666666666666, + "risk": 0.42625, + "threshold": 0.5606441699664559 + }, + { + "coverage": 0.267, + "risk": 0.42696629213483145, + "threshold": 0.5602627149866435 + }, + { + "coverage": 0.2673333333333333, + "risk": 0.42643391521197005, + "threshold": 0.5598093842722485 + }, + { + "coverage": 0.26766666666666666, + "risk": 0.42714819427148193, + "threshold": 0.559465114652877 + }, + { + "coverage": 0.268, + "risk": 0.42661691542288555, + "threshold": 0.5594381778591597 + }, + { + "coverage": 0.2683333333333333, + "risk": 0.4260869565217391, + "threshold": 0.5586979636730346 + }, + { + "coverage": 0.26866666666666666, + "risk": 0.4267990074441687, + "threshold": 0.5580301143991626 + }, + { + "coverage": 0.269, + "risk": 0.42627013630731103, + "threshold": 0.5580021811835564 + }, + { + "coverage": 0.2693333333333333, + "risk": 0.42574257425742573, + "threshold": 0.5578505780263514 + }, + { + "coverage": 0.26966666666666667, + "risk": 0.4264524103831891, + "threshold": 0.5578369331898335 + }, + { + "coverage": 0.27, + "risk": 0.42592592592592593, + "threshold": 0.5578083992650322 + }, + { + "coverage": 0.2703333333333333, + "risk": 0.4254007398273736, + "threshold": 0.5573821614080239 + }, + { + "coverage": 0.27066666666666667, + "risk": 0.4248768472906404, + "threshold": 0.5558554948285287 + }, + { + "coverage": 0.271, + "risk": 0.42435424354243545, + "threshold": 0.5554343560894628 + }, + { + "coverage": 0.2713333333333333, + "risk": 0.4238329238329238, + "threshold": 0.5553850108365151 + }, + { + "coverage": 0.27166666666666667, + "risk": 0.4233128834355828, + "threshold": 0.5552630812332078 + }, + { + "coverage": 0.272, + "risk": 0.4227941176470588, + "threshold": 0.5543531022945005 + }, + { + "coverage": 0.2723333333333333, + "risk": 0.423500611995104, + "threshold": 0.5535031260523595 + }, + { + "coverage": 0.27266666666666667, + "risk": 0.4229828850855746, + "threshold": 0.5533532460677151 + }, + { + "coverage": 0.273, + "risk": 0.42246642246642246, + "threshold": 0.5526556055479928 + }, + { + "coverage": 0.2733333333333333, + "risk": 0.4219512195121951, + "threshold": 0.5518405268607564 + }, + { + "coverage": 0.27366666666666667, + "risk": 0.42143727161997563, + "threshold": 0.55169979241499 + }, + { + "coverage": 0.274, + "risk": 0.4209245742092457, + "threshold": 0.5516667187090979 + }, + { + "coverage": 0.2743333333333333, + "risk": 0.4204131227217497, + "threshold": 0.5514725844478862 + }, + { + "coverage": 0.27466666666666667, + "risk": 0.4199029126213592, + "threshold": 0.5512563723240884 + }, + { + "coverage": 0.275, + "risk": 0.4206060606060606, + "threshold": 0.5511641748631718 + }, + { + "coverage": 0.2753333333333333, + "risk": 0.42009685230024213, + "threshold": 0.5508529944167584 + }, + { + "coverage": 0.27566666666666667, + "risk": 0.4195888754534462, + "threshold": 0.5499898977873129 + }, + { + "coverage": 0.276, + "risk": 0.4190821256038647, + "threshold": 0.5497508521276118 + }, + { + "coverage": 0.2763333333333333, + "risk": 0.41857659831121835, + "threshold": 0.5497335963320191 + }, + { + "coverage": 0.27666666666666667, + "risk": 0.41927710843373495, + "threshold": 0.548745309303907 + }, + { + "coverage": 0.277, + "risk": 0.4187725631768953, + "threshold": 0.5486754267081959 + }, + { + "coverage": 0.2773333333333333, + "risk": 0.4182692307692308, + "threshold": 0.5485361238855151 + }, + { + "coverage": 0.2776666666666667, + "risk": 0.4177671068427371, + "threshold": 0.5484491637728287 + }, + { + "coverage": 0.278, + "risk": 0.4172661870503597, + "threshold": 0.5483032281859062 + }, + { + "coverage": 0.2783333333333333, + "risk": 0.4167664670658683, + "threshold": 0.5476309337822406 + }, + { + "coverage": 0.2786666666666667, + "risk": 0.41626794258373206, + "threshold": 0.5474693444323518 + }, + { + "coverage": 0.279, + "risk": 0.4157706093189964, + "threshold": 0.5470018509265225 + }, + { + "coverage": 0.2793333333333333, + "risk": 0.4152744630071599, + "threshold": 0.5464093313460832 + }, + { + "coverage": 0.2796666666666667, + "risk": 0.41477949940405245, + "threshold": 0.5460920605612494 + }, + { + "coverage": 0.28, + "risk": 0.4142857142857143, + "threshold": 0.5455564917575917 + }, + { + "coverage": 0.2803333333333333, + "risk": 0.41379310344827586, + "threshold": 0.5448930650159765 + }, + { + "coverage": 0.2806666666666667, + "risk": 0.4144893111638955, + "threshold": 0.5448278887877978 + }, + { + "coverage": 0.281, + "risk": 0.4139976275207592, + "threshold": 0.5442617424733175 + }, + { + "coverage": 0.2813333333333333, + "risk": 0.4146919431279621, + "threshold": 0.5438541151620209 + }, + { + "coverage": 0.2816666666666667, + "risk": 0.41420118343195267, + "threshold": 0.5433803583382855 + }, + { + "coverage": 0.282, + "risk": 0.41371158392434987, + "threshold": 0.542777787525762 + }, + { + "coverage": 0.2823333333333333, + "risk": 0.4132231404958678, + "threshold": 0.5420358413092509 + }, + { + "coverage": 0.2826666666666667, + "risk": 0.41273584905660377, + "threshold": 0.5418821623337713 + }, + { + "coverage": 0.283, + "risk": 0.4122497055359246, + "threshold": 0.5418745129642152 + }, + { + "coverage": 0.2833333333333333, + "risk": 0.41294117647058826, + "threshold": 0.541794876261919 + }, + { + "coverage": 0.2836666666666667, + "risk": 0.41245593419506466, + "threshold": 0.5406842732737719 + }, + { + "coverage": 0.284, + "risk": 0.4131455399061033, + "threshold": 0.5403185767689835 + }, + { + "coverage": 0.2843333333333333, + "risk": 0.4138335287221571, + "threshold": 0.5399969635835283 + }, + { + "coverage": 0.2846666666666667, + "risk": 0.4133489461358314, + "threshold": 0.5396852673162933 + }, + { + "coverage": 0.285, + "risk": 0.4128654970760234, + "threshold": 0.5386524562505649 + }, + { + "coverage": 0.2853333333333333, + "risk": 0.41238317757009346, + "threshold": 0.538575931522575 + }, + { + "coverage": 0.2856666666666667, + "risk": 0.411901983663944, + "threshold": 0.5385674482657896 + }, + { + "coverage": 0.286, + "risk": 0.4125874125874126, + "threshold": 0.5378861892325774 + }, + { + "coverage": 0.28633333333333333, + "risk": 0.41327124563445866, + "threshold": 0.537687014842447 + }, + { + "coverage": 0.2866666666666667, + "risk": 0.4127906976744186, + "threshold": 0.5376718395170184 + }, + { + "coverage": 0.287, + "risk": 0.413472706155633, + "threshold": 0.5374777375419258 + }, + { + "coverage": 0.28733333333333333, + "risk": 0.41299303944315546, + "threshold": 0.5373858380263161 + }, + { + "coverage": 0.2876666666666667, + "risk": 0.41251448435689453, + "threshold": 0.5371448913953809 + }, + { + "coverage": 0.288, + "risk": 0.41203703703703703, + "threshold": 0.5369674917107463 + }, + { + "coverage": 0.28833333333333333, + "risk": 0.4115606936416185, + "threshold": 0.5366630767961891 + }, + { + "coverage": 0.2886666666666667, + "risk": 0.4110854503464203, + "threshold": 0.5365664055646171 + }, + { + "coverage": 0.289, + "risk": 0.41061130334486734, + "threshold": 0.5354228411707845 + }, + { + "coverage": 0.28933333333333333, + "risk": 0.41013824884792627, + "threshold": 0.535255787008879 + }, + { + "coverage": 0.2896666666666667, + "risk": 0.4096662830840046, + "threshold": 0.5349733629456848 + }, + { + "coverage": 0.29, + "risk": 0.4091954022988506, + "threshold": 0.5343942940623428 + }, + { + "coverage": 0.29033333333333333, + "risk": 0.40987370838117104, + "threshold": 0.5331888450996746 + }, + { + "coverage": 0.2906666666666667, + "risk": 0.4105504587155963, + "threshold": 0.5324517468319694 + }, + { + "coverage": 0.291, + "risk": 0.41008018327605955, + "threshold": 0.5310754650859469 + }, + { + "coverage": 0.29133333333333333, + "risk": 0.4107551487414188, + "threshold": 0.5308799992342889 + }, + { + "coverage": 0.2916666666666667, + "risk": 0.4102857142857143, + "threshold": 0.5307117556067054 + }, + { + "coverage": 0.292, + "risk": 0.4098173515981735, + "threshold": 0.5306382306021796 + }, + { + "coverage": 0.29233333333333333, + "risk": 0.40935005701254273, + "threshold": 0.5306206573957509 + }, + { + "coverage": 0.2926666666666667, + "risk": 0.40888382687927105, + "threshold": 0.5301746361467772 + }, + { + "coverage": 0.293, + "risk": 0.40841865756541523, + "threshold": 0.5299638088471879 + }, + { + "coverage": 0.29333333333333333, + "risk": 0.40795454545454546, + "threshold": 0.5289146331537942 + }, + { + "coverage": 0.2936666666666667, + "risk": 0.40749148694665155, + "threshold": 0.5286173763294488 + }, + { + "coverage": 0.294, + "risk": 0.4070294784580499, + "threshold": 0.5285459274637112 + }, + { + "coverage": 0.29433333333333334, + "risk": 0.40656851642129105, + "threshold": 0.528467179904648 + }, + { + "coverage": 0.2946666666666667, + "risk": 0.4072398190045249, + "threshold": 0.5282826951375043 + }, + { + "coverage": 0.295, + "risk": 0.4067796610169492, + "threshold": 0.5282668380777574 + }, + { + "coverage": 0.29533333333333334, + "risk": 0.40632054176072235, + "threshold": 0.5282171612437845 + }, + { + "coverage": 0.2956666666666667, + "risk": 0.40586245772266066, + "threshold": 0.5281768015585878 + }, + { + "coverage": 0.296, + "risk": 0.40653153153153154, + "threshold": 0.5276701118460783 + }, + { + "coverage": 0.29633333333333334, + "risk": 0.40607424071991, + "threshold": 0.5276054226178748 + }, + { + "coverage": 0.2966666666666667, + "risk": 0.40561797752808987, + "threshold": 0.5261635450868651 + }, + { + "coverage": 0.297, + "risk": 0.4051627384960718, + "threshold": 0.5252278625942097 + }, + { + "coverage": 0.29733333333333334, + "risk": 0.40582959641255606, + "threshold": 0.5252061265386044 + }, + { + "coverage": 0.2976666666666667, + "risk": 0.4053751399776036, + "threshold": 0.525117851524278 + }, + { + "coverage": 0.298, + "risk": 0.40492170022371365, + "threshold": 0.5250327566056241 + }, + { + "coverage": 0.29833333333333334, + "risk": 0.4044692737430168, + "threshold": 0.5233533034027339 + }, + { + "coverage": 0.2986666666666667, + "risk": 0.40401785714285715, + "threshold": 0.5219670890617797 + }, + { + "coverage": 0.299, + "risk": 0.4035674470457079, + "threshold": 0.5215917954782742 + }, + { + "coverage": 0.29933333333333334, + "risk": 0.40311804008908686, + "threshold": 0.5210093935645965 + }, + { + "coverage": 0.2996666666666667, + "risk": 0.40266963292547275, + "threshold": 0.5193055156996389 + }, + { + "coverage": 0.3, + "risk": 0.4022222222222222, + "threshold": 0.5192709717956768 + }, + { + "coverage": 0.30033333333333334, + "risk": 0.4017758046614872, + "threshold": 0.5188024656540183 + }, + { + "coverage": 0.3006666666666667, + "risk": 0.401330376940133, + "threshold": 0.518389855519006 + }, + { + "coverage": 0.301, + "risk": 0.4019933554817276, + "threshold": 0.5183180440982923 + }, + { + "coverage": 0.30133333333333334, + "risk": 0.40154867256637167, + "threshold": 0.5182982213128929 + }, + { + "coverage": 0.3016666666666667, + "risk": 0.4011049723756906, + "threshold": 0.5180480326849812 + }, + { + "coverage": 0.302, + "risk": 0.40066225165562913, + "threshold": 0.517831385037331 + }, + { + "coverage": 0.30233333333333334, + "risk": 0.4002205071664829, + "threshold": 0.5176726720542589 + }, + { + "coverage": 0.30266666666666664, + "risk": 0.3997797356828194, + "threshold": 0.5175979693434266 + }, + { + "coverage": 0.303, + "risk": 0.39933993399339934, + "threshold": 0.5175397015186707 + }, + { + "coverage": 0.30333333333333334, + "risk": 0.4, + "threshold": 0.5174714754151587 + }, + { + "coverage": 0.30366666666666664, + "risk": 0.3995609220636663, + "threshold": 0.516830856210182 + }, + { + "coverage": 0.304, + "risk": 0.3991228070175439, + "threshold": 0.5157795090686684 + }, + { + "coverage": 0.30433333333333334, + "risk": 0.39868565169769987, + "threshold": 0.5141989769297849 + }, + { + "coverage": 0.30466666666666664, + "risk": 0.3982494529540481, + "threshold": 0.513452076494398 + }, + { + "coverage": 0.305, + "risk": 0.3978142076502732, + "threshold": 0.5134261696565086 + }, + { + "coverage": 0.30533333333333335, + "risk": 0.39737991266375544, + "threshold": 0.5130557913989311 + }, + { + "coverage": 0.30566666666666664, + "risk": 0.3969465648854962, + "threshold": 0.5118929533799276 + }, + { + "coverage": 0.306, + "risk": 0.39760348583877997, + "threshold": 0.5112781994346687 + }, + { + "coverage": 0.30633333333333335, + "risk": 0.397170837867247, + "threshold": 0.5103774191823245 + }, + { + "coverage": 0.30666666666666664, + "risk": 0.3967391304347826, + "threshold": 0.510214046173639 + }, + { + "coverage": 0.307, + "risk": 0.3963083604777416, + "threshold": 0.5098060087306301 + }, + { + "coverage": 0.30733333333333335, + "risk": 0.3958785249457701, + "threshold": 0.5096882543473045 + }, + { + "coverage": 0.30766666666666664, + "risk": 0.39544962080173346, + "threshold": 0.5094348914187412 + }, + { + "coverage": 0.308, + "risk": 0.395021645021645, + "threshold": 0.5089916353745967 + }, + { + "coverage": 0.30833333333333335, + "risk": 0.3956756756756757, + "threshold": 0.5088829782117951 + }, + { + "coverage": 0.30866666666666664, + "risk": 0.3952483801295896, + "threshold": 0.5087719430363118 + }, + { + "coverage": 0.309, + "risk": 0.3948220064724919, + "threshold": 0.5087047875794756 + }, + { + "coverage": 0.30933333333333335, + "risk": 0.39439655172413796, + "threshold": 0.5080188957268795 + }, + { + "coverage": 0.30966666666666665, + "risk": 0.3939720129171152, + "threshold": 0.5079634080211258 + }, + { + "coverage": 0.31, + "risk": 0.3935483870967742, + "threshold": 0.507949665395125 + }, + { + "coverage": 0.31033333333333335, + "risk": 0.39312567132116005, + "threshold": 0.5064321165775233 + }, + { + "coverage": 0.31066666666666665, + "risk": 0.3927038626609442, + "threshold": 0.5062085066105797 + }, + { + "coverage": 0.311, + "risk": 0.39228295819935693, + "threshold": 0.5061497761470239 + }, + { + "coverage": 0.31133333333333335, + "risk": 0.3929336188436831, + "threshold": 0.506146565840727 + }, + { + "coverage": 0.31166666666666665, + "risk": 0.3925133689839572, + "threshold": 0.5058687292054236 + }, + { + "coverage": 0.312, + "risk": 0.3920940170940171, + "threshold": 0.5053698377887821 + }, + { + "coverage": 0.31233333333333335, + "risk": 0.39167556029882605, + "threshold": 0.5053538516329509 + }, + { + "coverage": 0.31266666666666665, + "risk": 0.39125799573560766, + "threshold": 0.5053265179850291 + }, + { + "coverage": 0.313, + "risk": 0.3908413205537806, + "threshold": 0.5050401593780016 + }, + { + "coverage": 0.31333333333333335, + "risk": 0.3904255319148936, + "threshold": 0.5048576791353023 + }, + { + "coverage": 0.31366666666666665, + "risk": 0.3900106269925611, + "threshold": 0.504224080870699 + }, + { + "coverage": 0.314, + "risk": 0.3895966029723992, + "threshold": 0.5036542106422942 + }, + { + "coverage": 0.31433333333333335, + "risk": 0.3902439024390244, + "threshold": 0.5033377250302554 + }, + { + "coverage": 0.31466666666666665, + "risk": 0.3898305084745763, + "threshold": 0.5031095809037944 + }, + { + "coverage": 0.315, + "risk": 0.38941798941798944, + "threshold": 0.502383654628966 + }, + { + "coverage": 0.31533333333333335, + "risk": 0.3890063424947146, + "threshold": 0.5021863033848447 + }, + { + "coverage": 0.31566666666666665, + "risk": 0.38965153115100315, + "threshold": 0.5021772821582022 + }, + { + "coverage": 0.316, + "risk": 0.38924050632911394, + "threshold": 0.5018816079566266 + }, + { + "coverage": 0.31633333333333336, + "risk": 0.38883034773445735, + "threshold": 0.5014136454302358 + }, + { + "coverage": 0.31666666666666665, + "risk": 0.38842105263157894, + "threshold": 0.5013781539972975 + }, + { + "coverage": 0.317, + "risk": 0.38801261829652994, + "threshold": 0.4991574997600846 + }, + { + "coverage": 0.31733333333333336, + "risk": 0.38865546218487396, + "threshold": 0.49897076826548437 + }, + { + "coverage": 0.31766666666666665, + "risk": 0.3882476390346275, + "threshold": 0.49665841769428415 + }, + { + "coverage": 0.318, + "risk": 0.38784067085953877, + "threshold": 0.4964654892819383 + }, + { + "coverage": 0.31833333333333336, + "risk": 0.387434554973822, + "threshold": 0.4961809660004591 + }, + { + "coverage": 0.31866666666666665, + "risk": 0.3880753138075314, + "threshold": 0.4953813957387901 + }, + { + "coverage": 0.319, + "risk": 0.3876698014629049, + "threshold": 0.4951225501739499 + }, + { + "coverage": 0.31933333333333336, + "risk": 0.3872651356993737, + "threshold": 0.4948320335480847 + }, + { + "coverage": 0.31966666666666665, + "risk": 0.38686131386861317, + "threshold": 0.49420344312961884 + }, + { + "coverage": 0.32, + "risk": 0.38645833333333335, + "threshold": 0.49415340203747643 + }, + { + "coverage": 0.32033333333333336, + "risk": 0.38605619146722164, + "threshold": 0.4937996704116893 + }, + { + "coverage": 0.32066666666666666, + "risk": 0.38565488565488565, + "threshold": 0.4934874176106982 + }, + { + "coverage": 0.321, + "risk": 0.3862928348909657, + "threshold": 0.4933969637179591 + }, + { + "coverage": 0.32133333333333336, + "risk": 0.38589211618257263, + "threshold": 0.4928307909951088 + }, + { + "coverage": 0.32166666666666666, + "risk": 0.3854922279792746, + "threshold": 0.4917288700147052 + }, + { + "coverage": 0.322, + "risk": 0.38509316770186336, + "threshold": 0.49151847780433167 + }, + { + "coverage": 0.32233333333333336, + "risk": 0.3846949327817994, + "threshold": 0.49123529270833927 + }, + { + "coverage": 0.32266666666666666, + "risk": 0.384297520661157, + "threshold": 0.4910277693975839 + }, + { + "coverage": 0.323, + "risk": 0.3849329205366357, + "threshold": 0.4898218605447648 + }, + { + "coverage": 0.3233333333333333, + "risk": 0.3845360824742268, + "threshold": 0.48968694092296416 + }, + { + "coverage": 0.32366666666666666, + "risk": 0.384140061791967, + "threshold": 0.48958495310983335 + }, + { + "coverage": 0.324, + "risk": 0.3837448559670782, + "threshold": 0.48920699619193636 + }, + { + "coverage": 0.3243333333333333, + "risk": 0.38335046248715315, + "threshold": 0.4891429053909841 + }, + { + "coverage": 0.32466666666666666, + "risk": 0.38295687885010266, + "threshold": 0.4883299520903134 + }, + { + "coverage": 0.325, + "risk": 0.38358974358974357, + "threshold": 0.4873923778572243 + }, + { + "coverage": 0.3253333333333333, + "risk": 0.3831967213114754, + "threshold": 0.4872359487209018 + }, + { + "coverage": 0.32566666666666666, + "risk": 0.3828045035823951, + "threshold": 0.4871678823812928 + }, + { + "coverage": 0.326, + "risk": 0.3824130879345603, + "threshold": 0.48667452162846586 + }, + { + "coverage": 0.3263333333333333, + "risk": 0.3830439223697651, + "threshold": 0.48607043145911855 + }, + { + "coverage": 0.32666666666666666, + "risk": 0.3826530612244898, + "threshold": 0.4858221091654851 + }, + { + "coverage": 0.327, + "risk": 0.382262996941896, + "threshold": 0.48471726781524943 + }, + { + "coverage": 0.3273333333333333, + "risk": 0.3818737270875764, + "threshold": 0.48471334065231747 + }, + { + "coverage": 0.32766666666666666, + "risk": 0.3814852492370295, + "threshold": 0.484579714992167 + }, + { + "coverage": 0.328, + "risk": 0.38109756097560976, + "threshold": 0.4844096206559953 + }, + { + "coverage": 0.3283333333333333, + "risk": 0.38071065989847713, + "threshold": 0.48391637048230984 + }, + { + "coverage": 0.32866666666666666, + "risk": 0.3803245436105477, + "threshold": 0.48368092806601665 + }, + { + "coverage": 0.329, + "risk": 0.3799392097264438, + "threshold": 0.4836718767520024 + }, + { + "coverage": 0.3293333333333333, + "risk": 0.37955465587044535, + "threshold": 0.4827241182407416 + }, + { + "coverage": 0.32966666666666666, + "risk": 0.37917087967644086, + "threshold": 0.481575548766995 + }, + { + "coverage": 0.33, + "risk": 0.3797979797979798, + "threshold": 0.48107525017865727 + }, + { + "coverage": 0.3303333333333333, + "risk": 0.3794147325933401, + "threshold": 0.48092020206234937 + }, + { + "coverage": 0.33066666666666666, + "risk": 0.3790322580645161, + "threshold": 0.47956723942470314 + }, + { + "coverage": 0.331, + "risk": 0.37865055387714, + "threshold": 0.47905407582637843 + }, + { + "coverage": 0.3313333333333333, + "risk": 0.37927565392354123, + "threshold": 0.478166999236739 + }, + { + "coverage": 0.33166666666666667, + "risk": 0.37889447236180906, + "threshold": 0.47804796987002823 + }, + { + "coverage": 0.332, + "risk": 0.3785140562248996, + "threshold": 0.47800605183416484 + }, + { + "coverage": 0.3323333333333333, + "risk": 0.3781344032096289, + "threshold": 0.4772949481617896 + }, + { + "coverage": 0.33266666666666667, + "risk": 0.3777555110220441, + "threshold": 0.4768766706301461 + }, + { + "coverage": 0.333, + "risk": 0.37737737737737737, + "threshold": 0.4766046355420371 + }, + { + "coverage": 0.3333333333333333, + "risk": 0.377, + "threshold": 0.4753358994046316 + }, + { + "coverage": 0.33366666666666667, + "risk": 0.37662337662337664, + "threshold": 0.4742882970961408 + }, + { + "coverage": 0.334, + "risk": 0.37624750499001997, + "threshold": 0.4739606324403536 + }, + { + "coverage": 0.3343333333333333, + "risk": 0.37587238285144564, + "threshold": 0.47382268008225836 + }, + { + "coverage": 0.33466666666666667, + "risk": 0.3754980079681275, + "threshold": 0.47337257487619017 + }, + { + "coverage": 0.335, + "risk": 0.3761194029850746, + "threshold": 0.4720962939676249 + }, + { + "coverage": 0.3353333333333333, + "risk": 0.3757455268389662, + "threshold": 0.4711833165660365 + }, + { + "coverage": 0.33566666666666667, + "risk": 0.3753723932472691, + "threshold": 0.4711137992891964 + }, + { + "coverage": 0.336, + "risk": 0.375, + "threshold": 0.47094867000222074 + }, + { + "coverage": 0.3363333333333333, + "risk": 0.37462834489593655, + "threshold": 0.4702965214455882 + }, + { + "coverage": 0.33666666666666667, + "risk": 0.37425742574257426, + "threshold": 0.46950992991384854 + }, + { + "coverage": 0.337, + "risk": 0.37388724035608306, + "threshold": 0.4693347506724048 + }, + { + "coverage": 0.3373333333333333, + "risk": 0.37351778656126483, + "threshold": 0.4691519078020648 + }, + { + "coverage": 0.33766666666666667, + "risk": 0.37314906219151034, + "threshold": 0.46832413206612766 + }, + { + "coverage": 0.338, + "risk": 0.3727810650887574, + "threshold": 0.46815165776154827 + }, + { + "coverage": 0.3383333333333333, + "risk": 0.3724137931034483, + "threshold": 0.4675981724047952 + }, + { + "coverage": 0.33866666666666667, + "risk": 0.37303149606299213, + "threshold": 0.46612231016738553 + }, + { + "coverage": 0.339, + "risk": 0.37364798426745327, + "threshold": 0.46610262267523495 + }, + { + "coverage": 0.3393333333333333, + "risk": 0.37328094302554027, + "threshold": 0.4656114090598945 + }, + { + "coverage": 0.3396666666666667, + "risk": 0.3729146221786065, + "threshold": 0.4655289911265825 + }, + { + "coverage": 0.34, + "risk": 0.37254901960784315, + "threshold": 0.46548834395589095 + }, + { + "coverage": 0.3403333333333333, + "risk": 0.3721841332027424, + "threshold": 0.4642330456478617 + }, + { + "coverage": 0.3406666666666667, + "risk": 0.37279843444227007, + "threshold": 0.463800044360702 + }, + { + "coverage": 0.341, + "risk": 0.3724340175953079, + "threshold": 0.46377601077167757 + }, + { + "coverage": 0.3413333333333333, + "risk": 0.3720703125, + "threshold": 0.463431413812465 + }, + { + "coverage": 0.3416666666666667, + "risk": 0.37170731707317073, + "threshold": 0.4633148413196678 + }, + { + "coverage": 0.342, + "risk": 0.3713450292397661, + "threshold": 0.4632822509417621 + }, + { + "coverage": 0.3423333333333333, + "risk": 0.37098344693281404, + "threshold": 0.46198023215168893 + }, + { + "coverage": 0.3426666666666667, + "risk": 0.37062256809338523, + "threshold": 0.4609281895674249 + }, + { + "coverage": 0.343, + "risk": 0.37026239067055394, + "threshold": 0.46039827804151034 + }, + { + "coverage": 0.3433333333333333, + "risk": 0.3699029126213592, + "threshold": 0.4603640610610096 + }, + { + "coverage": 0.3436666666666667, + "risk": 0.36954413191076624, + "threshold": 0.4595565557469808 + }, + { + "coverage": 0.344, + "risk": 0.3701550387596899, + "threshold": 0.45863586153086905 + }, + { + "coverage": 0.3443333333333333, + "risk": 0.3697967086156825, + "threshold": 0.45852076390243773 + }, + { + "coverage": 0.3446666666666667, + "risk": 0.3694390715667311, + "threshold": 0.45850801670542396 + }, + { + "coverage": 0.345, + "risk": 0.3690821256038647, + "threshold": 0.45812202134109836 + }, + { + "coverage": 0.3453333333333333, + "risk": 0.3687258687258687, + "threshold": 0.45716650598337166 + }, + { + "coverage": 0.3456666666666667, + "risk": 0.36933461909353904, + "threshold": 0.4571517168731632 + }, + { + "coverage": 0.346, + "risk": 0.3699421965317919, + "threshold": 0.4569124449911857 + }, + { + "coverage": 0.3463333333333333, + "risk": 0.3695861405197305, + "threshold": 0.4551716782586398 + }, + { + "coverage": 0.3466666666666667, + "risk": 0.3701923076923077, + "threshold": 0.4549269077418493 + }, + { + "coverage": 0.347, + "risk": 0.36983669548511044, + "threshold": 0.45460764313078356 + }, + { + "coverage": 0.3473333333333333, + "risk": 0.3694817658349328, + "threshold": 0.45451776800623694 + }, + { + "coverage": 0.3476666666666667, + "risk": 0.3691275167785235, + "threshold": 0.4541815189601192 + }, + { + "coverage": 0.348, + "risk": 0.36877394636015326, + "threshold": 0.4534346986831058 + }, + { + "coverage": 0.34833333333333333, + "risk": 0.3684210526315789, + "threshold": 0.4526608755143365 + }, + { + "coverage": 0.3486666666666667, + "risk": 0.36806883365200765, + "threshold": 0.45208322656948186 + }, + { + "coverage": 0.349, + "risk": 0.36771728748806115, + "threshold": 0.45163912530599815 + }, + { + "coverage": 0.34933333333333333, + "risk": 0.36736641221374045, + "threshold": 0.45153042931697535 + }, + { + "coverage": 0.3496666666666667, + "risk": 0.3679694947569113, + "threshold": 0.45067706425422094 + }, + { + "coverage": 0.35, + "risk": 0.3676190476190476, + "threshold": 0.448580934853001 + }, + { + "coverage": 0.35033333333333333, + "risk": 0.368220742150333, + "threshold": 0.44717361797862687 + }, + { + "coverage": 0.3506666666666667, + "risk": 0.3678707224334601, + "threshold": 0.4471414751730444 + }, + { + "coverage": 0.351, + "risk": 0.36752136752136755, + "threshold": 0.44693535829610215 + }, + { + "coverage": 0.35133333333333333, + "risk": 0.36717267552182165, + "threshold": 0.44648952936510783 + }, + { + "coverage": 0.3516666666666667, + "risk": 0.36682464454976305, + "threshold": 0.4459419529430434 + }, + { + "coverage": 0.352, + "risk": 0.3664772727272727, + "threshold": 0.44580088985077704 + }, + { + "coverage": 0.35233333333333333, + "risk": 0.36707663197729423, + "threshold": 0.4440426834666216 + }, + { + "coverage": 0.3526666666666667, + "risk": 0.3667296786389414, + "threshold": 0.4439352755417053 + }, + { + "coverage": 0.353, + "risk": 0.3663833805476865, + "threshold": 0.4439049653875755 + }, + { + "coverage": 0.35333333333333333, + "risk": 0.3660377358490566, + "threshold": 0.44280039038192126 + }, + { + "coverage": 0.3536666666666667, + "risk": 0.3656927426955702, + "threshold": 0.4422623029500752 + }, + { + "coverage": 0.354, + "risk": 0.3653483992467043, + "threshold": 0.44219699421701775 + }, + { + "coverage": 0.35433333333333333, + "risk": 0.3659454374412041, + "threshold": 0.4406049818906707 + }, + { + "coverage": 0.3546666666666667, + "risk": 0.3656015037593985, + "threshold": 0.44024357941710957 + }, + { + "coverage": 0.355, + "risk": 0.3652582159624413, + "threshold": 0.4401592848747515 + }, + { + "coverage": 0.35533333333333333, + "risk": 0.36585365853658536, + "threshold": 0.4400005407337285 + }, + { + "coverage": 0.3556666666666667, + "risk": 0.36644798500468606, + "threshold": 0.4399439003145885 + }, + { + "coverage": 0.356, + "risk": 0.3661048689138577, + "threshold": 0.4395859394171995 + }, + { + "coverage": 0.35633333333333334, + "risk": 0.36576239476145933, + "threshold": 0.43946817993863546 + }, + { + "coverage": 0.3566666666666667, + "risk": 0.36542056074766355, + "threshold": 0.4388034294177555 + }, + { + "coverage": 0.357, + "risk": 0.36507936507936506, + "threshold": 0.43800036699397676 + }, + { + "coverage": 0.35733333333333334, + "risk": 0.36473880597014924, + "threshold": 0.43773704622946286 + }, + { + "coverage": 0.3576666666666667, + "risk": 0.36439888164026096, + "threshold": 0.437476782979886 + }, + { + "coverage": 0.358, + "risk": 0.36405959031657353, + "threshold": 0.4373672619793713 + }, + { + "coverage": 0.35833333333333334, + "risk": 0.36372093023255814, + "threshold": 0.43734695637866 + }, + { + "coverage": 0.3586666666666667, + "risk": 0.36338289962825276, + "threshold": 0.4369206351979059 + }, + { + "coverage": 0.359, + "risk": 0.3630454967502321, + "threshold": 0.4358311531908861 + }, + { + "coverage": 0.35933333333333334, + "risk": 0.362708719851577, + "threshold": 0.435706546567199 + }, + { + "coverage": 0.3596666666666667, + "risk": 0.3623725671918443, + "threshold": 0.435614326515771 + }, + { + "coverage": 0.36, + "risk": 0.362962962962963, + "threshold": 0.43470218941716326 + }, + { + "coverage": 0.36033333333333334, + "risk": 0.36262719703977797, + "threshold": 0.4346507416550666 + }, + { + "coverage": 0.3606666666666667, + "risk": 0.3632162661737523, + "threshold": 0.4346498585925791 + }, + { + "coverage": 0.361, + "risk": 0.3628808864265928, + "threshold": 0.43286302117703307 + }, + { + "coverage": 0.36133333333333334, + "risk": 0.3634686346863469, + "threshold": 0.4327361992667589 + }, + { + "coverage": 0.3616666666666667, + "risk": 0.3631336405529954, + "threshold": 0.4324345619058218 + }, + { + "coverage": 0.362, + "risk": 0.36279926335174956, + "threshold": 0.4309700376149665 + }, + { + "coverage": 0.36233333333333334, + "risk": 0.3624655013799448, + "threshold": 0.43045592441603103 + }, + { + "coverage": 0.3626666666666667, + "risk": 0.3630514705882353, + "threshold": 0.43031312751849266 + }, + { + "coverage": 0.363, + "risk": 0.3627180899908173, + "threshold": 0.4302410698620275 + }, + { + "coverage": 0.36333333333333334, + "risk": 0.363302752293578, + "threshold": 0.4301475835126686 + }, + { + "coverage": 0.3636666666666667, + "risk": 0.3629697525206233, + "threshold": 0.42945867793920994 + }, + { + "coverage": 0.364, + "risk": 0.3626373626373626, + "threshold": 0.4281980007520233 + }, + { + "coverage": 0.36433333333333334, + "risk": 0.36322049405306495, + "threshold": 0.4281667287962384 + }, + { + "coverage": 0.36466666666666664, + "risk": 0.36288848263254114, + "threshold": 0.4278238060305971 + }, + { + "coverage": 0.365, + "risk": 0.36255707762557077, + "threshold": 0.426295791838415 + }, + { + "coverage": 0.36533333333333334, + "risk": 0.36222627737226276, + "threshold": 0.42409010823453414 + }, + { + "coverage": 0.36566666666666664, + "risk": 0.3628076572470374, + "threshold": 0.4235344284022235 + }, + { + "coverage": 0.366, + "risk": 0.36247723132969034, + "threshold": 0.42343420403824555 + }, + { + "coverage": 0.36633333333333334, + "risk": 0.362147406733394, + "threshold": 0.4233763643807138 + }, + { + "coverage": 0.36666666666666664, + "risk": 0.3618181818181818, + "threshold": 0.4231038120570406 + }, + { + "coverage": 0.367, + "risk": 0.3614895549500454, + "threshold": 0.4215526912756518 + }, + { + "coverage": 0.36733333333333335, + "risk": 0.3620689655172414, + "threshold": 0.42145219736295697 + }, + { + "coverage": 0.36766666666666664, + "risk": 0.3617407071622847, + "threshold": 0.42053141624798057 + }, + { + "coverage": 0.368, + "risk": 0.36141304347826086, + "threshold": 0.4204703622799895 + }, + { + "coverage": 0.36833333333333335, + "risk": 0.3610859728506787, + "threshold": 0.42013424781879477 + }, + { + "coverage": 0.36866666666666664, + "risk": 0.36075949367088606, + "threshold": 0.42012680337766856 + }, + { + "coverage": 0.369, + "risk": 0.36133694670280037, + "threshold": 0.4198823908698823 + }, + { + "coverage": 0.36933333333333335, + "risk": 0.36101083032490977, + "threshold": 0.4196928219728857 + }, + { + "coverage": 0.36966666666666664, + "risk": 0.3606853020739405, + "threshold": 0.41965224339055834 + }, + { + "coverage": 0.37, + "risk": 0.36036036036036034, + "threshold": 0.41922979401975374 + }, + { + "coverage": 0.37033333333333335, + "risk": 0.3609360936093609, + "threshold": 0.4184171673139715 + }, + { + "coverage": 0.37066666666666664, + "risk": 0.3606115107913669, + "threshold": 0.41826852231826195 + }, + { + "coverage": 0.371, + "risk": 0.36028751123090746, + "threshold": 0.4172758587119311 + }, + { + "coverage": 0.37133333333333335, + "risk": 0.3599640933572711, + "threshold": 0.4170898808652991 + }, + { + "coverage": 0.37166666666666665, + "risk": 0.3596412556053812, + "threshold": 0.4155514622282516 + }, + { + "coverage": 0.372, + "risk": 0.3593189964157706, + "threshold": 0.41544827105910487 + }, + { + "coverage": 0.37233333333333335, + "risk": 0.35899731423455683, + "threshold": 0.41456846496260796 + }, + { + "coverage": 0.37266666666666665, + "risk": 0.35867620751341683, + "threshold": 0.4135136258812259 + }, + { + "coverage": 0.373, + "risk": 0.3583556747095621, + "threshold": 0.41314240541250224 + }, + { + "coverage": 0.37333333333333335, + "risk": 0.3580357142857143, + "threshold": 0.4128658659256683 + }, + { + "coverage": 0.37366666666666665, + "risk": 0.3577163247100803, + "threshold": 0.4107746310551377 + }, + { + "coverage": 0.374, + "risk": 0.3582887700534759, + "threshold": 0.4095445744941084 + }, + { + "coverage": 0.37433333333333335, + "risk": 0.358860195903829, + "threshold": 0.40902107596325826 + }, + { + "coverage": 0.37466666666666665, + "risk": 0.3585409252669039, + "threshold": 0.4083768205045699 + }, + { + "coverage": 0.375, + "risk": 0.3582222222222222, + "threshold": 0.40694863934873166 + }, + { + "coverage": 0.37533333333333335, + "risk": 0.35790408525754885, + "threshold": 0.4067671463488057 + }, + { + "coverage": 0.37566666666666665, + "risk": 0.35758651286601595, + "threshold": 0.4066320777514363 + }, + { + "coverage": 0.376, + "risk": 0.35815602836879434, + "threshold": 0.40650419984216907 + }, + { + "coverage": 0.37633333333333335, + "risk": 0.3587245349867139, + "threshold": 0.405399560368969 + }, + { + "coverage": 0.37666666666666665, + "risk": 0.3584070796460177, + "threshold": 0.40516635277627144 + }, + { + "coverage": 0.377, + "risk": 0.358974358974359, + "threshold": 0.40472848564013003 + }, + { + "coverage": 0.37733333333333335, + "risk": 0.3586572438162544, + "threshold": 0.4043874496834277 + }, + { + "coverage": 0.37766666666666665, + "risk": 0.3592233009708738, + "threshold": 0.4037579009370802 + }, + { + "coverage": 0.378, + "risk": 0.35890652557319225, + "threshold": 0.4020000848201037 + }, + { + "coverage": 0.37833333333333335, + "risk": 0.3585903083700441, + "threshold": 0.40188342228588764 + }, + { + "coverage": 0.37866666666666665, + "risk": 0.35827464788732394, + "threshold": 0.4017220582567654 + }, + { + "coverage": 0.379, + "risk": 0.3579595426561126, + "threshold": 0.40131674126897304 + }, + { + "coverage": 0.37933333333333336, + "risk": 0.35764499121265375, + "threshold": 0.4005240352304889 + }, + { + "coverage": 0.37966666666666665, + "risk": 0.35733099209833186, + "threshold": 0.40010996096497725 + }, + { + "coverage": 0.38, + "risk": 0.3570175438596491, + "threshold": 0.3986634495466268 + }, + { + "coverage": 0.38033333333333336, + "risk": 0.3567046450482033, + "threshold": 0.3984463696599086 + }, + { + "coverage": 0.38066666666666665, + "risk": 0.3563922942206655, + "threshold": 0.39790302351527695 + }, + { + "coverage": 0.381, + "risk": 0.3560804899387577, + "threshold": 0.39779972463482915 + }, + { + "coverage": 0.38133333333333336, + "risk": 0.3557692307692308, + "threshold": 0.39757616331174916 + }, + { + "coverage": 0.38166666666666665, + "risk": 0.3554585152838428, + "threshold": 0.39749927361081067 + }, + { + "coverage": 0.382, + "risk": 0.3551483420593368, + "threshold": 0.39694187841209727 + }, + { + "coverage": 0.38233333333333336, + "risk": 0.3548387096774194, + "threshold": 0.3959392537262362 + }, + { + "coverage": 0.38266666666666665, + "risk": 0.35452961672473865, + "threshold": 0.39563315170242447 + }, + { + "coverage": 0.383, + "risk": 0.35422106179286333, + "threshold": 0.39559705923726424 + }, + { + "coverage": 0.38333333333333336, + "risk": 0.35391304347826086, + "threshold": 0.3955450374105864 + }, + { + "coverage": 0.38366666666666666, + "risk": 0.3536055603822763, + "threshold": 0.39541822857497677 + }, + { + "coverage": 0.384, + "risk": 0.3541666666666667, + "threshold": 0.3931026387776853 + }, + { + "coverage": 0.38433333333333336, + "risk": 0.3538594969644406, + "threshold": 0.39283747267308244 + }, + { + "coverage": 0.38466666666666666, + "risk": 0.3535528596187175, + "threshold": 0.3919236625068708 + }, + { + "coverage": 0.385, + "risk": 0.35324675324675325, + "threshold": 0.3911635508014401 + }, + { + "coverage": 0.38533333333333336, + "risk": 0.35294117647058826, + "threshold": 0.39104288247649904 + }, + { + "coverage": 0.38566666666666666, + "risk": 0.3526361279170268, + "threshold": 0.39098474287992596 + }, + { + "coverage": 0.386, + "risk": 0.35233160621761656, + "threshold": 0.3902487068906917 + }, + { + "coverage": 0.3863333333333333, + "risk": 0.35202761000862814, + "threshold": 0.3901344752168551 + }, + { + "coverage": 0.38666666666666666, + "risk": 0.35172413793103446, + "threshold": 0.38842171089676997 + }, + { + "coverage": 0.387, + "risk": 0.35142118863049093, + "threshold": 0.38829146496295536 + }, + { + "coverage": 0.3873333333333333, + "risk": 0.35111876075731496, + "threshold": 0.38826663068804185 + }, + { + "coverage": 0.38766666666666666, + "risk": 0.35081685296646603, + "threshold": 0.3872780769596539 + }, + { + "coverage": 0.388, + "risk": 0.35051546391752575, + "threshold": 0.3867241105390075 + }, + { + "coverage": 0.3883333333333333, + "risk": 0.3502145922746781, + "threshold": 0.38621012266184374 + }, + { + "coverage": 0.38866666666666666, + "risk": 0.34991423670668953, + "threshold": 0.3857477808627832 + }, + { + "coverage": 0.389, + "risk": 0.3496143958868895, + "threshold": 0.3856102264823255 + }, + { + "coverage": 0.3893333333333333, + "risk": 0.3493150684931507, + "threshold": 0.38557779784538826 + }, + { + "coverage": 0.38966666666666666, + "risk": 0.34901625320787, + "threshold": 0.38553019328030735 + }, + { + "coverage": 0.39, + "risk": 0.3487179487179487, + "threshold": 0.38436639402017775 + }, + { + "coverage": 0.3903333333333333, + "risk": 0.3484201537147737, + "threshold": 0.38377583368451684 + }, + { + "coverage": 0.39066666666666666, + "risk": 0.34812286689419797, + "threshold": 0.3831864016247042 + }, + { + "coverage": 0.391, + "risk": 0.34782608695652173, + "threshold": 0.3830667535095441 + }, + { + "coverage": 0.3913333333333333, + "risk": 0.3475298126064736, + "threshold": 0.381302035336726 + }, + { + "coverage": 0.39166666666666666, + "risk": 0.3472340425531915, + "threshold": 0.3812776021787045 + }, + { + "coverage": 0.392, + "risk": 0.3469387755102041, + "threshold": 0.38080892145728085 + }, + { + "coverage": 0.3923333333333333, + "risk": 0.3466440101954121, + "threshold": 0.3799174426988745 + }, + { + "coverage": 0.39266666666666666, + "risk": 0.3463497453310696, + "threshold": 0.3792515878082989 + }, + { + "coverage": 0.393, + "risk": 0.3460559796437659, + "threshold": 0.37789689854540187 + }, + { + "coverage": 0.3933333333333333, + "risk": 0.34576271186440677, + "threshold": 0.377835844023166 + }, + { + "coverage": 0.39366666666666666, + "risk": 0.3454699407281964, + "threshold": 0.3762012305771562 + }, + { + "coverage": 0.394, + "risk": 0.34517766497461927, + "threshold": 0.37615492772317155 + }, + { + "coverage": 0.3943333333333333, + "risk": 0.3448858833474218, + "threshold": 0.3760971252947059 + }, + { + "coverage": 0.39466666666666667, + "risk": 0.34459459459459457, + "threshold": 0.3759716947532177 + }, + { + "coverage": 0.395, + "risk": 0.34430379746835443, + "threshold": 0.37469520372161613 + }, + { + "coverage": 0.3953333333333333, + "risk": 0.3440134907251265, + "threshold": 0.37380804645227655 + }, + { + "coverage": 0.39566666666666667, + "risk": 0.34372367312552654, + "threshold": 0.37378938343959267 + }, + { + "coverage": 0.396, + "risk": 0.3434343434343434, + "threshold": 0.37364622191031077 + }, + { + "coverage": 0.3963333333333333, + "risk": 0.34314550042052144, + "threshold": 0.37322380214171597 + }, + { + "coverage": 0.39666666666666667, + "risk": 0.34285714285714286, + "threshold": 0.3728441585689929 + }, + { + "coverage": 0.397, + "risk": 0.3425692695214106, + "threshold": 0.3728323708186251 + }, + { + "coverage": 0.3973333333333333, + "risk": 0.3422818791946309, + "threshold": 0.37275800905943124 + }, + { + "coverage": 0.39766666666666667, + "risk": 0.34199497066219614, + "threshold": 0.37230995113434195 + }, + { + "coverage": 0.398, + "risk": 0.3417085427135678, + "threshold": 0.37195948662345274 + }, + { + "coverage": 0.3983333333333333, + "risk": 0.3414225941422594, + "threshold": 0.3719477785595494 + }, + { + "coverage": 0.39866666666666667, + "risk": 0.3419732441471572, + "threshold": 0.3715034259966564 + }, + { + "coverage": 0.399, + "risk": 0.341687552213868, + "threshold": 0.3710951314018447 + }, + { + "coverage": 0.3993333333333333, + "risk": 0.34140233722871455, + "threshold": 0.3708069846838097 + }, + { + "coverage": 0.39966666666666667, + "risk": 0.34111759799833197, + "threshold": 0.36898352006556273 + }, + { + "coverage": 0.4, + "risk": 0.3408333333333333, + "threshold": 0.36882207366014463 + }, + { + "coverage": 0.4003333333333333, + "risk": 0.3405495420482931, + "threshold": 0.36868171018084817 + }, + { + "coverage": 0.40066666666666667, + "risk": 0.3402662229617304, + "threshold": 0.36774762507456427 + }, + { + "coverage": 0.401, + "risk": 0.3399833748960931, + "threshold": 0.36743079373873694 + }, + { + "coverage": 0.4013333333333333, + "risk": 0.34053156146179403, + "threshold": 0.3670661177064502 + }, + { + "coverage": 0.40166666666666667, + "risk": 0.34024896265560167, + "threshold": 0.36695367614991503 + }, + { + "coverage": 0.402, + "risk": 0.33996683250414594, + "threshold": 0.36673818453083895 + }, + { + "coverage": 0.4023333333333333, + "risk": 0.3396851698425849, + "threshold": 0.36649763619775455 + }, + { + "coverage": 0.4026666666666667, + "risk": 0.3394039735099338, + "threshold": 0.36625810478821963 + }, + { + "coverage": 0.403, + "risk": 0.3391232423490488, + "threshold": 0.365148971866877 + }, + { + "coverage": 0.4033333333333333, + "risk": 0.33884297520661155, + "threshold": 0.36509332250242754 + }, + { + "coverage": 0.4036666666666667, + "risk": 0.3385631709331131, + "threshold": 0.3639590692170551 + }, + { + "coverage": 0.404, + "risk": 0.33828382838283827, + "threshold": 0.36102278787982767 + }, + { + "coverage": 0.4043333333333333, + "risk": 0.33800494641385, + "threshold": 0.36062076522704833 + }, + { + "coverage": 0.4046666666666667, + "risk": 0.33772652388797364, + "threshold": 0.3604228200886646 + }, + { + "coverage": 0.405, + "risk": 0.3374485596707819, + "threshold": 0.36002345636338007 + }, + { + "coverage": 0.4053333333333333, + "risk": 0.3371710526315789, + "threshold": 0.35983295115851593 + }, + { + "coverage": 0.4056666666666667, + "risk": 0.3368940016433854, + "threshold": 0.3587701680187263 + }, + { + "coverage": 0.406, + "risk": 0.3374384236453202, + "threshold": 0.3581062265740586 + }, + { + "coverage": 0.4063333333333333, + "risk": 0.3379819524200164, + "threshold": 0.3573800557521115 + }, + { + "coverage": 0.4066666666666667, + "risk": 0.3377049180327869, + "threshold": 0.3567374044964098 + }, + { + "coverage": 0.407, + "risk": 0.33742833742833744, + "threshold": 0.35605259730494127 + }, + { + "coverage": 0.4073333333333333, + "risk": 0.337152209492635, + "threshold": 0.3560098527562751 + }, + { + "coverage": 0.4076666666666667, + "risk": 0.33687653311529026, + "threshold": 0.35596948459223765 + }, + { + "coverage": 0.408, + "risk": 0.3374183006535948, + "threshold": 0.3558718576661721 + }, + { + "coverage": 0.4083333333333333, + "risk": 0.33714285714285713, + "threshold": 0.3555759765262354 + }, + { + "coverage": 0.4086666666666667, + "risk": 0.3368678629690049, + "threshold": 0.3543286704557199 + }, + { + "coverage": 0.409, + "risk": 0.33659331703341483, + "threshold": 0.35398721984348164 + }, + { + "coverage": 0.4093333333333333, + "risk": 0.33713355048859933, + "threshold": 0.35256199622085993 + }, + { + "coverage": 0.4096666666666667, + "risk": 0.3368592351505289, + "threshold": 0.3521283977165143 + }, + { + "coverage": 0.41, + "risk": 0.33658536585365856, + "threshold": 0.3514365295227724 + }, + { + "coverage": 0.4103333333333333, + "risk": 0.3363119415109667, + "threshold": 0.35106221479186994 + }, + { + "coverage": 0.4106666666666667, + "risk": 0.336038961038961, + "threshold": 0.3502942149881574 + }, + { + "coverage": 0.411, + "risk": 0.3357664233576642, + "threshold": 0.35024924534102836 + }, + { + "coverage": 0.41133333333333333, + "risk": 0.3354943273905997, + "threshold": 0.3500887477082033 + }, + { + "coverage": 0.4116666666666667, + "risk": 0.33522267206477735, + "threshold": 0.349452350094512 + }, + { + "coverage": 0.412, + "risk": 0.33495145631067963, + "threshold": 0.349338172298752 + }, + { + "coverage": 0.41233333333333333, + "risk": 0.33468067906224735, + "threshold": 0.348394130009702 + }, + { + "coverage": 0.4126666666666667, + "risk": 0.3344103392568659, + "threshold": 0.3477682469959572 + }, + { + "coverage": 0.413, + "risk": 0.3341404358353511, + "threshold": 0.347538590633057 + }, + { + "coverage": 0.41333333333333333, + "risk": 0.3338709677419355, + "threshold": 0.34684858057891566 + }, + { + "coverage": 0.4136666666666667, + "risk": 0.33360193392425463, + "threshold": 0.3459768556870008 + }, + { + "coverage": 0.414, + "risk": 0.33413848631239934, + "threshold": 0.345732399341485 + }, + { + "coverage": 0.41433333333333333, + "risk": 0.333869670152856, + "threshold": 0.34491692064122137 + }, + { + "coverage": 0.4146666666666667, + "risk": 0.33360128617363344, + "threshold": 0.3437911611728141 + }, + { + "coverage": 0.415, + "risk": 0.3333333333333333, + "threshold": 0.34370283106325367 + }, + { + "coverage": 0.41533333333333333, + "risk": 0.3330658105939005, + "threshold": 0.3432297293938163 + }, + { + "coverage": 0.4156666666666667, + "risk": 0.33360064153969526, + "threshold": 0.3430054878299228 + }, + { + "coverage": 0.416, + "risk": 0.3333333333333333, + "threshold": 0.34276629802269276 + }, + { + "coverage": 0.41633333333333333, + "risk": 0.33306645316253003, + "threshold": 0.3427303082974427 + }, + { + "coverage": 0.4166666666666667, + "risk": 0.3328, + "threshold": 0.3424886364391564 + }, + { + "coverage": 0.417, + "risk": 0.33253397282174263, + "threshold": 0.34214485311659815 + }, + { + "coverage": 0.41733333333333333, + "risk": 0.3330670926517572, + "threshold": 0.3416016558863773 + }, + { + "coverage": 0.4176666666666667, + "risk": 0.33280127693535516, + "threshold": 0.34135656724875363 + }, + { + "coverage": 0.418, + "risk": 0.33253588516746413, + "threshold": 0.3410374847483159 + }, + { + "coverage": 0.41833333333333333, + "risk": 0.33306772908366533, + "threshold": 0.3408386209467881 + }, + { + "coverage": 0.4186666666666667, + "risk": 0.3328025477707006, + "threshold": 0.34004459273482307 + }, + { + "coverage": 0.419, + "risk": 0.3325377883850438, + "threshold": 0.33959070409298836 + }, + { + "coverage": 0.41933333333333334, + "risk": 0.3322734499205087, + "threshold": 0.3395552473561565 + }, + { + "coverage": 0.4196666666666667, + "risk": 0.33200953137410644, + "threshold": 0.3392257947518925 + }, + { + "coverage": 0.42, + "risk": 0.33174603174603173, + "threshold": 0.3389752905975506 + }, + { + "coverage": 0.42033333333333334, + "risk": 0.33148295003965106, + "threshold": 0.33885017143261265 + }, + { + "coverage": 0.4206666666666667, + "risk": 0.3312202852614897, + "threshold": 0.3387765954877278 + }, + { + "coverage": 0.421, + "risk": 0.3309580364212193, + "threshold": 0.3384943315087271 + }, + { + "coverage": 0.42133333333333334, + "risk": 0.33069620253164556, + "threshold": 0.33734206854304616 + }, + { + "coverage": 0.4216666666666667, + "risk": 0.33043478260869563, + "threshold": 0.33512986370694814 + }, + { + "coverage": 0.422, + "risk": 0.330173775671406, + "threshold": 0.33494071751800564 + }, + { + "coverage": 0.42233333333333334, + "risk": 0.32991318074191, + "threshold": 0.3348959403511389 + }, + { + "coverage": 0.4226666666666667, + "risk": 0.32965299684542587, + "threshold": 0.3342444414035166 + }, + { + "coverage": 0.423, + "risk": 0.3301812450748621, + "threshold": 0.33384003729205264 + }, + { + "coverage": 0.42333333333333334, + "risk": 0.3299212598425197, + "threshold": 0.33369475042877966 + }, + { + "coverage": 0.4236666666666667, + "risk": 0.3296616837136113, + "threshold": 0.33355767861206387 + }, + { + "coverage": 0.424, + "risk": 0.330188679245283, + "threshold": 0.3333911909484 + }, + { + "coverage": 0.42433333333333334, + "risk": 0.32992930086410055, + "threshold": 0.33240007579078873 + }, + { + "coverage": 0.4246666666666667, + "risk": 0.32967032967032966, + "threshold": 0.33235085917541307 + }, + { + "coverage": 0.425, + "risk": 0.32941176470588235, + "threshold": 0.3322799495982411 + }, + { + "coverage": 0.42533333333333334, + "risk": 0.329153605015674, + "threshold": 0.3317454568098171 + }, + { + "coverage": 0.4256666666666667, + "risk": 0.3288958496476116, + "threshold": 0.3313350926600943 + }, + { + "coverage": 0.426, + "risk": 0.3286384976525822, + "threshold": 0.3299009658905619 + }, + { + "coverage": 0.42633333333333334, + "risk": 0.328381548084441, + "threshold": 0.3292057455584426 + }, + { + "coverage": 0.4266666666666667, + "risk": 0.328125, + "threshold": 0.32849753009484267 + }, + { + "coverage": 0.427, + "risk": 0.32786885245901637, + "threshold": 0.32811609559152255 + }, + { + "coverage": 0.42733333333333334, + "risk": 0.32761310452418096, + "threshold": 0.3280030322841614 + }, + { + "coverage": 0.42766666666666664, + "risk": 0.3273577552611068, + "threshold": 0.328000365597768 + }, + { + "coverage": 0.428, + "risk": 0.32710280373831774, + "threshold": 0.32764092249371896 + }, + { + "coverage": 0.42833333333333334, + "risk": 0.32684824902723736, + "threshold": 0.32696423448687234 + }, + { + "coverage": 0.42866666666666664, + "risk": 0.3265940902021773, + "threshold": 0.32694469872582865 + }, + { + "coverage": 0.429, + "risk": 0.32634032634032634, + "threshold": 0.32620601467087823 + }, + { + "coverage": 0.42933333333333334, + "risk": 0.32608695652173914, + "threshold": 0.3258559631804209 + }, + { + "coverage": 0.42966666666666664, + "risk": 0.32583397982932505, + "threshold": 0.32555145926201656 + }, + { + "coverage": 0.43, + "risk": 0.32558139534883723, + "threshold": 0.3251335670352464 + }, + { + "coverage": 0.43033333333333335, + "risk": 0.32532920216886135, + "threshold": 0.3244555623534736 + }, + { + "coverage": 0.43066666666666664, + "risk": 0.32507739938080493, + "threshold": 0.32374646025677395 + }, + { + "coverage": 0.431, + "risk": 0.3248259860788863, + "threshold": 0.32356818532293447 + }, + { + "coverage": 0.43133333333333335, + "risk": 0.32457496136012365, + "threshold": 0.3231025913792353 + }, + { + "coverage": 0.43166666666666664, + "risk": 0.32432432432432434, + "threshold": 0.3222162203031232 + }, + { + "coverage": 0.432, + "risk": 0.32407407407407407, + "threshold": 0.32157378514030105 + }, + { + "coverage": 0.43233333333333335, + "risk": 0.3238242097147263, + "threshold": 0.3210312324530337 + }, + { + "coverage": 0.43266666666666664, + "risk": 0.3235747303543914, + "threshold": 0.3205637979597471 + }, + { + "coverage": 0.433, + "risk": 0.3233256351039261, + "threshold": 0.32039289244379265 + }, + { + "coverage": 0.43333333333333335, + "risk": 0.3230769230769231, + "threshold": 0.3198930371706931 + }, + { + "coverage": 0.43366666666666664, + "risk": 0.3228285933897002, + "threshold": 0.3197478801083084 + }, + { + "coverage": 0.434, + "risk": 0.3225806451612903, + "threshold": 0.31911258995674435 + }, + { + "coverage": 0.43433333333333335, + "risk": 0.32233307751343054, + "threshold": 0.31788056868268105 + }, + { + "coverage": 0.43466666666666665, + "risk": 0.3220858895705521, + "threshold": 0.31779767061990727 + }, + { + "coverage": 0.435, + "risk": 0.3218390804597701, + "threshold": 0.3171114811435706 + }, + { + "coverage": 0.43533333333333335, + "risk": 0.3215926493108729, + "threshold": 0.31692315682030414 + }, + { + "coverage": 0.43566666666666665, + "risk": 0.32134659525631215, + "threshold": 0.3165188361086023 + }, + { + "coverage": 0.436, + "risk": 0.3211009174311927, + "threshold": 0.31643893856796806 + }, + { + "coverage": 0.43633333333333335, + "risk": 0.32161955691367455, + "threshold": 0.31591868914658083 + }, + { + "coverage": 0.43666666666666665, + "risk": 0.32213740458015266, + "threshold": 0.31584142436423085 + }, + { + "coverage": 0.437, + "risk": 0.32189168573607935, + "threshold": 0.31551473764889437 + }, + { + "coverage": 0.43733333333333335, + "risk": 0.32164634146341464, + "threshold": 0.3153661756517982 + }, + { + "coverage": 0.43766666666666665, + "risk": 0.3214013709063214, + "threshold": 0.31431303138814526 + }, + { + "coverage": 0.438, + "risk": 0.3211567732115677, + "threshold": 0.3135048895602332 + }, + { + "coverage": 0.43833333333333335, + "risk": 0.3209125475285171, + "threshold": 0.312083480917836 + }, + { + "coverage": 0.43866666666666665, + "risk": 0.32066869300911854, + "threshold": 0.31161639250598056 + }, + { + "coverage": 0.439, + "risk": 0.3204252088078967, + "threshold": 0.31138775933342405 + }, + { + "coverage": 0.43933333333333335, + "risk": 0.3201820940819423, + "threshold": 0.31069345184649466 + }, + { + "coverage": 0.43966666666666665, + "risk": 0.3199393479909022, + "threshold": 0.3100209782403118 + }, + { + "coverage": 0.44, + "risk": 0.3196969696969697, + "threshold": 0.30965777320680493 + }, + { + "coverage": 0.44033333333333335, + "risk": 0.3194549583648751, + "threshold": 0.30867640491800585 + }, + { + "coverage": 0.44066666666666665, + "risk": 0.31921331316187596, + "threshold": 0.30837271703890984 + }, + { + "coverage": 0.441, + "risk": 0.31897203325774753, + "threshold": 0.30786940289231496 + }, + { + "coverage": 0.44133333333333336, + "risk": 0.3187311178247734, + "threshold": 0.3069845895374702 + }, + { + "coverage": 0.44166666666666665, + "risk": 0.31849056603773584, + "threshold": 0.3065938695636335 + }, + { + "coverage": 0.442, + "risk": 0.31825037707390647, + "threshold": 0.3059907976813645 + }, + { + "coverage": 0.44233333333333336, + "risk": 0.3180105501130369, + "threshold": 0.30581189102523265 + }, + { + "coverage": 0.44266666666666665, + "risk": 0.3177710843373494, + "threshold": 0.3056119299366543 + }, + { + "coverage": 0.443, + "risk": 0.31753197893152746, + "threshold": 0.30452883995696467 + }, + { + "coverage": 0.44333333333333336, + "risk": 0.3172932330827068, + "threshold": 0.304169379197283 + }, + { + "coverage": 0.44366666666666665, + "risk": 0.3178061607813674, + "threshold": 0.30389027538209473 + }, + { + "coverage": 0.444, + "risk": 0.3183183183183183, + "threshold": 0.3037104672505202 + }, + { + "coverage": 0.44433333333333336, + "risk": 0.31807951987997, + "threshold": 0.3036959643933126 + }, + { + "coverage": 0.44466666666666665, + "risk": 0.31784107946026985, + "threshold": 0.3036164143403784 + }, + { + "coverage": 0.445, + "risk": 0.31760299625468164, + "threshold": 0.30340821700553916 + }, + { + "coverage": 0.44533333333333336, + "risk": 0.31736526946107785, + "threshold": 0.30311084507047953 + }, + { + "coverage": 0.44566666666666666, + "risk": 0.31712789827973076, + "threshold": 0.30271120605101354 + }, + { + "coverage": 0.446, + "risk": 0.31689088191330345, + "threshold": 0.30216748048189074 + }, + { + "coverage": 0.44633333333333336, + "risk": 0.3166542195668409, + "threshold": 0.30065737169097984 + }, + { + "coverage": 0.44666666666666666, + "risk": 0.3164179104477612, + "threshold": 0.30049243268330167 + }, + { + "coverage": 0.447, + "risk": 0.3161819537658464, + "threshold": 0.3003611144027892 + }, + { + "coverage": 0.44733333333333336, + "risk": 0.31594634873323396, + "threshold": 0.3002834985219143 + }, + { + "coverage": 0.44766666666666666, + "risk": 0.31571109456440805, + "threshold": 0.30019818164468315 + }, + { + "coverage": 0.448, + "risk": 0.31547619047619047, + "threshold": 0.30008946798376684 + }, + { + "coverage": 0.4483333333333333, + "risk": 0.31524163568773234, + "threshold": 0.2990715577893224 + }, + { + "coverage": 0.44866666666666666, + "risk": 0.3150074294205052, + "threshold": 0.2989853636485657 + }, + { + "coverage": 0.449, + "risk": 0.31551596139569416, + "threshold": 0.2985417266526833 + }, + { + "coverage": 0.4493333333333333, + "risk": 0.31528189910979226, + "threshold": 0.2978172746403845 + }, + { + "coverage": 0.44966666666666666, + "risk": 0.31504818383988137, + "threshold": 0.2972720569838377 + }, + { + "coverage": 0.45, + "risk": 0.31555555555555553, + "threshold": 0.2971964319381276 + }, + { + "coverage": 0.4503333333333333, + "risk": 0.3153219837157661, + "threshold": 0.2970614596042314 + }, + { + "coverage": 0.45066666666666666, + "risk": 0.3158284023668639, + "threshold": 0.29642768943373154 + }, + { + "coverage": 0.451, + "risk": 0.3155949741315595, + "threshold": 0.2949020575115567 + }, + { + "coverage": 0.4513333333333333, + "risk": 0.3153618906942393, + "threshold": 0.2943274218996774 + }, + { + "coverage": 0.45166666666666666, + "risk": 0.3151291512915129, + "threshold": 0.2940296211642093 + }, + { + "coverage": 0.452, + "risk": 0.3148967551622419, + "threshold": 0.2939608884420397 + }, + { + "coverage": 0.4523333333333333, + "risk": 0.31466470154753134, + "threshold": 0.29357607642710937 + }, + { + "coverage": 0.45266666666666666, + "risk": 0.31443298969072164, + "threshold": 0.29256921482772186 + }, + { + "coverage": 0.453, + "risk": 0.3142016188373804, + "threshold": 0.2925026284208009 + }, + { + "coverage": 0.4533333333333333, + "risk": 0.3139705882352941, + "threshold": 0.29243546884187616 + }, + { + "coverage": 0.45366666666666666, + "risk": 0.31373989713445993, + "threshold": 0.2912814654459217 + }, + { + "coverage": 0.454, + "risk": 0.3135095447870778, + "threshold": 0.2912133361911637 + }, + { + "coverage": 0.4543333333333333, + "risk": 0.3132795304475422, + "threshold": 0.2903130282730566 + }, + { + "coverage": 0.45466666666666666, + "risk": 0.313049853372434, + "threshold": 0.2886576388647762 + }, + { + "coverage": 0.455, + "risk": 0.3128205128205128, + "threshold": 0.28826760544876273 + }, + { + "coverage": 0.4553333333333333, + "risk": 0.31259150805270863, + "threshold": 0.28810077024919173 + }, + { + "coverage": 0.45566666666666666, + "risk": 0.31236283833211415, + "threshold": 0.28765837595164195 + }, + { + "coverage": 0.456, + "risk": 0.3128654970760234, + "threshold": 0.28707910343013937 + }, + { + "coverage": 0.4563333333333333, + "risk": 0.3126369612856099, + "threshold": 0.2869898206420755 + }, + { + "coverage": 0.45666666666666667, + "risk": 0.3124087591240876, + "threshold": 0.28613128176230934 + }, + { + "coverage": 0.457, + "risk": 0.312180889861415, + "threshold": 0.2852024384720596 + }, + { + "coverage": 0.4573333333333333, + "risk": 0.3119533527696793, + "threshold": 0.283519866699317 + }, + { + "coverage": 0.45766666666666667, + "risk": 0.3117261471230881, + "threshold": 0.28266394351134877 + }, + { + "coverage": 0.458, + "risk": 0.31149927219796214, + "threshold": 0.28260156605769865 + }, + { + "coverage": 0.4583333333333333, + "risk": 0.31127272727272726, + "threshold": 0.28208565372037003 + }, + { + "coverage": 0.45866666666666667, + "risk": 0.311046511627907, + "threshold": 0.2813943552942225 + }, + { + "coverage": 0.459, + "risk": 0.31082062454611475, + "threshold": 0.2805826463023063 + }, + { + "coverage": 0.4593333333333333, + "risk": 0.31059506531204645, + "threshold": 0.2798498747836021 + }, + { + "coverage": 0.45966666666666667, + "risk": 0.3103698332124728, + "threshold": 0.2794640119317744 + }, + { + "coverage": 0.46, + "risk": 0.3101449275362319, + "threshold": 0.2792325013255681 + }, + { + "coverage": 0.4603333333333333, + "risk": 0.3106444605358436, + "threshold": 0.27912163831768105 + }, + { + "coverage": 0.46066666666666667, + "risk": 0.3104196816208394, + "threshold": 0.27762843773020157 + }, + { + "coverage": 0.461, + "risk": 0.31019522776572667, + "threshold": 0.2774535441945176 + }, + { + "coverage": 0.4613333333333333, + "risk": 0.3106936416184971, + "threshold": 0.2773615559348668 + }, + { + "coverage": 0.46166666666666667, + "risk": 0.3104693140794224, + "threshold": 0.27660459944341953 + }, + { + "coverage": 0.462, + "risk": 0.31024531024531027, + "threshold": 0.27602660313133315 + }, + { + "coverage": 0.4623333333333333, + "risk": 0.31002162941600575, + "threshold": 0.27579224802482677 + }, + { + "coverage": 0.46266666666666667, + "risk": 0.30979827089337175, + "threshold": 0.27560839587119645 + }, + { + "coverage": 0.463, + "risk": 0.3095752339812815, + "threshold": 0.27456594640444715 + }, + { + "coverage": 0.4633333333333333, + "risk": 0.30935251798561153, + "threshold": 0.274562521507803 + }, + { + "coverage": 0.46366666666666667, + "risk": 0.30913012221423436, + "threshold": 0.2743114179633088 + }, + { + "coverage": 0.464, + "risk": 0.3089080459770115, + "threshold": 0.2742624460068519 + }, + { + "coverage": 0.4643333333333333, + "risk": 0.30868628858578606, + "threshold": 0.27415246466098214 + }, + { + "coverage": 0.4646666666666667, + "risk": 0.3084648493543759, + "threshold": 0.273922873263283 + }, + { + "coverage": 0.465, + "risk": 0.30824372759856633, + "threshold": 0.27391596891355213 + }, + { + "coverage": 0.4653333333333333, + "risk": 0.30802292263610315, + "threshold": 0.27332892370772116 + }, + { + "coverage": 0.4656666666666667, + "risk": 0.30780243378668576, + "threshold": 0.2727588461158794 + }, + { + "coverage": 0.466, + "risk": 0.30758226037195996, + "threshold": 0.27274048547051577 + }, + { + "coverage": 0.4663333333333333, + "risk": 0.3073624017155111, + "threshold": 0.2726585427689655 + }, + { + "coverage": 0.4666666666666667, + "risk": 0.30714285714285716, + "threshold": 0.271479260107863 + }, + { + "coverage": 0.467, + "risk": 0.30692362598144185, + "threshold": 0.2714388621194589 + }, + { + "coverage": 0.4673333333333333, + "risk": 0.3067047075606277, + "threshold": 0.27121212076269086 + }, + { + "coverage": 0.4676666666666667, + "risk": 0.30648610121168923, + "threshold": 0.2705552208516285 + }, + { + "coverage": 0.468, + "risk": 0.30626780626780625, + "threshold": 0.26965017020597887 + }, + { + "coverage": 0.4683333333333333, + "risk": 0.30604982206405695, + "threshold": 0.2694358894057713 + }, + { + "coverage": 0.4686666666666667, + "risk": 0.3058321479374111, + "threshold": 0.26936753323517565 + }, + { + "coverage": 0.469, + "risk": 0.30561478322672353, + "threshold": 0.2693511079294493 + }, + { + "coverage": 0.4693333333333333, + "risk": 0.30610795454545453, + "threshold": 0.26821550296227414 + }, + { + "coverage": 0.4696666666666667, + "risk": 0.3058907026259759, + "threshold": 0.26676755758988013 + }, + { + "coverage": 0.47, + "risk": 0.3056737588652482, + "threshold": 0.2667435747497267 + }, + { + "coverage": 0.4703333333333333, + "risk": 0.3054571226080794, + "threshold": 0.26641961280226895 + }, + { + "coverage": 0.4706666666666667, + "risk": 0.3052407932011331, + "threshold": 0.26615227077468495 + }, + { + "coverage": 0.471, + "risk": 0.3050247699929229, + "threshold": 0.26607344853787873 + }, + { + "coverage": 0.4713333333333333, + "risk": 0.3048090523338048, + "threshold": 0.2653534941716645 + }, + { + "coverage": 0.4716666666666667, + "risk": 0.3053003533568905, + "threshold": 0.2651658771844966 + }, + { + "coverage": 0.472, + "risk": 0.3050847457627119, + "threshold": 0.2651288667720923 + }, + { + "coverage": 0.4723333333333333, + "risk": 0.3048694424841214, + "threshold": 0.26488698390906606 + }, + { + "coverage": 0.4726666666666667, + "risk": 0.304654442877292, + "threshold": 0.2648299921334265 + }, + { + "coverage": 0.473, + "risk": 0.3044397463002114, + "threshold": 0.2644653246422742 + }, + { + "coverage": 0.47333333333333333, + "risk": 0.30422535211267604, + "threshold": 0.2643637144011228 + }, + { + "coverage": 0.4736666666666667, + "risk": 0.3047149894440535, + "threshold": 0.263347533552056 + }, + { + "coverage": 0.474, + "risk": 0.30450070323488043, + "threshold": 0.26290911712581455 + }, + { + "coverage": 0.47433333333333333, + "risk": 0.3049894588896697, + "threshold": 0.26217245328457195 + }, + { + "coverage": 0.4746666666666667, + "risk": 0.3047752808988764, + "threshold": 0.26198687147223054 + }, + { + "coverage": 0.475, + "risk": 0.3045614035087719, + "threshold": 0.26195673789543117 + }, + { + "coverage": 0.47533333333333333, + "risk": 0.30434782608695654, + "threshold": 0.26174133277355593 + }, + { + "coverage": 0.4756666666666667, + "risk": 0.3041345480028031, + "threshold": 0.26140415523083094 + }, + { + "coverage": 0.476, + "risk": 0.30392156862745096, + "threshold": 0.2609085239244854 + }, + { + "coverage": 0.47633333333333333, + "risk": 0.30370888733379986, + "threshold": 0.2604596300101022 + }, + { + "coverage": 0.4766666666666667, + "risk": 0.3034965034965035, + "threshold": 0.2601326078037926 + }, + { + "coverage": 0.477, + "risk": 0.30328441649196364, + "threshold": 0.2596446554751421 + }, + { + "coverage": 0.47733333333333333, + "risk": 0.30307262569832405, + "threshold": 0.2588764917361619 + }, + { + "coverage": 0.4776666666666667, + "risk": 0.30286113049546404, + "threshold": 0.25872296856176236 + }, + { + "coverage": 0.478, + "risk": 0.303347280334728, + "threshold": 0.25869698505544164 + }, + { + "coverage": 0.47833333333333333, + "risk": 0.30313588850174217, + "threshold": 0.25855555894866744 + }, + { + "coverage": 0.4786666666666667, + "risk": 0.30292479108635095, + "threshold": 0.2581308478719949 + }, + { + "coverage": 0.479, + "risk": 0.302713987473904, + "threshold": 0.2577109415690671 + }, + { + "coverage": 0.47933333333333333, + "risk": 0.30250347705146036, + "threshold": 0.2576186076578495 + }, + { + "coverage": 0.4796666666666667, + "risk": 0.30229325920778316, + "threshold": 0.25731169857304353 + }, + { + "coverage": 0.48, + "risk": 0.3020833333333333, + "threshold": 0.25620815559688853 + }, + { + "coverage": 0.48033333333333333, + "risk": 0.3018736988202637, + "threshold": 0.25510350540640897 + }, + { + "coverage": 0.4806666666666667, + "risk": 0.3016643550624133, + "threshold": 0.25509476077989435 + }, + { + "coverage": 0.481, + "risk": 0.30145530145530147, + "threshold": 0.25464825378194583 + }, + { + "coverage": 0.48133333333333334, + "risk": 0.3012465373961219, + "threshold": 0.25448307122019415 + }, + { + "coverage": 0.4816666666666667, + "risk": 0.30103806228373703, + "threshold": 0.2543754451783586 + }, + { + "coverage": 0.482, + "risk": 0.3008298755186722, + "threshold": 0.25418512020167366 + }, + { + "coverage": 0.48233333333333334, + "risk": 0.3006219765031099, + "threshold": 0.25413658236358544 + }, + { + "coverage": 0.4826666666666667, + "risk": 0.300414364640884, + "threshold": 0.25394207575946254 + }, + { + "coverage": 0.483, + "risk": 0.30089717046238784, + "threshold": 0.2538630424573998 + }, + { + "coverage": 0.48333333333333334, + "risk": 0.3006896551724138, + "threshold": 0.25274792177713545 + }, + { + "coverage": 0.4836666666666667, + "risk": 0.3004824259131633, + "threshold": 0.25202743027323127 + }, + { + "coverage": 0.484, + "risk": 0.3002754820936639, + "threshold": 0.2515324506579428 + }, + { + "coverage": 0.48433333333333334, + "risk": 0.30006882312456984, + "threshold": 0.2510331289278214 + }, + { + "coverage": 0.4846666666666667, + "risk": 0.2998624484181568, + "threshold": 0.2507746220205196 + }, + { + "coverage": 0.485, + "risk": 0.29965635738831614, + "threshold": 0.2506113556616347 + }, + { + "coverage": 0.48533333333333334, + "risk": 0.29945054945054944, + "threshold": 0.25054907387022685 + }, + { + "coverage": 0.4856666666666667, + "risk": 0.29924502402196296, + "threshold": 0.25036181758058373 + }, + { + "coverage": 0.486, + "risk": 0.299039780521262, + "threshold": 0.2494581445466425 + }, + { + "coverage": 0.48633333333333334, + "risk": 0.2988348183687457, + "threshold": 0.24944684773409523 + }, + { + "coverage": 0.4866666666666667, + "risk": 0.29863013698630136, + "threshold": 0.247989668040185 + }, + { + "coverage": 0.487, + "risk": 0.29842573579739906, + "threshold": 0.24622678691222208 + }, + { + "coverage": 0.48733333333333334, + "risk": 0.2982216142270862, + "threshold": 0.24603960763754723 + }, + { + "coverage": 0.4876666666666667, + "risk": 0.2987012987012987, + "threshold": 0.2459505007400001 + }, + { + "coverage": 0.488, + "risk": 0.29849726775956287, + "threshold": 0.2454574089306316 + }, + { + "coverage": 0.48833333333333334, + "risk": 0.2982935153583618, + "threshold": 0.2452710081182951 + }, + { + "coverage": 0.4886666666666667, + "risk": 0.2980900409276944, + "threshold": 0.2451980001134034 + }, + { + "coverage": 0.489, + "risk": 0.29788684389911385, + "threshold": 0.24393956003372283 + }, + { + "coverage": 0.48933333333333334, + "risk": 0.29768392370572205, + "threshold": 0.24319584695471422 + }, + { + "coverage": 0.48966666666666664, + "risk": 0.29748127978216476, + "threshold": 0.24307026722691344 + }, + { + "coverage": 0.49, + "risk": 0.2972789115646258, + "threshold": 0.24212403363517881 + }, + { + "coverage": 0.49033333333333334, + "risk": 0.29707681849082257, + "threshold": 0.24137287962237802 + }, + { + "coverage": 0.49066666666666664, + "risk": 0.296875, + "threshold": 0.241176120515337 + }, + { + "coverage": 0.491, + "risk": 0.296673455532926, + "threshold": 0.24097181673431117 + }, + { + "coverage": 0.49133333333333334, + "risk": 0.296472184531886, + "threshold": 0.23927028753811647 + }, + { + "coverage": 0.49166666666666664, + "risk": 0.29627118644067796, + "threshold": 0.2388877839588214 + }, + { + "coverage": 0.492, + "risk": 0.2967479674796748, + "threshold": 0.23803113282957258 + }, + { + "coverage": 0.49233333333333335, + "risk": 0.2965470548408937, + "threshold": 0.23790878250395894 + }, + { + "coverage": 0.49266666666666664, + "risk": 0.2963464140730717, + "threshold": 0.23787599252313996 + }, + { + "coverage": 0.493, + "risk": 0.2961460446247465, + "threshold": 0.23756272386261937 + }, + { + "coverage": 0.49333333333333335, + "risk": 0.2959459459459459, + "threshold": 0.23701712538158942 + }, + { + "coverage": 0.49366666666666664, + "risk": 0.29574611748818364, + "threshold": 0.23651663222338667 + }, + { + "coverage": 0.494, + "risk": 0.29554655870445345, + "threshold": 0.235924368172069 + }, + { + "coverage": 0.49433333333333335, + "risk": 0.29534726904922454, + "threshold": 0.23585888100002547 + }, + { + "coverage": 0.49466666666666664, + "risk": 0.29514824797843664, + "threshold": 0.23540312411648592 + }, + { + "coverage": 0.495, + "risk": 0.29494949494949496, + "threshold": 0.2353786309529171 + }, + { + "coverage": 0.49533333333333335, + "risk": 0.29475100942126514, + "threshold": 0.23534909990446579 + }, + { + "coverage": 0.49566666666666664, + "risk": 0.2945527908540686, + "threshold": 0.23524184686042388 + }, + { + "coverage": 0.496, + "risk": 0.29435483870967744, + "threshold": 0.23509495415580106 + }, + { + "coverage": 0.49633333333333335, + "risk": 0.2941571524513096, + "threshold": 0.2340142215774642 + }, + { + "coverage": 0.49666666666666665, + "risk": 0.29395973154362415, + "threshold": 0.23290550358572207 + }, + { + "coverage": 0.497, + "risk": 0.2937625754527163, + "threshold": 0.2327445437174106 + }, + { + "coverage": 0.49733333333333335, + "risk": 0.29423592493297585, + "threshold": 0.23228158498412527 + }, + { + "coverage": 0.49766666666666665, + "risk": 0.2940388479571333, + "threshold": 0.23201129638010412 + }, + { + "coverage": 0.498, + "risk": 0.2938420348058902, + "threshold": 0.23166257794951645 + }, + { + "coverage": 0.49833333333333335, + "risk": 0.2936454849498328, + "threshold": 0.2316562373685985 + }, + { + "coverage": 0.49866666666666665, + "risk": 0.2934491978609626, + "threshold": 0.23140317018337894 + }, + { + "coverage": 0.499, + "risk": 0.29325317301269205, + "threshold": 0.23095745871939738 + }, + { + "coverage": 0.49933333333333335, + "risk": 0.2930574098798398, + "threshold": 0.2302362262021857 + }, + { + "coverage": 0.49966666666666665, + "risk": 0.29286190793862577, + "threshold": 0.2299622904018642 + }, + { + "coverage": 0.5, + "risk": 0.2926666666666667, + "threshold": 0.2296270196351912 + }, + { + "coverage": 0.5003333333333333, + "risk": 0.29247168554297137, + "threshold": 0.22917881608086746 + }, + { + "coverage": 0.5006666666666667, + "risk": 0.29227696404793607, + "threshold": 0.22885459260758353 + }, + { + "coverage": 0.501, + "risk": 0.29208250166334, + "threshold": 0.2287084229119225 + }, + { + "coverage": 0.5013333333333333, + "risk": 0.29188829787234044, + "threshold": 0.22850600935478807 + }, + { + "coverage": 0.5016666666666667, + "risk": 0.29169435215946843, + "threshold": 0.22831426523941412 + }, + { + "coverage": 0.502, + "risk": 0.2915006640106242, + "threshold": 0.22737494582459566 + }, + { + "coverage": 0.5023333333333333, + "risk": 0.2913072329130723, + "threshold": 0.22605370507840208 + }, + { + "coverage": 0.5026666666666667, + "risk": 0.2911140583554377, + "threshold": 0.2258080851654503 + }, + { + "coverage": 0.503, + "risk": 0.29092113982770046, + "threshold": 0.2253484547567815 + }, + { + "coverage": 0.5033333333333333, + "risk": 0.29072847682119207, + "threshold": 0.22531008172482406 + }, + { + "coverage": 0.5036666666666667, + "risk": 0.29053606882859034, + "threshold": 0.22493369205693867 + }, + { + "coverage": 0.504, + "risk": 0.29034391534391535, + "threshold": 0.2244781424059579 + }, + { + "coverage": 0.5043333333333333, + "risk": 0.29015201586252476, + "threshold": 0.22334804825169643 + }, + { + "coverage": 0.5046666666666667, + "risk": 0.2899603698811096, + "threshold": 0.22329882223156677 + }, + { + "coverage": 0.505, + "risk": 0.28976897689768977, + "threshold": 0.22304956144808752 + }, + { + "coverage": 0.5053333333333333, + "risk": 0.29023746701846964, + "threshold": 0.22301579032395036 + }, + { + "coverage": 0.5056666666666667, + "risk": 0.2900461437046803, + "threshold": 0.22246231727643037 + }, + { + "coverage": 0.506, + "risk": 0.2898550724637681, + "threshold": 0.22213931303702109 + }, + { + "coverage": 0.5063333333333333, + "risk": 0.2896642527978934, + "threshold": 0.22201047253197143 + }, + { + "coverage": 0.5066666666666667, + "risk": 0.2901315789473684, + "threshold": 0.22162802870555495 + }, + { + "coverage": 0.507, + "risk": 0.28994082840236685, + "threshold": 0.2215271469037819 + }, + { + "coverage": 0.5073333333333333, + "risk": 0.2897503285151117, + "threshold": 0.22114004707469015 + }, + { + "coverage": 0.5076666666666667, + "risk": 0.2895600787918582, + "threshold": 0.2209308769521142 + }, + { + "coverage": 0.508, + "risk": 0.28937007874015747, + "threshold": 0.22083291414029754 + }, + { + "coverage": 0.5083333333333333, + "risk": 0.28918032786885245, + "threshold": 0.22046263076424594 + }, + { + "coverage": 0.5086666666666667, + "risk": 0.2889908256880734, + "threshold": 0.21924133338635654 + }, + { + "coverage": 0.509, + "risk": 0.28945645055664704, + "threshold": 0.2190181892055364 + }, + { + "coverage": 0.5093333333333333, + "risk": 0.2892670157068063, + "threshold": 0.21871175080489305 + }, + { + "coverage": 0.5096666666666667, + "risk": 0.289077828646174, + "threshold": 0.21832153331484766 + }, + { + "coverage": 0.51, + "risk": 0.28888888888888886, + "threshold": 0.21822778682454685 + }, + { + "coverage": 0.5103333333333333, + "risk": 0.28870019595035923, + "threshold": 0.21765453979271265 + }, + { + "coverage": 0.5106666666666667, + "risk": 0.2885117493472585, + "threshold": 0.217456044142909 + }, + { + "coverage": 0.511, + "risk": 0.2883235485975212, + "threshold": 0.21664329837644725 + }, + { + "coverage": 0.5113333333333333, + "risk": 0.288135593220339, + "threshold": 0.21657882767860423 + }, + { + "coverage": 0.5116666666666667, + "risk": 0.28794788273615635, + "threshold": 0.21631108242599573 + }, + { + "coverage": 0.512, + "risk": 0.2877604166666667, + "threshold": 0.21604756262905575 + }, + { + "coverage": 0.5123333333333333, + "risk": 0.28757319453480806, + "threshold": 0.21542240790760228 + }, + { + "coverage": 0.5126666666666667, + "risk": 0.28738621586475943, + "threshold": 0.21540606293840847 + }, + { + "coverage": 0.513, + "risk": 0.2871994801819363, + "threshold": 0.21522921128720207 + }, + { + "coverage": 0.5133333333333333, + "risk": 0.287012987012987, + "threshold": 0.2140663972704665 + }, + { + "coverage": 0.5136666666666667, + "risk": 0.28682673588578844, + "threshold": 0.21400063765406002 + }, + { + "coverage": 0.514, + "risk": 0.28664072632944226, + "threshold": 0.21367467351202607 + }, + { + "coverage": 0.5143333333333333, + "risk": 0.2864549578742709, + "threshold": 0.21330242267672914 + }, + { + "coverage": 0.5146666666666667, + "risk": 0.2862694300518135, + "threshold": 0.21322629734286538 + }, + { + "coverage": 0.515, + "risk": 0.286084142394822, + "threshold": 0.21313780067928825 + }, + { + "coverage": 0.5153333333333333, + "risk": 0.2858990944372574, + "threshold": 0.21065260683894171 + }, + { + "coverage": 0.5156666666666667, + "risk": 0.2857142857142857, + "threshold": 0.21023012901117194 + }, + { + "coverage": 0.516, + "risk": 0.28552971576227393, + "threshold": 0.21021644468837217 + }, + { + "coverage": 0.5163333333333333, + "risk": 0.2859909619109103, + "threshold": 0.20965495879805165 + }, + { + "coverage": 0.5166666666666667, + "risk": 0.28580645161290325, + "threshold": 0.20953741676987098 + }, + { + "coverage": 0.517, + "risk": 0.2856221792392005, + "threshold": 0.208929491516927 + }, + { + "coverage": 0.5173333333333333, + "risk": 0.2854381443298969, + "threshold": 0.20852378673474986 + }, + { + "coverage": 0.5176666666666667, + "risk": 0.28525434642627173, + "threshold": 0.20846093081162861 + }, + { + "coverage": 0.518, + "risk": 0.2857142857142857, + "threshold": 0.20826477320415684 + }, + { + "coverage": 0.5183333333333333, + "risk": 0.28553054662379423, + "threshold": 0.20765187463133375 + }, + { + "coverage": 0.5186666666666667, + "risk": 0.2853470437017995, + "threshold": 0.2074569881709165 + }, + { + "coverage": 0.519, + "risk": 0.28516377649325625, + "threshold": 0.2070805797776089 + }, + { + "coverage": 0.5193333333333333, + "risk": 0.28562259306803595, + "threshold": 0.20697739037031931 + }, + { + "coverage": 0.5196666666666667, + "risk": 0.2854393842206543, + "threshold": 0.20681228249689002 + }, + { + "coverage": 0.52, + "risk": 0.28525641025641024, + "threshold": 0.20671024399053922 + }, + { + "coverage": 0.5203333333333333, + "risk": 0.28507367072389495, + "threshold": 0.2066032616752961 + }, + { + "coverage": 0.5206666666666667, + "risk": 0.2848911651728553, + "threshold": 0.20646477080557493 + }, + { + "coverage": 0.521, + "risk": 0.2853486884197057, + "threshold": 0.20578728622508488 + }, + { + "coverage": 0.5213333333333333, + "risk": 0.2858056265984655, + "threshold": 0.20578064193821544 + }, + { + "coverage": 0.5216666666666666, + "risk": 0.2856230031948882, + "threshold": 0.20543801741824363 + }, + { + "coverage": 0.522, + "risk": 0.28544061302681994, + "threshold": 0.20527379818501174 + }, + { + "coverage": 0.5223333333333333, + "risk": 0.2852584556477345, + "threshold": 0.20526715227924192 + }, + { + "coverage": 0.5226666666666666, + "risk": 0.2850765306122449, + "threshold": 0.20516863025171478 + }, + { + "coverage": 0.523, + "risk": 0.28489483747609945, + "threshold": 0.2049441072659855 + }, + { + "coverage": 0.5233333333333333, + "risk": 0.28471337579617834, + "threshold": 0.20463114970839513 + }, + { + "coverage": 0.5236666666666666, + "risk": 0.28453214513049013, + "threshold": 0.20358922753600392 + }, + { + "coverage": 0.524, + "risk": 0.28498727735368956, + "threshold": 0.20348654274035802 + }, + { + "coverage": 0.5243333333333333, + "risk": 0.2848061029879212, + "threshold": 0.20284780690824952 + }, + { + "coverage": 0.5246666666666666, + "risk": 0.2852604828462516, + "threshold": 0.2021413797010178 + }, + { + "coverage": 0.525, + "risk": 0.2850793650793651, + "threshold": 0.2018802402095806 + }, + { + "coverage": 0.5253333333333333, + "risk": 0.2848984771573604, + "threshold": 0.20118207372611202 + }, + { + "coverage": 0.5256666666666666, + "risk": 0.284717818642993, + "threshold": 0.20109351484594448 + }, + { + "coverage": 0.526, + "risk": 0.28453738910012677, + "threshold": 0.20094996369822632 + }, + { + "coverage": 0.5263333333333333, + "risk": 0.28435718809373023, + "threshold": 0.2007800738028508 + }, + { + "coverage": 0.5266666666666666, + "risk": 0.28417721518987343, + "threshold": 0.2003265454124652 + }, + { + "coverage": 0.527, + "risk": 0.2839974699557242, + "threshold": 0.20027710956061115 + }, + { + "coverage": 0.5273333333333333, + "risk": 0.2838179519595449, + "threshold": 0.19993910348585184 + }, + { + "coverage": 0.5276666666666666, + "risk": 0.28363866077068856, + "threshold": 0.19916720349413422 + }, + { + "coverage": 0.528, + "risk": 0.28345959595959597, + "threshold": 0.19892271147956492 + }, + { + "coverage": 0.5283333333333333, + "risk": 0.2832807570977918, + "threshold": 0.1983481519487542 + }, + { + "coverage": 0.5286666666666666, + "risk": 0.28310214375788145, + "threshold": 0.19779406872623803 + }, + { + "coverage": 0.529, + "risk": 0.2829237555135476, + "threshold": 0.19752524768189372 + }, + { + "coverage": 0.5293333333333333, + "risk": 0.2827455919395466, + "threshold": 0.19741932872164233 + }, + { + "coverage": 0.5296666666666666, + "risk": 0.28256765261170547, + "threshold": 0.19739079319587266 + }, + { + "coverage": 0.53, + "risk": 0.28238993710691823, + "threshold": 0.19726831937576536 + }, + { + "coverage": 0.5303333333333333, + "risk": 0.2822124450031427, + "threshold": 0.1952593829401635 + }, + { + "coverage": 0.5306666666666666, + "risk": 0.28203517587939697, + "threshold": 0.19492893559548233 + }, + { + "coverage": 0.531, + "risk": 0.28185812931575643, + "threshold": 0.19422773969730214 + }, + { + "coverage": 0.5313333333333333, + "risk": 0.2816813048933501, + "threshold": 0.1938485093173174 + }, + { + "coverage": 0.5316666666666666, + "risk": 0.28150470219435736, + "threshold": 0.1937064574738385 + }, + { + "coverage": 0.532, + "risk": 0.28132832080200504, + "threshold": 0.19253419474274927 + }, + { + "coverage": 0.5323333333333333, + "risk": 0.2817783343769568, + "threshold": 0.19222985235562365 + }, + { + "coverage": 0.5326666666666666, + "risk": 0.2816020025031289, + "threshold": 0.19182147727941626 + }, + { + "coverage": 0.533, + "risk": 0.28205128205128205, + "threshold": 0.1916213503051944 + }, + { + "coverage": 0.5333333333333333, + "risk": 0.2825, + "threshold": 0.1915267609313944 + }, + { + "coverage": 0.5336666666666666, + "risk": 0.28232354778263585, + "threshold": 0.19152416824473414 + }, + { + "coverage": 0.534, + "risk": 0.28214731585518105, + "threshold": 0.19144580037275022 + }, + { + "coverage": 0.5343333333333333, + "risk": 0.28197130380536495, + "threshold": 0.1909225186452637 + }, + { + "coverage": 0.5346666666666666, + "risk": 0.2817955112219451, + "threshold": 0.1907725315067129 + }, + { + "coverage": 0.535, + "risk": 0.28161993769470406, + "threshold": 0.19055464477799886 + }, + { + "coverage": 0.5353333333333333, + "risk": 0.28144458281444584, + "threshold": 0.18987776372741746 + }, + { + "coverage": 0.5356666666666666, + "risk": 0.28126944617299315, + "threshold": 0.18934198441652683 + }, + { + "coverage": 0.536, + "risk": 0.2810945273631841, + "threshold": 0.1892539649840344 + }, + { + "coverage": 0.5363333333333333, + "risk": 0.2809198259788689, + "threshold": 0.18813930790320846 + }, + { + "coverage": 0.5366666666666666, + "risk": 0.28074534161490683, + "threshold": 0.1879553272669865 + }, + { + "coverage": 0.537, + "risk": 0.28057107386716323, + "threshold": 0.18685928682254704 + }, + { + "coverage": 0.5373333333333333, + "risk": 0.2803970223325062, + "threshold": 0.18657741350996032 + }, + { + "coverage": 0.5376666666666666, + "risk": 0.28022318660880347, + "threshold": 0.18632778894433003 + }, + { + "coverage": 0.538, + "risk": 0.28004956629491945, + "threshold": 0.18606110568543882 + }, + { + "coverage": 0.5383333333333333, + "risk": 0.27987616099071205, + "threshold": 0.18586901394164942 + }, + { + "coverage": 0.5386666666666666, + "risk": 0.28032178217821785, + "threshold": 0.1858666677789133 + }, + { + "coverage": 0.539, + "risk": 0.28014842300556586, + "threshold": 0.1858311905953917 + }, + { + "coverage": 0.5393333333333333, + "risk": 0.2799752781211372, + "threshold": 0.1857112849877835 + }, + { + "coverage": 0.5396666666666666, + "risk": 0.2798023471278567, + "threshold": 0.18563383174127207 + }, + { + "coverage": 0.54, + "risk": 0.2796296296296296, + "threshold": 0.18561731124551734 + }, + { + "coverage": 0.5403333333333333, + "risk": 0.2794571252313387, + "threshold": 0.18545837709076785 + }, + { + "coverage": 0.5406666666666666, + "risk": 0.2792848335388409, + "threshold": 0.18531442821274907 + }, + { + "coverage": 0.541, + "risk": 0.27911275415896486, + "threshold": 0.18527987821736294 + }, + { + "coverage": 0.5413333333333333, + "risk": 0.2789408866995074, + "threshold": 0.185009720127808 + }, + { + "coverage": 0.5416666666666666, + "risk": 0.27876923076923077, + "threshold": 0.18477627751115866 + }, + { + "coverage": 0.542, + "risk": 0.2785977859778598, + "threshold": 0.18432653513472144 + }, + { + "coverage": 0.5423333333333333, + "risk": 0.2784265519360787, + "threshold": 0.18380894979775603 + }, + { + "coverage": 0.5426666666666666, + "risk": 0.27825552825552824, + "threshold": 0.18373874818151628 + }, + { + "coverage": 0.543, + "risk": 0.27808471454880296, + "threshold": 0.18368561593764843 + }, + { + "coverage": 0.5433333333333333, + "risk": 0.27791411042944786, + "threshold": 0.18344250763109257 + }, + { + "coverage": 0.5436666666666666, + "risk": 0.27774371551195587, + "threshold": 0.18339761773922844 + }, + { + "coverage": 0.544, + "risk": 0.2775735294117647, + "threshold": 0.18327583140199022 + }, + { + "coverage": 0.5443333333333333, + "risk": 0.2774035517452541, + "threshold": 0.18325542678664122 + }, + { + "coverage": 0.5446666666666666, + "risk": 0.277233782129743, + "threshold": 0.18262716369090415 + }, + { + "coverage": 0.545, + "risk": 0.27706422018348625, + "threshold": 0.18172245058860462 + }, + { + "coverage": 0.5453333333333333, + "risk": 0.2768948655256724, + "threshold": 0.18156760828777624 + }, + { + "coverage": 0.5456666666666666, + "risk": 0.2773365913255956, + "threshold": 0.1812177737733067 + }, + { + "coverage": 0.546, + "risk": 0.2771672771672772, + "threshold": 0.18114142723194135 + }, + { + "coverage": 0.5463333333333333, + "risk": 0.2769981696156193, + "threshold": 0.18077371286572505 + }, + { + "coverage": 0.5466666666666666, + "risk": 0.27682926829268295, + "threshold": 0.1805338817590811 + }, + { + "coverage": 0.547, + "risk": 0.27666057282145035, + "threshold": 0.18018981365438386 + }, + { + "coverage": 0.5473333333333333, + "risk": 0.27649208282582216, + "threshold": 0.18015261775623614 + }, + { + "coverage": 0.5476666666666666, + "risk": 0.27632379793061473, + "threshold": 0.17897889615808207 + }, + { + "coverage": 0.548, + "risk": 0.2761557177615572, + "threshold": 0.17893126483428975 + }, + { + "coverage": 0.5483333333333333, + "risk": 0.27598784194528875, + "threshold": 0.1787874271797605 + }, + { + "coverage": 0.5486666666666666, + "risk": 0.275820170109356, + "threshold": 0.1783343989067819 + }, + { + "coverage": 0.549, + "risk": 0.2756527018822101, + "threshold": 0.17776787809735378 + }, + { + "coverage": 0.5493333333333333, + "risk": 0.27548543689320387, + "threshold": 0.17774726566478932 + }, + { + "coverage": 0.5496666666666666, + "risk": 0.2759248029108551, + "threshold": 0.17746223335854225 + }, + { + "coverage": 0.55, + "risk": 0.27575757575757576, + "threshold": 0.17728943295136257 + }, + { + "coverage": 0.5503333333333333, + "risk": 0.2755905511811024, + "threshold": 0.1770470866316474 + }, + { + "coverage": 0.5506666666666666, + "risk": 0.2754237288135593, + "threshold": 0.1769086401316504 + }, + { + "coverage": 0.551, + "risk": 0.2752571082879613, + "threshold": 0.17639029694989564 + }, + { + "coverage": 0.5513333333333333, + "risk": 0.2750906892382104, + "threshold": 0.17624687446807455 + }, + { + "coverage": 0.5516666666666666, + "risk": 0.27492447129909364, + "threshold": 0.17568359938042644 + }, + { + "coverage": 0.552, + "risk": 0.2747584541062802, + "threshold": 0.17563660781461948 + }, + { + "coverage": 0.5523333333333333, + "risk": 0.27459263729631866, + "threshold": 0.17552436135349175 + }, + { + "coverage": 0.5526666666666666, + "risk": 0.2744270205066345, + "threshold": 0.17522824047514446 + }, + { + "coverage": 0.553, + "risk": 0.2742616033755274, + "threshold": 0.17519039718469984 + }, + { + "coverage": 0.5533333333333333, + "risk": 0.2740963855421687, + "threshold": 0.17466628814123591 + }, + { + "coverage": 0.5536666666666666, + "risk": 0.27393136664659845, + "threshold": 0.17442224121514652 + }, + { + "coverage": 0.554, + "risk": 0.27376654632972325, + "threshold": 0.17429659846233359 + }, + { + "coverage": 0.5543333333333333, + "risk": 0.2736019242333133, + "threshold": 0.17428712193927895 + }, + { + "coverage": 0.5546666666666666, + "risk": 0.2734375, + "threshold": 0.17426898322036377 + }, + { + "coverage": 0.555, + "risk": 0.2732732732732733, + "threshold": 0.174152191162409 + }, + { + "coverage": 0.5553333333333333, + "risk": 0.27310924369747897, + "threshold": 0.17398446023446518 + }, + { + "coverage": 0.5556666666666666, + "risk": 0.27294541091781643, + "threshold": 0.17388829645696827 + }, + { + "coverage": 0.556, + "risk": 0.27278177458033576, + "threshold": 0.17277210884313718 + }, + { + "coverage": 0.5563333333333333, + "risk": 0.2726183343319353, + "threshold": 0.17261860966914702 + }, + { + "coverage": 0.5566666666666666, + "risk": 0.27245508982035926, + "threshold": 0.17183737110284678 + }, + { + "coverage": 0.557, + "risk": 0.2722920406941951, + "threshold": 0.17172342856374095 + }, + { + "coverage": 0.5573333333333333, + "risk": 0.27212918660287083, + "threshold": 0.17090460478519337 + }, + { + "coverage": 0.5576666666666666, + "risk": 0.2719665271966527, + "threshold": 0.17076074701086105 + }, + { + "coverage": 0.558, + "risk": 0.27180406212664276, + "threshold": 0.17064645918402435 + }, + { + "coverage": 0.5583333333333333, + "risk": 0.2716417910447761, + "threshold": 0.16981843158658194 + }, + { + "coverage": 0.5586666666666666, + "risk": 0.2714797136038186, + "threshold": 0.16949483076329785 + }, + { + "coverage": 0.559, + "risk": 0.2713178294573643, + "threshold": 0.16897546274735314 + }, + { + "coverage": 0.5593333333333333, + "risk": 0.27115613825983315, + "threshold": 0.168908611737261 + }, + { + "coverage": 0.5596666666666666, + "risk": 0.27099463966646814, + "threshold": 0.1685327693586755 + }, + { + "coverage": 0.56, + "risk": 0.2714285714285714, + "threshold": 0.1680705265969631 + }, + { + "coverage": 0.5603333333333333, + "risk": 0.2712671029149316, + "threshold": 0.1678050293207241 + }, + { + "coverage": 0.5606666666666666, + "risk": 0.27110582639714625, + "threshold": 0.1677642061882679 + }, + { + "coverage": 0.561, + "risk": 0.2709447415329768, + "threshold": 0.16775496111735888 + }, + { + "coverage": 0.5613333333333334, + "risk": 0.27078384798099764, + "threshold": 0.16768444837414223 + }, + { + "coverage": 0.5616666666666666, + "risk": 0.27062314540059346, + "threshold": 0.16766873259816645 + }, + { + "coverage": 0.562, + "risk": 0.2704626334519573, + "threshold": 0.1672159901013094 + }, + { + "coverage": 0.5623333333333334, + "risk": 0.2703023117960877, + "threshold": 0.16656699877931536 + }, + { + "coverage": 0.5626666666666666, + "risk": 0.27014218009478674, + "threshold": 0.16554956055338055 + }, + { + "coverage": 0.563, + "risk": 0.2699822380106572, + "threshold": 0.1652949484413944 + }, + { + "coverage": 0.5633333333333334, + "risk": 0.2698224852071006, + "threshold": 0.16487204806525022 + }, + { + "coverage": 0.5636666666666666, + "risk": 0.2696629213483146, + "threshold": 0.16486264088064478 + }, + { + "coverage": 0.564, + "risk": 0.2695035460992908, + "threshold": 0.1648525844133472 + }, + { + "coverage": 0.5643333333333334, + "risk": 0.26934435912581217, + "threshold": 0.1644288887707133 + }, + { + "coverage": 0.5646666666666667, + "risk": 0.269185360094451, + "threshold": 0.16417774578661895 + }, + { + "coverage": 0.565, + "risk": 0.26902654867256637, + "threshold": 0.1639549951674152 + }, + { + "coverage": 0.5653333333333334, + "risk": 0.2688679245283019, + "threshold": 0.16386384453141326 + }, + { + "coverage": 0.5656666666666667, + "risk": 0.26870948733058336, + "threshold": 0.16379981370912497 + }, + { + "coverage": 0.566, + "risk": 0.26855123674911663, + "threshold": 0.16378514483944107 + }, + { + "coverage": 0.5663333333333334, + "risk": 0.26839317245438493, + "threshold": 0.16376272464840216 + }, + { + "coverage": 0.5666666666666667, + "risk": 0.26823529411764707, + "threshold": 0.16369532565299816 + }, + { + "coverage": 0.567, + "risk": 0.26807760141093473, + "threshold": 0.1632055410661342 + }, + { + "coverage": 0.5673333333333334, + "risk": 0.2679200940070505, + "threshold": 0.1628082013661803 + }, + { + "coverage": 0.5676666666666667, + "risk": 0.26776277157956546, + "threshold": 0.16113129734562684 + }, + { + "coverage": 0.568, + "risk": 0.2676056338028169, + "threshold": 0.1593749103328179 + }, + { + "coverage": 0.5683333333333334, + "risk": 0.26744868035190617, + "threshold": 0.15925568801809 + }, + { + "coverage": 0.5686666666666667, + "risk": 0.2672919109026964, + "threshold": 0.15894969675742435 + }, + { + "coverage": 0.569, + "risk": 0.2671353251318102, + "threshold": 0.15779578558975804 + }, + { + "coverage": 0.5693333333333334, + "risk": 0.26697892271662765, + "threshold": 0.1575505142995399 + }, + { + "coverage": 0.5696666666666667, + "risk": 0.2668227033352838, + "threshold": 0.15722292300576324 + }, + { + "coverage": 0.57, + "risk": 0.26666666666666666, + "threshold": 0.1565216588782624 + }, + { + "coverage": 0.5703333333333334, + "risk": 0.26651081239041496, + "threshold": 0.15638102967383397 + }, + { + "coverage": 0.5706666666666667, + "risk": 0.26635514018691586, + "threshold": 0.15627877227256906 + }, + { + "coverage": 0.571, + "risk": 0.266199649737303, + "threshold": 0.15604102577112414 + }, + { + "coverage": 0.5713333333333334, + "risk": 0.2660443407234539, + "threshold": 0.15568832157266146 + }, + { + "coverage": 0.5716666666666667, + "risk": 0.26588921282798833, + "threshold": 0.15556204892007275 + }, + { + "coverage": 0.572, + "risk": 0.26573426573426573, + "threshold": 0.155545686191842 + }, + { + "coverage": 0.5723333333333334, + "risk": 0.26557949912638323, + "threshold": 0.15524780128472795 + }, + { + "coverage": 0.5726666666666667, + "risk": 0.26542491268917345, + "threshold": 0.15521965843793703 + }, + { + "coverage": 0.573, + "risk": 0.26527050610820246, + "threshold": 0.155029061706178 + }, + { + "coverage": 0.5733333333333334, + "risk": 0.2651162790697674, + "threshold": 0.15470983901690555 + }, + { + "coverage": 0.5736666666666667, + "risk": 0.26496223126089485, + "threshold": 0.15466428440745814 + }, + { + "coverage": 0.574, + "risk": 0.26480836236933797, + "threshold": 0.1542426535804146 + }, + { + "coverage": 0.5743333333333334, + "risk": 0.2646546720835752, + "threshold": 0.15376090969988374 + }, + { + "coverage": 0.5746666666666667, + "risk": 0.26450116009280744, + "threshold": 0.1534753769330408 + }, + { + "coverage": 0.575, + "risk": 0.2643478260869565, + "threshold": 0.1523264771947608 + }, + { + "coverage": 0.5753333333333334, + "risk": 0.2641946697566628, + "threshold": 0.1521245917240903 + }, + { + "coverage": 0.5756666666666667, + "risk": 0.26404169079328316, + "threshold": 0.15189305857409888 + }, + { + "coverage": 0.576, + "risk": 0.2638888888888889, + "threshold": 0.15171800642645636 + }, + { + "coverage": 0.5763333333333334, + "risk": 0.26373626373626374, + "threshold": 0.1510689770269915 + }, + { + "coverage": 0.5766666666666667, + "risk": 0.2635838150289017, + "threshold": 0.1501079682262435 + }, + { + "coverage": 0.577, + "risk": 0.2634315424610052, + "threshold": 0.14985911844327857 + }, + { + "coverage": 0.5773333333333334, + "risk": 0.2632794457274827, + "threshold": 0.14909704564393442 + }, + { + "coverage": 0.5776666666666667, + "risk": 0.2631275245239469, + "threshold": 0.14895347127332753 + }, + { + "coverage": 0.578, + "risk": 0.2629757785467128, + "threshold": 0.14859819384392906 + }, + { + "coverage": 0.5783333333333334, + "risk": 0.2628242074927954, + "threshold": 0.14828269337592512 + }, + { + "coverage": 0.5786666666666667, + "risk": 0.2626728110599078, + "threshold": 0.14790663044057667 + }, + { + "coverage": 0.579, + "risk": 0.26252158894645944, + "threshold": 0.1478181409183494 + }, + { + "coverage": 0.5793333333333334, + "risk": 0.262945914844649, + "threshold": 0.14757569812617047 + }, + { + "coverage": 0.5796666666666667, + "risk": 0.26279470960322027, + "threshold": 0.14717249850013242 + }, + { + "coverage": 0.58, + "risk": 0.26264367816091955, + "threshold": 0.1458820447943593 + }, + { + "coverage": 0.5803333333333334, + "risk": 0.26249282021826537, + "threshold": 0.14558242979302594 + }, + { + "coverage": 0.5806666666666667, + "risk": 0.26234213547646384, + "threshold": 0.14544493080577883 + }, + { + "coverage": 0.581, + "risk": 0.2621916236374068, + "threshold": 0.1453869205130785 + }, + { + "coverage": 0.5813333333333334, + "risk": 0.26204128440366975, + "threshold": 0.14517282598187414 + }, + { + "coverage": 0.5816666666666667, + "risk": 0.26189111747851, + "threshold": 0.1451565251264426 + }, + { + "coverage": 0.582, + "risk": 0.26174112256586485, + "threshold": 0.14507103976669122 + }, + { + "coverage": 0.5823333333333334, + "risk": 0.2615912993703492, + "threshold": 0.14460726392204165 + }, + { + "coverage": 0.5826666666666667, + "risk": 0.261441647597254, + "threshold": 0.14452517260988185 + }, + { + "coverage": 0.583, + "risk": 0.2612921669525443, + "threshold": 0.14448303003699756 + }, + { + "coverage": 0.5833333333333334, + "risk": 0.2611428571428571, + "threshold": 0.14363967430716476 + }, + { + "coverage": 0.5836666666666667, + "risk": 0.2609937178754997, + "threshold": 0.14320320466416056 + }, + { + "coverage": 0.584, + "risk": 0.2608447488584475, + "threshold": 0.14309028953371666 + }, + { + "coverage": 0.5843333333333334, + "risk": 0.2606959498003423, + "threshold": 0.14189506165753832 + }, + { + "coverage": 0.5846666666666667, + "risk": 0.26054732041049034, + "threshold": 0.14161983130428907 + }, + { + "coverage": 0.585, + "risk": 0.2603988603988604, + "threshold": 0.1414999977038731 + }, + { + "coverage": 0.5853333333333334, + "risk": 0.260250569476082, + "threshold": 0.1411859603971747 + }, + { + "coverage": 0.5856666666666667, + "risk": 0.26010244735344334, + "threshold": 0.14086406512787011 + }, + { + "coverage": 0.586, + "risk": 0.25995449374288965, + "threshold": 0.14073741348667035 + }, + { + "coverage": 0.5863333333333334, + "risk": 0.259806708357021, + "threshold": 0.14036212575243565 + }, + { + "coverage": 0.5866666666666667, + "risk": 0.2596590909090909, + "threshold": 0.1399290330404126 + }, + { + "coverage": 0.587, + "risk": 0.25951164111300395, + "threshold": 0.1398227859817662 + }, + { + "coverage": 0.5873333333333334, + "risk": 0.2593643586833144, + "threshold": 0.1398117492843097 + }, + { + "coverage": 0.5876666666666667, + "risk": 0.25921724333522406, + "threshold": 0.1398032339860501 + }, + { + "coverage": 0.588, + "risk": 0.2590702947845805, + "threshold": 0.13973466098695256 + }, + { + "coverage": 0.5883333333333334, + "risk": 0.25892351274787534, + "threshold": 0.1394967634818839 + }, + { + "coverage": 0.5886666666666667, + "risk": 0.2587768969422424, + "threshold": 0.13918796537763856 + }, + { + "coverage": 0.589, + "risk": 0.2586304470854556, + "threshold": 0.138994053575396 + }, + { + "coverage": 0.5893333333333334, + "risk": 0.2584841628959276, + "threshold": 0.13887105024462953 + }, + { + "coverage": 0.5896666666666667, + "risk": 0.2583380440927077, + "threshold": 0.13871537232713962 + }, + { + "coverage": 0.59, + "risk": 0.25819209039548024, + "threshold": 0.13863917523317607 + }, + { + "coverage": 0.5903333333333334, + "risk": 0.25804630152456237, + "threshold": 0.13862570781023542 + }, + { + "coverage": 0.5906666666666667, + "risk": 0.2584650112866817, + "threshold": 0.13843525168868184 + }, + { + "coverage": 0.591, + "risk": 0.25831923293852227, + "threshold": 0.13797439304448744 + }, + { + "coverage": 0.5913333333333334, + "risk": 0.25817361894024804, + "threshold": 0.13751220408409773 + }, + { + "coverage": 0.5916666666666667, + "risk": 0.2580281690140845, + "threshold": 0.13746819086029413 + }, + { + "coverage": 0.592, + "risk": 0.25788288288288286, + "threshold": 0.13676302712114904 + }, + { + "coverage": 0.5923333333333334, + "risk": 0.2577377602701182, + "threshold": 0.13609167448377513 + }, + { + "coverage": 0.5926666666666667, + "risk": 0.2575928008998875, + "threshold": 0.13605512031953917 + }, + { + "coverage": 0.593, + "risk": 0.25744800449690836, + "threshold": 0.1359951272102343 + }, + { + "coverage": 0.5933333333333334, + "risk": 0.2573033707865168, + "threshold": 0.13576316902267196 + }, + { + "coverage": 0.5936666666666667, + "risk": 0.2571588994946659, + "threshold": 0.1354958314590068 + }, + { + "coverage": 0.594, + "risk": 0.2570145903479237, + "threshold": 0.13540464934202304 + }, + { + "coverage": 0.5943333333333334, + "risk": 0.2568704430734717, + "threshold": 0.13526545195138998 + }, + { + "coverage": 0.5946666666666667, + "risk": 0.25672645739910316, + "threshold": 0.13516086619239065 + }, + { + "coverage": 0.595, + "risk": 0.25658263305322127, + "threshold": 0.13501415112867546 + }, + { + "coverage": 0.5953333333333334, + "risk": 0.2564389697648376, + "threshold": 0.13468238300388385 + }, + { + "coverage": 0.5956666666666667, + "risk": 0.2562954672635702, + "threshold": 0.13421701604080144 + }, + { + "coverage": 0.596, + "risk": 0.25615212527964204, + "threshold": 0.13380426485706257 + }, + { + "coverage": 0.5963333333333334, + "risk": 0.25600894354387926, + "threshold": 0.1336810597496255 + }, + { + "coverage": 0.5966666666666667, + "risk": 0.2558659217877095, + "threshold": 0.1335922011895433 + }, + { + "coverage": 0.597, + "risk": 0.25572305974316023, + "threshold": 0.1329682355457644 + }, + { + "coverage": 0.5973333333333334, + "risk": 0.25558035714285715, + "threshold": 0.13281079302672266 + }, + { + "coverage": 0.5976666666666667, + "risk": 0.25543781372002233, + "threshold": 0.13261811981706256 + }, + { + "coverage": 0.598, + "risk": 0.2552954292084727, + "threshold": 0.13258117265232464 + }, + { + "coverage": 0.5983333333333334, + "risk": 0.2551532033426184, + "threshold": 0.13244646270160226 + }, + { + "coverage": 0.5986666666666667, + "risk": 0.25501113585746105, + "threshold": 0.13135224740100562 + }, + { + "coverage": 0.599, + "risk": 0.2548692264885921, + "threshold": 0.13115109521359422 + }, + { + "coverage": 0.5993333333333334, + "risk": 0.25472747497219134, + "threshold": 0.13071625258578565 + }, + { + "coverage": 0.5996666666666667, + "risk": 0.254585881045025, + "threshold": 0.13066488429386358 + }, + { + "coverage": 0.6, + "risk": 0.2544444444444444, + "threshold": 0.13065583106499115 + }, + { + "coverage": 0.6003333333333334, + "risk": 0.2543031649083842, + "threshold": 0.13059296517866872 + }, + { + "coverage": 0.6006666666666667, + "risk": 0.2541620421753607, + "threshold": 0.13026945191147832 + }, + { + "coverage": 0.601, + "risk": 0.2540210759844703, + "threshold": 0.130210626037208 + }, + { + "coverage": 0.6013333333333334, + "risk": 0.25388026607538805, + "threshold": 0.12986608478016057 + }, + { + "coverage": 0.6016666666666667, + "risk": 0.25373961218836566, + "threshold": 0.12946707843812974 + }, + { + "coverage": 0.602, + "risk": 0.25359911406423036, + "threshold": 0.12944156936206377 + }, + { + "coverage": 0.6023333333333334, + "risk": 0.25345877144438295, + "threshold": 0.12883506920195126 + }, + { + "coverage": 0.6026666666666667, + "risk": 0.25331858407079644, + "threshold": 0.12882370810002136 + }, + { + "coverage": 0.603, + "risk": 0.2531785516860144, + "threshold": 0.12836397542124495 + }, + { + "coverage": 0.6033333333333334, + "risk": 0.25303867403314917, + "threshold": 0.1276967891386897 + }, + { + "coverage": 0.6036666666666667, + "risk": 0.2528989508558807, + "threshold": 0.12735540185461072 + }, + { + "coverage": 0.604, + "risk": 0.25275938189845476, + "threshold": 0.12676168235205582 + }, + { + "coverage": 0.6043333333333333, + "risk": 0.2531715388858246, + "threshold": 0.12667524854581824 + }, + { + "coverage": 0.6046666666666667, + "risk": 0.25303197353914003, + "threshold": 0.12646583203581604 + }, + { + "coverage": 0.605, + "risk": 0.2528925619834711, + "threshold": 0.1254700467703553 + }, + { + "coverage": 0.6053333333333333, + "risk": 0.2527533039647577, + "threshold": 0.1252785764396964 + }, + { + "coverage": 0.6056666666666667, + "risk": 0.2526141992294992, + "threshold": 0.12450309062166096 + }, + { + "coverage": 0.606, + "risk": 0.2524752475247525, + "threshold": 0.12427352637150932 + }, + { + "coverage": 0.6063333333333333, + "risk": 0.2523364485981308, + "threshold": 0.12399974808202247 + }, + { + "coverage": 0.6066666666666667, + "risk": 0.2521978021978022, + "threshold": 0.12352808905157452 + }, + { + "coverage": 0.607, + "risk": 0.25205930807248766, + "threshold": 0.12337506578969319 + }, + { + "coverage": 0.6073333333333333, + "risk": 0.2519209659714599, + "threshold": 0.1227644021459242 + }, + { + "coverage": 0.6076666666666667, + "risk": 0.251782775644542, + "threshold": 0.12263204243269293 + }, + { + "coverage": 0.608, + "risk": 0.25164473684210525, + "threshold": 0.12223452175576595 + }, + { + "coverage": 0.6083333333333333, + "risk": 0.2515068493150685, + "threshold": 0.12204129513766904 + }, + { + "coverage": 0.6086666666666667, + "risk": 0.25136911281489593, + "threshold": 0.12196400642958306 + }, + { + "coverage": 0.609, + "risk": 0.2512315270935961, + "threshold": 0.12146827573458081 + }, + { + "coverage": 0.6093333333333333, + "risk": 0.2510940919037199, + "threshold": 0.12132252755085178 + }, + { + "coverage": 0.6096666666666667, + "risk": 0.25095680699835976, + "threshold": 0.12125808744639777 + }, + { + "coverage": 0.61, + "risk": 0.25136612021857924, + "threshold": 0.12103681524597082 + }, + { + "coverage": 0.6103333333333333, + "risk": 0.2512288367012561, + "threshold": 0.12063970643268168 + }, + { + "coverage": 0.6106666666666667, + "risk": 0.25109170305676853, + "threshold": 0.12054358464505757 + }, + { + "coverage": 0.611, + "risk": 0.2509547190398254, + "threshold": 0.12051340793137712 + }, + { + "coverage": 0.6113333333333333, + "risk": 0.25081788440567065, + "threshold": 0.12042669913591376 + }, + { + "coverage": 0.6116666666666667, + "risk": 0.2506811989100817, + "threshold": 0.12040007232537558 + }, + { + "coverage": 0.612, + "risk": 0.25054466230936817, + "threshold": 0.11988457295929089 + }, + { + "coverage": 0.6123333333333333, + "risk": 0.25040827436037016, + "threshold": 0.1198151408244855 + }, + { + "coverage": 0.6126666666666667, + "risk": 0.250272034820457, + "threshold": 0.11959551385059983 + }, + { + "coverage": 0.613, + "risk": 0.2501359434475258, + "threshold": 0.11942326295204236 + }, + { + "coverage": 0.6133333333333333, + "risk": 0.25, + "threshold": 0.11929256757624486 + }, + { + "coverage": 0.6136666666666667, + "risk": 0.2498642042368278, + "threshold": 0.11901527503234116 + }, + { + "coverage": 0.614, + "risk": 0.249728555917481, + "threshold": 0.11770898209205528 + }, + { + "coverage": 0.6143333333333333, + "risk": 0.24959305480195335, + "threshold": 0.11757264273233838 + }, + { + "coverage": 0.6146666666666667, + "risk": 0.24945770065075923, + "threshold": 0.11747487329692358 + }, + { + "coverage": 0.615, + "risk": 0.24932249322493225, + "threshold": 0.11678633645534775 + }, + { + "coverage": 0.6153333333333333, + "risk": 0.24918743228602383, + "threshold": 0.11662554492794526 + }, + { + "coverage": 0.6156666666666667, + "risk": 0.24905251759610178, + "threshold": 0.11558822194140718 + }, + { + "coverage": 0.616, + "risk": 0.24891774891774893, + "threshold": 0.1155408126006233 + }, + { + "coverage": 0.6163333333333333, + "risk": 0.24878312601406166, + "threshold": 0.11530761167472425 + }, + { + "coverage": 0.6166666666666667, + "risk": 0.24864864864864866, + "threshold": 0.11483308170198804 + }, + { + "coverage": 0.617, + "risk": 0.2485143165856294, + "threshold": 0.11457192162345615 + }, + { + "coverage": 0.6173333333333333, + "risk": 0.24838012958963282, + "threshold": 0.11442198869843458 + }, + { + "coverage": 0.6176666666666667, + "risk": 0.248246087425796, + "threshold": 0.11387545606444294 + }, + { + "coverage": 0.618, + "risk": 0.2481121898597627, + "threshold": 0.11309733373306396 + }, + { + "coverage": 0.6183333333333333, + "risk": 0.24797843665768193, + "threshold": 0.11253090144433996 + }, + { + "coverage": 0.6186666666666667, + "risk": 0.2478448275862069, + "threshold": 0.11231497489973157 + }, + { + "coverage": 0.619, + "risk": 0.24771136241249328, + "threshold": 0.11214871424616155 + }, + { + "coverage": 0.6193333333333333, + "risk": 0.24757804090419805, + "threshold": 0.1118453292984843 + }, + { + "coverage": 0.6196666666666667, + "risk": 0.24744486282947822, + "threshold": 0.11137557930354706 + }, + { + "coverage": 0.62, + "risk": 0.24731182795698925, + "threshold": 0.11117376489481165 + }, + { + "coverage": 0.6203333333333333, + "risk": 0.24717893605588392, + "threshold": 0.1111644733456511 + }, + { + "coverage": 0.6206666666666667, + "risk": 0.24704618689581095, + "threshold": 0.11068832310043002 + }, + { + "coverage": 0.621, + "risk": 0.24691358024691357, + "threshold": 0.11056092362342529 + }, + { + "coverage": 0.6213333333333333, + "risk": 0.24678111587982832, + "threshold": 0.11039390556999444 + }, + { + "coverage": 0.6216666666666667, + "risk": 0.24664879356568364, + "threshold": 0.11037130323655783 + }, + { + "coverage": 0.622, + "risk": 0.2465166130760986, + "threshold": 0.11035997067533779 + }, + { + "coverage": 0.6223333333333333, + "risk": 0.24638457418318158, + "threshold": 0.1091155318538247 + }, + { + "coverage": 0.6226666666666667, + "risk": 0.2462526766595289, + "threshold": 0.10911205656497544 + }, + { + "coverage": 0.623, + "risk": 0.24612092027822366, + "threshold": 0.10909896931050087 + }, + { + "coverage": 0.6233333333333333, + "risk": 0.24598930481283424, + "threshold": 0.10894240028612696 + }, + { + "coverage": 0.6236666666666667, + "risk": 0.24585783003741316, + "threshold": 0.10858638383767415 + }, + { + "coverage": 0.624, + "risk": 0.24572649572649571, + "threshold": 0.10856113206964843 + }, + { + "coverage": 0.6243333333333333, + "risk": 0.24559530165509877, + "threshold": 0.10848312616810235 + }, + { + "coverage": 0.6246666666666667, + "risk": 0.24546424759871932, + "threshold": 0.10823637084695296 + }, + { + "coverage": 0.625, + "risk": 0.24533333333333332, + "threshold": 0.10813202957345272 + }, + { + "coverage": 0.6253333333333333, + "risk": 0.24520255863539445, + "threshold": 0.10808710952624292 + }, + { + "coverage": 0.6256666666666667, + "risk": 0.24507192328183272, + "threshold": 0.1080240115741888 + }, + { + "coverage": 0.626, + "risk": 0.24494142705005326, + "threshold": 0.10782939675343597 + }, + { + "coverage": 0.6263333333333333, + "risk": 0.24481106971793506, + "threshold": 0.10741899261674161 + }, + { + "coverage": 0.6266666666666667, + "risk": 0.24468085106382978, + "threshold": 0.10702241960422007 + }, + { + "coverage": 0.627, + "risk": 0.24508240297713982, + "threshold": 0.10693591339362278 + }, + { + "coverage": 0.6273333333333333, + "risk": 0.24495217853347503, + "threshold": 0.1064661477251818 + }, + { + "coverage": 0.6276666666666667, + "risk": 0.24482209240573552, + "threshold": 0.10598778468637766 + }, + { + "coverage": 0.628, + "risk": 0.24469214437367304, + "threshold": 0.10583360734026515 + }, + { + "coverage": 0.6283333333333333, + "risk": 0.24456233421750664, + "threshold": 0.1055089408701427 + }, + { + "coverage": 0.6286666666666667, + "risk": 0.24443266171792152, + "threshold": 0.10548009582072479 + }, + { + "coverage": 0.629, + "risk": 0.24430312665606782, + "threshold": 0.1051539447475871 + }, + { + "coverage": 0.6293333333333333, + "risk": 0.2441737288135593, + "threshold": 0.10460454048768347 + }, + { + "coverage": 0.6296666666666667, + "risk": 0.2440444679724722, + "threshold": 0.10455359426991943 + }, + { + "coverage": 0.63, + "risk": 0.24391534391534392, + "threshold": 0.10412134797181213 + }, + { + "coverage": 0.6303333333333333, + "risk": 0.24378635642517188, + "threshold": 0.10396682114209559 + }, + { + "coverage": 0.6306666666666667, + "risk": 0.24365750528541227, + "threshold": 0.10390104265338965 + }, + { + "coverage": 0.631, + "risk": 0.24352879027997887, + "threshold": 0.10363803489941913 + }, + { + "coverage": 0.6313333333333333, + "risk": 0.24340021119324182, + "threshold": 0.10362590669198046 + }, + { + "coverage": 0.6316666666666667, + "risk": 0.24327176781002638, + "threshold": 0.1033373775179084 + }, + { + "coverage": 0.632, + "risk": 0.24367088607594936, + "threshold": 0.10320112696783035 + }, + { + "coverage": 0.6323333333333333, + "risk": 0.24354243542435425, + "threshold": 0.10307808289687291 + }, + { + "coverage": 0.6326666666666667, + "risk": 0.2434141201264489, + "threshold": 0.10277355601637764 + }, + { + "coverage": 0.633, + "risk": 0.24328593996840442, + "threshold": 0.10227794185210497 + }, + { + "coverage": 0.6333333333333333, + "risk": 0.2431578947368421, + "threshold": 0.10223208435883249 + }, + { + "coverage": 0.6336666666666667, + "risk": 0.2430299842188322, + "threshold": 0.10204987055929712 + }, + { + "coverage": 0.634, + "risk": 0.24290220820189273, + "threshold": 0.10195325512563076 + }, + { + "coverage": 0.6343333333333333, + "risk": 0.24277456647398843, + "threshold": 0.10124620035607934 + }, + { + "coverage": 0.6346666666666667, + "risk": 0.2426470588235294, + "threshold": 0.10109697564359997 + }, + { + "coverage": 0.635, + "risk": 0.24251968503937008, + "threshold": 0.10090584612021443 + }, + { + "coverage": 0.6353333333333333, + "risk": 0.24239244491080797, + "threshold": 0.1001255633336108 + }, + { + "coverage": 0.6356666666666667, + "risk": 0.2422653382275826, + "threshold": 0.10011947423699608 + }, + { + "coverage": 0.636, + "risk": 0.24213836477987422, + "threshold": 0.09962072900535661 + }, + { + "coverage": 0.6363333333333333, + "risk": 0.24201152435830278, + "threshold": 0.09957427541400454 + }, + { + "coverage": 0.6366666666666667, + "risk": 0.2418848167539267, + "threshold": 0.09918800633917858 + }, + { + "coverage": 0.637, + "risk": 0.2422815279958137, + "threshold": 0.09777705976007105 + }, + { + "coverage": 0.6373333333333333, + "risk": 0.24215481171548117, + "threshold": 0.09761684015710485 + }, + { + "coverage": 0.6376666666666667, + "risk": 0.24202822791427078, + "threshold": 0.09717839590590162 + }, + { + "coverage": 0.638, + "risk": 0.241901776384535, + "threshold": 0.09701476080597407 + }, + { + "coverage": 0.6383333333333333, + "risk": 0.24177545691906005, + "threshold": 0.09634374536534857 + }, + { + "coverage": 0.6386666666666667, + "risk": 0.24164926931106473, + "threshold": 0.0959274992857123 + }, + { + "coverage": 0.639, + "risk": 0.24152321335419927, + "threshold": 0.09570849976972826 + }, + { + "coverage": 0.6393333333333333, + "risk": 0.24139728884254433, + "threshold": 0.09570085122224223 + }, + { + "coverage": 0.6396666666666667, + "risk": 0.2412714955706097, + "threshold": 0.09470666046530303 + }, + { + "coverage": 0.64, + "risk": 0.24114583333333334, + "threshold": 0.09462116040732914 + }, + { + "coverage": 0.6403333333333333, + "risk": 0.24102030192608018, + "threshold": 0.09453545638723893 + }, + { + "coverage": 0.6406666666666667, + "risk": 0.240894901144641, + "threshold": 0.09443356751135829 + }, + { + "coverage": 0.641, + "risk": 0.24076963078523142, + "threshold": 0.09432846417887382 + }, + { + "coverage": 0.6413333333333333, + "risk": 0.24064449064449064, + "threshold": 0.09425224756663945 + }, + { + "coverage": 0.6416666666666667, + "risk": 0.24051948051948052, + "threshold": 0.0934284543674296 + }, + { + "coverage": 0.642, + "risk": 0.24039460020768433, + "threshold": 0.09310416326088196 + }, + { + "coverage": 0.6423333333333333, + "risk": 0.2402698495070057, + "threshold": 0.09275478406295394 + }, + { + "coverage": 0.6426666666666667, + "risk": 0.24014522821576764, + "threshold": 0.09259925962361533 + }, + { + "coverage": 0.643, + "risk": 0.24002073613271124, + "threshold": 0.09250106032471804 + }, + { + "coverage": 0.6433333333333333, + "risk": 0.23989637305699482, + "threshold": 0.09190418212057824 + }, + { + "coverage": 0.6436666666666667, + "risk": 0.23977213878819265, + "threshold": 0.09186262006739954 + }, + { + "coverage": 0.644, + "risk": 0.23964803312629399, + "threshold": 0.09131510660636286 + }, + { + "coverage": 0.6443333333333333, + "risk": 0.23952405587170203, + "threshold": 0.09129990985175526 + }, + { + "coverage": 0.6446666666666667, + "risk": 0.2394002068252327, + "threshold": 0.09046885636178431 + }, + { + "coverage": 0.645, + "risk": 0.2392764857881137, + "threshold": 0.09031467533272755 + }, + { + "coverage": 0.6453333333333333, + "risk": 0.23915289256198347, + "threshold": 0.08995881620602923 + }, + { + "coverage": 0.6456666666666667, + "risk": 0.23902942694889004, + "threshold": 0.08993078341463165 + }, + { + "coverage": 0.646, + "risk": 0.23890608875129, + "threshold": 0.08984352508819089 + }, + { + "coverage": 0.6463333333333333, + "risk": 0.23878287777204746, + "threshold": 0.0896015238513805 + }, + { + "coverage": 0.6466666666666666, + "risk": 0.238659793814433, + "threshold": 0.08945900317846531 + }, + { + "coverage": 0.647, + "risk": 0.2385368366821226, + "threshold": 0.08938567927490065 + }, + { + "coverage": 0.6473333333333333, + "risk": 0.2384140061791967, + "threshold": 0.08915479234266961 + }, + { + "coverage": 0.6476666666666666, + "risk": 0.23829130211013896, + "threshold": 0.0887626930937747 + }, + { + "coverage": 0.648, + "risk": 0.2381687242798354, + "threshold": 0.08869506556913877 + }, + { + "coverage": 0.6483333333333333, + "risk": 0.23804627249357327, + "threshold": 0.08863286279678637 + }, + { + "coverage": 0.6486666666666666, + "risk": 0.23792394655704008, + "threshold": 0.08842147243563836 + }, + { + "coverage": 0.649, + "risk": 0.23780174627632256, + "threshold": 0.08761556455240334 + }, + { + "coverage": 0.6493333333333333, + "risk": 0.23767967145790556, + "threshold": 0.0874793262866133 + }, + { + "coverage": 0.6496666666666666, + "risk": 0.2375577219086711, + "threshold": 0.08738620705469297 + }, + { + "coverage": 0.65, + "risk": 0.23743589743589744, + "threshold": 0.08650816585147691 + }, + { + "coverage": 0.6503333333333333, + "risk": 0.23731419784725782, + "threshold": 0.08647260211041642 + }, + { + "coverage": 0.6506666666666666, + "risk": 0.23719262295081966, + "threshold": 0.08589623403033164 + }, + { + "coverage": 0.651, + "risk": 0.23707117255504354, + "threshold": 0.08571134779389675 + }, + { + "coverage": 0.6513333333333333, + "risk": 0.23694984646878198, + "threshold": 0.08558355650011949 + }, + { + "coverage": 0.6516666666666666, + "risk": 0.23682864450127877, + "threshold": 0.08554491961847537 + }, + { + "coverage": 0.652, + "risk": 0.2367075664621677, + "threshold": 0.08552336393399837 + }, + { + "coverage": 0.6523333333333333, + "risk": 0.23658661216147164, + "threshold": 0.08502898483820434 + }, + { + "coverage": 0.6526666666666666, + "risk": 0.23646578140960164, + "threshold": 0.08497933699185647 + }, + { + "coverage": 0.653, + "risk": 0.2363450740173558, + "threshold": 0.08467892931317565 + }, + { + "coverage": 0.6533333333333333, + "risk": 0.23622448979591837, + "threshold": 0.08401422903218404 + }, + { + "coverage": 0.6536666666666666, + "risk": 0.23610402855685875, + "threshold": 0.08384676420840156 + }, + { + "coverage": 0.654, + "risk": 0.23598369011213047, + "threshold": 0.08372086089876178 + }, + { + "coverage": 0.6543333333333333, + "risk": 0.2358634742740703, + "threshold": 0.08356217240765017 + }, + { + "coverage": 0.6546666666666666, + "risk": 0.23574338085539714, + "threshold": 0.08340842756306223 + }, + { + "coverage": 0.655, + "risk": 0.2356234096692112, + "threshold": 0.08331755598580531 + }, + { + "coverage": 0.6553333333333333, + "risk": 0.23550356052899288, + "threshold": 0.08305727480975596 + }, + { + "coverage": 0.6556666666666666, + "risk": 0.23538383324860193, + "threshold": 0.08301655693085963 + }, + { + "coverage": 0.656, + "risk": 0.23526422764227642, + "threshold": 0.08298232164558307 + }, + { + "coverage": 0.6563333333333333, + "risk": 0.2351447435246318, + "threshold": 0.08293106840004283 + }, + { + "coverage": 0.6566666666666666, + "risk": 0.2350253807106599, + "threshold": 0.08286478766968836 + }, + { + "coverage": 0.657, + "risk": 0.23490613901572804, + "threshold": 0.08284234195463668 + }, + { + "coverage": 0.6573333333333333, + "risk": 0.2347870182555781, + "threshold": 0.08255385832356887 + }, + { + "coverage": 0.6576666666666666, + "risk": 0.23466801824632538, + "threshold": 0.08247207160624398 + }, + { + "coverage": 0.658, + "risk": 0.23454913880445796, + "threshold": 0.08246674347276246 + }, + { + "coverage": 0.6583333333333333, + "risk": 0.23443037974683545, + "threshold": 0.082238894131406 + }, + { + "coverage": 0.6586666666666666, + "risk": 0.23431174089068826, + "threshold": 0.08205607503493492 + }, + { + "coverage": 0.659, + "risk": 0.2341932220536166, + "threshold": 0.08204796028105668 + }, + { + "coverage": 0.6593333333333333, + "risk": 0.23407482305358948, + "threshold": 0.0819148136314922 + }, + { + "coverage": 0.6596666666666666, + "risk": 0.23395654370894392, + "threshold": 0.08180465969236787 + }, + { + "coverage": 0.66, + "risk": 0.23383838383838385, + "threshold": 0.08166289825430718 + }, + { + "coverage": 0.6603333333333333, + "risk": 0.2337203432609793, + "threshold": 0.08155052573665401 + }, + { + "coverage": 0.6606666666666666, + "risk": 0.2341069626639758, + "threshold": 0.08136322026482844 + }, + { + "coverage": 0.661, + "risk": 0.23398890569843672, + "threshold": 0.08085701178259928 + }, + { + "coverage": 0.6613333333333333, + "risk": 0.23387096774193547, + "threshold": 0.08068081541260505 + }, + { + "coverage": 0.6616666666666666, + "risk": 0.23375314861460958, + "threshold": 0.08041020651303028 + }, + { + "coverage": 0.662, + "risk": 0.23363544813695872, + "threshold": 0.07993640817724285 + }, + { + "coverage": 0.6623333333333333, + "risk": 0.233517866129844, + "threshold": 0.07984450656505641 + }, + { + "coverage": 0.6626666666666666, + "risk": 0.23340040241448692, + "threshold": 0.07976805788589983 + }, + { + "coverage": 0.663, + "risk": 0.23328305681246858, + "threshold": 0.0794658109688179 + }, + { + "coverage": 0.6633333333333333, + "risk": 0.23316582914572864, + "threshold": 0.07942305012677717 + }, + { + "coverage": 0.6636666666666666, + "risk": 0.23304871923656453, + "threshold": 0.0792410923055513 + }, + { + "coverage": 0.664, + "risk": 0.23293172690763053, + "threshold": 0.07889734495641237 + }, + { + "coverage": 0.6643333333333333, + "risk": 0.23281485198193677, + "threshold": 0.07867700088142847 + }, + { + "coverage": 0.6646666666666666, + "risk": 0.23269809428284854, + "threshold": 0.07839121247881821 + }, + { + "coverage": 0.665, + "risk": 0.23258145363408522, + "threshold": 0.07835766157937828 + }, + { + "coverage": 0.6653333333333333, + "risk": 0.23246492985971945, + "threshold": 0.07827586969025566 + }, + { + "coverage": 0.6656666666666666, + "risk": 0.23234852278417625, + "threshold": 0.07810666379229181 + }, + { + "coverage": 0.666, + "risk": 0.23223223223223224, + "threshold": 0.07795116036382015 + }, + { + "coverage": 0.6663333333333333, + "risk": 0.2321160580290145, + "threshold": 0.07785759041706378 + }, + { + "coverage": 0.6666666666666666, + "risk": 0.232, + "threshold": 0.07776541912014519 + }, + { + "coverage": 0.667, + "risk": 0.2318840579710145, + "threshold": 0.07771501375224307 + }, + { + "coverage": 0.6673333333333333, + "risk": 0.23176823176823177, + "threshold": 0.07761415459814772 + }, + { + "coverage": 0.6676666666666666, + "risk": 0.23165252121817273, + "threshold": 0.0775816636424036 + }, + { + "coverage": 0.668, + "risk": 0.2315369261477046, + "threshold": 0.07750253948549883 + }, + { + "coverage": 0.6683333333333333, + "risk": 0.2314214463840399, + "threshold": 0.0771862495399327 + }, + { + "coverage": 0.6686666666666666, + "risk": 0.2313060817547358, + "threshold": 0.07718355014268483 + }, + { + "coverage": 0.669, + "risk": 0.2311908320876931, + "threshold": 0.07701318874862334 + }, + { + "coverage": 0.6693333333333333, + "risk": 0.23107569721115537, + "threshold": 0.07680484583776427 + }, + { + "coverage": 0.6696666666666666, + "risk": 0.2309606769537083, + "threshold": 0.07651749683473841 + }, + { + "coverage": 0.67, + "risk": 0.2308457711442786, + "threshold": 0.07645911914969404 + }, + { + "coverage": 0.6703333333333333, + "risk": 0.23073097961213326, + "threshold": 0.07600143510167254 + }, + { + "coverage": 0.6706666666666666, + "risk": 0.23061630218687873, + "threshold": 0.07574590648521491 + }, + { + "coverage": 0.671, + "risk": 0.23050173869846002, + "threshold": 0.0756628871354678 + }, + { + "coverage": 0.6713333333333333, + "risk": 0.23038728897715988, + "threshold": 0.07560854797052702 + }, + { + "coverage": 0.6716666666666666, + "risk": 0.23027295285359803, + "threshold": 0.07558191793553763 + }, + { + "coverage": 0.672, + "risk": 0.23015873015873015, + "threshold": 0.07535431021173691 + }, + { + "coverage": 0.6723333333333333, + "risk": 0.2300446207238473, + "threshold": 0.07531012383042585 + }, + { + "coverage": 0.6726666666666666, + "risk": 0.22993062438057482, + "threshold": 0.07527059643375215 + }, + { + "coverage": 0.673, + "risk": 0.2298167409608717, + "threshold": 0.07526020814276185 + }, + { + "coverage": 0.6733333333333333, + "risk": 0.2297029702970297, + "threshold": 0.07522566728026149 + }, + { + "coverage": 0.6736666666666666, + "risk": 0.22958931222167245, + "threshold": 0.07516710373915052 + }, + { + "coverage": 0.674, + "risk": 0.2294757665677547, + "threshold": 0.07477265720771323 + }, + { + "coverage": 0.6743333333333333, + "risk": 0.22936233316856156, + "threshold": 0.07420598124443226 + }, + { + "coverage": 0.6746666666666666, + "risk": 0.22924901185770752, + "threshold": 0.0738038852136667 + }, + { + "coverage": 0.675, + "risk": 0.2291358024691358, + "threshold": 0.07363864431070116 + }, + { + "coverage": 0.6753333333333333, + "risk": 0.22902270483711748, + "threshold": 0.07358431878861069 + }, + { + "coverage": 0.6756666666666666, + "risk": 0.2289097187962506, + "threshold": 0.07357396506053332 + }, + { + "coverage": 0.676, + "risk": 0.22879684418145957, + "threshold": 0.07332665402963358 + }, + { + "coverage": 0.6763333333333333, + "risk": 0.22868408082799407, + "threshold": 0.07313290039478883 + }, + { + "coverage": 0.6766666666666666, + "risk": 0.22857142857142856, + "threshold": 0.0730073106901303 + }, + { + "coverage": 0.677, + "risk": 0.22845888724766125, + "threshold": 0.07260488552847841 + }, + { + "coverage": 0.6773333333333333, + "risk": 0.2283464566929134, + "threshold": 0.0724348424952129 + }, + { + "coverage": 0.6776666666666666, + "risk": 0.22823413674372847, + "threshold": 0.07238910696303238 + }, + { + "coverage": 0.678, + "risk": 0.22812192723697147, + "threshold": 0.07238271538607549 + }, + { + "coverage": 0.6783333333333333, + "risk": 0.228009828009828, + "threshold": 0.07235782418837639 + }, + { + "coverage": 0.6786666666666666, + "risk": 0.22789783889980353, + "threshold": 0.07227222061412666 + }, + { + "coverage": 0.679, + "risk": 0.22778595974472263, + "threshold": 0.07213162784761384 + }, + { + "coverage": 0.6793333333333333, + "risk": 0.22767419038272815, + "threshold": 0.07183666459228658 + }, + { + "coverage": 0.6796666666666666, + "risk": 0.22756253065228052, + "threshold": 0.07179983033521173 + }, + { + "coverage": 0.68, + "risk": 0.22745098039215686, + "threshold": 0.07171750758653929 + }, + { + "coverage": 0.6803333333333333, + "risk": 0.22733953944145027, + "threshold": 0.07159442174448337 + }, + { + "coverage": 0.6806666666666666, + "risk": 0.22722820763956905, + "threshold": 0.0712982189235151 + }, + { + "coverage": 0.681, + "risk": 0.22711698482623593, + "threshold": 0.07114876647611339 + }, + { + "coverage": 0.6813333333333333, + "risk": 0.22700587084148727, + "threshold": 0.07109073960695253 + }, + { + "coverage": 0.6816666666666666, + "risk": 0.22689486552567237, + "threshold": 0.07091479867957111 + }, + { + "coverage": 0.682, + "risk": 0.22678396871945258, + "threshold": 0.07052402850704227 + }, + { + "coverage": 0.6823333333333333, + "risk": 0.22667318026380068, + "threshold": 0.07034197739281207 + }, + { + "coverage": 0.6826666666666666, + "risk": 0.2265625, + "threshold": 0.07028606985957367 + }, + { + "coverage": 0.683, + "risk": 0.22645192776964373, + "threshold": 0.06969718685354803 + }, + { + "coverage": 0.6833333333333333, + "risk": 0.22634146341463415, + "threshold": 0.06963156662186593 + }, + { + "coverage": 0.6836666666666666, + "risk": 0.22623110677718186, + "threshold": 0.06939740522760998 + }, + { + "coverage": 0.684, + "risk": 0.22612085769980506, + "threshold": 0.06916786045149136 + }, + { + "coverage": 0.6843333333333333, + "risk": 0.2260107160253288, + "threshold": 0.06913687332863379 + }, + { + "coverage": 0.6846666666666666, + "risk": 0.22590068159688412, + "threshold": 0.0688339186062628 + }, + { + "coverage": 0.685, + "risk": 0.22579075425790754, + "threshold": 0.06865236684178115 + }, + { + "coverage": 0.6853333333333333, + "risk": 0.22568093385214008, + "threshold": 0.06856121630207426 + }, + { + "coverage": 0.6856666666666666, + "risk": 0.22557122022362663, + "threshold": 0.06848971661499408 + }, + { + "coverage": 0.686, + "risk": 0.22546161321671526, + "threshold": 0.06845780593819695 + }, + { + "coverage": 0.6863333333333334, + "risk": 0.22535211267605634, + "threshold": 0.06811342530502143 + }, + { + "coverage": 0.6866666666666666, + "risk": 0.22524271844660193, + "threshold": 0.06762145381981342 + }, + { + "coverage": 0.687, + "risk": 0.22513343037360506, + "threshold": 0.06754827302072453 + }, + { + "coverage": 0.6873333333333334, + "risk": 0.22502424830261883, + "threshold": 0.06753154879984402 + }, + { + "coverage": 0.6876666666666666, + "risk": 0.22491517207949588, + "threshold": 0.06751752009289531 + }, + { + "coverage": 0.688, + "risk": 0.2248062015503876, + "threshold": 0.06732604160330573 + }, + { + "coverage": 0.6883333333333334, + "risk": 0.22469733656174334, + "threshold": 0.06730431733992201 + }, + { + "coverage": 0.6886666666666666, + "risk": 0.2245885769603098, + "threshold": 0.06730414461772333 + }, + { + "coverage": 0.689, + "risk": 0.22447992259313013, + "threshold": 0.06714813295842295 + }, + { + "coverage": 0.6893333333333334, + "risk": 0.22437137330754353, + "threshold": 0.06679520442959133 + }, + { + "coverage": 0.6896666666666667, + "risk": 0.22426292895118416, + "threshold": 0.06675937506636098 + }, + { + "coverage": 0.69, + "risk": 0.22415458937198068, + "threshold": 0.06662373408239425 + }, + { + "coverage": 0.6903333333333334, + "risk": 0.22404635441815549, + "threshold": 0.06625642466615614 + }, + { + "coverage": 0.6906666666666667, + "risk": 0.22393822393822393, + "threshold": 0.06618868363303447 + }, + { + "coverage": 0.691, + "risk": 0.22383019778099372, + "threshold": 0.06611418552988145 + }, + { + "coverage": 0.6913333333333334, + "risk": 0.22420443587270975, + "threshold": 0.06609977264846459 + }, + { + "coverage": 0.6916666666666667, + "risk": 0.22457831325301206, + "threshold": 0.06563072107196471 + }, + { + "coverage": 0.692, + "risk": 0.22447013487475914, + "threshold": 0.06553584775010814 + }, + { + "coverage": 0.6923333333333334, + "risk": 0.22436206066441983, + "threshold": 0.06539346459933792 + }, + { + "coverage": 0.6926666666666667, + "risk": 0.22425409047160733, + "threshold": 0.06536811968784441 + }, + { + "coverage": 0.693, + "risk": 0.22414622414622415, + "threshold": 0.06523710037134457 + }, + { + "coverage": 0.6933333333333334, + "risk": 0.22403846153846155, + "threshold": 0.0650660524305536 + }, + { + "coverage": 0.6936666666666667, + "risk": 0.22393080249879865, + "threshold": 0.06502670112526869 + }, + { + "coverage": 0.694, + "risk": 0.22382324687800192, + "threshold": 0.0649541188341944 + }, + { + "coverage": 0.6943333333333334, + "risk": 0.22371579452712434, + "threshold": 0.0646926311138541 + }, + { + "coverage": 0.6946666666666667, + "risk": 0.2236084452975048, + "threshold": 0.06430348341891139 + }, + { + "coverage": 0.695, + "risk": 0.22350119904076737, + "threshold": 0.06379154166447297 + }, + { + "coverage": 0.6953333333333334, + "risk": 0.2233940556088207, + "threshold": 0.06369092122007738 + }, + { + "coverage": 0.6956666666666667, + "risk": 0.2232870148538572, + "threshold": 0.06367287025362163 + }, + { + "coverage": 0.696, + "risk": 0.2231800766283525, + "threshold": 0.06354418511595859 + }, + { + "coverage": 0.6963333333333334, + "risk": 0.22307324078506463, + "threshold": 0.06332387637299075 + }, + { + "coverage": 0.6966666666666667, + "risk": 0.22296650717703348, + "threshold": 0.06296992893578456 + }, + { + "coverage": 0.697, + "risk": 0.22285987565758011, + "threshold": 0.06252980258551401 + }, + { + "coverage": 0.6973333333333334, + "risk": 0.22275334608030592, + "threshold": 0.06239555485202587 + }, + { + "coverage": 0.6976666666666667, + "risk": 0.22264691829909222, + "threshold": 0.062263459196750304 + }, + { + "coverage": 0.698, + "risk": 0.22254059216809932, + "threshold": 0.062222353204029225 + }, + { + "coverage": 0.6983333333333334, + "risk": 0.2224343675417661, + "threshold": 0.06205201114260193 + }, + { + "coverage": 0.6986666666666667, + "risk": 0.22232824427480916, + "threshold": 0.0617868507817548 + }, + { + "coverage": 0.699, + "risk": 0.2222222222222222, + "threshold": 0.061733796461669516 + }, + { + "coverage": 0.6993333333333334, + "risk": 0.2221163012392755, + "threshold": 0.06117371997248825 + }, + { + "coverage": 0.6996666666666667, + "risk": 0.222010481181515, + "threshold": 0.06091450072449844 + }, + { + "coverage": 0.7, + "risk": 0.2219047619047619, + "threshold": 0.06081759384433477 + }, + { + "coverage": 0.7003333333333334, + "risk": 0.22179914326511185, + "threshold": 0.060801148938424504 + }, + { + "coverage": 0.7006666666666667, + "risk": 0.22169362511893434, + "threshold": 0.060606357435176325 + }, + { + "coverage": 0.701, + "risk": 0.2215882073228721, + "threshold": 0.06027825470294823 + }, + { + "coverage": 0.7013333333333334, + "risk": 0.2214828897338403, + "threshold": 0.06019085099078013 + }, + { + "coverage": 0.7016666666666667, + "risk": 0.22137767220902613, + "threshold": 0.059792416912202884 + }, + { + "coverage": 0.702, + "risk": 0.22127255460588793, + "threshold": 0.05978569291621152 + }, + { + "coverage": 0.7023333333333334, + "risk": 0.22116753678215473, + "threshold": 0.059577173966468194 + }, + { + "coverage": 0.7026666666666667, + "risk": 0.22106261859582543, + "threshold": 0.0591392478735815 + }, + { + "coverage": 0.703, + "risk": 0.22095779990516834, + "threshold": 0.05891485630557271 + }, + { + "coverage": 0.7033333333333334, + "risk": 0.2213270142180095, + "threshold": 0.05890763469578268 + }, + { + "coverage": 0.7036666666666667, + "risk": 0.22122216958787305, + "threshold": 0.05889276162252396 + }, + { + "coverage": 0.704, + "risk": 0.22111742424242425, + "threshold": 0.05881009502451265 + }, + { + "coverage": 0.7043333333333334, + "risk": 0.22101277804070044, + "threshold": 0.05856115216919596 + }, + { + "coverage": 0.7046666666666667, + "risk": 0.22090823084200567, + "threshold": 0.058527803219394416 + }, + { + "coverage": 0.705, + "risk": 0.22080378250591018, + "threshold": 0.05851625766778868 + }, + { + "coverage": 0.7053333333333334, + "risk": 0.22069943289224953, + "threshold": 0.058507072604971296 + }, + { + "coverage": 0.7056666666666667, + "risk": 0.22059518186112423, + "threshold": 0.05835653141148124 + }, + { + "coverage": 0.706, + "risk": 0.22049102927289896, + "threshold": 0.058053437113128235 + }, + { + "coverage": 0.7063333333333334, + "risk": 0.22038697498820198, + "threshold": 0.05782567989575249 + }, + { + "coverage": 0.7066666666666667, + "risk": 0.22028301886792453, + "threshold": 0.05781163474581857 + }, + { + "coverage": 0.707, + "risk": 0.22017916077322017, + "threshold": 0.05748975695841805 + }, + { + "coverage": 0.7073333333333334, + "risk": 0.22007540056550423, + "threshold": 0.05748609402105099 + }, + { + "coverage": 0.7076666666666667, + "risk": 0.22044276966556758, + "threshold": 0.05721032408141546 + }, + { + "coverage": 0.708, + "risk": 0.22033898305084745, + "threshold": 0.05714151394488489 + }, + { + "coverage": 0.7083333333333334, + "risk": 0.22023529411764706, + "threshold": 0.05702112708125046 + }, + { + "coverage": 0.7086666666666667, + "risk": 0.22013170272812793, + "threshold": 0.0569923222621251 + }, + { + "coverage": 0.709, + "risk": 0.22002820874471085, + "threshold": 0.05669320013293434 + }, + { + "coverage": 0.7093333333333334, + "risk": 0.2199248120300752, + "threshold": 0.05665912896760696 + }, + { + "coverage": 0.7096666666666667, + "risk": 0.21982151244715828, + "threshold": 0.05659353171200851 + }, + { + "coverage": 0.71, + "risk": 0.21971830985915494, + "threshold": 0.056249422243363906 + }, + { + "coverage": 0.7103333333333334, + "risk": 0.21961520412951666, + "threshold": 0.05614044500393749 + }, + { + "coverage": 0.7106666666666667, + "risk": 0.21951219512195122, + "threshold": 0.05587826062131868 + }, + { + "coverage": 0.711, + "risk": 0.21940928270042195, + "threshold": 0.05584780888749085 + }, + { + "coverage": 0.7113333333333334, + "risk": 0.21930646672914714, + "threshold": 0.05565858586410335 + }, + { + "coverage": 0.7116666666666667, + "risk": 0.21920374707259954, + "threshold": 0.05562685871335498 + }, + { + "coverage": 0.712, + "risk": 0.21910112359550563, + "threshold": 0.05557871921140984 + }, + { + "coverage": 0.7123333333333334, + "risk": 0.21899859616284512, + "threshold": 0.055576425859921195 + }, + { + "coverage": 0.7126666666666667, + "risk": 0.21889616463985032, + "threshold": 0.05551527717851008 + }, + { + "coverage": 0.713, + "risk": 0.2187938288920056, + "threshold": 0.05542934147538046 + }, + { + "coverage": 0.7133333333333334, + "risk": 0.21869158878504674, + "threshold": 0.05542249397786731 + }, + { + "coverage": 0.7136666666666667, + "risk": 0.2185894441849603, + "threshold": 0.05539962801763544 + }, + { + "coverage": 0.714, + "risk": 0.2184873949579832, + "threshold": 0.05515403667301588 + }, + { + "coverage": 0.7143333333333334, + "risk": 0.21838544097060195, + "threshold": 0.055034831330934814 + }, + { + "coverage": 0.7146666666666667, + "risk": 0.21828358208955223, + "threshold": 0.05481806763188018 + }, + { + "coverage": 0.715, + "risk": 0.21818181818181817, + "threshold": 0.05467541494989313 + }, + { + "coverage": 0.7153333333333334, + "risk": 0.21808014911463186, + "threshold": 0.05449528678881087 + }, + { + "coverage": 0.7156666666666667, + "risk": 0.21797857475547275, + "threshold": 0.054440394263307094 + }, + { + "coverage": 0.716, + "risk": 0.21787709497206703, + "threshold": 0.0542053454583223 + }, + { + "coverage": 0.7163333333333334, + "risk": 0.21777570963238715, + "threshold": 0.054188889192742375 + }, + { + "coverage": 0.7166666666666667, + "risk": 0.21767441860465117, + "threshold": 0.0540144030472062 + }, + { + "coverage": 0.717, + "risk": 0.2175732217573222, + "threshold": 0.053969385556004196 + }, + { + "coverage": 0.7173333333333334, + "risk": 0.2174721189591078, + "threshold": 0.05393218760996871 + }, + { + "coverage": 0.7176666666666667, + "risk": 0.2173711100789596, + "threshold": 0.053927364627579376 + }, + { + "coverage": 0.718, + "risk": 0.21727019498607242, + "threshold": 0.053844544024431146 + }, + { + "coverage": 0.7183333333333334, + "risk": 0.217169373549884, + "threshold": 0.05346663010421324 + }, + { + "coverage": 0.7186666666666667, + "risk": 0.21706864564007422, + "threshold": 0.053336253469318404 + }, + { + "coverage": 0.719, + "risk": 0.21696801112656466, + "threshold": 0.0532279979033882 + }, + { + "coverage": 0.7193333333333334, + "risk": 0.21686746987951808, + "threshold": 0.05321827813177993 + }, + { + "coverage": 0.7196666666666667, + "risk": 0.21676702176933765, + "threshold": 0.053151384056692515 + }, + { + "coverage": 0.72, + "risk": 0.21666666666666667, + "threshold": 0.052970379406588344 + }, + { + "coverage": 0.7203333333333334, + "risk": 0.21656640444238778, + "threshold": 0.05283529845016146 + }, + { + "coverage": 0.7206666666666667, + "risk": 0.21646623496762257, + "threshold": 0.05275161876750975 + }, + { + "coverage": 0.721, + "risk": 0.21636615811373092, + "threshold": 0.05275104670815557 + }, + { + "coverage": 0.7213333333333334, + "risk": 0.21626617375231053, + "threshold": 0.05268092994394554 + }, + { + "coverage": 0.7216666666666667, + "risk": 0.2161662817551963, + "threshold": 0.05258029904278523 + }, + { + "coverage": 0.722, + "risk": 0.21606648199445982, + "threshold": 0.05240658328738427 + }, + { + "coverage": 0.7223333333333334, + "risk": 0.21596677434240885, + "threshold": 0.052297610359508466 + }, + { + "coverage": 0.7226666666666667, + "risk": 0.2158671586715867, + "threshold": 0.0522065384782912 + }, + { + "coverage": 0.723, + "risk": 0.2157676348547718, + "threshold": 0.052129979697949184 + }, + { + "coverage": 0.7233333333333334, + "risk": 0.21566820276497695, + "threshold": 0.05211273814587473 + }, + { + "coverage": 0.7236666666666667, + "risk": 0.2155688622754491, + "threshold": 0.05199020087006199 + }, + { + "coverage": 0.724, + "risk": 0.2154696132596685, + "threshold": 0.05167345831072866 + }, + { + "coverage": 0.7243333333333334, + "risk": 0.21537045559134838, + "threshold": 0.05166678805342263 + }, + { + "coverage": 0.7246666666666667, + "risk": 0.21527138914443422, + "threshold": 0.05151868445918581 + }, + { + "coverage": 0.725, + "risk": 0.21517241379310345, + "threshold": 0.051505830301944666 + }, + { + "coverage": 0.7253333333333334, + "risk": 0.21507352941176472, + "threshold": 0.05146846663835694 + }, + { + "coverage": 0.7256666666666667, + "risk": 0.21497473587505742, + "threshold": 0.05140110627617698 + }, + { + "coverage": 0.726, + "risk": 0.21487603305785125, + "threshold": 0.05134858098485863 + }, + { + "coverage": 0.7263333333333334, + "risk": 0.21477742083524554, + "threshold": 0.05122044771592699 + }, + { + "coverage": 0.7266666666666667, + "risk": 0.21467889908256882, + "threshold": 0.051147400202074436 + }, + { + "coverage": 0.727, + "risk": 0.21458046767537828, + "threshold": 0.05105333909351644 + }, + { + "coverage": 0.7273333333333334, + "risk": 0.21448212648945922, + "threshold": 0.05060462759033986 + }, + { + "coverage": 0.7276666666666667, + "risk": 0.21438387540082454, + "threshold": 0.050551388380788716 + }, + { + "coverage": 0.728, + "risk": 0.21428571428571427, + "threshold": 0.050476403550651465 + }, + { + "coverage": 0.7283333333333334, + "risk": 0.21418764302059495, + "threshold": 0.05042625118585267 + }, + { + "coverage": 0.7286666666666667, + "risk": 0.2140896614821592, + "threshold": 0.05042016504086616 + }, + { + "coverage": 0.729, + "risk": 0.2139917695473251, + "threshold": 0.05035485523106024 + }, + { + "coverage": 0.7293333333333333, + "risk": 0.21389396709323583, + "threshold": 0.0502820941739359 + }, + { + "coverage": 0.7296666666666667, + "risk": 0.21379625399725902, + "threshold": 0.05023066823708428 + }, + { + "coverage": 0.73, + "risk": 0.2136986301369863, + "threshold": 0.050156115435610875 + }, + { + "coverage": 0.7303333333333333, + "risk": 0.21360109539023278, + "threshold": 0.04957749268217491 + }, + { + "coverage": 0.7306666666666667, + "risk": 0.21350364963503649, + "threshold": 0.049371385136218415 + }, + { + "coverage": 0.731, + "risk": 0.213406292749658, + "threshold": 0.049188547377565006 + }, + { + "coverage": 0.7313333333333333, + "risk": 0.21330902461257975, + "threshold": 0.04904232278852663 + }, + { + "coverage": 0.7316666666666667, + "risk": 0.21321184510250568, + "threshold": 0.04883465408168466 + }, + { + "coverage": 0.732, + "risk": 0.21311475409836064, + "threshold": 0.04855789095550411 + }, + { + "coverage": 0.7323333333333333, + "risk": 0.21301775147928995, + "threshold": 0.04848409973800358 + }, + { + "coverage": 0.7326666666666667, + "risk": 0.21292083712465878, + "threshold": 0.04812486527721527 + }, + { + "coverage": 0.733, + "risk": 0.21282401091405184, + "threshold": 0.04802281016676849 + }, + { + "coverage": 0.7333333333333333, + "risk": 0.21272727272727274, + "threshold": 0.04796502816633911 + }, + { + "coverage": 0.7336666666666667, + "risk": 0.21263062244434347, + "threshold": 0.04778260892969561 + }, + { + "coverage": 0.734, + "risk": 0.2125340599455041, + "threshold": 0.047518312076087046 + }, + { + "coverage": 0.7343333333333333, + "risk": 0.212437585111212, + "threshold": 0.04749422118274793 + }, + { + "coverage": 0.7346666666666667, + "risk": 0.21234119782214156, + "threshold": 0.04705878694550372 + }, + { + "coverage": 0.735, + "risk": 0.21224489795918366, + "threshold": 0.04704439151524065 + }, + { + "coverage": 0.7353333333333333, + "risk": 0.21214868540344514, + "threshold": 0.04694434498687019 + }, + { + "coverage": 0.7356666666666667, + "risk": 0.2120525600362483, + "threshold": 0.04692761263860705 + }, + { + "coverage": 0.736, + "risk": 0.21195652173913043, + "threshold": 0.046898092394862735 + }, + { + "coverage": 0.7363333333333333, + "risk": 0.21186057039384337, + "threshold": 0.04657219065428347 + }, + { + "coverage": 0.7366666666666667, + "risk": 0.21176470588235294, + "threshold": 0.046422392402496286 + }, + { + "coverage": 0.737, + "risk": 0.21166892808683854, + "threshold": 0.04638615432060219 + }, + { + "coverage": 0.7373333333333333, + "risk": 0.2115732368896926, + "threshold": 0.0463340886224556 + }, + { + "coverage": 0.7376666666666667, + "risk": 0.2114776321735201, + "threshold": 0.046136968624622625 + }, + { + "coverage": 0.738, + "risk": 0.21138211382113822, + "threshold": 0.04571633164348297 + }, + { + "coverage": 0.7383333333333333, + "risk": 0.21128668171557563, + "threshold": 0.0455601517060252 + }, + { + "coverage": 0.7386666666666667, + "risk": 0.2111913357400722, + "threshold": 0.04551578114445733 + }, + { + "coverage": 0.739, + "risk": 0.21109607577807848, + "threshold": 0.045333618782937325 + }, + { + "coverage": 0.7393333333333333, + "risk": 0.2110009017132552, + "threshold": 0.04527399632758152 + }, + { + "coverage": 0.7396666666666667, + "risk": 0.21090581342947273, + "threshold": 0.04525003050287974 + }, + { + "coverage": 0.74, + "risk": 0.21081081081081082, + "threshold": 0.04506705667187101 + }, + { + "coverage": 0.7403333333333333, + "risk": 0.21071589374155786, + "threshold": 0.045062548567015914 + }, + { + "coverage": 0.7406666666666667, + "risk": 0.21062106210621062, + "threshold": 0.04478393566303808 + }, + { + "coverage": 0.741, + "risk": 0.21052631578947367, + "threshold": 0.04470612289289544 + }, + { + "coverage": 0.7413333333333333, + "risk": 0.210431654676259, + "threshold": 0.044531323226797814 + }, + { + "coverage": 0.7416666666666667, + "risk": 0.21033707865168538, + "threshold": 0.04436665763027983 + }, + { + "coverage": 0.742, + "risk": 0.21024258760107817, + "threshold": 0.04435243288792868 + }, + { + "coverage": 0.7423333333333333, + "risk": 0.21014818140996858, + "threshold": 0.04422368110911154 + }, + { + "coverage": 0.7426666666666667, + "risk": 0.21005385996409337, + "threshold": 0.04411280223754827 + }, + { + "coverage": 0.743, + "risk": 0.20995962314939434, + "threshold": 0.04410564261830386 + }, + { + "coverage": 0.7433333333333333, + "risk": 0.20986547085201793, + "threshold": 0.044095920210354966 + }, + { + "coverage": 0.7436666666666667, + "risk": 0.20977140295831465, + "threshold": 0.04407554594288004 + }, + { + "coverage": 0.744, + "risk": 0.20967741935483872, + "threshold": 0.04400347233563096 + }, + { + "coverage": 0.7443333333333333, + "risk": 0.2095835199283475, + "threshold": 0.04394039471888454 + }, + { + "coverage": 0.7446666666666667, + "risk": 0.20948970456580127, + "threshold": 0.04379962421547952 + }, + { + "coverage": 0.745, + "risk": 0.20939597315436242, + "threshold": 0.04312809076118057 + }, + { + "coverage": 0.7453333333333333, + "risk": 0.20930232558139536, + "threshold": 0.043093754106963394 + }, + { + "coverage": 0.7456666666666667, + "risk": 0.20920876173446581, + "threshold": 0.042850384497832766 + }, + { + "coverage": 0.746, + "risk": 0.20911528150134048, + "threshold": 0.04282058103619931 + }, + { + "coverage": 0.7463333333333333, + "risk": 0.2090218847699866, + "threshold": 0.042727385882724706 + }, + { + "coverage": 0.7466666666666667, + "risk": 0.20892857142857144, + "threshold": 0.04260401816785609 + }, + { + "coverage": 0.747, + "risk": 0.20883534136546184, + "threshold": 0.042218585332340824 + }, + { + "coverage": 0.7473333333333333, + "risk": 0.2087421944692239, + "threshold": 0.0420146748417766 + }, + { + "coverage": 0.7476666666666667, + "risk": 0.2086491306286224, + "threshold": 0.04196243739643514 + }, + { + "coverage": 0.748, + "risk": 0.20855614973262032, + "threshold": 0.041841461427327385 + }, + { + "coverage": 0.7483333333333333, + "risk": 0.20846325167037863, + "threshold": 0.04181698886955291 + }, + { + "coverage": 0.7486666666666667, + "risk": 0.20837043633125557, + "threshold": 0.041701013502880906 + }, + { + "coverage": 0.749, + "risk": 0.2082777036048064, + "threshold": 0.04166094303318285 + }, + { + "coverage": 0.7493333333333333, + "risk": 0.20818505338078291, + "threshold": 0.04160106297573475 + }, + { + "coverage": 0.7496666666666667, + "risk": 0.20809248554913296, + "threshold": 0.04154034617149745 + }, + { + "coverage": 0.75, + "risk": 0.208, + "threshold": 0.04153756045252555 + }, + { + "coverage": 0.7503333333333333, + "risk": 0.2079075966237228, + "threshold": 0.04150196276336917 + }, + { + "coverage": 0.7506666666666667, + "risk": 0.20781527531083482, + "threshold": 0.041450484684743695 + }, + { + "coverage": 0.751, + "risk": 0.20772303595206393, + "threshold": 0.041248434008602765 + }, + { + "coverage": 0.7513333333333333, + "risk": 0.20763087843833186, + "threshold": 0.04102500170225332 + }, + { + "coverage": 0.7516666666666667, + "risk": 0.2075388026607539, + "threshold": 0.040855622338511685 + }, + { + "coverage": 0.752, + "risk": 0.2074468085106383, + "threshold": 0.04083951993177736 + }, + { + "coverage": 0.7523333333333333, + "risk": 0.20735489587948605, + "threshold": 0.040679464498213636 + }, + { + "coverage": 0.7526666666666667, + "risk": 0.20726306465899025, + "threshold": 0.040648090888764886 + }, + { + "coverage": 0.753, + "risk": 0.20717131474103587, + "threshold": 0.04061263904370932 + }, + { + "coverage": 0.7533333333333333, + "risk": 0.20707964601769913, + "threshold": 0.04051172965813328 + }, + { + "coverage": 0.7536666666666667, + "risk": 0.20698805838124723, + "threshold": 0.0404423608349996 + }, + { + "coverage": 0.754, + "risk": 0.20689655172413793, + "threshold": 0.040184261271585114 + }, + { + "coverage": 0.7543333333333333, + "risk": 0.206805125939019, + "threshold": 0.04014980687681867 + }, + { + "coverage": 0.7546666666666667, + "risk": 0.2067137809187279, + "threshold": 0.040133923418550356 + }, + { + "coverage": 0.755, + "risk": 0.20662251655629138, + "threshold": 0.04001000206682328 + }, + { + "coverage": 0.7553333333333333, + "risk": 0.20653133274492497, + "threshold": 0.039996217545672 + }, + { + "coverage": 0.7556666666666667, + "risk": 0.20644022937803264, + "threshold": 0.0398742286340364 + }, + { + "coverage": 0.756, + "risk": 0.20634920634920634, + "threshold": 0.03977363104300234 + }, + { + "coverage": 0.7563333333333333, + "risk": 0.20625826355222565, + "threshold": 0.03930098836201594 + }, + { + "coverage": 0.7566666666666667, + "risk": 0.20616740088105726, + "threshold": 0.039283294590057756 + }, + { + "coverage": 0.757, + "risk": 0.20607661822985468, + "threshold": 0.03898081816857227 + }, + { + "coverage": 0.7573333333333333, + "risk": 0.20598591549295775, + "threshold": 0.03883680957815968 + }, + { + "coverage": 0.7576666666666667, + "risk": 0.20589529256489222, + "threshold": 0.03881118120147916 + }, + { + "coverage": 0.758, + "risk": 0.20624450307827616, + "threshold": 0.038733032863397486 + }, + { + "coverage": 0.7583333333333333, + "risk": 0.20615384615384616, + "threshold": 0.038626368835548645 + }, + { + "coverage": 0.7586666666666667, + "risk": 0.2060632688927944, + "threshold": 0.03859163667667707 + }, + { + "coverage": 0.759, + "risk": 0.2059727711901625, + "threshold": 0.03831535631143535 + }, + { + "coverage": 0.7593333333333333, + "risk": 0.20588235294117646, + "threshold": 0.037929447016169474 + }, + { + "coverage": 0.7596666666666667, + "risk": 0.20579201404124617, + "threshold": 0.03789873839203011 + }, + { + "coverage": 0.76, + "risk": 0.2057017543859649, + "threshold": 0.03784152552735859 + }, + { + "coverage": 0.7603333333333333, + "risk": 0.20561157387110915, + "threshold": 0.0377718367346761 + }, + { + "coverage": 0.7606666666666667, + "risk": 0.20552147239263804, + "threshold": 0.03743246589603131 + }, + { + "coverage": 0.761, + "risk": 0.20543144984669295, + "threshold": 0.03723132996588082 + }, + { + "coverage": 0.7613333333333333, + "risk": 0.2053415061295972, + "threshold": 0.03685049088661744 + }, + { + "coverage": 0.7616666666666667, + "risk": 0.20525164113785557, + "threshold": 0.036776327574011676 + }, + { + "coverage": 0.762, + "risk": 0.205161854768154, + "threshold": 0.036756720365969796 + }, + { + "coverage": 0.7623333333333333, + "risk": 0.20507214691735898, + "threshold": 0.03663078323885111 + }, + { + "coverage": 0.7626666666666667, + "risk": 0.20498251748251747, + "threshold": 0.03648045716888661 + }, + { + "coverage": 0.763, + "risk": 0.20489296636085627, + "threshold": 0.0362468411394596 + }, + { + "coverage": 0.7633333333333333, + "risk": 0.20480349344978166, + "threshold": 0.036229960637202165 + }, + { + "coverage": 0.7636666666666667, + "risk": 0.2047140986468791, + "threshold": 0.03618615064059179 + }, + { + "coverage": 0.764, + "risk": 0.20462478184991273, + "threshold": 0.0360852825217616 + }, + { + "coverage": 0.7643333333333333, + "risk": 0.20453554295682513, + "threshold": 0.03585243753675002 + }, + { + "coverage": 0.7646666666666667, + "risk": 0.20444638186573671, + "threshold": 0.03566305931826325 + }, + { + "coverage": 0.765, + "risk": 0.20435729847494552, + "threshold": 0.035656174100818165 + }, + { + "coverage": 0.7653333333333333, + "risk": 0.20426829268292682, + "threshold": 0.03550178714139137 + }, + { + "coverage": 0.7656666666666667, + "risk": 0.20417936438833262, + "threshold": 0.035382880778351875 + }, + { + "coverage": 0.766, + "risk": 0.2040905134899913, + "threshold": 0.034742286420776286 + }, + { + "coverage": 0.7663333333333333, + "risk": 0.20400173988690734, + "threshold": 0.03466711011927961 + }, + { + "coverage": 0.7666666666666667, + "risk": 0.20391304347826086, + "threshold": 0.03463053795805291 + }, + { + "coverage": 0.767, + "risk": 0.2038244241634072, + "threshold": 0.03458943319218932 + }, + { + "coverage": 0.7673333333333333, + "risk": 0.20373588184187663, + "threshold": 0.034569165306512016 + }, + { + "coverage": 0.7676666666666667, + "risk": 0.20364741641337386, + "threshold": 0.03453330515458641 + }, + { + "coverage": 0.768, + "risk": 0.2035590277777778, + "threshold": 0.03441051893562072 + }, + { + "coverage": 0.7683333333333333, + "risk": 0.203470715835141, + "threshold": 0.0344029593941874 + }, + { + "coverage": 0.7686666666666667, + "risk": 0.2033824804856895, + "threshold": 0.03430864508982165 + }, + { + "coverage": 0.769, + "risk": 0.20329432162982228, + "threshold": 0.0342342650932023 + }, + { + "coverage": 0.7693333333333333, + "risk": 0.20320623916811092, + "threshold": 0.03409412494755652 + }, + { + "coverage": 0.7696666666666667, + "risk": 0.20311823300129928, + "threshold": 0.034053716149677976 + }, + { + "coverage": 0.77, + "risk": 0.20303030303030303, + "threshold": 0.03397071611416105 + }, + { + "coverage": 0.7703333333333333, + "risk": 0.20294244915620943, + "threshold": 0.03386608057502031 + }, + { + "coverage": 0.7706666666666667, + "risk": 0.20285467128027682, + "threshold": 0.033725722324630024 + }, + { + "coverage": 0.771, + "risk": 0.20276696930393429, + "threshold": 0.033691492223571484 + }, + { + "coverage": 0.7713333333333333, + "risk": 0.20267934312878133, + "threshold": 0.03368043791218693 + }, + { + "coverage": 0.7716666666666666, + "risk": 0.20259179265658747, + "threshold": 0.03364418338851629 + }, + { + "coverage": 0.772, + "risk": 0.20250431778929187, + "threshold": 0.033568199315977466 + }, + { + "coverage": 0.7723333333333333, + "risk": 0.20241691842900303, + "threshold": 0.033547274670310295 + }, + { + "coverage": 0.7726666666666666, + "risk": 0.20232959447799828, + "threshold": 0.033514372076039016 + }, + { + "coverage": 0.773, + "risk": 0.20224234583872358, + "threshold": 0.033504974417295656 + }, + { + "coverage": 0.7733333333333333, + "risk": 0.2021551724137931, + "threshold": 0.03345619985927274 + }, + { + "coverage": 0.7736666666666666, + "risk": 0.2020680741059888, + "threshold": 0.03326960370921992 + }, + { + "coverage": 0.774, + "risk": 0.20198105081826012, + "threshold": 0.03319837671554615 + }, + { + "coverage": 0.7743333333333333, + "risk": 0.20189410245372363, + "threshold": 0.03308582886730102 + }, + { + "coverage": 0.7746666666666666, + "risk": 0.20180722891566266, + "threshold": 0.03293476173309496 + }, + { + "coverage": 0.775, + "risk": 0.20172043010752688, + "threshold": 0.032843260110497166 + }, + { + "coverage": 0.7753333333333333, + "risk": 0.20163370593293206, + "threshold": 0.03267900272737898 + }, + { + "coverage": 0.7756666666666666, + "risk": 0.20154705629565964, + "threshold": 0.03261565367116983 + }, + { + "coverage": 0.776, + "risk": 0.20146048109965636, + "threshold": 0.032510132284231864 + }, + { + "coverage": 0.7763333333333333, + "risk": 0.2013739802490339, + "threshold": 0.032445150731554045 + }, + { + "coverage": 0.7766666666666666, + "risk": 0.20128755364806866, + "threshold": 0.03221328149056867 + }, + { + "coverage": 0.777, + "risk": 0.2012012012012012, + "threshold": 0.03220954125040242 + }, + { + "coverage": 0.7773333333333333, + "risk": 0.20111492281303603, + "threshold": 0.032067977632551724 + }, + { + "coverage": 0.7776666666666666, + "risk": 0.2010287183883412, + "threshold": 0.032050888575098836 + }, + { + "coverage": 0.778, + "risk": 0.200942587832048, + "threshold": 0.031797000054712386 + }, + { + "coverage": 0.7783333333333333, + "risk": 0.20085653104925053, + "threshold": 0.03155605107095773 + }, + { + "coverage": 0.7786666666666666, + "risk": 0.2007705479452055, + "threshold": 0.031525506877385964 + }, + { + "coverage": 0.779, + "risk": 0.20068463842533163, + "threshold": 0.03150914645250246 + }, + { + "coverage": 0.7793333333333333, + "risk": 0.20059880239520958, + "threshold": 0.0314267788813008 + }, + { + "coverage": 0.7796666666666666, + "risk": 0.20051303976058144, + "threshold": 0.031311783723253196 + }, + { + "coverage": 0.78, + "risk": 0.20042735042735044, + "threshold": 0.03116518755323638 + }, + { + "coverage": 0.7803333333333333, + "risk": 0.20034173430158053, + "threshold": 0.031162412256457488 + }, + { + "coverage": 0.7806666666666666, + "risk": 0.20025619128949615, + "threshold": 0.031141944011013112 + }, + { + "coverage": 0.781, + "risk": 0.20017072129748187, + "threshold": 0.031117990742013244 + }, + { + "coverage": 0.7813333333333333, + "risk": 0.20008532423208192, + "threshold": 0.03087157726282823 + }, + { + "coverage": 0.7816666666666666, + "risk": 0.2, + "threshold": 0.03074618831851909 + }, + { + "coverage": 0.782, + "risk": 0.1999147485080989, + "threshold": 0.030745353988147987 + }, + { + "coverage": 0.7823333333333333, + "risk": 0.1998295696634001, + "threshold": 0.03061605318592112 + }, + { + "coverage": 0.7826666666666666, + "risk": 0.19974446337308346, + "threshold": 0.0304780278048978 + }, + { + "coverage": 0.783, + "risk": 0.199659429544487, + "threshold": 0.030387118671338413 + }, + { + "coverage": 0.7833333333333333, + "risk": 0.19957446808510637, + "threshold": 0.030361283596935993 + }, + { + "coverage": 0.7836666666666666, + "risk": 0.19948957890259464, + "threshold": 0.03031008638330847 + }, + { + "coverage": 0.784, + "risk": 0.19940476190476192, + "threshold": 0.030139754771174025 + }, + { + "coverage": 0.7843333333333333, + "risk": 0.19932001699957502, + "threshold": 0.03012510381358348 + }, + { + "coverage": 0.7846666666666666, + "risk": 0.19923534409515717, + "threshold": 0.030030606295292455 + }, + { + "coverage": 0.785, + "risk": 0.1991507430997877, + "threshold": 0.029918913736034735 + }, + { + "coverage": 0.7853333333333333, + "risk": 0.19906621392190152, + "threshold": 0.029772550632491967 + }, + { + "coverage": 0.7856666666666666, + "risk": 0.1989817564700891, + "threshold": 0.02972433552412634 + }, + { + "coverage": 0.786, + "risk": 0.19889737065309585, + "threshold": 0.029677658878635153 + }, + { + "coverage": 0.7863333333333333, + "risk": 0.19881305637982197, + "threshold": 0.02966920745504489 + }, + { + "coverage": 0.7866666666666666, + "risk": 0.19872881355932204, + "threshold": 0.029626558234544006 + }, + { + "coverage": 0.787, + "risk": 0.19864464210080474, + "threshold": 0.02942588620837945 + }, + { + "coverage": 0.7873333333333333, + "risk": 0.1985605419136325, + "threshold": 0.029297968344336543 + }, + { + "coverage": 0.7876666666666666, + "risk": 0.1984765129073212, + "threshold": 0.029100889620230698 + }, + { + "coverage": 0.788, + "risk": 0.19839255499153977, + "threshold": 0.02889669648440151 + }, + { + "coverage": 0.7883333333333333, + "risk": 0.19830866807610995, + "threshold": 0.028856634295980916 + }, + { + "coverage": 0.7886666666666666, + "risk": 0.19822485207100593, + "threshold": 0.028767627825344075 + }, + { + "coverage": 0.789, + "risk": 0.19814110688635403, + "threshold": 0.028620783126268163 + }, + { + "coverage": 0.7893333333333333, + "risk": 0.19805743243243243, + "threshold": 0.028480485253485045 + }, + { + "coverage": 0.7896666666666666, + "risk": 0.19797382861967075, + "threshold": 0.02847054239134747 + }, + { + "coverage": 0.79, + "risk": 0.1978902953586498, + "threshold": 0.02839987121382752 + }, + { + "coverage": 0.7903333333333333, + "risk": 0.19780683256010123, + "threshold": 0.02832147301307487 + }, + { + "coverage": 0.7906666666666666, + "risk": 0.19772344013490725, + "threshold": 0.028282343354434906 + }, + { + "coverage": 0.791, + "risk": 0.1976401179941003, + "threshold": 0.028232464184443404 + }, + { + "coverage": 0.7913333333333333, + "risk": 0.1975568660488627, + "threshold": 0.028156522280905167 + }, + { + "coverage": 0.7916666666666666, + "risk": 0.19747368421052633, + "threshold": 0.028030214253209392 + }, + { + "coverage": 0.792, + "risk": 0.1973905723905724, + "threshold": 0.028015033836014377 + }, + { + "coverage": 0.7923333333333333, + "risk": 0.19730753050063105, + "threshold": 0.027868226670484057 + }, + { + "coverage": 0.7926666666666666, + "risk": 0.19764507989907484, + "threshold": 0.027862364500326055 + }, + { + "coverage": 0.793, + "risk": 0.19756200084068937, + "threshold": 0.027830131239318736 + }, + { + "coverage": 0.7933333333333333, + "risk": 0.19747899159663865, + "threshold": 0.027771602444178232 + }, + { + "coverage": 0.7936666666666666, + "risk": 0.1973960520789584, + "threshold": 0.027716669081195338 + }, + { + "coverage": 0.794, + "risk": 0.19731318219983207, + "threshold": 0.027710689824983332 + }, + { + "coverage": 0.7943333333333333, + "risk": 0.19723038187159042, + "threshold": 0.02753313086939895 + }, + { + "coverage": 0.7946666666666666, + "risk": 0.19714765100671142, + "threshold": 0.027514052632756872 + }, + { + "coverage": 0.795, + "risk": 0.1970649895178197, + "threshold": 0.027369512021896022 + }, + { + "coverage": 0.7953333333333333, + "risk": 0.1969823973176865, + "threshold": 0.02735848848478754 + }, + { + "coverage": 0.7956666666666666, + "risk": 0.19689987431922915, + "threshold": 0.02705158410579878 + }, + { + "coverage": 0.796, + "risk": 0.1968174204355109, + "threshold": 0.026866533337509563 + }, + { + "coverage": 0.7963333333333333, + "risk": 0.19673503557974048, + "threshold": 0.026854838932163318 + }, + { + "coverage": 0.7966666666666666, + "risk": 0.19665271966527198, + "threshold": 0.02684922698527143 + }, + { + "coverage": 0.797, + "risk": 0.19657047260560434, + "threshold": 0.026827477437104105 + }, + { + "coverage": 0.7973333333333333, + "risk": 0.19690635451505017, + "threshold": 0.026818034278155232 + }, + { + "coverage": 0.7976666666666666, + "risk": 0.1968240702047639, + "threshold": 0.026792005002607998 + }, + { + "coverage": 0.798, + "risk": 0.19674185463659147, + "threshold": 0.026777453798750784 + }, + { + "coverage": 0.7983333333333333, + "risk": 0.1966597077244259, + "threshold": 0.02673378324927197 + }, + { + "coverage": 0.7986666666666666, + "risk": 0.19657762938230383, + "threshold": 0.02671506964835899 + }, + { + "coverage": 0.799, + "risk": 0.1964956195244055, + "threshold": 0.026640548554322595 + }, + { + "coverage": 0.7993333333333333, + "risk": 0.19641367806505422, + "threshold": 0.026637424244138435 + }, + { + "coverage": 0.7996666666666666, + "risk": 0.19633180491871613, + "threshold": 0.026593377495449915 + }, + { + "coverage": 0.8, + "risk": 0.19625, + "threshold": 0.026466068541994434 + }, + { + "coverage": 0.8003333333333333, + "risk": 0.1961682632236568, + "threshold": 0.026297666329593548 + }, + { + "coverage": 0.8006666666666666, + "risk": 0.19608659450457952, + "threshold": 0.026229330301424016 + }, + { + "coverage": 0.801, + "risk": 0.19600499375780275, + "threshold": 0.026200447889605377 + }, + { + "coverage": 0.8013333333333333, + "risk": 0.1959234608985025, + "threshold": 0.02617641368812919 + }, + { + "coverage": 0.8016666666666666, + "risk": 0.19584199584199585, + "threshold": 0.025661863070736774 + }, + { + "coverage": 0.802, + "risk": 0.19576059850374064, + "threshold": 0.02551987624877671 + }, + { + "coverage": 0.8023333333333333, + "risk": 0.19567926879933528, + "threshold": 0.02539503827936158 + }, + { + "coverage": 0.8026666666666666, + "risk": 0.19559800664451826, + "threshold": 0.02533401379629774 + }, + { + "coverage": 0.803, + "risk": 0.19551681195516812, + "threshold": 0.025252359455997506 + }, + { + "coverage": 0.8033333333333333, + "risk": 0.1954356846473029, + "threshold": 0.025225977237538985 + }, + { + "coverage": 0.8036666666666666, + "risk": 0.19535462463708006, + "threshold": 0.02516700743605143 + }, + { + "coverage": 0.804, + "risk": 0.19527363184079602, + "threshold": 0.02490349397635815 + }, + { + "coverage": 0.8043333333333333, + "risk": 0.19519270617488604, + "threshold": 0.024640206066237534 + }, + { + "coverage": 0.8046666666666666, + "risk": 0.19511184755592378, + "threshold": 0.024636664169524192 + }, + { + "coverage": 0.805, + "risk": 0.19503105590062111, + "threshold": 0.024500400353727844 + }, + { + "coverage": 0.8053333333333333, + "risk": 0.19495033112582782, + "threshold": 0.024433760049497245 + }, + { + "coverage": 0.8056666666666666, + "risk": 0.19486967314853124, + "threshold": 0.024394545390108154 + }, + { + "coverage": 0.806, + "risk": 0.19478908188585609, + "threshold": 0.02436198673668769 + }, + { + "coverage": 0.8063333333333333, + "risk": 0.19470855725506409, + "threshold": 0.024245838490902286 + }, + { + "coverage": 0.8066666666666666, + "risk": 0.19462809917355373, + "threshold": 0.024233746972897135 + }, + { + "coverage": 0.807, + "risk": 0.19454770755885997, + "threshold": 0.024172994503964304 + }, + { + "coverage": 0.8073333333333333, + "risk": 0.19446738232865401, + "threshold": 0.02416729099923081 + }, + { + "coverage": 0.8076666666666666, + "risk": 0.19438712340074288, + "threshold": 0.023897450139639145 + }, + { + "coverage": 0.808, + "risk": 0.1943069306930693, + "threshold": 0.02389271399146847 + }, + { + "coverage": 0.8083333333333333, + "risk": 0.19422680412371135, + "threshold": 0.023792128425977138 + }, + { + "coverage": 0.8086666666666666, + "risk": 0.1941467436108821, + "threshold": 0.023776996821677297 + }, + { + "coverage": 0.809, + "risk": 0.19406674907292953, + "threshold": 0.02371243254004733 + }, + { + "coverage": 0.8093333333333333, + "risk": 0.19398682042833607, + "threshold": 0.023671925764454038 + }, + { + "coverage": 0.8096666666666666, + "risk": 0.1939069575957184, + "threshold": 0.023629362645490863 + }, + { + "coverage": 0.81, + "risk": 0.19382716049382717, + "threshold": 0.023598571672758365 + }, + { + "coverage": 0.8103333333333333, + "risk": 0.19374742904154668, + "threshold": 0.02342266379490282 + }, + { + "coverage": 0.8106666666666666, + "risk": 0.19366776315789475, + "threshold": 0.023073043713672015 + }, + { + "coverage": 0.811, + "risk": 0.1935881627620222, + "threshold": 0.022931430395143948 + }, + { + "coverage": 0.8113333333333334, + "risk": 0.19350862777321282, + "threshold": 0.02290092638779691 + }, + { + "coverage": 0.8116666666666666, + "risk": 0.19342915811088296, + "threshold": 0.02286904816715814 + }, + { + "coverage": 0.812, + "risk": 0.1933497536945813, + "threshold": 0.02284743614612057 + }, + { + "coverage": 0.8123333333333334, + "risk": 0.19327041444398851, + "threshold": 0.022830966671792297 + }, + { + "coverage": 0.8126666666666666, + "risk": 0.19319114027891715, + "threshold": 0.022823110206958114 + }, + { + "coverage": 0.813, + "risk": 0.1931119311193112, + "threshold": 0.02272724408116167 + }, + { + "coverage": 0.8133333333333334, + "risk": 0.1930327868852459, + "threshold": 0.0226412120986243 + }, + { + "coverage": 0.8136666666666666, + "risk": 0.1929537074969275, + "threshold": 0.02261876765577169 + }, + { + "coverage": 0.814, + "risk": 0.1928746928746929, + "threshold": 0.02258079194392011 + }, + { + "coverage": 0.8143333333333334, + "risk": 0.19279574293900942, + "threshold": 0.02251652261839974 + }, + { + "coverage": 0.8146666666666667, + "risk": 0.19271685761047463, + "threshold": 0.022332998506883313 + }, + { + "coverage": 0.815, + "risk": 0.19263803680981595, + "threshold": 0.02232998910009044 + }, + { + "coverage": 0.8153333333333334, + "risk": 0.19255928045789045, + "threshold": 0.022314282304273066 + }, + { + "coverage": 0.8156666666666667, + "risk": 0.1924805884756845, + "threshold": 0.02189227024075361 + }, + { + "coverage": 0.816, + "risk": 0.19240196078431374, + "threshold": 0.02187505906278512 + }, + { + "coverage": 0.8163333333333334, + "risk": 0.19232339730502246, + "threshold": 0.021609969839650352 + }, + { + "coverage": 0.8166666666666667, + "risk": 0.19224489795918367, + "threshold": 0.021571045093206835 + }, + { + "coverage": 0.817, + "risk": 0.19216646266829865, + "threshold": 0.021391465886145454 + }, + { + "coverage": 0.8173333333333334, + "risk": 0.19208809135399674, + "threshold": 0.021358958283115605 + }, + { + "coverage": 0.8176666666666667, + "risk": 0.19200978393803506, + "threshold": 0.021219021627407422 + }, + { + "coverage": 0.818, + "risk": 0.19193154034229828, + "threshold": 0.02121586272366868 + }, + { + "coverage": 0.8183333333333334, + "risk": 0.19185336048879836, + "threshold": 0.021157422866979067 + }, + { + "coverage": 0.8186666666666667, + "risk": 0.19177524429967427, + "threshold": 0.021083552231344185 + }, + { + "coverage": 0.819, + "risk": 0.1916971916971917, + "threshold": 0.021047910819297765 + }, + { + "coverage": 0.8193333333333334, + "risk": 0.19161920260374288, + "threshold": 0.020950333322305007 + }, + { + "coverage": 0.8196666666666667, + "risk": 0.19154127694184628, + "threshold": 0.020910549721360232 + }, + { + "coverage": 0.82, + "risk": 0.19146341463414634, + "threshold": 0.02084859332910804 + }, + { + "coverage": 0.8203333333333334, + "risk": 0.19138561560341324, + "threshold": 0.020841942683962826 + }, + { + "coverage": 0.8206666666666667, + "risk": 0.19130787977254265, + "threshold": 0.02083839036358381 + }, + { + "coverage": 0.821, + "risk": 0.19123020706455543, + "threshold": 0.020748125529580514 + }, + { + "coverage": 0.8213333333333334, + "risk": 0.1911525974025974, + "threshold": 0.020611388679815677 + }, + { + "coverage": 0.8216666666666667, + "risk": 0.19107505070993916, + "threshold": 0.020049516408189003 + }, + { + "coverage": 0.822, + "risk": 0.19099756690997566, + "threshold": 0.01993646685363636 + }, + { + "coverage": 0.8223333333333334, + "risk": 0.1909201459262262, + "threshold": 0.01992370907397446 + }, + { + "coverage": 0.8226666666666667, + "risk": 0.19084278768233387, + "threshold": 0.019875646340442952 + }, + { + "coverage": 0.823, + "risk": 0.19076549210206561, + "threshold": 0.01972937909143281 + }, + { + "coverage": 0.8233333333333334, + "risk": 0.19068825910931175, + "threshold": 0.01956447251822671 + }, + { + "coverage": 0.8236666666666667, + "risk": 0.1906110886280858, + "threshold": 0.019512638748876186 + }, + { + "coverage": 0.824, + "risk": 0.19053398058252427, + "threshold": 0.019482170550761856 + }, + { + "coverage": 0.8243333333333334, + "risk": 0.19045693489688637, + "threshold": 0.019461650726342785 + }, + { + "coverage": 0.8246666666666667, + "risk": 0.19037995149555376, + "threshold": 0.019455309645929612 + }, + { + "coverage": 0.825, + "risk": 0.1903030303030303, + "threshold": 0.0194227468690826 + }, + { + "coverage": 0.8253333333333334, + "risk": 0.19022617124394184, + "threshold": 0.019414740395831468 + }, + { + "coverage": 0.8256666666666667, + "risk": 0.19014937424303593, + "threshold": 0.019175876469398365 + }, + { + "coverage": 0.826, + "risk": 0.1900726392251816, + "threshold": 0.019025144016592437 + }, + { + "coverage": 0.8263333333333334, + "risk": 0.1899959661153691, + "threshold": 0.019003286961378284 + }, + { + "coverage": 0.8266666666666667, + "risk": 0.18991935483870967, + "threshold": 0.018982170514201927 + }, + { + "coverage": 0.827, + "risk": 0.18984280532043532, + "threshold": 0.018951300954264436 + }, + { + "coverage": 0.8273333333333334, + "risk": 0.18976631748589848, + "threshold": 0.018885215792414797 + }, + { + "coverage": 0.8276666666666667, + "risk": 0.1896898912605719, + "threshold": 0.018836519311983404 + }, + { + "coverage": 0.828, + "risk": 0.18961352657004832, + "threshold": 0.018805869176668235 + }, + { + "coverage": 0.8283333333333334, + "risk": 0.18953722334004025, + "threshold": 0.018639769635035226 + }, + { + "coverage": 0.8286666666666667, + "risk": 0.18946098149637972, + "threshold": 0.0185732082638491 + }, + { + "coverage": 0.829, + "risk": 0.18938480096501809, + "threshold": 0.018527048610857962 + }, + { + "coverage": 0.8293333333333334, + "risk": 0.18930868167202572, + "threshold": 0.018509476863916356 + }, + { + "coverage": 0.8296666666666667, + "risk": 0.1892326235435918, + "threshold": 0.018440649951348243 + }, + { + "coverage": 0.83, + "risk": 0.1891566265060241, + "threshold": 0.018342375121338697 + }, + { + "coverage": 0.8303333333333334, + "risk": 0.1890806904857487, + "threshold": 0.018257159984143616 + }, + { + "coverage": 0.8306666666666667, + "risk": 0.1890048154093098, + "threshold": 0.018242801058588574 + }, + { + "coverage": 0.831, + "risk": 0.18892900120336945, + "threshold": 0.018131702781471602 + }, + { + "coverage": 0.8313333333333334, + "risk": 0.1888532477947073, + "threshold": 0.018103802690995093 + }, + { + "coverage": 0.8316666666666667, + "risk": 0.18877755511022043, + "threshold": 0.018071683972586607 + }, + { + "coverage": 0.832, + "risk": 0.18870192307692307, + "threshold": 0.018043725457513107 + }, + { + "coverage": 0.8323333333333334, + "risk": 0.18862635162194633, + "threshold": 0.01803673872801791 + }, + { + "coverage": 0.8326666666666667, + "risk": 0.18855084067253802, + "threshold": 0.01796779789086672 + }, + { + "coverage": 0.833, + "risk": 0.18847539015606243, + "threshold": 0.017918639819762272 + }, + { + "coverage": 0.8333333333333334, + "risk": 0.1884, + "threshold": 0.01785403509867921 + }, + { + "coverage": 0.8336666666666667, + "risk": 0.18832467013194723, + "threshold": 0.017641011197572817 + }, + { + "coverage": 0.834, + "risk": 0.1882494004796163, + "threshold": 0.017613653211252942 + }, + { + "coverage": 0.8343333333333334, + "risk": 0.188174190970835, + "threshold": 0.017580808891924652 + }, + { + "coverage": 0.8346666666666667, + "risk": 0.1880990415335463, + "threshold": 0.017579903895168374 + }, + { + "coverage": 0.835, + "risk": 0.18802395209580838, + "threshold": 0.017568476341383407 + }, + { + "coverage": 0.8353333333333334, + "risk": 0.1879489225857941, + "threshold": 0.01748468007820673 + }, + { + "coverage": 0.8356666666666667, + "risk": 0.187873952931791, + "threshold": 0.01744490023719787 + }, + { + "coverage": 0.836, + "risk": 0.18779904306220097, + "threshold": 0.0174448586185377 + }, + { + "coverage": 0.8363333333333334, + "risk": 0.18772419290554004, + "threshold": 0.01738989407212558 + }, + { + "coverage": 0.8366666666666667, + "risk": 0.18764940239043826, + "threshold": 0.017357796337524586 + }, + { + "coverage": 0.837, + "risk": 0.1875746714456392, + "threshold": 0.017348674132805855 + }, + { + "coverage": 0.8373333333333334, + "risk": 0.1875, + "threshold": 0.017318299342394088 + }, + { + "coverage": 0.8376666666666667, + "risk": 0.18742538798249106, + "threshold": 0.017299491011524042 + }, + { + "coverage": 0.838, + "risk": 0.1873508353221957, + "threshold": 0.01728387925348392 + }, + { + "coverage": 0.8383333333333334, + "risk": 0.18727634194831014, + "threshold": 0.016880342045710393 + }, + { + "coverage": 0.8386666666666667, + "risk": 0.1872019077901431, + "threshold": 0.01677430202065566 + }, + { + "coverage": 0.839, + "risk": 0.1871275327771156, + "threshold": 0.01672850816663214 + }, + { + "coverage": 0.8393333333333334, + "risk": 0.18705321683876092, + "threshold": 0.01664852642929944 + }, + { + "coverage": 0.8396666666666667, + "risk": 0.1869789599047241, + "threshold": 0.016560116293773224 + }, + { + "coverage": 0.84, + "risk": 0.1869047619047619, + "threshold": 0.016476018034550916 + }, + { + "coverage": 0.8403333333333334, + "risk": 0.18683062276874257, + "threshold": 0.016450577073257675 + }, + { + "coverage": 0.8406666666666667, + "risk": 0.18675654242664552, + "threshold": 0.01645008206257059 + }, + { + "coverage": 0.841, + "risk": 0.18668252080856124, + "threshold": 0.016448713910919545 + }, + { + "coverage": 0.8413333333333334, + "risk": 0.18660855784469096, + "threshold": 0.016379739694947517 + }, + { + "coverage": 0.8416666666666667, + "risk": 0.18653465346534653, + "threshold": 0.01622724837718433 + }, + { + "coverage": 0.842, + "risk": 0.18646080760095013, + "threshold": 0.01619292483254267 + }, + { + "coverage": 0.8423333333333334, + "risk": 0.18638702018203404, + "threshold": 0.01611686661978338 + }, + { + "coverage": 0.8426666666666667, + "risk": 0.1863132911392405, + "threshold": 0.016011238711238412 + }, + { + "coverage": 0.843, + "risk": 0.18623962040332148, + "threshold": 0.0159952053918892 + }, + { + "coverage": 0.8433333333333334, + "risk": 0.18616600790513835, + "threshold": 0.01596432494182145 + }, + { + "coverage": 0.8436666666666667, + "risk": 0.1860924535756618, + "threshold": 0.015937891892239337 + }, + { + "coverage": 0.844, + "risk": 0.18601895734597157, + "threshold": 0.015759029545788013 + }, + { + "coverage": 0.8443333333333334, + "risk": 0.1859455191472562, + "threshold": 0.01575826225944962 + }, + { + "coverage": 0.8446666666666667, + "risk": 0.18587213891081295, + "threshold": 0.015724462735497914 + }, + { + "coverage": 0.845, + "risk": 0.18579881656804734, + "threshold": 0.015723345442059975 + }, + { + "coverage": 0.8453333333333334, + "risk": 0.1857255520504732, + "threshold": 0.015639045808341658 + }, + { + "coverage": 0.8456666666666667, + "risk": 0.18565234528971225, + "threshold": 0.015612728711579088 + }, + { + "coverage": 0.846, + "risk": 0.18557919621749408, + "threshold": 0.015554629022882132 + }, + { + "coverage": 0.8463333333333334, + "risk": 0.18550610476565577, + "threshold": 0.015540014171442879 + }, + { + "coverage": 0.8466666666666667, + "risk": 0.18543307086614172, + "threshold": 0.015403662386020193 + }, + { + "coverage": 0.847, + "risk": 0.18536009445100354, + "threshold": 0.015385866593206792 + }, + { + "coverage": 0.8473333333333334, + "risk": 0.18528717545239967, + "threshold": 0.015233086563845193 + }, + { + "coverage": 0.8476666666666667, + "risk": 0.18521431380259537, + "threshold": 0.015171071744552525 + }, + { + "coverage": 0.848, + "risk": 0.18514150943396226, + "threshold": 0.015130122614513082 + }, + { + "coverage": 0.8483333333333334, + "risk": 0.1850687622789784, + "threshold": 0.015048495274493114 + }, + { + "coverage": 0.8486666666666667, + "risk": 0.1849960722702278, + "threshold": 0.014964784372774124 + }, + { + "coverage": 0.849, + "risk": 0.18492343934040048, + "threshold": 0.014939549305644282 + }, + { + "coverage": 0.8493333333333334, + "risk": 0.18485086342229198, + "threshold": 0.014862253249951805 + }, + { + "coverage": 0.8496666666666667, + "risk": 0.18477834444880345, + "threshold": 0.014833261965198649 + }, + { + "coverage": 0.85, + "risk": 0.18470588235294116, + "threshold": 0.014805101251192178 + }, + { + "coverage": 0.8503333333333334, + "risk": 0.18463347706781655, + "threshold": 0.014801467594706387 + }, + { + "coverage": 0.8506666666666667, + "risk": 0.18456112852664577, + "threshold": 0.014736102568521824 + }, + { + "coverage": 0.851, + "risk": 0.1844888366627497, + "threshold": 0.014683870283012285 + }, + { + "coverage": 0.8513333333333334, + "risk": 0.18441660140955365, + "threshold": 0.01461136495029592 + }, + { + "coverage": 0.8516666666666667, + "risk": 0.1843444227005871, + "threshold": 0.014457578235028296 + }, + { + "coverage": 0.852, + "risk": 0.18427230046948356, + "threshold": 0.014441821932706506 + }, + { + "coverage": 0.8523333333333334, + "risk": 0.18420023464998045, + "threshold": 0.014291740871300637 + }, + { + "coverage": 0.8526666666666667, + "risk": 0.18412822517591867, + "threshold": 0.014163894745809574 + }, + { + "coverage": 0.853, + "risk": 0.18405627198124266, + "threshold": 0.014142454603284239 + }, + { + "coverage": 0.8533333333333334, + "risk": 0.183984375, + "threshold": 0.014037889407635276 + }, + { + "coverage": 0.8536666666666667, + "risk": 0.18391253416634126, + "threshold": 0.013809565479047914 + }, + { + "coverage": 0.854, + "risk": 0.18384074941451992, + "threshold": 0.013809553009192855 + }, + { + "coverage": 0.8543333333333333, + "risk": 0.18376902067889192, + "threshold": 0.01369459284908823 + }, + { + "coverage": 0.8546666666666667, + "risk": 0.18369734789391576, + "threshold": 0.013533770604593293 + }, + { + "coverage": 0.855, + "risk": 0.18362573099415205, + "threshold": 0.013533021542496112 + }, + { + "coverage": 0.8553333333333333, + "risk": 0.18355416991426343, + "threshold": 0.013514890360988647 + }, + { + "coverage": 0.8556666666666667, + "risk": 0.1834826645890144, + "threshold": 0.013449764963407364 + }, + { + "coverage": 0.856, + "risk": 0.18341121495327103, + "threshold": 0.01344414851268798 + }, + { + "coverage": 0.8563333333333333, + "risk": 0.18333982094200077, + "threshold": 0.0133703645765706 + }, + { + "coverage": 0.8566666666666667, + "risk": 0.18326848249027236, + "threshold": 0.013308972035971624 + }, + { + "coverage": 0.857, + "risk": 0.18319719953325556, + "threshold": 0.013254502073148815 + }, + { + "coverage": 0.8573333333333333, + "risk": 0.18312597200622083, + "threshold": 0.013252174193050517 + }, + { + "coverage": 0.8576666666666667, + "risk": 0.18305479984453946, + "threshold": 0.013248886504220554 + }, + { + "coverage": 0.858, + "risk": 0.18298368298368298, + "threshold": 0.013216666626521708 + }, + { + "coverage": 0.8583333333333333, + "risk": 0.1829126213592233, + "threshold": 0.01321078890295783 + }, + { + "coverage": 0.8586666666666667, + "risk": 0.1828416149068323, + "threshold": 0.013173678776932467 + }, + { + "coverage": 0.859, + "risk": 0.18277066356228172, + "threshold": 0.01315764013186295 + }, + { + "coverage": 0.8593333333333333, + "risk": 0.182699767261443, + "threshold": 0.01314691644797739 + }, + { + "coverage": 0.8596666666666667, + "risk": 0.18262892594028693, + "threshold": 0.013110181343956766 + }, + { + "coverage": 0.86, + "risk": 0.18255813953488373, + "threshold": 0.013108943415482802 + }, + { + "coverage": 0.8603333333333333, + "risk": 0.18248740798140256, + "threshold": 0.013104625893532743 + }, + { + "coverage": 0.8606666666666667, + "risk": 0.18241673121611154, + "threshold": 0.01308004901063665 + }, + { + "coverage": 0.861, + "risk": 0.18234610917537747, + "threshold": 0.013038905944395919 + }, + { + "coverage": 0.8613333333333333, + "risk": 0.18227554179566563, + "threshold": 0.012991016813755299 + }, + { + "coverage": 0.8616666666666667, + "risk": 0.18220502901353966, + "threshold": 0.012912954715547357 + }, + { + "coverage": 0.862, + "risk": 0.18213457076566125, + "threshold": 0.012875336084361916 + }, + { + "coverage": 0.8623333333333333, + "risk": 0.1820641669887901, + "threshold": 0.012864628749030617 + }, + { + "coverage": 0.8626666666666667, + "risk": 0.18199381761978362, + "threshold": 0.012816440762354381 + }, + { + "coverage": 0.863, + "risk": 0.18192352259559674, + "threshold": 0.012803689195546676 + }, + { + "coverage": 0.8633333333333333, + "risk": 0.18185328185328184, + "threshold": 0.012682474246768272 + }, + { + "coverage": 0.8636666666666667, + "risk": 0.18178309532998843, + "threshold": 0.012592703812995789 + }, + { + "coverage": 0.864, + "risk": 0.18171296296296297, + "threshold": 0.01259225261394009 + }, + { + "coverage": 0.8643333333333333, + "risk": 0.18164288468954878, + "threshold": 0.012485362329566003 + }, + { + "coverage": 0.8646666666666667, + "risk": 0.18157286044718582, + "threshold": 0.012432499169030213 + }, + { + "coverage": 0.865, + "risk": 0.1815028901734104, + "threshold": 0.012402768781232712 + }, + { + "coverage": 0.8653333333333333, + "risk": 0.18143297380585516, + "threshold": 0.012346556238247836 + }, + { + "coverage": 0.8656666666666667, + "risk": 0.18136311128224875, + "threshold": 0.012339003394106754 + }, + { + "coverage": 0.866, + "risk": 0.1812933025404157, + "threshold": 0.012302455898618178 + }, + { + "coverage": 0.8663333333333333, + "risk": 0.18122354751827627, + "threshold": 0.01225556435078873 + }, + { + "coverage": 0.8666666666666667, + "risk": 0.18115384615384617, + "threshold": 0.012209496314548282 + }, + { + "coverage": 0.867, + "risk": 0.18108419838523646, + "threshold": 0.012160268750577541 + }, + { + "coverage": 0.8673333333333333, + "risk": 0.18101460415065335, + "threshold": 0.012098582583522623 + }, + { + "coverage": 0.8676666666666667, + "risk": 0.180945063388398, + "threshold": 0.012069685900788767 + }, + { + "coverage": 0.868, + "risk": 0.18087557603686635, + "threshold": 0.012037798761073363 + }, + { + "coverage": 0.8683333333333333, + "risk": 0.18080614203454895, + "threshold": 0.012003605817359455 + }, + { + "coverage": 0.8686666666666667, + "risk": 0.1807367613200307, + "threshold": 0.011918709989304005 + }, + { + "coverage": 0.869, + "risk": 0.1806674338319908, + "threshold": 0.011848673763719628 + }, + { + "coverage": 0.8693333333333333, + "risk": 0.18059815950920247, + "threshold": 0.011809614422984858 + }, + { + "coverage": 0.8696666666666667, + "risk": 0.18052893829053276, + "threshold": 0.011681202772621297 + }, + { + "coverage": 0.87, + "risk": 0.18045977011494252, + "threshold": 0.011671536184184789 + }, + { + "coverage": 0.8703333333333333, + "risk": 0.180390654921486, + "threshold": 0.011668316309989244 + }, + { + "coverage": 0.8706666666666667, + "risk": 0.18032159264931089, + "threshold": 0.011604989386329617 + }, + { + "coverage": 0.871, + "risk": 0.18025258323765786, + "threshold": 0.011544872051041032 + }, + { + "coverage": 0.8713333333333333, + "risk": 0.18018362662586074, + "threshold": 0.011510987010947321 + }, + { + "coverage": 0.8716666666666667, + "risk": 0.18011472275334608, + "threshold": 0.011413543036130215 + }, + { + "coverage": 0.872, + "risk": 0.18004587155963303, + "threshold": 0.011412074508007018 + }, + { + "coverage": 0.8723333333333333, + "risk": 0.17997707298433321, + "threshold": 0.011357713004620597 + }, + { + "coverage": 0.8726666666666667, + "risk": 0.1799083269671505, + "threshold": 0.011340039105870607 + }, + { + "coverage": 0.873, + "risk": 0.17983963344788087, + "threshold": 0.011304697635273126 + }, + { + "coverage": 0.8733333333333333, + "risk": 0.17977099236641222, + "threshold": 0.01120485717213885 + }, + { + "coverage": 0.8736666666666667, + "risk": 0.17970240366272416, + "threshold": 0.011154654651322763 + }, + { + "coverage": 0.874, + "risk": 0.17963386727688788, + "threshold": 0.011093446519741616 + }, + { + "coverage": 0.8743333333333333, + "risk": 0.17956538314906595, + "threshold": 0.011041013135374627 + }, + { + "coverage": 0.8746666666666667, + "risk": 0.1794969512195122, + "threshold": 0.01095230646851094 + }, + { + "coverage": 0.875, + "risk": 0.17942857142857144, + "threshold": 0.010907917535687879 + }, + { + "coverage": 0.8753333333333333, + "risk": 0.17936024371667936, + "threshold": 0.010880381143000521 + }, + { + "coverage": 0.8756666666666667, + "risk": 0.1792919680243624, + "threshold": 0.010788860557706817 + }, + { + "coverage": 0.876, + "risk": 0.17922374429223745, + "threshold": 0.010729228957393129 + }, + { + "coverage": 0.8763333333333333, + "risk": 0.1791555724610118, + "threshold": 0.010679368185880604 + }, + { + "coverage": 0.8766666666666667, + "risk": 0.1790874524714829, + "threshold": 0.010597037346111676 + }, + { + "coverage": 0.877, + "risk": 0.1790193842645382, + "threshold": 0.010517287606082233 + }, + { + "coverage": 0.8773333333333333, + "risk": 0.178951367781155, + "threshold": 0.010472876202847075 + }, + { + "coverage": 0.8776666666666667, + "risk": 0.1788834029624003, + "threshold": 0.010461292095887935 + }, + { + "coverage": 0.878, + "risk": 0.1788154897494305, + "threshold": 0.010419432562746555 + }, + { + "coverage": 0.8783333333333333, + "risk": 0.17874762808349146, + "threshold": 0.010392475443728214 + }, + { + "coverage": 0.8786666666666667, + "risk": 0.17867981790591805, + "threshold": 0.010381709945008265 + }, + { + "coverage": 0.879, + "risk": 0.17861205915813425, + "threshold": 0.010357519283551539 + }, + { + "coverage": 0.8793333333333333, + "risk": 0.17854435178165276, + "threshold": 0.01033732446391246 + }, + { + "coverage": 0.8796666666666667, + "risk": 0.17847669571807503, + "threshold": 0.010294244650431673 + }, + { + "coverage": 0.88, + "risk": 0.1784090909090909, + "threshold": 0.01021192141370687 + }, + { + "coverage": 0.8803333333333333, + "risk": 0.1783415372964786, + "threshold": 0.010149695679557823 + }, + { + "coverage": 0.8806666666666667, + "risk": 0.17827403482210447, + "threshold": 0.010142223662885544 + }, + { + "coverage": 0.881, + "risk": 0.1782065834279228, + "threshold": 0.010087215233770704 + }, + { + "coverage": 0.8813333333333333, + "risk": 0.17813918305597579, + "threshold": 0.010086417119039641 + }, + { + "coverage": 0.8816666666666667, + "risk": 0.17807183364839319, + "threshold": 0.01008214933545094 + }, + { + "coverage": 0.882, + "risk": 0.17800453514739228, + "threshold": 0.01007582258429899 + }, + { + "coverage": 0.8823333333333333, + "risk": 0.17793728749527768, + "threshold": 0.010047586705030458 + }, + { + "coverage": 0.8826666666666667, + "risk": 0.1778700906344411, + "threshold": 0.0100456363394586 + }, + { + "coverage": 0.883, + "risk": 0.17780294450736125, + "threshold": 0.010015199145284889 + }, + { + "coverage": 0.8833333333333333, + "risk": 0.17773584905660378, + "threshold": 0.009987044819135558 + }, + { + "coverage": 0.8836666666666667, + "risk": 0.17766880422482081, + "threshold": 0.009978677879905652 + }, + { + "coverage": 0.884, + "risk": 0.17760180995475114, + "threshold": 0.009911718275465583 + }, + { + "coverage": 0.8843333333333333, + "risk": 0.17753486618921976, + "threshold": 0.009908493270767508 + }, + { + "coverage": 0.8846666666666667, + "risk": 0.1774679728711379, + "threshold": 0.009857292847274957 + }, + { + "coverage": 0.885, + "risk": 0.17740112994350282, + "threshold": 0.009843633733392684 + }, + { + "coverage": 0.8853333333333333, + "risk": 0.1773343373493976, + "threshold": 0.009841402340038326 + }, + { + "coverage": 0.8856666666666667, + "risk": 0.17726759503199097, + "threshold": 0.009806883476424937 + }, + { + "coverage": 0.886, + "risk": 0.17720090293453725, + "threshold": 0.009773741359433416 + }, + { + "coverage": 0.8863333333333333, + "risk": 0.17713426100037608, + "threshold": 0.009764626128574922 + }, + { + "coverage": 0.8866666666666667, + "risk": 0.17706766917293232, + "threshold": 0.009739769029007937 + }, + { + "coverage": 0.887, + "risk": 0.17700112739571588, + "threshold": 0.009707558009532673 + }, + { + "coverage": 0.8873333333333333, + "risk": 0.17693463561232156, + "threshold": 0.009588734643443718 + }, + { + "coverage": 0.8876666666666667, + "risk": 0.17686819376642884, + "threshold": 0.009529947871915977 + }, + { + "coverage": 0.888, + "risk": 0.1768018018018018, + "threshold": 0.009426600279931942 + }, + { + "coverage": 0.8883333333333333, + "risk": 0.17673545966228893, + "threshold": 0.009339675002764622 + }, + { + "coverage": 0.8886666666666667, + "risk": 0.17666916729182294, + "threshold": 0.009309856113013886 + }, + { + "coverage": 0.889, + "risk": 0.1766029246344207, + "threshold": 0.009305902950246903 + }, + { + "coverage": 0.8893333333333333, + "risk": 0.1765367316341829, + "threshold": 0.009274235158733178 + }, + { + "coverage": 0.8896666666666667, + "risk": 0.17647058823529413, + "threshold": 0.009265269027199494 + }, + { + "coverage": 0.89, + "risk": 0.17640449438202246, + "threshold": 0.009199028934969536 + }, + { + "coverage": 0.8903333333333333, + "risk": 0.1763384500187196, + "threshold": 0.009197837469745751 + }, + { + "coverage": 0.8906666666666667, + "risk": 0.17627245508982037, + "threshold": 0.009173696728676644 + }, + { + "coverage": 0.891, + "risk": 0.17620650953984288, + "threshold": 0.009161744548798203 + }, + { + "coverage": 0.8913333333333333, + "risk": 0.17614061331338818, + "threshold": 0.00908109745604257 + }, + { + "coverage": 0.8916666666666667, + "risk": 0.1760747663551402, + "threshold": 0.009013402906845672 + }, + { + "coverage": 0.892, + "risk": 0.17600896860986548, + "threshold": 0.0090075960965522 + }, + { + "coverage": 0.8923333333333333, + "risk": 0.17594322002241314, + "threshold": 0.008929520943241823 + }, + { + "coverage": 0.8926666666666667, + "risk": 0.17587752053771472, + "threshold": 0.00889573962028354 + }, + { + "coverage": 0.893, + "risk": 0.17581187010078386, + "threshold": 0.008882794975731316 + }, + { + "coverage": 0.8933333333333333, + "risk": 0.17574626865671641, + "threshold": 0.008866558613955048 + }, + { + "coverage": 0.8936666666666667, + "risk": 0.17568071615069003, + "threshold": 0.008831267210734971 + }, + { + "coverage": 0.894, + "risk": 0.1756152125279642, + "threshold": 0.008816036481459158 + }, + { + "coverage": 0.8943333333333333, + "risk": 0.17554975773387999, + "threshold": 0.008732553083949723 + }, + { + "coverage": 0.8946666666666667, + "risk": 0.1754843517138599, + "threshold": 0.00872549450903068 + }, + { + "coverage": 0.895, + "risk": 0.17541899441340783, + "threshold": 0.008722066168937793 + }, + { + "coverage": 0.8953333333333333, + "risk": 0.17535368577810873, + "threshold": 0.008658342190694819 + }, + { + "coverage": 0.8956666666666667, + "risk": 0.17528842575362857, + "threshold": 0.008617196059785379 + }, + { + "coverage": 0.896, + "risk": 0.17522321428571427, + "threshold": 0.008475665724737425 + }, + { + "coverage": 0.8963333333333333, + "risk": 0.17515805132019338, + "threshold": 0.008474640657081017 + }, + { + "coverage": 0.8966666666666666, + "risk": 0.17509293680297397, + "threshold": 0.00843643738909095 + }, + { + "coverage": 0.897, + "risk": 0.1750278706800446, + "threshold": 0.008436282021135286 + }, + { + "coverage": 0.8973333333333333, + "risk": 0.17496285289747399, + "threshold": 0.008415699827047502 + }, + { + "coverage": 0.8976666666666666, + "risk": 0.17489788340141108, + "threshold": 0.008362603265927512 + }, + { + "coverage": 0.898, + "risk": 0.17483296213808464, + "threshold": 0.008339418447954694 + }, + { + "coverage": 0.8983333333333333, + "risk": 0.17476808905380334, + "threshold": 0.008307750776388997 + }, + { + "coverage": 0.8986666666666666, + "risk": 0.17470326409495548, + "threshold": 0.008276423854642582 + }, + { + "coverage": 0.899, + "risk": 0.1746384872080089, + "threshold": 0.008264718473338834 + }, + { + "coverage": 0.8993333333333333, + "risk": 0.17457375833951075, + "threshold": 0.008186197610399484 + }, + { + "coverage": 0.8996666666666666, + "risk": 0.17450907743608743, + "threshold": 0.00814401423892887 + }, + { + "coverage": 0.9, + "risk": 0.17444444444444446, + "threshold": 0.008141185915011817 + }, + { + "coverage": 0.9003333333333333, + "risk": 0.17437985931136615, + "threshold": 0.008123667768691861 + }, + { + "coverage": 0.9006666666666666, + "risk": 0.17431532198371577, + "threshold": 0.008122282200464525 + }, + { + "coverage": 0.901, + "risk": 0.17425083240843509, + "threshold": 0.00811607503131272 + }, + { + "coverage": 0.9013333333333333, + "risk": 0.17418639053254437, + "threshold": 0.00811239130043026 + }, + { + "coverage": 0.9016666666666666, + "risk": 0.17412199630314232, + "threshold": 0.008078366191177513 + }, + { + "coverage": 0.902, + "risk": 0.17405764966740578, + "threshold": 0.008045028219894326 + }, + { + "coverage": 0.9023333333333333, + "risk": 0.17399335057258958, + "threshold": 0.00796088334943566 + }, + { + "coverage": 0.9026666666666666, + "risk": 0.1739290989660266, + "threshold": 0.007907117617573556 + }, + { + "coverage": 0.903, + "risk": 0.17386489479512734, + "threshold": 0.00785173312018727 + }, + { + "coverage": 0.9033333333333333, + "risk": 0.17380073800738008, + "threshold": 0.007824419554858356 + }, + { + "coverage": 0.9036666666666666, + "risk": 0.17373662855035044, + "threshold": 0.007820871267805656 + }, + { + "coverage": 0.904, + "risk": 0.1736725663716814, + "threshold": 0.0077702216240964584 + }, + { + "coverage": 0.9043333333333333, + "risk": 0.17360855141909326, + "threshold": 0.007765239188875266 + }, + { + "coverage": 0.9046666666666666, + "risk": 0.1735445836403832, + "threshold": 0.0077553842601055635 + }, + { + "coverage": 0.905, + "risk": 0.1734806629834254, + "threshold": 0.0077258469132628 + }, + { + "coverage": 0.9053333333333333, + "risk": 0.17341678939617083, + "threshold": 0.007641872473120442 + }, + { + "coverage": 0.9056666666666666, + "risk": 0.17335296282664703, + "threshold": 0.007602737358056042 + }, + { + "coverage": 0.906, + "risk": 0.17328918322295805, + "threshold": 0.00760076094411617 + }, + { + "coverage": 0.9063333333333333, + "risk": 0.1732254505332843, + "threshold": 0.007594318274540247 + }, + { + "coverage": 0.9066666666666666, + "risk": 0.17316176470588235, + "threshold": 0.007565044470086994 + }, + { + "coverage": 0.907, + "risk": 0.1730981256890849, + "threshold": 0.00753815427582928 + }, + { + "coverage": 0.9073333333333333, + "risk": 0.1730345334313005, + "threshold": 0.007532645463300797 + }, + { + "coverage": 0.9076666666666666, + "risk": 0.1729709878810136, + "threshold": 0.007524004325325058 + }, + { + "coverage": 0.908, + "risk": 0.17290748898678415, + "threshold": 0.0075100654536097424 + }, + { + "coverage": 0.9083333333333333, + "risk": 0.1728440366972477, + "threshold": 0.007506482743720387 + }, + { + "coverage": 0.9086666666666666, + "risk": 0.17278063096111518, + "threshold": 0.007479657872076092 + }, + { + "coverage": 0.909, + "risk": 0.17271727172717272, + "threshold": 0.007452527987874575 + }, + { + "coverage": 0.9093333333333333, + "risk": 0.17265395894428154, + "threshold": 0.007399496279493263 + }, + { + "coverage": 0.9096666666666666, + "risk": 0.1725906925613778, + "threshold": 0.007337389253940746 + }, + { + "coverage": 0.91, + "risk": 0.17252747252747253, + "threshold": 0.00732771173499994 + }, + { + "coverage": 0.9103333333333333, + "risk": 0.1724642987916514, + "threshold": 0.007321437631630878 + }, + { + "coverage": 0.9106666666666666, + "risk": 0.17240117130307467, + "threshold": 0.007297880246133776 + }, + { + "coverage": 0.911, + "risk": 0.17233809001097694, + "threshold": 0.007283682577881579 + }, + { + "coverage": 0.9113333333333333, + "risk": 0.17227505486466715, + "threshold": 0.007280298671540853 + }, + { + "coverage": 0.9116666666666666, + "risk": 0.17221206581352833, + "threshold": 0.007227118776316746 + }, + { + "coverage": 0.912, + "risk": 0.17214912280701755, + "threshold": 0.007209824916811147 + }, + { + "coverage": 0.9123333333333333, + "risk": 0.1720862257946657, + "threshold": 0.007206027940011596 + }, + { + "coverage": 0.9126666666666666, + "risk": 0.17202337472607743, + "threshold": 0.007201350734672463 + }, + { + "coverage": 0.913, + "risk": 0.171960569550931, + "threshold": 0.007162763242485124 + }, + { + "coverage": 0.9133333333333333, + "risk": 0.1718978102189781, + "threshold": 0.007120766424196632 + }, + { + "coverage": 0.9136666666666666, + "risk": 0.1718350966800438, + "threshold": 0.007096124569046615 + }, + { + "coverage": 0.914, + "risk": 0.17177242888402625, + "threshold": 0.007050861459723323 + }, + { + "coverage": 0.9143333333333333, + "risk": 0.17170980678089684, + "threshold": 0.007024850971320987 + }, + { + "coverage": 0.9146666666666666, + "risk": 0.1716472303206997, + "threshold": 0.0070206625905012456 + }, + { + "coverage": 0.915, + "risk": 0.17158469945355193, + "threshold": 0.0069515215654783 + }, + { + "coverage": 0.9153333333333333, + "risk": 0.1715222141296431, + "threshold": 0.006902502881325545 + }, + { + "coverage": 0.9156666666666666, + "risk": 0.17145977429923553, + "threshold": 0.006901639789039878 + }, + { + "coverage": 0.916, + "risk": 0.17139737991266377, + "threshold": 0.006890955767042082 + }, + { + "coverage": 0.9163333333333333, + "risk": 0.17133503092033467, + "threshold": 0.006809006949258474 + }, + { + "coverage": 0.9166666666666666, + "risk": 0.17127272727272727, + "threshold": 0.006797091039169866 + }, + { + "coverage": 0.917, + "risk": 0.17121046892039257, + "threshold": 0.006788910396369861 + }, + { + "coverage": 0.9173333333333333, + "risk": 0.1711482558139535, + "threshold": 0.006729789008064317 + }, + { + "coverage": 0.9176666666666666, + "risk": 0.17108608790410462, + "threshold": 0.0066589132718395064 + }, + { + "coverage": 0.918, + "risk": 0.1710239651416122, + "threshold": 0.006538529767029127 + }, + { + "coverage": 0.9183333333333333, + "risk": 0.17096188747731397, + "threshold": 0.006498770389050794 + }, + { + "coverage": 0.9186666666666666, + "risk": 0.170899854862119, + "threshold": 0.006498206599080509 + }, + { + "coverage": 0.919, + "risk": 0.1708378672470076, + "threshold": 0.006466647623092752 + }, + { + "coverage": 0.9193333333333333, + "risk": 0.17077592458303117, + "threshold": 0.006393120570240183 + }, + { + "coverage": 0.9196666666666666, + "risk": 0.17071402682131206, + "threshold": 0.006367680450231176 + }, + { + "coverage": 0.92, + "risk": 0.17065217391304346, + "threshold": 0.006366062007578066 + }, + { + "coverage": 0.9203333333333333, + "risk": 0.1705903658094893, + "threshold": 0.006330723332902697 + }, + { + "coverage": 0.9206666666666666, + "risk": 0.17052860246198406, + "threshold": 0.006328791506483392 + }, + { + "coverage": 0.921, + "risk": 0.17046688382193267, + "threshold": 0.006270780177386871 + }, + { + "coverage": 0.9213333333333333, + "risk": 0.17040520984081042, + "threshold": 0.006211911776384649 + }, + { + "coverage": 0.9216666666666666, + "risk": 0.17034358047016274, + "threshold": 0.006119153899269675 + }, + { + "coverage": 0.922, + "risk": 0.1702819956616052, + "threshold": 0.006100984772685378 + }, + { + "coverage": 0.9223333333333333, + "risk": 0.1702204553668233, + "threshold": 0.006070771993521774 + }, + { + "coverage": 0.9226666666666666, + "risk": 0.17015895953757226, + "threshold": 0.006066852168015363 + }, + { + "coverage": 0.923, + "risk": 0.17009750812567714, + "threshold": 0.005931781658246468 + }, + { + "coverage": 0.9233333333333333, + "risk": 0.1700361010830325, + "threshold": 0.005858788383663375 + }, + { + "coverage": 0.9236666666666666, + "risk": 0.16997473836160232, + "threshold": 0.005842993470183493 + }, + { + "coverage": 0.924, + "risk": 0.1699134199134199, + "threshold": 0.0058399483409718195 + }, + { + "coverage": 0.9243333333333333, + "risk": 0.16985214569058782, + "threshold": 0.005808946673573824 + }, + { + "coverage": 0.9246666666666666, + "risk": 0.16979091564527757, + "threshold": 0.005701411193346741 + }, + { + "coverage": 0.925, + "risk": 0.16972972972972972, + "threshold": 0.005693751339639105 + }, + { + "coverage": 0.9253333333333333, + "risk": 0.1696685878962536, + "threshold": 0.005611989103886774 + }, + { + "coverage": 0.9256666666666666, + "risk": 0.16960749009722723, + "threshold": 0.0055690925586514934 + }, + { + "coverage": 0.926, + "risk": 0.1695464362850972, + "threshold": 0.0055606695844835595 + }, + { + "coverage": 0.9263333333333333, + "risk": 0.16948542641237854, + "threshold": 0.005528993986237355 + }, + { + "coverage": 0.9266666666666666, + "risk": 0.16942446043165468, + "threshold": 0.005526304383845268 + }, + { + "coverage": 0.927, + "risk": 0.16936353829557713, + "threshold": 0.005421520336449478 + }, + { + "coverage": 0.9273333333333333, + "risk": 0.16930265995686555, + "threshold": 0.0053902599038992615 + }, + { + "coverage": 0.9276666666666666, + "risk": 0.1692418253683076, + "threshold": 0.00538818646353272 + }, + { + "coverage": 0.928, + "risk": 0.16918103448275862, + "threshold": 0.0053558620459955575 + }, + { + "coverage": 0.9283333333333333, + "risk": 0.16912028725314182, + "threshold": 0.005293062537031715 + }, + { + "coverage": 0.9286666666666666, + "risk": 0.16905958363244794, + "threshold": 0.005236017615355507 + }, + { + "coverage": 0.929, + "risk": 0.1689989235737352, + "threshold": 0.00523499737093882 + }, + { + "coverage": 0.9293333333333333, + "risk": 0.16893830703012913, + "threshold": 0.005152676679455214 + }, + { + "coverage": 0.9296666666666666, + "risk": 0.16887773395482253, + "threshold": 0.005117248368782016 + }, + { + "coverage": 0.93, + "risk": 0.16881720430107527, + "threshold": 0.005056342442202157 + }, + { + "coverage": 0.9303333333333333, + "risk": 0.16875671802221426, + "threshold": 0.005023246932483242 + }, + { + "coverage": 0.9306666666666666, + "risk": 0.16869627507163323, + "threshold": 0.005013247452328005 + }, + { + "coverage": 0.931, + "risk": 0.1686358754027927, + "threshold": 0.004960397969318415 + }, + { + "coverage": 0.9313333333333333, + "risk": 0.16857551896921977, + "threshold": 0.004954339680421958 + }, + { + "coverage": 0.9316666666666666, + "risk": 0.16851520572450804, + "threshold": 0.0049499262909502816 + }, + { + "coverage": 0.932, + "risk": 0.1684549356223176, + "threshold": 0.004910298812352772 + }, + { + "coverage": 0.9323333333333333, + "risk": 0.16839470861637468, + "threshold": 0.00490903410498145 + }, + { + "coverage": 0.9326666666666666, + "risk": 0.16833452466047177, + "threshold": 0.0048754513199321175 + }, + { + "coverage": 0.933, + "risk": 0.1682743837084673, + "threshold": 0.0048663069263627035 + }, + { + "coverage": 0.9333333333333333, + "risk": 0.1682142857142857, + "threshold": 0.00485678166062373 + }, + { + "coverage": 0.9336666666666666, + "risk": 0.16815423063191717, + "threshold": 0.004842116066498368 + }, + { + "coverage": 0.934, + "risk": 0.16809421841541755, + "threshold": 0.00481558999294465 + }, + { + "coverage": 0.9343333333333333, + "risk": 0.16803424901890832, + "threshold": 0.00481135605999603 + }, + { + "coverage": 0.9346666666666666, + "risk": 0.1679743223965763, + "threshold": 0.004802931072191962 + }, + { + "coverage": 0.935, + "risk": 0.1679144385026738, + "threshold": 0.004754534678059052 + }, + { + "coverage": 0.9353333333333333, + "risk": 0.16785459729151816, + "threshold": 0.004753043762549487 + }, + { + "coverage": 0.9356666666666666, + "risk": 0.16779479871749198, + "threshold": 0.004748192250558176 + }, + { + "coverage": 0.936, + "risk": 0.16773504273504272, + "threshold": 0.0047450405141114095 + }, + { + "coverage": 0.9363333333333334, + "risk": 0.1676753292986828, + "threshold": 0.00470099058743971 + }, + { + "coverage": 0.9366666666666666, + "risk": 0.16761565836298933, + "threshold": 0.00460090241622727 + }, + { + "coverage": 0.937, + "risk": 0.16755602988260407, + "threshold": 0.004593054371119325 + }, + { + "coverage": 0.9373333333333334, + "risk": 0.16749644381223328, + "threshold": 0.004580379543483278 + }, + { + "coverage": 0.9376666666666666, + "risk": 0.16743690010664772, + "threshold": 0.004566277083448449 + }, + { + "coverage": 0.938, + "risk": 0.1673773987206823, + "threshold": 0.004544046089632509 + }, + { + "coverage": 0.9383333333333334, + "risk": 0.16731793960923624, + "threshold": 0.004489885067794008 + }, + { + "coverage": 0.9386666666666666, + "risk": 0.16725852272727273, + "threshold": 0.004486464984494579 + }, + { + "coverage": 0.939, + "risk": 0.16719914802981894, + "threshold": 0.004481411454523439 + }, + { + "coverage": 0.9393333333333334, + "risk": 0.16713981547196594, + "threshold": 0.004481242141063271 + }, + { + "coverage": 0.9396666666666667, + "risk": 0.16708052500886839, + "threshold": 0.004446908658849108 + }, + { + "coverage": 0.94, + "risk": 0.16702127659574467, + "threshold": 0.004428155752802877 + }, + { + "coverage": 0.9403333333333334, + "risk": 0.16696207018787665, + "threshold": 0.0044266082525923935 + }, + { + "coverage": 0.9406666666666667, + "risk": 0.1669029057406095, + "threshold": 0.004407229539165909 + }, + { + "coverage": 0.941, + "risk": 0.16684378320935175, + "threshold": 0.004374964519129131 + }, + { + "coverage": 0.9413333333333334, + "risk": 0.16678470254957506, + "threshold": 0.0043659428062329635 + }, + { + "coverage": 0.9416666666666667, + "risk": 0.16672566371681416, + "threshold": 0.00434597801772496 + }, + { + "coverage": 0.942, + "risk": 0.16666666666666666, + "threshold": 0.004308772588912088 + }, + { + "coverage": 0.9423333333333334, + "risk": 0.16696144322603468, + "threshold": 0.004296457791033729 + }, + { + "coverage": 0.9426666666666667, + "risk": 0.1669024045261669, + "threshold": 0.004284143492855139 + }, + { + "coverage": 0.943, + "risk": 0.16684340756451044, + "threshold": 0.00426803745511424 + }, + { + "coverage": 0.9433333333333334, + "risk": 0.1667844522968198, + "threshold": 0.004241307482743955 + }, + { + "coverage": 0.9436666666666667, + "risk": 0.16672553867891204, + "threshold": 0.004226619231973342 + }, + { + "coverage": 0.944, + "risk": 0.16666666666666666, + "threshold": 0.004097135749100501 + }, + { + "coverage": 0.9443333333333334, + "risk": 0.1666078362160254, + "threshold": 0.00408403195258518 + }, + { + "coverage": 0.9446666666666667, + "risk": 0.16654904728299225, + "threshold": 0.004019190154123686 + }, + { + "coverage": 0.945, + "risk": 0.16649029982363317, + "threshold": 0.004003600588443868 + }, + { + "coverage": 0.9453333333333334, + "risk": 0.16643159379407615, + "threshold": 0.003914621977011966 + }, + { + "coverage": 0.9456666666666667, + "risk": 0.1663729291505111, + "threshold": 0.003888490781445094 + }, + { + "coverage": 0.946, + "risk": 0.16631430584918958, + "threshold": 0.003858131908746952 + }, + { + "coverage": 0.9463333333333334, + "risk": 0.16625572384642479, + "threshold": 0.003857433027089694 + }, + { + "coverage": 0.9466666666666667, + "risk": 0.16619718309859155, + "threshold": 0.003831749006332521 + }, + { + "coverage": 0.947, + "risk": 0.16613868356212602, + "threshold": 0.0038100652239783836 + }, + { + "coverage": 0.9473333333333334, + "risk": 0.1660802251935257, + "threshold": 0.003733258092971334 + }, + { + "coverage": 0.9476666666666667, + "risk": 0.1660218079493493, + "threshold": 0.0037105015181583675 + }, + { + "coverage": 0.948, + "risk": 0.1659634317862166, + "threshold": 0.0037013147627030624 + }, + { + "coverage": 0.9483333333333334, + "risk": 0.16590509666080844, + "threshold": 0.003694047623773454 + }, + { + "coverage": 0.9486666666666667, + "risk": 0.16584680252986647, + "threshold": 0.00366148412780341 + }, + { + "coverage": 0.949, + "risk": 0.1657885493501932, + "threshold": 0.0036493094433768373 + }, + { + "coverage": 0.9493333333333334, + "risk": 0.16573033707865167, + "threshold": 0.003648110227990455 + }, + { + "coverage": 0.9496666666666667, + "risk": 0.16567216567216567, + "threshold": 0.003641885004689361 + }, + { + "coverage": 0.95, + "risk": 0.1656140350877193, + "threshold": 0.0036327889088644066 + }, + { + "coverage": 0.9503333333333334, + "risk": 0.16555594528235706, + "threshold": 0.0036259333866524796 + }, + { + "coverage": 0.9506666666666667, + "risk": 0.16549789621318373, + "threshold": 0.0035812716211436656 + }, + { + "coverage": 0.951, + "risk": 0.16543988783736419, + "threshold": 0.0035488292676232555 + }, + { + "coverage": 0.9513333333333334, + "risk": 0.16538192011212333, + "threshold": 0.0035095204839267926 + }, + { + "coverage": 0.9516666666666667, + "risk": 0.16532399299474607, + "threshold": 0.0034928281403754794 + }, + { + "coverage": 0.952, + "risk": 0.16526610644257703, + "threshold": 0.00345354530888017 + }, + { + "coverage": 0.9523333333333334, + "risk": 0.16520826041302064, + "threshold": 0.0033536215426328924 + }, + { + "coverage": 0.9526666666666667, + "risk": 0.16515045486354094, + "threshold": 0.0033372640600322794 + }, + { + "coverage": 0.953, + "risk": 0.16509268975166141, + "threshold": 0.0033097347396485208 + }, + { + "coverage": 0.9533333333333334, + "risk": 0.16503496503496504, + "threshold": 0.003284284256213439 + }, + { + "coverage": 0.9536666666666667, + "risk": 0.16497728067109402, + "threshold": 0.003224346932552248 + }, + { + "coverage": 0.954, + "risk": 0.16491963661774983, + "threshold": 0.003110911838661939 + }, + { + "coverage": 0.9543333333333334, + "risk": 0.164862032832693, + "threshold": 0.003091753061513015 + }, + { + "coverage": 0.9546666666666667, + "risk": 0.164804469273743, + "threshold": 0.003078854403022238 + }, + { + "coverage": 0.955, + "risk": 0.16474694589877836, + "threshold": 0.003046237186795454 + }, + { + "coverage": 0.9553333333333334, + "risk": 0.16468946266573622, + "threshold": 0.003036481985717429 + }, + { + "coverage": 0.9556666666666667, + "risk": 0.1646320195326125, + "threshold": 0.0030338946587222873 + }, + { + "coverage": 0.956, + "risk": 0.16457461645746166, + "threshold": 0.0030094650508193294 + }, + { + "coverage": 0.9563333333333334, + "risk": 0.16451725339839665, + "threshold": 0.0030018285309353315 + }, + { + "coverage": 0.9566666666666667, + "risk": 0.16445993031358885, + "threshold": 0.002983264237111365 + }, + { + "coverage": 0.957, + "risk": 0.16440264716126785, + "threshold": 0.0029382756766892374 + }, + { + "coverage": 0.9573333333333334, + "risk": 0.16434540389972144, + "threshold": 0.0029301479755715885 + }, + { + "coverage": 0.9576666666666667, + "risk": 0.1642882004872955, + "threshold": 0.0029299833035314983 + }, + { + "coverage": 0.958, + "risk": 0.16423103688239388, + "threshold": 0.0028361742284451806 + }, + { + "coverage": 0.9583333333333334, + "risk": 0.16417391304347825, + "threshold": 0.0028021811748230176 + }, + { + "coverage": 0.9586666666666667, + "risk": 0.16411682892906815, + "threshold": 0.002780087571627088 + }, + { + "coverage": 0.959, + "risk": 0.1640597844977407, + "threshold": 0.0027529532161176917 + }, + { + "coverage": 0.9593333333333334, + "risk": 0.16400277970813065, + "threshold": 0.002747289233330579 + }, + { + "coverage": 0.9596666666666667, + "risk": 0.16394581451893017, + "threshold": 0.0027160899266184123 + }, + { + "coverage": 0.96, + "risk": 0.1638888888888889, + "threshold": 0.002711296997226679 + }, + { + "coverage": 0.9603333333333334, + "risk": 0.16383200277681362, + "threshold": 0.002702384262982106 + }, + { + "coverage": 0.9606666666666667, + "risk": 0.16377515614156835, + "threshold": 0.0026965111497358548 + }, + { + "coverage": 0.961, + "risk": 0.16371834894207424, + "threshold": 0.0026855349710259413 + }, + { + "coverage": 0.9613333333333334, + "risk": 0.1636615811373093, + "threshold": 0.0026535119021904754 + }, + { + "coverage": 0.9616666666666667, + "risk": 0.1636048526863085, + "threshold": 0.002619836599806427 + }, + { + "coverage": 0.962, + "risk": 0.16354816354816354, + "threshold": 0.0026083873465212928 + }, + { + "coverage": 0.9623333333333334, + "risk": 0.16349151368202286, + "threshold": 0.0025856310945670457 + }, + { + "coverage": 0.9626666666666667, + "risk": 0.1634349030470914, + "threshold": 0.0025825617615304742 + }, + { + "coverage": 0.963, + "risk": 0.16337833160263066, + "threshold": 0.002525991419376773 + }, + { + "coverage": 0.9633333333333334, + "risk": 0.16332179930795848, + "threshold": 0.0025177736802929718 + }, + { + "coverage": 0.9636666666666667, + "risk": 0.16326530612244897, + "threshold": 0.0025040676945011155 + }, + { + "coverage": 0.964, + "risk": 0.1632088520055325, + "threshold": 0.0024840813726643343 + }, + { + "coverage": 0.9643333333333334, + "risk": 0.16315243691669548, + "threshold": 0.0024470596003731892 + }, + { + "coverage": 0.9646666666666667, + "risk": 0.1630960608154803, + "threshold": 0.002410363187595116 + }, + { + "coverage": 0.965, + "risk": 0.1630397236614853, + "threshold": 0.0024083409039426505 + }, + { + "coverage": 0.9653333333333334, + "risk": 0.16298342541436464, + "threshold": 0.002337843921246831 + }, + { + "coverage": 0.9656666666666667, + "risk": 0.1629271660338281, + "threshold": 0.002302896372176843 + }, + { + "coverage": 0.966, + "risk": 0.16287094547964112, + "threshold": 0.002290312825577913 + }, + { + "coverage": 0.9663333333333334, + "risk": 0.1628147637116247, + "threshold": 0.0022789718176329616 + }, + { + "coverage": 0.9666666666666667, + "risk": 0.16275862068965516, + "threshold": 0.002278187964502 + }, + { + "coverage": 0.967, + "risk": 0.16270251637366426, + "threshold": 0.0022494589318513987 + }, + { + "coverage": 0.9673333333333334, + "risk": 0.16264645072363887, + "threshold": 0.002192368751529628 + }, + { + "coverage": 0.9676666666666667, + "risk": 0.1625904236996211, + "threshold": 0.002185017603627565 + }, + { + "coverage": 0.968, + "risk": 0.162534435261708, + "threshold": 0.002181862492912007 + }, + { + "coverage": 0.9683333333333334, + "risk": 0.16247848537005163, + "threshold": 0.002158243378949143 + }, + { + "coverage": 0.9686666666666667, + "risk": 0.1624225739848589, + "threshold": 0.0021394593800819614 + }, + { + "coverage": 0.969, + "risk": 0.16236670106639148, + "threshold": 0.0021289559298578223 + }, + { + "coverage": 0.9693333333333334, + "risk": 0.1623108665749656, + "threshold": 0.0021094539825969125 + }, + { + "coverage": 0.9696666666666667, + "risk": 0.16225507047095222, + "threshold": 0.002094813916206539 + }, + { + "coverage": 0.97, + "risk": 0.16219931271477664, + "threshold": 0.002055794282522837 + }, + { + "coverage": 0.9703333333333334, + "risk": 0.16214359326691857, + "threshold": 0.0020201647906744205 + }, + { + "coverage": 0.9706666666666667, + "risk": 0.1620879120879121, + "threshold": 0.0019975093664058472 + }, + { + "coverage": 0.971, + "risk": 0.16203226913834534, + "threshold": 0.0019923921246443295 + }, + { + "coverage": 0.9713333333333334, + "risk": 0.16197666437886069, + "threshold": 0.001966205161953862 + }, + { + "coverage": 0.9716666666666667, + "risk": 0.16192109777015437, + "threshold": 0.0019573049737833617 + }, + { + "coverage": 0.972, + "risk": 0.16186556927297668, + "threshold": 0.0019107262897453577 + }, + { + "coverage": 0.9723333333333334, + "risk": 0.16181007884813164, + "threshold": 0.0018914952802735025 + }, + { + "coverage": 0.9726666666666667, + "risk": 0.16175462645647704, + "threshold": 0.0018561070574980357 + }, + { + "coverage": 0.973, + "risk": 0.1616992120589243, + "threshold": 0.0018372730982366703 + }, + { + "coverage": 0.9733333333333334, + "risk": 0.16164383561643836, + "threshold": 0.0018153561448684984 + }, + { + "coverage": 0.9736666666666667, + "risk": 0.16158849709003767, + "threshold": 0.0018142001117767805 + }, + { + "coverage": 0.974, + "risk": 0.16153319644079397, + "threshold": 0.001786310816306509 + }, + { + "coverage": 0.9743333333333334, + "risk": 0.16147793362983237, + "threshold": 0.0017656701801423622 + }, + { + "coverage": 0.9746666666666667, + "risk": 0.16142270861833105, + "threshold": 0.001729682224565612 + }, + { + "coverage": 0.975, + "risk": 0.16136752136752136, + "threshold": 0.001721533890228994 + }, + { + "coverage": 0.9753333333333334, + "risk": 0.16131237183868763, + "threshold": 0.0016744589794665855 + }, + { + "coverage": 0.9756666666666667, + "risk": 0.16125725999316706, + "threshold": 0.0016707251201903011 + }, + { + "coverage": 0.976, + "risk": 0.16120218579234974, + "threshold": 0.0016560800141358584 + }, + { + "coverage": 0.9763333333333334, + "risk": 0.1611471491976784, + "threshold": 0.001654475978735936 + }, + { + "coverage": 0.9766666666666667, + "risk": 0.16109215017064846, + "threshold": 0.0016152298590979572 + }, + { + "coverage": 0.977, + "risk": 0.16103718867280792, + "threshold": 0.0016095128325175334 + }, + { + "coverage": 0.9773333333333334, + "risk": 0.16098226466575716, + "threshold": 0.0015725065939398188 + }, + { + "coverage": 0.9776666666666667, + "risk": 0.160927378111149, + "threshold": 0.001568303301583963 + }, + { + "coverage": 0.978, + "risk": 0.1608725289706885, + "threshold": 0.0015669437526563725 + }, + { + "coverage": 0.9783333333333334, + "risk": 0.1608177172061329, + "threshold": 0.0015601953979998836 + }, + { + "coverage": 0.9786666666666667, + "risk": 0.16076294277929154, + "threshold": 0.0015392509181539672 + }, + { + "coverage": 0.979, + "risk": 0.16070820565202587, + "threshold": 0.0015323262577797377 + }, + { + "coverage": 0.9793333333333333, + "risk": 0.16065350578624915, + "threshold": 0.0015278716224825684 + }, + { + "coverage": 0.9796666666666667, + "risk": 0.1605988431439265, + "threshold": 0.0014970458510148057 + }, + { + "coverage": 0.98, + "risk": 0.16054421768707483, + "threshold": 0.0014885802889295409 + }, + { + "coverage": 0.9803333333333333, + "risk": 0.16048962937776268, + "threshold": 0.0014793039817521125 + }, + { + "coverage": 0.9806666666666667, + "risk": 0.16043507817811012, + "threshold": 0.001470279968829618 + }, + { + "coverage": 0.981, + "risk": 0.16038056405028883, + "threshold": 0.0014556456715881319 + }, + { + "coverage": 0.9813333333333333, + "risk": 0.16032608695652173, + "threshold": 0.001450162609424576 + }, + { + "coverage": 0.9816666666666667, + "risk": 0.16027164685908318, + "threshold": 0.001405710657819093 + }, + { + "coverage": 0.982, + "risk": 0.16021724372029872, + "threshold": 0.0013898215399006768 + }, + { + "coverage": 0.9823333333333333, + "risk": 0.16016287750254496, + "threshold": 0.0013729150762158526 + }, + { + "coverage": 0.9826666666666667, + "risk": 0.16010854816824965, + "threshold": 0.001369676499773991 + }, + { + "coverage": 0.983, + "risk": 0.16005425567989148, + "threshold": 0.0013410221325110955 + }, + { + "coverage": 0.9833333333333333, + "risk": 0.16, + "threshold": 0.0013399831985271221 + }, + { + "coverage": 0.9836666666666667, + "risk": 0.15994578109115554, + "threshold": 0.0013126176277437898 + }, + { + "coverage": 0.984, + "risk": 0.15989159891598917, + "threshold": 0.001296048322269419 + }, + { + "coverage": 0.9843333333333333, + "risk": 0.15983745343718253, + "threshold": 0.0012939565348248337 + }, + { + "coverage": 0.9846666666666667, + "risk": 0.15978334461746785, + "threshold": 0.0012527921470073258 + }, + { + "coverage": 0.985, + "risk": 0.15972927241962775, + "threshold": 0.0011886092805492726 + }, + { + "coverage": 0.9853333333333333, + "risk": 0.15967523680649526, + "threshold": 0.0011860513413085288 + }, + { + "coverage": 0.9856666666666667, + "risk": 0.15962123774095366, + "threshold": 0.001145659418948021 + }, + { + "coverage": 0.986, + "risk": 0.15956727518593644, + "threshold": 0.0011398751847559166 + }, + { + "coverage": 0.9863333333333333, + "risk": 0.15951334910442716, + "threshold": 0.0011175115864390807 + }, + { + "coverage": 0.9866666666666667, + "risk": 0.15945945945945947, + "threshold": 0.001108476359496547 + }, + { + "coverage": 0.987, + "risk": 0.15940560621411684, + "threshold": 0.0011025534075981113 + }, + { + "coverage": 0.9873333333333333, + "risk": 0.15935178933153274, + "threshold": 0.0010778191532712553 + }, + { + "coverage": 0.9876666666666667, + "risk": 0.1592980087748903, + "threshold": 0.0010201430942746517 + }, + { + "coverage": 0.988, + "risk": 0.1592442645074224, + "threshold": 0.000998900859040098 + }, + { + "coverage": 0.9883333333333333, + "risk": 0.15919055649241146, + "threshold": 0.0009571807510975239 + }, + { + "coverage": 0.9886666666666667, + "risk": 0.15913688469318948, + "threshold": 0.0009416512525685474 + }, + { + "coverage": 0.989, + "risk": 0.15908324907313784, + "threshold": 0.0009343740011867195 + }, + { + "coverage": 0.9893333333333333, + "risk": 0.15902964959568733, + "threshold": 0.0009230995843876619 + }, + { + "coverage": 0.9896666666666667, + "risk": 0.15897608622431794, + "threshold": 0.0009209772733476913 + }, + { + "coverage": 0.99, + "risk": 0.1589225589225589, + "threshold": 0.0008985707959738009 + }, + { + "coverage": 0.9903333333333333, + "risk": 0.15886906765398856, + "threshold": 0.0008955000769669429 + }, + { + "coverage": 0.9906666666666667, + "risk": 0.15881561238223418, + "threshold": 0.0008514159015959165 + }, + { + "coverage": 0.991, + "risk": 0.15876219307097209, + "threshold": 0.0007831324481554095 + }, + { + "coverage": 0.9913333333333333, + "risk": 0.15870880968392737, + "threshold": 0.000760160026150846 + }, + { + "coverage": 0.9916666666666667, + "risk": 0.15865546218487395, + "threshold": 0.0007123117807113041 + }, + { + "coverage": 0.992, + "risk": 0.1586021505376344, + "threshold": 0.0006999337213578478 + }, + { + "coverage": 0.9923333333333333, + "risk": 0.15854887470607995, + "threshold": 0.0006834044169813822 + }, + { + "coverage": 0.9926666666666667, + "risk": 0.1584956346541303, + "threshold": 0.000683184497532922 + }, + { + "coverage": 0.993, + "risk": 0.15844243034575362, + "threshold": 0.0006545229199005305 + }, + { + "coverage": 0.9933333333333333, + "risk": 0.15838926174496645, + "threshold": 0.0006347236617978296 + }, + { + "coverage": 0.9936666666666667, + "risk": 0.15833612881583362, + "threshold": 0.0006280833394729042 + }, + { + "coverage": 0.994, + "risk": 0.15828303152246814, + "threshold": 0.0006064108522709659 + }, + { + "coverage": 0.9943333333333333, + "risk": 0.15822996982903118, + "threshold": 0.0005996328893833714 + }, + { + "coverage": 0.9946666666666667, + "risk": 0.1581769436997319, + "threshold": 0.0005563796123576412 + }, + { + "coverage": 0.995, + "risk": 0.15812395309882746, + "threshold": 0.0005269920686088171 + }, + { + "coverage": 0.9953333333333333, + "risk": 0.1580709979906229, + "threshold": 0.0004647959706020991 + }, + { + "coverage": 0.9956666666666667, + "risk": 0.15801807833947104, + "threshold": 0.0004635220169096875 + }, + { + "coverage": 0.996, + "risk": 0.15796519410977242, + "threshold": 0.00045425868518249996 + }, + { + "coverage": 0.9963333333333333, + "risk": 0.15791234526597525, + "threshold": 0.0004337108397364496 + }, + { + "coverage": 0.9966666666666667, + "risk": 0.15785953177257525, + "threshold": 0.00041810444995262326 + }, + { + "coverage": 0.997, + "risk": 0.15780675359411567, + "threshold": 0.0003898704996254181 + }, + { + "coverage": 0.9973333333333333, + "risk": 0.15775401069518716, + "threshold": 0.0003876736493226494 + }, + { + "coverage": 0.9976666666666667, + "risk": 0.15770130304042768, + "threshold": 0.00037593886886055 + }, + { + "coverage": 0.998, + "risk": 0.1576486305945224, + "threshold": 0.0003131397647017947 + }, + { + "coverage": 0.9983333333333333, + "risk": 0.15759599332220367, + "threshold": 0.00027916933640686944 + }, + { + "coverage": 0.9986666666666667, + "risk": 0.157543391188251, + "threshold": 0.00026639007488888787 + }, + { + "coverage": 0.999, + "risk": 0.15749082415749083, + "threshold": 0.0002401424161400307 + }, + { + "coverage": 0.9993333333333333, + "risk": 0.15743829219479652, + "threshold": 0.00023342592405766174 + }, + { + "coverage": 0.9996666666666667, + "risk": 0.15738579526508836, + "threshold": 0.00017991571917858734 + }, + { + "coverage": 1.0, + "risk": 0.15733333333333333, + "threshold": 0.00014517113061610106 + } + ], + "style_accuracy": { + "lower": 0.7076666666666667, + "mean": 0.7256666666666667, + "speaker_count": 1500.0, + "upper": 0.7426666666666667 + }, + "style_brier": 0.18148320570541257, + "style_ece": 0.03822516584210098, + "support": { + "accuracy": 0.801, + "brier": 0.1397579431180587, + "count": 3000, + "ece": 0.08753392520064882, + "f1": 0.7352549889135255, + "precision": 0.6605577689243027, + "recall": 0.829 + } + }, + "task_scope": "style+support+conflict+citation", + "threshold": 0.9645485858739319 +} diff --git a/supplement/language_provenance/artifacts/snli_semantic_mlp_seed1.json b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed1.json new file mode 100644 index 0000000..d5b33ed --- /dev/null +++ b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed1.json @@ -0,0 +1,137825 @@ +{ + "accept_temperature": 1.25, + "acceptance_mode": "selective_head", + "calibration_file": "checkpoints/snli_semantic_mlp_seed1.calibration.json", + "checkpoint_data_fingerprint": "sha256:0f4c8bb05648174215d0c23ab9ea29d88c97116c40b33b95e05b70f566c51a6c", + "checkpoint_epoch": 2, + "citation_threshold": 0.6850651502609253, + "citation_top_k": 2, + "evaluation_data_fingerprint": "sha256:0b6094089901215bf51c37e8c38ec81617fe96bbf5c512363f94b1d84827effe", + "evaluation_splits": [ + "test" + ], + "gate_threshold": 0.5, + "rows": [ + { + "accept_correct": 0, + "accept_probability": 0.6650759184251909, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17503459751605988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.311349481344223, + 0.49217817187309265, + 0.9316077828407288, + 0.38197940587997437, + 0.05773615464568138 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 1, + "style_probability": 0.6099550724029541, + "style_target": 1, + "support_probability": 0.7002346515655518, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7670234110580697, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15061084926128387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2786084711551666, + 0.4903106987476349, + 0.9642768502235413, + 0.32759329676628113, + 0.08963877707719803 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 0, + "style_probability": 0.7528350949287415, + "style_target": 0, + "support_probability": 0.7317819595336914, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.75772938719404, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2657783329486847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21616660058498383, + 0.9214860200881958, + 0.1474199742078781, + 0.41950923204421997, + 0.1472736895084381, + 0.08101523667573929 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 1, + "style_probability": 0.6294142007827759, + "style_target": 1, + "support_probability": 0.8276510238647461, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6490041678803178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23833471536636353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21422305703163147, + 0.9167251586914062, + 0.1452573835849762, + 0.3921842575073242, + 0.10105223208665848, + 0.23343554139137268 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 1, + "style_probability": 0.437211811542511, + "style_target": 0, + "support_probability": 0.7475558519363403, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.26502077140141206, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44406744837760925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2693605124950409, + 0.36538898944854736, + 0.8547930121421814, + 0.1174006536602974, + 0.10208979994058609, + 0.12680858373641968, + 0.10347670316696167, + 0.2339915931224823, + 0.38244232535362244 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 0, + "style_probability": 0.4998643696308136, + "style_target": 1, + "support_probability": 0.29606500267982483, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03738766904526154, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2689516544342041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36121851205825806, + 0.3820970058441162, + 0.9165709018707275, + 0.1074819341301918, + 0.13183699548244476, + 0.3010035455226898, + 0.11958038806915283, + 0.14718766510486603, + 0.2020280659198761 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 1, + "style_probability": 0.11305984109640121, + "style_target": 0, + "support_probability": 0.48716652393341064, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18270429429878157, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7893476486206055, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4378941059112549, + 0.4744361340999603, + 0.7775974869728088, + 0.31023845076560974, + 0.09483229368925095, + 0.06037720665335655, + 0.06924030184745789, + 0.08782405406236649, + 0.17078512907028198, + 0.12631481885910034, + 0.09927998483181 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 1, + "style_probability": 0.55999356508255, + "style_target": 1, + "support_probability": 0.20195262134075165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46898157611802893, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5721890926361084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4349204897880554, + 0.4627038538455963, + 0.8241270780563354, + 0.283795565366745, + 0.20401614904403687, + 0.11902762204408646, + 0.12425868213176727, + 0.06678835302591324, + 0.0645446926355362, + 0.05239643529057503, + 0.11504863202571869 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 0, + "style_probability": 0.774347722530365, + "style_target": 0, + "support_probability": 0.39139124751091003, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8350436460717271, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20412033796310425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9338026642799377, + 0.4192667305469513, + 0.4404350221157074, + 0.612748384475708, + 0.08012497425079346 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.859178900718689, + "style_target": 1, + "support_probability": 0.841849684715271, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38395279939130833, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30231308937072754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9401294589042664, + 0.48713549971580505, + 0.46766698360443115, + 0.6429224014282227, + 0.09618985652923584 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.41893404722213745, + "style_target": 0, + "support_probability": 0.7924761176109314, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1551596271689842, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3987278938293457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14430446922779083, + 0.5162221789360046, + 0.7433833479881287, + 0.1444443017244339, + 0.045102719217538834, + 0.11945091933012009 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.7381625771522522, + "style_target": 1, + "support_probability": 0.20109687745571136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05723410008685526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.407133549451828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1912970095872879, + 0.5320635437965393, + 0.7223460674285889, + 0.1334322690963745, + 0.042146701365709305, + 0.04778868332505226 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.27590811252593994, + "style_target": 0, + "support_probability": 0.20839408040046692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8151020857551246, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16628360748291016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.988246500492096, + 0.23880814015865326, + 0.3627106547355652, + 0.329434335231781, + 0.08643496036529541, + 0.04163043946027756, + 0.05748654529452324, + 0.10246603935956955, + 0.12536394596099854 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.7786011099815369, + "style_target": 1, + "support_probability": 0.596642017364502, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5361509858143029, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15922723710536957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9783912301063538, + 0.23472364246845245, + 0.3863058388233185, + 0.3056098222732544, + 0.13109122216701508, + 0.08382879942655563, + 0.12145747244358063, + 0.045787353068590164, + 0.08955684304237366 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.3139556646347046, + "style_target": 0, + "support_probability": 0.5111121535301208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7664555532248202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22305572032928467, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2256164252758026, + 0.2661043405532837, + 0.9211268424987793, + 0.3920604884624481, + 0.2885887920856476, + 0.14967045187950134, + 0.14584526419639587, + 0.09829028695821762, + 0.1689128577709198, + 0.10322675108909607, + 0.14997607469558716 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.8376944065093994, + "style_target": 1, + "support_probability": 0.6602149605751038, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5081960675299803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20320342481136322, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28634634613990784, + 0.287221223115921, + 0.9580504298210144, + 0.44091880321502686, + 0.10946202278137207, + 0.11180927604436874, + 0.09911621361970901, + 0.14432714879512787, + 0.1984039843082428, + 0.21581175923347473 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.45358312129974365, + "style_target": 0, + "support_probability": 0.7373768091201782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13217350725602048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20349843800067902, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7936053276062012, + 0.29923418164253235, + 0.3484713137149811, + 0.5344157814979553, + 0.08336061239242554 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 0, + "style_probability": 0.32043659687042236, + "style_target": 1, + "support_probability": 0.4754081964492798, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014413435761233107, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33224618434906006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8942718505859375, + 0.39804476499557495, + 0.3790024518966675, + 0.6280476450920105, + 0.1270068883895874 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 1, + "style_probability": 0.09143713116645813, + "style_target": 0, + "support_probability": 0.4250575602054596, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8470767487883062, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05060466378927231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:48:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9363553524017334, + 0.2726863920688629, + 0.7781825065612793, + 0.17771869897842407, + 0.0686943531036377, + 0.0525231771171093 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 1, + "style_probability": 0.695795476436615, + "style_target": 1, + "support_probability": 0.9262133836746216, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.905905409910487, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04705999791622162, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:48:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.965021550655365, + 0.355229914188385, + 0.8008208274841309, + 0.16607631742954254, + 0.09414994716644287, + 0.09317214787006378 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 0, + "style_probability": 0.8305851221084595, + "style_target": 0, + "support_probability": 0.937220573425293, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012283114729549984, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7660568952560425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17347101867198944, + 0.1531762331724167, + 0.45326265692710876, + 0.42343631386756897, + 0.07220415025949478, + 0.21695902943611145, + 0.06355985999107361, + 0.05621727555990219, + 0.04447579011321068 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 0, + "style_probability": 0.4489670991897583, + "style_target": 1, + "support_probability": 0.020882003009319305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013825278484435334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8496221303939819, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13223858177661896, + 0.16223202645778656, + 0.3476145565509796, + 0.45190316438674927, + 0.07737354189157486, + 0.06563951075077057, + 0.10678186267614365, + 0.0646350234746933, + 0.04273522272706032 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 1, + "style_probability": 0.22501708567142487, + "style_target": 0, + "support_probability": 0.026005789637565613, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8780960954633116, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10718438774347305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.969337522983551, + 0.5574308037757874, + 0.4207235276699066, + 0.5843679904937744, + 0.07813657075166702, + 0.04114735871553421, + 0.1214756965637207, + 0.3884914815425873, + 0.049201976507902145, + 0.07097148150205612, + 0.06490572541952133 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.6337077021598816, + "style_target": 1, + "support_probability": 0.9723390340805054, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4013805894376643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09518970549106598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9910394549369812, + 0.46016979217529297, + 0.45878076553344727, + 0.6644245386123657, + 0.1311255842447281, + 0.15442802011966705, + 0.22148986160755157, + 0.11714278906583786, + 0.09073057025671005, + 0.17572996020317078 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.14930912852287292, + "style_target": 0, + "support_probability": 0.9685858488082886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6121535559675415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11449664831161499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6075611710548401, + 0.23565316200256348, + 0.9156646728515625, + 0.3902359902858734, + 0.079701729118824 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 1, + "style_probability": 0.5748734474182129, + "style_target": 1, + "support_probability": 0.5538483262062073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7077237668758231, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1583377867937088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6154507994651794, + 0.2094283401966095, + 0.9473262429237366, + 0.38159555196762085, + 0.2080843597650528 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 0, + "style_probability": 0.7955572605133057, + "style_target": 0, + "support_probability": 0.5025107264518738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2747818845968582, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6734172701835632, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7016249895095825, + 0.28615450859069824, + 0.29600101709365845, + 0.44655051827430725, + 0.10278482735157013, + 0.080963633954525 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.5632894039154053, + "style_target": 1, + "support_probability": 0.1782299280166626, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13548545739576306, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8486937880516052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6323834657669067, + 0.32834771275520325, + 0.27365371584892273, + 0.5386292338371277, + 0.10918932408094406, + 0.13682809472084045 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.2570330798625946, + "style_target": 0, + "support_probability": 0.11294417828321457, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4448719309164728, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47853294014930725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9184397459030151, + 0.3528003990650177, + 0.34777092933654785, + 0.2754126191139221, + 0.05306839197874069, + 0.07613665610551834, + 0.08047439903020859, + 0.048675697296857834, + 0.05125498026609421 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 0, + "style_probability": 0.43765172362327576, + "style_target": 1, + "support_probability": 0.7394444346427917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03681022223404854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31806668639183044, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9709692001342773, + 0.4574365019798279, + 0.4341917335987091, + 0.33617961406707764, + 0.09903894364833832, + 0.2904866635799408, + 0.13636282086372375, + 0.09259376674890518, + 0.1001199260354042 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 1, + "style_probability": 0.08588758856058121, + "style_target": 0, + "support_probability": 0.7753857970237732, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8375042463444143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0677027478814125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4764719009399414, + 0.5060123205184937, + 0.9869808554649353, + 0.5345864295959473, + 0.04454450681805611, + 0.0480915866792202, + 0.06918016076087952, + 0.03446953371167183, + 0.06653290241956711, + 0.04136650636792183, + 0.04438285902142525 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 1, + "style_probability": 0.6674131155014038, + "style_target": 1, + "support_probability": 0.9132714867591858, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8111139117503275, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08069457113742828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5427479147911072, + 0.5928978323936462, + 0.994228720664978, + 0.5378932952880859, + 0.03769584745168686, + 0.03731471300125122, + 0.04671965911984444, + 0.09910649061203003, + 0.058859892189502716, + 0.1574723720550537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 0, + "style_probability": 0.7270655035972595, + "style_target": 0, + "support_probability": 0.905767560005188, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8205185655276555, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08107907325029373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.186626598238945, + 0.8944337368011475, + 0.400604248046875, + 0.3059978783130646, + 0.1128728836774826 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 1, + "style_probability": 0.5628772974014282, + "style_target": 1, + "support_probability": 0.9572323560714722, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8270570305565016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08278028666973114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17895285785198212, + 0.9375371336936951, + 0.4369269609451294, + 0.33624395728111267, + 0.09217776358127594 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 1, + "style_probability": 0.3621598780155182, + "style_target": 0, + "support_probability": 0.9632986783981323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002246091790704538, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9274495840072632, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3065127432346344, + 0.3034420609474182, + 0.28900620341300964, + 0.5185656547546387, + 0.13194657862186432, + 0.08589204400777817 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 0, + "style_probability": 0.3091351389884949, + "style_target": 1, + "support_probability": 0.002878227736800909, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008208035312067058, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9370405077934265, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3864915668964386, + 0.3833911716938019, + 0.3081052005290985, + 0.5767991542816162, + 0.14380986988544464, + 0.21384753286838531 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 1, + "style_probability": 0.09636639058589935, + "style_target": 0, + "support_probability": 0.00776715949177742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10652962041550046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27304086089134216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26353588700294495, + 0.3528960645198822, + 0.8983328342437744, + 0.26353588700294495, + 0.14573337137699127, + 0.08132770657539368, + 0.11535971611738205, + 0.1106116846203804, + 0.0867735743522644 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 1, + "style_probability": 0.5039941668510437, + "style_target": 1, + "support_probability": 0.09829320013523102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29841399540399277, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16928981244564056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2107669711112976, + 0.30882492661476135, + 0.9249740242958069, + 0.2107669711112976, + 0.13766416907310486, + 0.09754451364278793, + 0.0943533256649971, + 0.16482852399349213, + 0.12637998163700104 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 0, + "style_probability": 0.8192257881164551, + "style_target": 0, + "support_probability": 0.1865939497947693, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6981050015554457, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1487451195716858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30442580580711365, + 0.9124030470848083, + 0.23737971484661102, + 0.30442580580711365, + 0.07180427759885788, + 0.09319295734167099, + 0.09378959238529205, + 0.11388903111219406, + 0.07606383413076401, + 0.08859964460134506, + 0.0572984404861927 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 0, + "style_probability": 0.3629351556301117, + "style_target": 1, + "support_probability": 0.8403535485267639, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4757826956596848, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2132318615913391, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3004053831100464, + 0.9076111912727356, + 0.21985283493995667, + 0.3004053831100464, + 0.08569132536649704, + 0.14523817598819733, + 0.0937589630484581, + 0.05867040529847145, + 0.06977217644453049, + 0.0453551821410656, + 0.0844535231590271 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 1, + "style_probability": 0.2370721995830536, + "style_target": 0, + "support_probability": 0.7124794721603394, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02827994024784048, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.807810366153717, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6323089599609375, + 0.6189859509468079, + 0.1825929433107376, + 0.21376219391822815, + 0.053627196699380875 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 0, + "style_probability": 0.4916188716888428, + "style_target": 1, + "support_probability": 0.0422949455678463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004375601674275334, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.854176938533783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7121548056602478, + 0.7070327401161194, + 0.21880505979061127, + 0.22779394686222076, + 0.12180525064468384 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 1, + "style_probability": 0.14982165396213531, + "style_target": 0, + "support_probability": 0.057773739099502563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06794070878297398, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5437865257263184, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5999702215194702, + 0.3408166766166687, + 0.5292596817016602, + 0.2373284548521042, + 0.11341223865747452, + 0.09066867083311081 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 1, + "style_probability": 0.620486319065094, + "style_target": 1, + "support_probability": 0.08417858183383942, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1359747197763795, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5961631536483765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6606808304786682, + 0.31922295689582825, + 0.566717803478241, + 0.19894808530807495, + 0.10407498478889465, + 0.29689091444015503 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 0, + "style_probability": 0.8323918581008911, + "style_target": 0, + "support_probability": 0.09444772452116013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01405800167780165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9192337989807129, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5221885442733765, + 0.5795931220054626, + 0.22626642882823944, + 0.5521326065063477, + 0.15885420143604279, + 0.0440724678337574, + 0.0927666649222374, + 0.0913592055439949, + 0.090032659471035 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 0, + "style_probability": 0.3989095985889435, + "style_target": 1, + "support_probability": 0.019011477008461952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03065522337835013, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9087638258934021, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4758366346359253, + 0.5859345197677612, + 0.21869197487831116, + 0.5378602743148804, + 0.10778871923685074, + 0.14493656158447266, + 0.16495798528194427, + 0.10396233946084976, + 0.21500258147716522 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 1, + "style_probability": 0.3450910151004791, + "style_target": 0, + "support_probability": 0.03244619444012642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021864935609178147, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8108710646629333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2786139249801636, + 0.19451949000358582, + 0.6407608389854431, + 0.28717541694641113, + 0.14497454464435577, + 0.06588948518037796, + 0.1648695021867752, + 0.10207846015691757, + 0.07950595766305923, + 0.12082378566265106, + 0.1237015500664711 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 0, + "style_probability": 0.31350773572921753, + "style_target": 1, + "support_probability": 0.02633388713002205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013077649855340838, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7242898941040039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2881827652454376, + 0.3326685130596161, + 0.8731310367584229, + 0.2643606662750244, + 0.08370550721883774, + 0.10296788066625595, + 0.07260332256555557, + 0.07868627458810806, + 0.10368379205465317, + 0.08876381814479828 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 1, + "style_probability": 0.126453697681427, + "style_target": 0, + "support_probability": 0.06818472594022751, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8788577185549881, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08367215842008591, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5845493078231812, + 0.9743412137031555, + 0.15734583139419556, + 0.15726853907108307, + 0.07938594371080399 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.8612719774246216, + "style_target": 1, + "support_probability": 0.7552598118782043, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6416044626270953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11347135901451111, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5330556035041809, + 0.9512178897857666, + 0.1761138141155243, + 0.16651803255081177, + 0.05689215287566185 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.44509464502334595, + "style_target": 0, + "support_probability": 0.741950273513794, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.471589616378926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02676863595843315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9988070726394653, + 0.14825491607189178, + 0.2632637023925781, + 0.1693437695503235, + 0.09877821803092957, + 0.04669347032904625 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.5842106342315674, + "style_target": 1, + "support_probability": 0.26267871260643005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3533086237799431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021575158461928368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9995819926261902, + 0.1847602277994156, + 0.2554187476634979, + 0.16180245578289032, + 0.07800435274839401, + 0.23986542224884033 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.33721643686294556, + "style_target": 0, + "support_probability": 0.35786953568458557, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7437003142363617, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16458570957183838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1770298182964325, + 0.23443889617919922, + 0.9605921506881714, + 0.4825085997581482, + 0.05163993313908577, + 0.05624251067638397, + 0.04738449677824974, + 0.0451197475194931, + 0.04482894018292427 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 1, + "style_probability": 0.8274039030075073, + "style_target": 1, + "support_probability": 0.4451749622821808, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6647611354996531, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11589251458644867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15592898428440094, + 0.2321700155735016, + 0.9411158561706543, + 0.4838316738605499, + 0.1891455054283142, + 0.22273769974708557, + 0.18552400171756744, + 0.05955125018954277, + 0.09998263418674469 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 0, + "style_probability": 0.7103452086448669, + "style_target": 0, + "support_probability": 0.5138142704963684, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8113085908351154, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14423318207263947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3818851411342621, + 0.22045786678791046, + 0.9840710163116455, + 0.30184122920036316, + 0.09867687523365021, + 0.12616963684558868, + 0.08538904786109924, + 0.15787093341350555, + 0.08974852412939072, + 0.0459761917591095, + 0.09793254733085632 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.8490118980407715, + "style_target": 1, + "support_probability": 0.7801740765571594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5284423531115581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1796037256717682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3684373199939728, + 0.2461121380329132, + 0.974109947681427, + 0.28411218523979187, + 0.06703005731105804, + 0.10735316574573517, + 0.08103425055742264, + 0.06964879482984543, + 0.1417517513036728, + 0.10837429016828537, + 0.21293269097805023 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.4221363663673401, + "style_target": 0, + "support_probability": 0.7351444363594055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02633587559135226, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9453496336936951, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27342769503593445, + 0.26391348242759705, + 0.22963425517082214, + 0.46120578050613403, + 0.11381996423006058 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.836566686630249, + "style_target": 1, + "support_probability": 0.018489547073841095, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002907507618371468, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.941386878490448, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5331204533576965, + 0.30539557337760925, + 0.37478896975517273, + 0.40997281670570374, + 0.12315507978200912 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.28921276330947876, + "style_target": 0, + "support_probability": 0.012281562201678753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12456269349191301, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8503551483154297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11754324287176132, + 0.47636184096336365, + 0.3844795525074005, + 0.11754324287176132, + 0.11291394382715225, + 0.13090816140174866 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 1, + "style_probability": 0.8625853657722473, + "style_target": 1, + "support_probability": 0.0501905232667923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011091389898213023, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9207866191864014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1320357769727707, + 0.4525439441204071, + 0.36512449383735657, + 0.1320357769727707, + 0.04781409353017807, + 0.08157569915056229 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 0, + "style_probability": 0.5521154403686523, + "style_target": 0, + "support_probability": 0.009005841799080372, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9201613112421213, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11160662025213242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34413251280784607, + 0.2573954164981842, + 0.9934515953063965, + 0.35423991084098816, + 0.04848431795835495, + 0.04463818669319153, + 0.12735344469547272, + 0.05386904999613762, + 0.06119616702198982 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.7545961737632751, + "style_target": 1, + "support_probability": 0.9180012345314026, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7734305266330996, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14463214576244354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3193321228027344, + 0.24469007551670074, + 0.9877869486808777, + 0.36705613136291504, + 0.043304357677698135, + 0.07423369586467743, + 0.04743961989879608, + 0.12938308715820312, + 0.0740741714835167 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.39760708808898926, + "style_target": 0, + "support_probability": 0.884629487991333, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014307098673560475, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.893746554851532, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.514708399772644, + 0.770879864692688, + 0.45080846548080444, + 0.514708399772644, + 0.1776731014251709, + 0.1641758233308792, + 0.08241414278745651, + 0.08115358650684357, + 0.14731302857398987, + 0.15059906244277954, + 0.19358789920806885 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.7645202875137329, + "style_target": 1, + "support_probability": 0.005420883186161518, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002939023018927831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8333437442779541, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5044295191764832, + 0.7863883376121521, + 0.4540660083293915, + 0.5044295191764832, + 0.0790126770734787, + 0.06617837399244308, + 0.182628333568573, + 0.08982688933610916, + 0.07922887802124023, + 0.1660885214805603 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.26212459802627563, + "style_target": 0, + "support_probability": 0.00697499280795455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5874097112671482, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14852647483348846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8294720649719238, + 0.7927922606468201, + 0.18680858612060547, + 0.17161248624324799, + 0.08990246057510376 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 0, + "style_probability": 0.39641842246055603, + "style_target": 1, + "support_probability": 0.8882042765617371, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20839887260370826, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18080511689186096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8621537089347839, + 0.8675686717033386, + 0.19063659012317657, + 0.15525858104228973, + 0.10363420099020004 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 1, + "style_probability": 0.1507882922887802, + "style_target": 0, + "support_probability": 0.8463033437728882, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7790344867699481, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13154949247837067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28798162937164307, + 0.17473186552524567, + 0.18642233312129974, + 0.9922069907188416, + 0.05914921686053276, + 0.08931516855955124 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.48170170187950134, + "style_target": 1, + "support_probability": 0.8981508612632751, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8759020066715724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12578806281089783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28541359305381775, + 0.15083113312721252, + 0.15761783719062805, + 0.9975316524505615, + 0.13270796835422516, + 0.04892418906092644 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.8009485602378845, + "style_target": 0, + "support_probability": 0.8424474596977234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10828308720522152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7788823246955872, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24566929042339325, + 0.8310033082962036, + 0.30667203664779663, + 0.3597027063369751, + 0.08791019022464752, + 0.09640883654356003, + 0.043944504112005234, + 0.08188502490520477, + 0.15949906408786774 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 0, + "style_probability": 0.4073247015476227, + "style_target": 1, + "support_probability": 0.08444610983133316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19223686435959084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7525556087493896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2495168000459671, + 0.8908873796463013, + 0.3146776854991913, + 0.40958696603775024, + 0.062440093606710434, + 0.0771636813879013, + 0.04279816150665283, + 0.08789768815040588, + 0.07141803205013275 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 1, + "style_probability": 0.38821297883987427, + "style_target": 0, + "support_probability": 0.18834246695041656, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4343786049040177, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5227131843566895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5112707018852234, + 0.7554839849472046, + 0.7485225200653076, + 0.5112707018852234, + 0.18167968094348907, + 0.09045245498418808, + 0.08514364808797836, + 0.1259172260761261, + 0.19057103991508484, + 0.11487691104412079, + 0.15221703052520752 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.5375092029571533, + "style_target": 1, + "support_probability": 0.5311629772186279, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01640426306181593, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5192770957946777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5753200650215149, + 0.7794665694236755, + 0.7469432950019836, + 0.5753200650215149, + 0.20480868220329285, + 0.11805670708417892, + 0.29096585512161255, + 0.0777505561709404, + 0.1263933926820755, + 0.16102363169193268 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.1150379553437233, + "style_target": 0, + "support_probability": 0.4976155459880829, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9222163398538663, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05434002727270126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9808927774429321, + 0.488325297832489, + 0.7888715863227844, + 0.2622777819633484, + 0.3127792775630951 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.7931915521621704, + "style_target": 1, + "support_probability": 0.9043577313423157, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8034736323316392, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07179121673107147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9707105755805969, + 0.4282943308353424, + 0.751514196395874, + 0.2907652258872986, + 0.19216619431972504 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.36129525303840637, + "style_target": 0, + "support_probability": 0.8785432577133179, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7118183781433383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23368623852729797, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22074192762374878, + 0.36610013246536255, + 0.9750956892967224, + 0.22074192762374878, + 0.16172820329666138, + 0.22264233231544495 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.8324710130691528, + "style_target": 1, + "support_probability": 0.4163146913051605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14849935816261134, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31652700901031494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23216842114925385, + 0.38448476791381836, + 0.9894410371780396, + 0.23216842114925385, + 0.10181562602519989, + 0.1353619247674942 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.28059104084968567, + "style_target": 0, + "support_probability": 0.32460951805114746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1642216522817908, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5135264992713928, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6789391040802002, + 0.3736451268196106, + 0.19977137446403503, + 0.5004364252090454, + 0.266720175743103, + 0.1409739851951599, + 0.12726280093193054, + 0.1109614297747612, + 0.13545778393745422 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 1, + "style_probability": 0.8423048853874207, + "style_target": 1, + "support_probability": 0.08057307451963425, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06657677811372674, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4879317283630371, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6062215566635132, + 0.31416329741477966, + 0.19810110330581665, + 0.4794468879699707, + 0.3134300112724304, + 0.1502445936203003, + 0.09315621107816696, + 0.07451179623603821, + 0.09466581046581268 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 1, + "style_probability": 0.4415203630924225, + "style_target": 0, + "support_probability": 0.11906944215297699, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07338674193645033, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.780601441860199, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.29174405336380005, + 0.22014561295509338, + 0.30081385374069214, + 0.5711381435394287, + 0.06999082118272781, + 0.10559102147817612, + 0.08815015852451324, + 0.1583014875650406, + 0.12761911749839783, + 0.1514548808336258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.835438072681427, + "style_target": 1, + "support_probability": 0.04510907828807831, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03233610581962417, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7428233623504639, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3032618463039398, + 0.24869659543037415, + 0.27027469873428345, + 0.5398529171943665, + 0.09325432032346725, + 0.10725785791873932, + 0.18325044214725494, + 0.11040261387825012, + 0.07735176384449005, + 0.16769881546497345 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.39636141061782837, + "style_target": 0, + "support_probability": 0.047036413103342056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21280862750375845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.19040600955486298, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43585216999053955, + 0.3689352571964264, + 0.29172220826148987, + 0.8079925775527954, + 0.08740494400262833 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.6791770458221436, + "style_target": 1, + "support_probability": 0.3650296628475189, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01883044607546156, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3002219498157501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.46430933475494385, + 0.41470709443092346, + 0.2578895688056946, + 0.8418421745300293, + 0.11363743245601654 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.20631061494350433, + "style_target": 0, + "support_probability": 0.29339858889579773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10504244826183128, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4773414731025696, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.500082790851593, + 0.7898801565170288, + 0.5974000096321106, + 0.44434359669685364, + 0.06707077473402023, + 0.20701909065246582 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 1, + "style_probability": 0.5920937061309814, + "style_target": 1, + "support_probability": 0.12318281829357147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11754075630186978, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3818275034427643, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4867965281009674, + 0.7492512464523315, + 0.5758720636367798, + 0.45365700125694275, + 0.06205415725708008, + 0.13842928409576416 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 1, + "style_probability": 0.4665989875793457, + "style_target": 0, + "support_probability": 0.15632495284080505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20263827562977355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24017757177352905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21842871606349945, + 0.8044426441192627, + 0.09205330163240433, + 0.26436248421669006, + 0.10523476451635361, + 0.07878708094358444, + 0.14375674724578857, + 0.13964420557022095, + 0.047828108072280884 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.7134791016578674, + "style_target": 1, + "support_probability": 0.10281297564506531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18241991698121668, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19423635303974152, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22399817407131195, + 0.7925265431404114, + 0.10230295360088348, + 0.2969706654548645, + 0.10592567920684814, + 0.15008525550365448, + 0.11482665687799454, + 0.07772300392389297, + 0.061204925179481506 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.356857031583786, + "style_target": 0, + "support_probability": 0.20221781730651855, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5929962164483915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1751934289932251, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6830560564994812, + 0.6831168532371521, + 0.7164211273193359, + 0.24656300246715546, + 0.08930041640996933, + 0.07788387686014175, + 0.031038576737046242, + 0.052907511591911316, + 0.15091091394424438, + 0.07229474186897278, + 0.06584484875202179 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.8553810119628906, + "style_target": 1, + "support_probability": 0.6814603805541992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12676703666679376, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16954202950000763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7401435375213623, + 0.7463422417640686, + 0.7168510556221008, + 0.2663939893245697, + 0.07050672173500061, + 0.06654313206672668, + 0.2556745707988739, + 0.08289998769760132, + 0.0376543290913105, + 0.11313515901565552 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.2043088674545288, + "style_target": 0, + "support_probability": 0.7751124501228333, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6647419528251756, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.199336975812912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9840988516807556, + 0.1959659606218338, + 0.4778294861316681, + 0.22917044162750244, + 0.0598183311522007 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 1, + "style_probability": 0.6747397780418396, + "style_target": 1, + "support_probability": 0.8058220744132996, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.601891233270859, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2882576286792755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9645006060600281, + 0.18435439467430115, + 0.4570689797401428, + 0.21541185677051544, + 0.033143818378448486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 0, + "style_probability": 0.5808756947517395, + "style_target": 0, + "support_probability": 0.7763786315917969, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41252055169830976, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4130091667175293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46680495142936707, + 0.6838993430137634, + 0.16520221531391144, + 0.4764184355735779, + 0.07609699666500092, + 0.0960155576467514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7644251585006714, + "style_target": 1, + "support_probability": 0.403695672750473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37554516723431985, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4182889759540558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44844481348991394, + 0.6606809496879578, + 0.1674906611442566, + 0.42103010416030884, + 0.07530780881643295, + 0.10800237953662872 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 0, + "style_probability": 0.5007571578025818, + "style_target": 0, + "support_probability": 0.44730886816978455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2334242259513932, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48803287744522095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49169352650642395, + 0.31478288769721985, + 0.5414694547653198, + 0.6449329257011414, + 0.03233693912625313, + 0.0702253207564354, + 0.03944599628448486, + 0.05867535620927811, + 0.07595863938331604 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.7614513039588928, + "style_target": 1, + "support_probability": 0.24085083603858948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011897745152481765, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.632835865020752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49563467502593994, + 0.3388381004333496, + 0.5497453808784485, + 0.5792048573493958, + 0.07695461809635162, + 0.14457440376281738, + 0.08707762509584427, + 0.0665767639875412, + 0.12892082333564758 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.18520180881023407, + "style_target": 0, + "support_probability": 0.1524338573217392, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8371850574252954, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0685516968369484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1752527952194214, + 0.46830517053604126, + 0.24839749932289124, + 0.9357537031173706, + 0.09474162012338638, + 0.028569575399160385, + 0.05939090624451637, + 0.08549897372722626, + 0.05609441176056862, + 0.023668542504310608, + 0.11602836847305298 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 1, + "style_probability": 0.7470627427101135, + "style_target": 1, + "support_probability": 0.8806939125061035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.583933999477932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11779612302780151, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17377369105815887, + 0.4347853660583496, + 0.2891811728477478, + 0.884897768497467, + 0.06335888803005219, + 0.16113214194774628, + 0.10271044820547104, + 0.11261985450983047, + 0.07888513058423996, + 0.06389697641134262, + 0.09483413398265839 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 1, + "style_probability": 0.46158674359321594, + "style_target": 0, + "support_probability": 0.7620443105697632, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5454367421122924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10616506636142731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9961556792259216, + 0.44568583369255066, + 0.39218616485595703, + 0.35481980443000793, + 0.13945193588733673 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 1, + "style_probability": 0.5613234639167786, + "style_target": 1, + "support_probability": 0.35514411330223083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6280717133333427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11417771875858307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9994962215423584, + 0.4355834126472473, + 0.39624473452568054, + 0.3633726239204407, + 0.06788824498653412 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 0, + "style_probability": 0.6811792254447937, + "style_target": 0, + "support_probability": 0.38488397002220154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28806007457519806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4117080867290497, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2622092366218567, + 0.9894877672195435, + 0.3380754292011261, + 0.24039283394813538, + 0.16952580213546753, + 0.12519237399101257 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 1, + "style_probability": 0.613402783870697, + "style_target": 1, + "support_probability": 0.11845269799232483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3034257497848066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49603602290153503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2642242908477783, + 0.9943680167198181, + 0.31233811378479004, + 0.1907530277967453, + 0.10159769654273987, + 0.09231060743331909 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 0, + "style_probability": 0.5419596433639526, + "style_target": 0, + "support_probability": 0.12147843837738037, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39837077859759595, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22726298868656158, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09334152936935425, + 0.8942907452583313, + 0.09334152936935425, + 0.23657450079917908, + 0.18708106875419617, + 0.13633866608142853, + 0.20418338477611542, + 0.12380136549472809, + 0.18756575882434845 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 1, + "style_probability": 0.6676462292671204, + "style_target": 1, + "support_probability": 0.3805994391441345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06889583578337416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26538628339767456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09728588908910751, + 0.943844199180603, + 0.09728588908910751, + 0.24428072571754456, + 0.05653616786003113, + 0.14447297155857086, + 0.06610236316919327, + 0.13778430223464966, + 0.09547525644302368 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 1, + "style_probability": 0.40852662920951843, + "style_target": 0, + "support_probability": 0.3120560944080353, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2647351130179425, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4209124445915222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.10467253625392914, + 0.20627112686634064, + 0.918427050113678, + 0.10467253625392914, + 0.044801272451877594, + 0.2873118221759796, + 0.22523564100265503, + 0.2087439000606537, + 0.13234743475914001, + 0.19377192854881287 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 0, + "style_probability": 0.47529420256614685, + "style_target": 1, + "support_probability": 0.3684409558773041, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23090919078407227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4379490911960602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.09301572293043137, + 0.209153950214386, + 0.941989004611969, + 0.09301572293043137, + 0.14111833274364471, + 0.16642862558364868, + 0.0745905414223671, + 0.11767277866601944, + 0.1252991110086441, + 0.09295179694890976 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 0, + "style_probability": 0.6137241125106812, + "style_target": 0, + "support_probability": 0.394837886095047, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020700840381741362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9055328369140625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1807844042778015, + 0.21509526669979095, + 0.211144357919693, + 0.7926232814788818, + 0.3226792514324188 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.5324038863182068, + "style_target": 1, + "support_probability": 0.0373288057744503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005849704448174114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9195052981376648, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23054338991641998, + 0.2268286943435669, + 0.23660455644130707, + 0.8355857133865356, + 0.06303926557302475 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.17301277816295624, + "style_target": 0, + "support_probability": 0.0420820489525795, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6424328346717909, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31412985920906067, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5609714388847351, + 0.3491869866847992, + 0.7576440572738647, + 0.1896490603685379, + 0.036832503974437714, + 0.15380722284317017 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 1, + "style_probability": 0.906345009803772, + "style_target": 1, + "support_probability": 0.6881686449050903, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2809877806997482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6158677935600281, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47783491015434265, + 0.3352675139904022, + 0.7535010576248169, + 0.19240489602088928, + 0.2795754671096802, + 0.22957849502563477 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 0, + "style_probability": 0.5177834033966064, + "style_target": 0, + "support_probability": 0.39887887239456177, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7813304060029291, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21579301357269287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21014831960201263, + 0.24504150450229645, + 0.26057201623916626, + 0.9144853353500366, + 0.07952531427145004, + 0.0901041179895401, + 0.1374427229166031, + 0.10432551056146622, + 0.10352178663015366 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 1, + "style_probability": 0.862678587436676, + "style_target": 1, + "support_probability": 0.7444677948951721, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26537964251812085, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5116078853607178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2200230211019516, + 0.23323795199394226, + 0.2569584250450134, + 0.8901689052581787, + 0.11037725955247879, + 0.0796767920255661, + 0.1116623729467392, + 0.11896419525146484, + 0.129364013671875 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 1, + "style_probability": 0.4780203104019165, + "style_target": 0, + "support_probability": 0.3345794081687927, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9150564255769958, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05989283695816994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6033181548118591, + 0.9965932965278625, + 0.7978106737136841, + 0.40222904086112976, + 0.06309554725885391, + 0.07900035381317139, + 0.12588581442832947, + 0.07937999814748764, + 0.060684312134981155, + 0.08514264225959778 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 1, + "style_probability": 0.8820498585700989, + "style_target": 1, + "support_probability": 0.9543818235397339, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6842960546783807, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060059696435928345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.631335973739624, + 0.996965229511261, + 0.821019172668457, + 0.42612260580062866, + 0.17522241175174713, + 0.09308483451604843, + 0.16171744465827942, + 0.13865923881530762, + 0.05617913976311684, + 0.12412099540233612 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 0, + "style_probability": 0.5126960277557373, + "style_target": 0, + "support_probability": 0.9598069190979004, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19614938016098254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6182724237442017, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24898181855678558, + 0.4732045531272888, + 0.8230711817741394, + 0.19874708354473114, + 0.16020338237285614 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.6172454953193665, + "style_target": 1, + "support_probability": 0.21558798849582672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18614362345947902, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.41890567541122437, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28295814990997314, + 0.43148770928382874, + 0.7863130569458008, + 0.21083414554595947, + 0.08422309160232544 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.29539060592651367, + "style_target": 0, + "support_probability": 0.3422505557537079, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4461084800000765, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4573475420475006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.747480571269989, + 0.4858105778694153, + 0.2601924240589142, + 0.5684092044830322, + 0.08148565143346786, + 0.20083947479724884 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.8059675693511963, + "style_target": 1, + "support_probability": 0.2045000046491623, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3501724118461752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3843701183795929, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.760871171951294, + 0.462838351726532, + 0.3141268193721771, + 0.5917211174964905, + 0.12348083406686783, + 0.06765282899141312 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.4749135971069336, + "style_target": 0, + "support_probability": 0.2228316068649292, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15909654519252658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6608228087425232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34766462445259094, + 0.4756057560443878, + 0.25407031178474426, + 0.36135268211364746, + 0.1896285116672516, + 0.10652489215135574, + 0.11484803259372711, + 0.2626093029975891, + 0.057666681706905365 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.8050514459609985, + "style_target": 1, + "support_probability": 0.17909841239452362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012437520465783577, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5610203742980957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4171116352081299, + 0.5955588221549988, + 0.3704681694507599, + 0.27488356828689575, + 0.08514802902936935, + 0.11131353676319122, + 0.11912906914949417, + 0.1654394268989563, + 0.13119514286518097 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.15992452204227448, + "style_target": 0, + "support_probability": 0.2000514417886734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002688452699994671, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9336009621620178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2790110111236572, + 0.3273433446884155, + 0.36438891291618347, + 0.2790110111236572, + 0.07338467985391617, + 0.1556534618139267, + 0.07125749439001083, + 0.10302136838436127, + 0.15216046571731567, + 0.1706491857767105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 1, + "style_probability": 0.7474221587181091, + "style_target": 1, + "support_probability": 0.0019417429575696588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001900569951178951, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9203323721885681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24553519487380981, + 0.318406343460083, + 0.3448505103588104, + 0.24553519487380981, + 0.09489753097295761, + 0.09525484591722488, + 0.11954440921545029, + 0.07811019569635391, + 0.08603738993406296, + 0.06994855403900146, + 0.07934031635522842 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 0, + "style_probability": 0.6147844195365906, + "style_target": 0, + "support_probability": 0.0021062293089926243, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1447697905723894, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15437853336334229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21374371647834778, + 0.3475230038166046, + 0.8824248909950256, + 0.2689853608608246, + 0.1322493851184845 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 0, + "style_probability": 0.35866862535476685, + "style_target": 1, + "support_probability": 0.2122974991798401, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25472839903103534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23268689215183258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2202964872121811, + 0.36642614006996155, + 0.8960011005401611, + 0.36726483702659607, + 0.0736701637506485 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 0, + "style_probability": 0.7517470121383667, + "style_target": 0, + "support_probability": 0.12960796058177948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06735982705636986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.366514652967453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7018977999687195, + 0.3661287724971771, + 0.24705487489700317, + 0.3494952619075775, + 0.08736307919025421, + 0.12241727858781815 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 0, + "style_probability": 0.307361364364624, + "style_target": 1, + "support_probability": 0.14554272592067719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.054019074597478324, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34375184774398804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6648712158203125, + 0.41575637459754944, + 0.21613004803657532, + 0.3378651738166809, + 0.10012297332286835, + 0.13225635886192322 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 1, + "style_probability": 0.20984606444835663, + "style_target": 0, + "support_probability": 0.09281445294618607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10902100316146214, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6163182854652405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49719366431236267, + 0.2970687747001648, + 0.7157875895500183, + 0.5762478113174438, + 0.07806625217199326, + 0.17616385221481323, + 0.7050694227218628, + 0.0953051969408989, + 0.09458143264055252 + ], + "selected_evidence_indices": [ + 2, + 6 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 0, + "style_probability": 0.4479800760746002, + "style_target": 1, + "support_probability": 0.16729886829853058, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03353553991401304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35148173570632935, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45730164647102356, + 0.3186396360397339, + 0.9219927191734314, + 0.6535847783088684, + 0.17919088900089264, + 0.1937248706817627, + 0.3610536456108093, + 0.15158143639564514, + 0.1322932094335556 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 1, + "style_probability": 0.1365594118833542, + "style_target": 0, + "support_probability": 0.4411434829235077, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2147475601681376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3572547137737274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10187429189682007, + 0.21572451293468475, + 0.4809766411781311, + 0.9209506511688232, + 0.09222777187824249, + 0.12262386083602905, + 0.14373862743377686, + 0.17616069316864014, + 0.48502233624458313, + 0.13580583035945892, + 0.2276366651058197 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 0, + "style_probability": 0.41462242603302, + "style_target": 1, + "support_probability": 0.3346432149410248, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4360549893427084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3755579888820648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.0840035080909729, + 0.1989041566848755, + 0.5151278972625732, + 0.9541220664978027, + 0.11260021477937698, + 0.24519886076450348, + 0.2267409712076187, + 0.15402786433696747, + 0.14390917122364044, + 0.18959997594356537, + 0.10566934198141098 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 0, + "style_probability": 0.6989803910255432, + "style_target": 0, + "support_probability": 0.5752684473991394, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.18622274774664566, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.32607516646385193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33585652709007263, + 0.292250394821167, + 0.7052431702613831, + 0.5031216144561768, + 0.04001550376415253 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 0, + "style_probability": 0.3062680661678314, + "style_target": 1, + "support_probability": 0.4578362703323364, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19988246118907657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3470528721809387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3619385063648224, + 0.24611467123031616, + 0.7096871733665466, + 0.4502253532409668, + 0.1018911823630333 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 1, + "style_probability": 0.22422797977924347, + "style_target": 0, + "support_probability": 0.43108344078063965, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020220417133150623, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.728463888168335, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28571775555610657, + 0.33768001198768616, + 0.27117645740509033, + 0.5554720759391785, + 0.03772209584712982, + 0.06920463591814041 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 1, + "style_probability": 0.500360369682312, + "style_target": 1, + "support_probability": 0.04380898177623749, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008500932884138012, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7423776984214783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3545604944229126, + 0.32539719343185425, + 0.3526659905910492, + 0.7258332967758179, + 0.11210932582616806, + 0.09545277059078217 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 1, + "style_probability": 0.31891727447509766, + "style_target": 0, + "support_probability": 0.07097852975130081, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8452674184143995, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20443615317344666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:386:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27705273032188416, + 0.5251883864402771, + 0.9654338955879211, + 0.4640703499317169, + 0.16238217055797577, + 0.060684360563755035, + 0.04826267436146736, + 0.06952420622110367, + 0.0847984254360199 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 1, + "style_probability": 0.5341544151306152, + "style_target": 1, + "support_probability": 0.8826481103897095, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9366927066188065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19572153687477112, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:386:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26437294483184814, + 0.468512624502182, + 0.9946731925010681, + 0.4814854562282562, + 0.09442221373319626, + 0.09361174702644348, + 0.10372929275035858, + 0.14987623691558838, + 0.035633258521556854 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 0, + "style_probability": 0.7779298424720764, + "style_target": 0, + "support_probability": 0.8876219391822815, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08519735104871946, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.655976414680481, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19159048795700073, + 0.2746305465698242, + 0.27281439304351807, + 0.6345731616020203, + 0.08155452460050583, + 0.08765632659196854, + 0.09432592988014221, + 0.11259397864341736, + 0.060947902500629425, + 0.06577193737030029, + 0.1296442151069641 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 0, + "style_probability": 0.4268096089363098, + "style_target": 1, + "support_probability": 0.06999755650758743, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03946855928857184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7065253257751465, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19346202909946442, + 0.30393779277801514, + 0.25724753737449646, + 0.6475165486335754, + 0.08093196153640747, + 0.17849290370941162, + 0.0818992555141449, + 0.12676627933979034, + 0.05095833167433739, + 0.046684008091688156, + 0.0909527838230133 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 1, + "style_probability": 0.1818205863237381, + "style_target": 0, + "support_probability": 0.07552189379930496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3067067740218575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23386801779270172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18423433601856232, + 0.975138783454895, + 0.36054813861846924, + 0.18423433601856232, + 0.18253101408481598 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 0, + "style_probability": 0.4626021385192871, + "style_target": 1, + "support_probability": 0.2169220894575119, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40114833002993777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17237527668476105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20005440711975098, + 0.9455953240394592, + 0.32496434450149536, + 0.20005440711975098, + 0.09989674389362335 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 1, + "style_probability": 0.370249480009079, + "style_target": 0, + "support_probability": 0.39301249384880066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14234867629857, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8504258990287781, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17988909780979156, + 0.3785342872142792, + 0.23898999392986298, + 0.1581897735595703, + 0.20977844297885895, + 0.1281338632106781 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.7314358353614807, + "style_target": 1, + "support_probability": 0.106754370033741, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024301152633056735, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9215056896209717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16569136083126068, + 0.41242414712905884, + 0.27982303500175476, + 0.1538177877664566, + 0.1617405265569687, + 0.14031833410263062 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.33827468752861023, + "style_target": 0, + "support_probability": 0.07080180197954178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1828004091905277, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3606818914413452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8112658262252808, + 0.2137991040945053, + 0.20911745727062225, + 0.20074079930782318, + 0.10084362328052521, + 0.08924730122089386, + 0.07931540161371231, + 0.19405502080917358, + 0.10329682379961014 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.8001886606216431, + "style_target": 1, + "support_probability": 0.0655321478843689, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01978588213967454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3300371468067169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8625921010971069, + 0.26329919695854187, + 0.34500065445899963, + 0.22877588868141174, + 0.10925170034170151, + 0.10764667391777039, + 0.1376272737979889, + 0.11730790883302689, + 0.12199115753173828 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.1516755223274231, + "style_target": 0, + "support_probability": 0.12923161685466766, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5370037326979176, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39624664187431335, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.766021192073822, + 0.7252988815307617, + 0.5255628228187561, + 0.5785122513771057, + 0.09057243168354034, + 0.057248763740062714, + 0.050789907574653625, + 0.043906185775995255, + 0.13699528574943542, + 0.08619891852140427 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 1, + "style_probability": 0.6484631896018982, + "style_target": 1, + "support_probability": 0.6324303150177002, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2577544022854102, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5034231543540955, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6899614930152893, + 0.7029436826705933, + 0.4562872648239136, + 0.5186818242073059, + 0.09284491091966629, + 0.05211886018514633, + 0.0403459370136261, + 0.07928316295146942, + 0.10078614950180054, + 0.1420706957578659, + 0.08273310214281082 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 1, + "style_probability": 0.4302797317504883, + "style_target": 0, + "support_probability": 0.4422449767589569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3365697718835956, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5444656014442444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7263685464859009, + 0.3389858603477478, + 0.4353202283382416, + 0.4116913378238678, + 0.04120996594429016 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.804578423500061, + "style_target": 1, + "support_probability": 0.218978151679039, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0654230216913685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8438855409622192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7115587592124939, + 0.31136247515678406, + 0.430541455745697, + 0.4161302447319031, + 0.10354539752006531 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.29854604601860046, + "style_target": 0, + "support_probability": 0.06028556451201439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5850714734633125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.367122620344162, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.946632981300354, + 0.10760197788476944, + 0.13471460342407227, + 0.47245433926582336, + 0.05658580735325813, + 0.05294781178236008 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.8412273526191711, + "style_target": 1, + "support_probability": 0.3725239634513855, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07022810011195804, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3682827055454254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9682902097702026, + 0.12308215349912643, + 0.16173554956912994, + 0.4476949870586395, + 0.0987926721572876, + 0.06332217901945114 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.18689511716365814, + "style_target": 0, + "support_probability": 0.37661388516426086, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.845937333796648, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1376590132713318, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1661880910396576, + 0.9348179697990417, + 0.7490492463111877, + 0.1126193255186081, + 0.17188788950443268, + 0.06578804552555084, + 0.04712370038032532, + 0.03519712761044502, + 0.04715954139828682 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 1, + "style_probability": 0.8144320845603943, + "style_target": 1, + "support_probability": 0.8659769892692566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7875351514826078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16202127933502197, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1946648210287094, + 0.9121227860450745, + 0.6066858768463135, + 0.10825391858816147, + 0.027959248051047325, + 0.07790601253509521, + 0.11144348978996277, + 0.054570894688367844, + 0.06538325548171997 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 0, + "style_probability": 0.572429358959198, + "style_target": 0, + "support_probability": 0.8927250504493713, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.025868779844610373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9767799973487854, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47193601727485657, + 0.3424687683582306, + 0.7662432789802551, + 0.2225707471370697, + 0.11696254462003708, + 0.18346551060676575, + 0.24002473056316376, + 0.10924775153398514, + 0.11597560346126556, + 0.08042788505554199, + 0.07360327988862991 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.7199467420578003, + "style_target": 1, + "support_probability": 0.012091787531971931, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010574528621004128, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9938125610351562, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4622524082660675, + 0.3567783236503601, + 0.7685811519622803, + 0.20456798374652863, + 0.06641300767660141, + 0.07493455708026886, + 0.07104968279600143, + 0.08180761337280273, + 0.07005481421947479, + 0.07235470414161682, + 0.08236914873123169 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.4495592415332794, + "style_target": 0, + "support_probability": 0.005300461780279875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10396602600010377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5867865681648254, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6769782900810242, + 0.1921158879995346, + 0.34186285734176636, + 0.29010945558547974, + 0.1484098881483078 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.6608548164367676, + "style_target": 1, + "support_probability": 0.12167897820472717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008538673407023986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8038204312324524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.682640016078949, + 0.18133682012557983, + 0.35891833901405334, + 0.3451197147369385, + 0.21373623609542847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.17305207252502441, + "style_target": 0, + "support_probability": 0.07700750976800919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013495174469438266, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.959439754486084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17399902641773224, + 0.4918910562992096, + 0.1657128781080246, + 0.17399902641773224, + 0.040676165372133255, + 0.11228100210428238 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 1, + "style_probability": 0.7144500613212585, + "style_target": 1, + "support_probability": 0.025035517290234566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01784595927690371, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9132795929908752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19751599431037903, + 0.4772926867008209, + 0.15468183159828186, + 0.19751599431037903, + 0.07901548594236374, + 0.15693725645542145 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 1, + "style_probability": 0.4552345871925354, + "style_target": 0, + "support_probability": 0.02862500585615635, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7862120229482885, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1679837703704834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20686471462249756, + 0.26212558150291443, + 0.2613426148891449, + 0.9578210115432739, + 0.11429194360971451, + 0.07904448360204697, + 0.07681797444820404, + 0.06882160156965256, + 0.07866591215133667 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.8780702352523804, + "style_target": 1, + "support_probability": 0.7749176621437073, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4571287892853647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2220451384782791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21432308852672577, + 0.23621216416358948, + 0.2670419216156006, + 0.9262109398841858, + 0.0908241793513298, + 0.054814696311950684, + 0.16708062589168549, + 0.1089676171541214, + 0.14952179789543152 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.45070505142211914, + "style_target": 0, + "support_probability": 0.7000462412834167, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7765223944440695, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24740153551101685, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4997880756855011, + 0.1844409555196762, + 0.9575690031051636, + 0.3329307436943054, + 0.12528152763843536, + 0.11087505519390106, + 0.05582006275653839, + 0.1026003509759903, + 0.06483844667673111, + 0.10679830610752106 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.7645910382270813, + "style_target": 1, + "support_probability": 0.7263155579566956, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04489998406551932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28319165110588074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5509465336799622, + 0.24585852026939392, + 0.9777326583862305, + 0.3711908161640167, + 0.0890173390507698, + 0.15660670399665833, + 0.14668676257133484, + 0.1646595299243927, + 0.12044431269168854, + 0.11908520758152008 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.056629620492458344, + "style_target": 0, + "support_probability": 0.5478214025497437, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1266904190319236, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7957721948623657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5063349604606628, + 0.3929442763328552, + 0.10295864194631577, + 0.30197015404701233, + 0.0449644960463047 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 1, + "style_probability": 0.8016179800033569, + "style_target": 1, + "support_probability": 0.25763604044914246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015623700972306448, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9491636753082275, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44478368759155273, + 0.39250993728637695, + 0.15199734270572662, + 0.28827565908432007, + 0.13066956400871277 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 1, + "style_probability": 0.3666060268878937, + "style_target": 0, + "support_probability": 0.05276303365826607, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04635832220978407, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9374150037765503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3024964928627014, + 0.23768216371536255, + 0.5665478706359863, + 0.31150898337364197, + 0.04275430738925934, + 0.062392737716436386 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.6849091053009033, + "style_target": 1, + "support_probability": 0.02640553191304207, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006818277479151654, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9495013356208801, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3064019978046417, + 0.24354583024978638, + 0.5358760356903076, + 0.33964765071868896, + 0.07101044803857803, + 0.08093881607055664 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.21717660129070282, + "style_target": 0, + "support_probability": 0.008209483698010445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4480972084070836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40967896580696106, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38687747716903687, + 0.5061765909194946, + 0.12369991838932037, + 0.7398446798324585, + 0.10120654106140137, + 0.0543343760073185, + 0.06523361057043076, + 0.03324989974498749, + 0.035092391073703766 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.9181419014930725, + "style_target": 1, + "support_probability": 0.15642130374908447, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1153569123789038, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12336193770170212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4577835500240326, + 0.5930120348930359, + 0.09328589588403702, + 0.7869523167610168, + 0.06101078540086746, + 0.2720765769481659, + 0.0625828355550766, + 0.06451284140348434, + 0.08009050041437149 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.3148172199726105, + "style_target": 0, + "support_probability": 0.4401059150695801, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6040018760959361, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19424311816692352, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.515947699546814, + 0.41589289903640747, + 0.6422128081321716, + 0.39876148104667664, + 0.041092704981565475, + 0.028339024633169174, + 0.08228813111782074, + 0.0650235116481781, + 0.02561604045331478, + 0.028374018147587776, + 0.028432561084628105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.8522785902023315, + "style_target": 1, + "support_probability": 0.918056070804596, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20833321204271316, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7145970463752747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34573328495025635, + 0.35702967643737793, + 0.5466904640197754, + 0.29619693756103516, + 0.0544230155646801, + 0.059030260890722275, + 0.034554608166217804, + 0.033467330038547516, + 0.04296194389462471, + 0.09530901908874512, + 0.10133001208305359 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.36867982149124146, + "style_target": 0, + "support_probability": 0.5342786312103271, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33029059669660227, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5196031928062439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5525766015052795, + 0.36358970403671265, + 0.42252445220947266, + 0.36358970403671265, + 0.2591557502746582 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.3721246123313904, + "style_target": 1, + "support_probability": 0.6192825436592102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5555303522586507, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29486560821533203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5961262583732605, + 0.3952523171901703, + 0.4554121494293213, + 0.3952523171901703, + 0.13010740280151367 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.8714244365692139, + "style_target": 0, + "support_probability": 0.8379395008087158, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04522675213923307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8695400953292847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4662621319293976, + 0.2126065492630005, + 0.2029988169670105, + 0.26550206542015076, + 0.11718377470970154, + 0.20755554735660553 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 0, + "style_probability": 0.34364598989486694, + "style_target": 1, + "support_probability": 0.07704829424619675, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003461722378083765, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9330672025680542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5099157691001892, + 0.31541910767555237, + 0.24222078919410706, + 0.32708117365837097, + 0.13583222031593323, + 0.1475728452205658 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 1, + "style_probability": 0.15023474395275116, + "style_target": 0, + "support_probability": 0.056067731231451035, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7116075236725686, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06146426871418953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9427040815353394, + 0.27213042974472046, + 0.3507784605026245, + 0.2054058313369751, + 0.10374508053064346, + 0.10517454147338867, + 0.08336798846721649, + 0.12339036166667938, + 0.1924068033695221 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.4042028784751892, + "style_target": 1, + "support_probability": 0.8336514234542847, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8487040973337437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06186502054333687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9469284415245056, + 0.3282162547111511, + 0.3521234691143036, + 0.20578527450561523, + 0.057168859988451004, + 0.1417625993490219, + 0.0953715518116951, + 0.07743383198976517, + 0.0637987032532692 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.6731491684913635, + "style_target": 0, + "support_probability": 0.8976097106933594, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2735700592545576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4480547308921814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14164766669273376, + 0.14164766669273376, + 0.17433564364910126, + 0.8781705498695374, + 0.06867631524801254, + 0.12127300351858139, + 0.06866952776908875, + 0.07549984008073807, + 0.07937512546777725, + 0.10948225855827332, + 0.259802907705307 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 1, + "style_probability": 0.5875492691993713, + "style_target": 1, + "support_probability": 0.09187202900648117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6408760502060964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15097849071025848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13758613169193268, + 0.13758613169193268, + 0.16406551003456116, + 0.9199713468551636, + 0.2097376585006714, + 0.07194040715694427, + 0.03894464671611786, + 0.04815787822008133, + 0.1318998634815216, + 0.18203292787075043, + 0.13489791750907898 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 0, + "style_probability": 0.8905037045478821, + "style_target": 0, + "support_probability": 0.24049516022205353, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.42029685906041125, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.160610169172287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2175537496805191, + 0.8886424899101257, + 0.2133486568927765, + 0.24391648173332214, + 0.11484050005674362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 0, + "style_probability": 0.41634801030158997, + "style_target": 1, + "support_probability": 0.563848614692688, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7265776819236632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07258831709623337, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.259630411863327, + 0.9517794251441956, + 0.18633385002613068, + 0.2405264973640442, + 0.07854695618152618 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 1, + "style_probability": 0.42038753628730774, + "style_target": 0, + "support_probability": 0.7642495036125183, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6697748815943311, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12289085239171982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37101489305496216, + 0.17432674765586853, + 0.37101489305496216, + 0.8727452158927917, + 0.06642453372478485, + 0.08478303253650665 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 0, + "style_probability": 0.3582192063331604, + "style_target": 1, + "support_probability": 0.8842456936836243, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5323577737178055, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12114239484071732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42582497000694275, + 0.24119623005390167, + 0.42582497000694275, + 0.9593529105186462, + 0.05401263013482094, + 0.089838907122612 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 1, + "style_probability": 0.28301429748535156, + "style_target": 0, + "support_probability": 0.9121800065040588, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12204937418039154, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6329478025436401, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4056423604488373, + 0.6503419280052185, + 0.3971578776836395, + 0.37677690386772156, + 0.23188580572605133, + 0.09159848839044571, + 0.10053588449954987, + 0.11187388002872467, + 0.08674940466880798 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6207872629165649, + "style_target": 1, + "support_probability": 0.08058059960603714, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08768518772471268, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.647308886051178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49928611516952515, + 0.6487388014793396, + 0.3976533114910126, + 0.4359537363052368, + 0.09867337346076965, + 0.11698809266090393, + 0.08563495427370071, + 0.11638189852237701, + 0.17329561710357666 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 0, + "style_probability": 0.7055526971817017, + "style_target": 0, + "support_probability": 0.0618281215429306, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1180551110976027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2801573872566223, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4081953465938568, + 0.7748748064041138, + 0.1433524489402771, + 0.21301881968975067, + 0.07749587297439575, + 0.2639631927013397, + 0.45417162775993347, + 0.05402654409408569, + 0.07191190123558044, + 0.06521747261285782, + 0.08082769066095352 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 0, + "style_probability": 0.38268762826919556, + "style_target": 1, + "support_probability": 0.20997048914432526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0907934908246105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6490617394447327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41939836740493774, + 0.6989603638648987, + 0.14590656757354736, + 0.1738288700580597, + 0.11612369120121002, + 0.10658629238605499, + 0.23753544688224792, + 0.06973379850387573, + 0.05796249210834503, + 0.042964689433574677, + 0.06948758661746979 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 1, + "style_probability": 0.312526136636734, + "style_target": 0, + "support_probability": 0.10171633213758469, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10607615508525056, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1668524444103241, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7161328792572021, + 0.13499118387699127, + 0.20745521783828735, + 0.2912818491458893, + 0.07318329811096191 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 1, + "style_probability": 0.5064892768859863, + "style_target": 1, + "support_probability": 0.17649950087070465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16588607790309948, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1363430619239807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6724750995635986, + 0.13279874622821808, + 0.2301161289215088, + 0.31119799613952637, + 0.14424161612987518 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 1, + "style_probability": 0.45905742049217224, + "style_target": 0, + "support_probability": 0.197369784116745, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25378232326422906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.47070083022117615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4072435200214386, + 0.33236685395240784, + 0.4072435200214386, + 0.7121648192405701, + 0.10516897588968277, + 0.06274887174367905 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.625795304775238, + "style_target": 1, + "support_probability": 0.25131604075431824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19472901895061523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.34617528319358826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43273845314979553, + 0.29911211133003235, + 0.43273845314979553, + 0.7118525505065918, + 0.10330216586589813, + 0.08862752467393875 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.24103280901908875, + "style_target": 0, + "support_probability": 0.30721932649612427, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5241953489143961, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30910176038742065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3046494722366333, + 0.4621566832065582, + 0.5960245728492737, + 0.17587479948997498, + 0.10206383466720581, + 0.03858741372823715, + 0.04128431901335716, + 0.05766066908836365, + 0.038443323224782944 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.665086030960083, + "style_target": 1, + "support_probability": 0.8596633672714233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2019294387314158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3970103859901428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38586392998695374, + 0.5727167129516602, + 0.7536787986755371, + 0.21170908212661743, + 0.08538205176591873, + 0.13110783696174622, + 0.06981122493743896, + 0.04631432518362999, + 0.08341792225837708 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.11390498280525208, + "style_target": 0, + "support_probability": 0.8824775218963623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02237470670010106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.604462742805481, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4302527606487274, + 0.7080109715461731, + 0.32701191306114197, + 0.20662060379981995, + 0.08500535041093826, + 0.12505307793617249, + 0.0855916440486908, + 0.09220927208662033, + 0.13192661106586456, + 0.06792263686656952 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 1, + "style_probability": 0.5154754519462585, + "style_target": 1, + "support_probability": 0.029034797102212906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03657771865789051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5818080306053162, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4356711208820343, + 0.700801432132721, + 0.3354583978652954, + 0.20242124795913696, + 0.12590378522872925, + 0.08625787496566772, + 0.06374906748533249, + 0.05326205864548683, + 0.07223819196224213, + 0.14822737872600555 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 0, + "style_probability": 0.5400056838989258, + "style_target": 0, + "support_probability": 0.02208002284169197, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6554378926021108, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0749349519610405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3196520507335663, + 0.061939045786857605, + 0.8994888663291931, + 0.09160761535167694, + 0.17115136981010437 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.8002271056175232, + "style_target": 1, + "support_probability": 0.6385942697525024, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21886252249719146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07727141678333282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31639400124549866, + 0.06581546366214752, + 0.8586747050285339, + 0.07538808882236481, + 0.24263213574886322 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.1648196280002594, + "style_target": 0, + "support_probability": 0.5381274819374084, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4671731016990259, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16361087560653687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8185336589813232, + 0.6471708416938782, + 0.49565911293029785, + 0.3421110510826111, + 0.05802086740732193, + 0.1278160810470581 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 1, + "style_probability": 0.7221248149871826, + "style_target": 1, + "support_probability": 0.6768279671669006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14479455131950275, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22252385318279266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8427377343177795, + 0.6861266493797302, + 0.4895898401737213, + 0.387521356344223, + 0.05253756418824196, + 0.1384717971086502 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 1, + "style_probability": 0.40470290184020996, + "style_target": 0, + "support_probability": 0.6334095597267151, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.047366016326561435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.904554545879364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8545054197311401, + 0.17872245609760284, + 0.29541876912117004, + 0.20137687027454376, + 0.05648493766784668, + 0.10644575208425522, + 0.11091580241918564, + 0.11680788546800613, + 0.23520143330097198 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 1, + "style_probability": 0.7257189750671387, + "style_target": 1, + "support_probability": 0.05212880298495293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023087028739790766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9132277965545654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8012415766716003, + 0.13955838978290558, + 0.25389739871025085, + 0.2109229415655136, + 0.11552707850933075, + 0.0917852595448494, + 0.0866907387971878, + 0.06443923711776733, + 0.07449880987405777 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 0, + "style_probability": 0.6587644815444946, + "style_target": 0, + "support_probability": 0.02316039614379406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2762702907783216, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7226630449295044, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32837867736816406, + 0.0836389884352684, + 0.3952378034591675, + 0.1413663774728775, + 0.04615520313382149, + 0.04197761416435242, + 0.04009074345231056, + 0.04326438158750534, + 0.04170282930135727, + 0.07370118051767349 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 1, + "style_probability": 0.8581372499465942, + "style_target": 1, + "support_probability": 0.2774064838886261, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0964332793188707, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8947339057922363, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3213152289390564, + 0.10085175186395645, + 0.30518147349357605, + 0.15662488341331482, + 0.039383791387081146, + 0.042094986885786057, + 0.03417547792196274, + 0.03312671557068825, + 0.029443005099892616, + 0.052849870175123215, + 0.12577705085277557 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 0, + "style_probability": 0.5365644693374634, + "style_target": 0, + "support_probability": 0.08795830607414246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10576380555226716, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5117428302764893, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4408525824546814, + 0.21742883324623108, + 0.5113682746887207, + 0.4408525824546814, + 0.05397365614771843 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 1, + "style_probability": 0.8804198503494263, + "style_target": 1, + "support_probability": 0.05625824257731438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01852532132336899, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5333983302116394, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.465778648853302, + 0.28233402967453003, + 0.5294126272201538, + 0.465778648853302, + 0.07886628806591034 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 0, + "style_probability": 0.64398193359375, + "style_target": 0, + "support_probability": 0.040008723735809326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21835624015063096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6427731513977051, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3936242461204529, + 0.2967892587184906, + 0.4247210621833801, + 0.9415165185928345, + 0.10974247753620148, + 0.09086216241121292 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 1, + "style_probability": 0.7623215913772583, + "style_target": 1, + "support_probability": 0.07529507577419281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14882496061598627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6748390793800354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34237733483314514, + 0.2758246958255768, + 0.4271126985549927, + 0.926681399345398, + 0.11772359907627106, + 0.14483262598514557 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 0, + "style_probability": 0.588689923286438, + "style_target": 0, + "support_probability": 0.08056624233722687, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8612875593924985, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10355305671691895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9453503489494324, + 0.20692303776741028, + 0.1569543331861496, + 0.20692303776741028, + 0.12422288209199905, + 0.09273739904165268, + 0.05689090117812157, + 0.04577738791704178, + 0.11452636122703552 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.8899522423744202, + "style_target": 1, + "support_probability": 0.8734586834907532, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5781179662134004, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2790958285331726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.901704728603363, + 0.18496911227703094, + 0.17427562177181244, + 0.18496911227703094, + 0.06253042817115784, + 0.1256403774023056, + 0.0339193232357502, + 0.07119176536798477, + 0.12827780842781067 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.4428345561027527, + "style_target": 0, + "support_probability": 0.7217151522636414, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1329266251263882, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7587092518806458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.544750452041626, + 0.28306037187576294, + 0.544750452041626, + 0.4238457679748535, + 0.08094516396522522, + 0.08446157723665237, + 0.10097448527812958, + 0.07011053711175919, + 0.06740190833806992, + 0.06569667905569077, + 0.03927407041192055 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.7608639001846313, + "style_target": 1, + "support_probability": 0.08018500357866287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006139444103613369, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8680785894393921, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5615468621253967, + 0.32244226336479187, + 0.5615468621253967, + 0.527973473072052, + 0.1087406650185585, + 0.07047705352306366, + 0.16585297882556915, + 0.1062203198671341, + 0.11481563001871109, + 0.11224063485860825 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.07043468952178955, + "style_target": 0, + "support_probability": 0.09356725215911865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11465922305529921, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8369492888450623, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5145509243011475, + 0.2960490584373474, + 0.44578900933265686, + 0.5896474719047546, + 0.15176653861999512 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.7154301404953003, + "style_target": 1, + "support_probability": 0.08269799500703812, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008724006997037704, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9696812629699707, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5251294374465942, + 0.32069528102874756, + 0.4561194181442261, + 0.521442711353302, + 0.12447872012853622 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.28980350494384766, + "style_target": 0, + "support_probability": 0.03311803564429283, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4561956834176207, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4309525489807129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.553585410118103, + 0.4316937029361725, + 0.29923033714294434, + 0.05023752152919769, + 0.04238242655992508, + 0.06489162147045135 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.3378659784793854, + "style_target": 1, + "support_probability": 0.8405382633209229, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4916396558627869, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3484087288379669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.511349618434906, + 0.5140082240104675, + 0.27810561656951904, + 0.03235514461994171, + 0.049784015864133835, + 0.019141022115945816 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.8980172276496887, + "style_target": 0, + "support_probability": 0.7190261483192444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7053781181586889, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21968558430671692, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3261750638484955, + 0.9830753803253174, + 0.31264469027519226, + 0.28049251437187195, + 0.12064255774021149, + 0.11399111896753311, + 0.09355738013982773, + 0.11616029590368271, + 0.10478213429450989 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 0, + "style_probability": 0.4611499011516571, + "style_target": 1, + "support_probability": 0.6039469242095947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5758594750329907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2840002179145813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37730279564857483, + 0.9854096174240112, + 0.26458877325057983, + 0.28020381927490234, + 0.1247992292046547, + 0.05256445333361626, + 0.0395895391702652, + 0.08568628132343292, + 0.09708241373300552 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 1, + "style_probability": 0.3745923638343811, + "style_target": 0, + "support_probability": 0.47069406509399414, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7141406820982525, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1490139216184616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7291557192802429, + 0.8984091281890869, + 0.28289371728897095, + 0.25839152932167053, + 0.1253170669078827, + 0.06249596178531647, + 0.24301782250404358, + 0.1634974181652069, + 0.23109963536262512, + 0.1296321302652359, + 0.10383323580026627 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 1, + "style_probability": 0.5473655462265015, + "style_target": 1, + "support_probability": 0.8432179689407349, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17610199894387432, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13574746251106262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.776799201965332, + 0.981208324432373, + 0.4304269552230835, + 0.310700386762619, + 0.127279132604599, + 0.19627134501934052, + 0.1389777511358261, + 0.15447689592838287, + 0.10384511947631836, + 0.1348128765821457 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 1, + "style_probability": 0.10590091347694397, + "style_target": 0, + "support_probability": 0.8285111784934998, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.800027473012421, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05562562867999077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.317218542098999, + 0.12390335649251938, + 0.24396653473377228, + 0.9237178564071655, + 0.07617329806089401 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 1, + "style_probability": 0.7998121380805969, + "style_target": 1, + "support_probability": 0.8995303511619568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5743631026860165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1288595348596573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.335569828748703, + 0.14768490195274353, + 0.3125655949115753, + 0.8703016638755798, + 0.07099051773548126 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 0, + "style_probability": 0.5081360340118408, + "style_target": 0, + "support_probability": 0.7853981256484985, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7840528305688175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10288310796022415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24528679251670837, + 0.28940442204475403, + 0.9809811115264893, + 0.36708882451057434, + 0.19863083958625793, + 0.05781792104244232 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.7953348755836487, + "style_target": 1, + "support_probability": 0.7168193459510803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44587970794979825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09068604558706284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22780832648277283, + 0.32943132519721985, + 0.9924747347831726, + 0.31756913661956787, + 0.044198792427778244, + 0.08509447425603867 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.3825043737888336, + "style_target": 0, + "support_probability": 0.8015051484107971, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15683919002881555, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7041081786155701, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3748915493488312, + 0.6016466021537781, + 0.16908451914787292, + 0.4038182497024536, + 0.06761522591114044, + 0.11946014314889908, + 0.1064942255616188, + 0.17750868201255798, + 0.12266666442155838 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 1, + "style_probability": 0.7743958830833435, + "style_target": 1, + "support_probability": 0.08756569027900696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07899131457183561, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.56417316198349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38506314158439636, + 0.4884621798992157, + 0.20144052803516388, + 0.388202041387558, + 0.1649676114320755, + 0.08757475763559341, + 0.08368634432554245, + 0.20976082980632782, + 0.14475446939468384 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 1, + "style_probability": 0.49289897084236145, + "style_target": 0, + "support_probability": 0.07671723514795303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029299502638380548, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7274733781814575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4522188901901245, + 0.2608560025691986, + 0.4600227177143097, + 0.6718592643737793, + 0.05655206739902496, + 0.08068342506885529, + 0.11124841868877411, + 0.09238908439874649, + 0.1727360486984253, + 0.10786929726600647, + 0.11561137437820435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.8164401054382324, + "style_target": 1, + "support_probability": 0.016944991424679756, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015001512711249233, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6439904570579529, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37165918946266174, + 0.2237500101327896, + 0.4025319218635559, + 0.6695982813835144, + 0.079923614859581, + 0.07358332723379135, + 0.20776230096817017, + 0.11018269509077072, + 0.1852576732635498, + 0.09591693431138992, + 0.09230407327413559 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.2818319797515869, + "style_target": 0, + "support_probability": 0.03407858684659004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050269652483256044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7024048566818237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25056877732276917, + 0.3666223883628845, + 0.7708404064178467, + 0.25056877732276917, + 0.05633379518985748 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 1, + "style_probability": 0.5309168100357056, + "style_target": 1, + "support_probability": 0.055517010390758514, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017578090996918278, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8725870847702026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22111202776432037, + 0.3392752707004547, + 0.7360141277313232, + 0.22111202776432037, + 0.10009852796792984 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 1, + "style_probability": 0.42853689193725586, + "style_target": 0, + "support_probability": 0.018378708511590958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013595241628332028, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9652830362319946, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.572257399559021, + 0.23870132863521576, + 0.37004604935646057, + 0.38414502143859863, + 0.1046457439661026, + 0.02932361513376236 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 0, + "style_probability": 0.2863521873950958, + "style_target": 1, + "support_probability": 0.05264570191502571, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002488942709947486, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9754246473312378, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6537809371948242, + 0.26745548844337463, + 0.4232758581638336, + 0.40581628680229187, + 0.12955322861671448, + 0.21169012784957886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 1, + "style_probability": 0.06186949089169502, + "style_target": 0, + "support_probability": 0.03702854737639427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3080124109654501, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36697712540626526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49455055594444275, + 0.5173238515853882, + 0.11947133392095566, + 0.1815820187330246, + 0.16830018162727356, + 0.13149385154247284, + 0.12033136934041977, + 0.10916321724653244, + 0.052918434143066406 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 1, + "style_probability": 0.5505862832069397, + "style_target": 1, + "support_probability": 0.5480971336364746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40836457157771583, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4574357271194458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5063917636871338, + 0.5407556295394897, + 0.18975944817066193, + 0.13429537415504456, + 0.17068220674991608, + 0.035874783992767334, + 0.11929154396057129, + 0.07424495369195938, + 0.06471268832683563 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 0, + "style_probability": 0.8942698240280151, + "style_target": 0, + "support_probability": 0.3068130910396576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3922913209918408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2588798701763153, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.923065185546875, + 0.2868906259536743, + 0.21326176822185516, + 0.3470228910446167, + 0.0943838357925415, + 0.12582416832447052, + 0.10145897418260574, + 0.17343224585056305, + 0.054381392896175385, + 0.07422129064798355, + 0.07688885182142258 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 1, + "style_probability": 0.5154026746749878, + "style_target": 1, + "support_probability": 0.41623958945274353, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3850245150009337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20912668108940125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9336585998535156, + 0.2711486220359802, + 0.23120373487472534, + 0.3272891938686371, + 0.15649254620075226, + 0.06185077503323555, + 0.0701054260134697, + 0.09160100668668747, + 0.12532617151737213, + 0.09976621717214584, + 0.11971693485975266 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 1, + "style_probability": 0.414379358291626, + "style_target": 0, + "support_probability": 0.42670661211013794, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7370768401013089, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20685245096683502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9332464337348938, + 0.19107896089553833, + 0.4628193974494934, + 0.26499587297439575, + 0.03869834169745445 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 1, + "style_probability": 0.5165367722511292, + "style_target": 1, + "support_probability": 0.7878391742706299, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5052692068683386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2885386645793915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8944494724273682, + 0.22983066737651825, + 0.4338107109069824, + 0.24901023507118225, + 0.10342758893966675 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 1, + "style_probability": 0.4203294515609741, + "style_target": 0, + "support_probability": 0.6645781993865967, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3229276750937931, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6892911195755005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22650720179080963, + 0.19856683909893036, + 0.332084596157074, + 0.9077451229095459, + 0.03206077963113785, + 0.031645189970731735 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 0, + "style_probability": 0.3881518840789795, + "style_target": 1, + "support_probability": 0.3009289503097534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2591259223560261, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7385312914848328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27372804284095764, + 0.197850301861763, + 0.35843318700790405, + 0.9248690009117126, + 0.11933550238609314, + 0.08516889065504074 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 0, + "style_probability": 0.7996410727500916, + "style_target": 0, + "support_probability": 0.12421378493309021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.363520803910956, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4562387466430664, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6316913366317749, + 0.2783641517162323, + 0.7131597399711609, + 0.6316913366317749, + 0.0762394517660141, + 0.07258389890193939, + 0.1185402125120163, + 0.027520660310983658, + 0.05788427218794823 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 0, + "style_probability": 0.320376992225647, + "style_target": 1, + "support_probability": 0.5075077414512634, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044523160592929205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5713426470756531, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6812812089920044, + 0.24494251608848572, + 0.7582051157951355, + 0.6812812089920044, + 0.05314820259809494, + 0.08501116186380386, + 0.12072446197271347, + 0.11715862154960632, + 0.09973114728927612 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 1, + "style_probability": 0.06685873121023178, + "style_target": 0, + "support_probability": 0.3917412757873535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18125638985953507, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.571090042591095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21816661953926086, + 0.30271002650260925, + 0.3420441150665283, + 0.1419253647327423, + 0.06693533062934875, + 0.2749515771865845, + 0.0545819029211998, + 0.05088416114449501, + 0.07908759266138077, + 0.02996836230158806, + 0.03957933187484741 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 1, + "style_probability": 0.602965772151947, + "style_target": 1, + "support_probability": 0.27459222078323364, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0869768232436173, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7848445773124695, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19448702037334442, + 0.31927502155303955, + 0.30864888429641724, + 0.1483387053012848, + 0.04076656326651573, + 0.10881272703409195, + 0.07988692820072174, + 0.058085665106773376, + 0.10991876572370529, + 0.0327664390206337, + 0.03772759810090065 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 0, + "style_probability": 0.6830577850341797, + "style_target": 0, + "support_probability": 0.11436425149440765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5249593917092739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21859177947044373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5411199331283569, + 0.9892788529396057, + 0.2556181252002716, + 0.17257611453533173, + 0.08003076910972595 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 1, + "style_probability": 0.8370344638824463, + "style_target": 1, + "support_probability": 0.4933904707431793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44527171031656465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1379079669713974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5187225341796875, + 0.9821228384971619, + 0.2909441888332367, + 0.16225041449069977, + 0.12042291462421417 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 0, + "style_probability": 0.5039121508598328, + "style_target": 0, + "support_probability": 0.5734524726867676, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6454814076237174, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15052349865436554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22328047454357147, + 0.7105750441551208, + 0.5172522664070129, + 0.3701958954334259, + 0.05816967040300369, + 0.02844531461596489 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.8473693132400513, + "style_target": 1, + "support_probability": 0.8594005703926086, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11280802974779508, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2227540910243988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3570808172225952, + 0.8443946242332458, + 0.4477284550666809, + 0.41042694449424744, + 0.0884738489985466, + 0.09930961579084396 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.10971041023731232, + "style_target": 0, + "support_probability": 0.7815279364585876, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018066256280513487, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8857209086418152, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25181376934051514, + 0.21821466088294983, + 0.3422950208187103, + 0.38305479288101196, + 0.07731536030769348, + 0.08661255985498428, + 0.06962605565786362, + 0.1467546969652176, + 0.08838953822851181 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 1, + "style_probability": 0.8149278163909912, + "style_target": 1, + "support_probability": 0.009803974069654942, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020857438137292576, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7010128498077393, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2755834460258484, + 0.24499937891960144, + 0.3388625681400299, + 0.35652098059654236, + 0.10639843344688416, + 0.14352229237556458, + 0.13506238162517548, + 0.16243457794189453, + 0.0974065512418747 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 1, + "style_probability": 0.3525240421295166, + "style_target": 0, + "support_probability": 0.02585679478943348, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.061070574523290976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4620918333530426, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4496181309223175, + 0.11124870181083679, + 0.8889349699020386, + 0.27366453409194946, + 0.0466756634414196, + 0.0768614113330841, + 0.13550879061222076, + 0.07691863924264908, + 0.06061020866036415, + 0.07102174311876297, + 0.04676314815878868 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.6898247003555298, + "style_target": 1, + "support_probability": 0.026210719719529152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022104586251519717, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8071336150169373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44837266206741333, + 0.14416415989398956, + 0.8396390676498413, + 0.26849427819252014, + 0.12484648078680038, + 0.11907068639993668, + 0.0913965180516243, + 0.058439020067453384, + 0.05862275883555412, + 0.08572062104940414 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.34571197628974915, + "style_target": 0, + "support_probability": 0.006997514050453901, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12980123082681658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.40543118119239807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3922345042228699, + 0.6239860653877258, + 0.44681355357170105, + 0.3922345042228699, + 0.05868186801671982 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.8337867856025696, + "style_target": 1, + "support_probability": 0.08675314486026764, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00594309539320495, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5154662728309631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40752166509628296, + 0.5821495652198792, + 0.4461807310581207, + 0.40752166509628296, + 0.10118028521537781 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.15125349164009094, + "style_target": 0, + "support_probability": 0.06129055470228195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10558343064718226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30884286761283875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21466778218746185, + 0.24166327714920044, + 0.28669342398643494, + 0.9217804670333862, + 0.07765444368124008, + 0.11799847334623337 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 1, + "style_probability": 0.652341365814209, + "style_target": 1, + "support_probability": 0.08269528299570084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10951676699296416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3684675693511963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21607817709445953, + 0.29164278507232666, + 0.27778419852256775, + 0.8831069469451904, + 0.24277740716934204, + 0.1313224881887436 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 0, + "style_probability": 0.6435091495513916, + "style_target": 0, + "support_probability": 0.05339866504073143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8579095502532922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14992810785770416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20187167823314667, + 0.4118458926677704, + 0.9657544493675232, + 0.23366989195346832, + 0.051804400980472565, + 0.23799832165241241, + 0.2299901694059372, + 0.07690746337175369, + 0.04695234075188637 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.9174832701683044, + "style_target": 1, + "support_probability": 0.795433759689331, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4566128368224171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22499790787696838, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.215727299451828, + 0.42532065510749817, + 0.9521051645278931, + 0.21374759078025818, + 0.0983908548951149, + 0.0883282795548439, + 0.3901430666446686, + 0.24816113710403442, + 0.1427048295736313 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.47398948669433594, + "style_target": 0, + "support_probability": 0.5530486106872559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30650755707350014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6669563055038452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21737411618232727, + 0.7859871983528137, + 0.31082871556282043, + 0.10977551341056824, + 0.14438174664974213, + 0.28173983097076416, + 0.05865536630153656, + 0.22078588604927063, + 0.1701183021068573, + 0.08802901953458786, + 0.11826491355895996 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.8303400874137878, + "style_target": 1, + "support_probability": 0.229145348072052, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12357904699275052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1856568604707718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32230597734451294, + 0.8524248600006104, + 0.3507528007030487, + 0.18902696669101715, + 0.12241159379482269, + 0.1353108435869217, + 0.17172732949256897, + 0.12303030490875244, + 0.1567838042974472, + 0.1293342262506485 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.1429503858089447, + "style_target": 0, + "support_probability": 0.8071221709251404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01192379841942079, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.882844865322113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6485904455184937, + 0.47266846895217896, + 0.37007737159729004, + 0.49974334239959717, + 0.09226684272289276 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 0, + "style_probability": 0.31806308031082153, + "style_target": 1, + "support_probability": 0.03669774904847145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011767655267132639, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8294976949691772, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.653334379196167, + 0.5194766521453857, + 0.4121641218662262, + 0.5520420670509338, + 0.22161293029785156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 1, + "style_probability": 0.40298551321029663, + "style_target": 0, + "support_probability": 0.060725513845682144, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7856578442129387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1045275405049324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9387505054473877, + 0.3131236135959625, + 0.3112109303474426, + 0.3131236135959625, + 0.02908104658126831, + 0.08282176405191422 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.37459293007850647, + "style_target": 1, + "support_probability": 0.94607013463974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.82403123285283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09728289395570755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.958267092704773, + 0.29789724946022034, + 0.2907715141773224, + 0.29789724946022034, + 0.06198230758309364, + 0.14245954155921936 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.5916375517845154, + "style_target": 0, + "support_probability": 0.9516664743423462, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22066795443676168, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17796802520751953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11166568100452423, + 0.21581877768039703, + 0.3245285749435425, + 0.9165056943893433, + 0.1086975559592247, + 0.07008933275938034, + 0.05329858511686325, + 0.04263831302523613, + 0.11034730076789856 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 0, + "style_probability": 0.49947601556777954, + "style_target": 1, + "support_probability": 0.24530868232250214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1786881450531461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17431119084358215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12621301412582397, + 0.21930760145187378, + 0.35621899366378784, + 0.9627166986465454, + 0.06424517929553986, + 0.10025010257959366, + 0.06932155787944794, + 0.059989120811223984, + 0.07297662645578384 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 1, + "style_probability": 0.4763464033603668, + "style_target": 0, + "support_probability": 0.30485591292381287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15693571217405664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32053500413894653, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33645203709602356, + 0.6937316060066223, + 0.3035535514354706, + 0.35142457485198975, + 0.06442344188690186, + 0.09114111214876175, + 0.05945277586579323, + 0.06145615503191948, + 0.05045530945062637, + 0.028034117072820663, + 0.04588789865374565 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 0, + "style_probability": 0.30183321237564087, + "style_target": 1, + "support_probability": 0.330011248588562, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12425314227603589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36001163721084595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4301261901855469, + 0.7176627516746521, + 0.3287849426269531, + 0.4054460823535919, + 0.05684997886419296, + 0.05139279365539551, + 0.08959105610847473, + 0.036709338426589966, + 0.05602337047457695, + 0.05303094908595085, + 0.1999601125717163 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 1, + "style_probability": 0.2730836868286133, + "style_target": 0, + "support_probability": 0.41237595677375793, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7515836077823719, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31036609411239624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:856:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9123364686965942, + 0.18546825647354126, + 0.3216795325279236, + 0.2991768419742584, + 0.16108989715576172 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 1, + "style_probability": 0.6879574060440063, + "style_target": 1, + "support_probability": 0.7319867610931396, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8524633912226678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34051617980003357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:856:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9515112638473511, + 0.216551274061203, + 0.3011174201965332, + 0.2778753638267517, + 0.048057522624731064 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 0, + "style_probability": 0.8647246956825256, + "style_target": 0, + "support_probability": 0.7164644002914429, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7253912605193052, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05192941054701805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9529322385787964, + 0.27851352095603943, + 0.26051637530326843, + 0.3088688850402832, + 0.24619793891906738, + 0.1894027590751648 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 0, + "style_probability": 0.4600839614868164, + "style_target": 1, + "support_probability": 0.4216606616973877, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4892392420686221, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10672170668840408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9412818551063538, + 0.27303528785705566, + 0.23557157814502716, + 0.3201605975627899, + 0.1625131368637085, + 0.19926945865154266 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 1, + "style_probability": 0.24762848019599915, + "style_target": 0, + "support_probability": 0.3579521179199219, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05542599073377763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6789370179176331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48186641931533813, + 0.8153612613677979, + 0.25030189752578735, + 0.34981557726860046, + 0.056021831929683685, + 0.05251169949769974, + 0.07387178391218185, + 0.14131991565227509, + 0.1048467680811882 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 0, + "style_probability": 0.3965340554714203, + "style_target": 1, + "support_probability": 0.07399032264947891, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013654239269823381, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6644092202186584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5970532298088074, + 0.9369823336601257, + 0.2720383107662201, + 0.45140013098716736, + 0.07431326806545258, + 0.19700637459754944, + 0.13637307286262512, + 0.08577103167772293, + 0.1887107640504837 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 1, + "style_probability": 0.07571578025817871, + "style_target": 0, + "support_probability": 0.08658114075660706, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.29165073328828656, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4144589602947235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7321234345436096, + 0.23932577669620514, + 0.4183468818664551, + 0.28902819752693176, + 0.0944165289402008, + 0.11707672476768494, + 0.08579467982053757, + 0.0698610469698906, + 0.09159675985574722, + 0.0939796045422554, + 0.09598265588283539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.3288772404193878, + "style_target": 1, + "support_probability": 0.473354309797287, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5468399741729248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.303834468126297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7607036828994751, + 0.2955322861671448, + 0.45387959480285645, + 0.31957346200942993, + 0.09115477651357651, + 0.06756124645471573, + 0.07965438067913055, + 0.0296632070094347, + 0.07920782268047333, + 0.08482050150632858, + 0.08078623563051224 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.7919400930404663, + "style_target": 0, + "support_probability": 0.5065947771072388, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11485302100088957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8669216632843018, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.807912290096283, + 0.11306187510490417, + 0.20068210363388062, + 0.3579651117324829, + 0.09967229515314102 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.8160725235939026, + "style_target": 1, + "support_probability": 0.10136257857084274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01866906525215442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6812661290168762, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8628436923027039, + 0.1477717161178589, + 0.16997428238391876, + 0.3644459843635559, + 0.13623903691768646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.2360287457704544, + "style_target": 0, + "support_probability": 0.21289901435375214, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.763374322311578, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10619909316301346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14872676134109497, + 0.25983473658561707, + 0.21008381247520447, + 0.8997323513031006, + 0.05883980914950371, + 0.10015229135751724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 1, + "style_probability": 0.7953016757965088, + "style_target": 1, + "support_probability": 0.7892932891845703, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6274074309124678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2060023993253708, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1630939543247223, + 0.293861448764801, + 0.17020848393440247, + 0.8654310703277588, + 0.4001920819282532, + 0.0709528774023056 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 0, + "style_probability": 0.5168163776397705, + "style_target": 0, + "support_probability": 0.7114065289497375, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5480517578496565, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44325941801071167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2732555866241455, + 0.41585826873779297, + 0.8109911680221558, + 0.602013349533081, + 0.059013769030570984, + 0.13095882534980774, + 0.1066756621003151, + 0.052172232419252396, + 0.13648734986782074 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.8118870854377747, + "style_target": 1, + "support_probability": 0.44032949209213257, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3971706875983068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45430779457092285, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3293783962726593, + 0.4591532051563263, + 0.7845070958137512, + 0.6013796925544739, + 0.0721941664814949, + 0.17009571194648743, + 0.08501364290714264, + 0.13712015748023987, + 0.13609440624713898 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.4542466700077057, + "style_target": 0, + "support_probability": 0.4538441300392151, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7310391804030308, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28949517011642456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8975003361701965, + 0.5076674222946167, + 0.1937730461359024, + 0.2648361623287201, + 0.03376356512308121, + 0.0908685252070427, + 0.053709644824266434, + 0.06983023881912231, + 0.055698178708553314, + 0.06458421796560287, + 0.0742063894867897 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.6665106415748596, + "style_target": 1, + "support_probability": 0.8059003353118896, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15807898750279303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2618226110935211, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9201878309249878, + 0.5798362493515015, + 0.21666675806045532, + 0.2779774069786072, + 0.062027424573898315, + 0.0889555886387825, + 0.13507521152496338, + 0.11246295273303986, + 0.059916574507951736, + 0.07780246436595917 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.11917934566736221, + "style_target": 0, + "support_probability": 0.8257519006729126, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6361969100006164, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04759950190782547, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9876798391342163, + 0.5859758853912354, + 0.20872856676578522, + 0.5328518748283386, + 0.07639266550540924 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 1, + "style_probability": 0.6536015868186951, + "style_target": 1, + "support_probability": 0.6653119325637817, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6747624958527317, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04970565438270569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9688168168067932, + 0.5320886373519897, + 0.2573316991329193, + 0.536557674407959, + 0.36696821451187134 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 0, + "style_probability": 0.625338077545166, + "style_target": 0, + "support_probability": 0.6769157648086548, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3668056557927197, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43578362464904785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19555920362472534, + 0.5906931161880493, + 0.3695067763328552, + 0.37034377455711365, + 0.09120450168848038, + 0.19013239443302155 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.8854263424873352, + "style_target": 1, + "support_probability": 0.18300513923168182, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03893854511024127, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5416092872619629, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18224065005779266, + 0.5824474096298218, + 0.28639477491378784, + 0.2815488278865814, + 0.14360901713371277, + 0.10795355588197708 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.2975565195083618, + "style_target": 0, + "support_probability": 0.07219681888818741, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7528578794483151, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09518063068389893, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5626368522644043, + 0.8452920913696289, + 0.3767005205154419, + 0.3693794906139374, + 0.074761301279068, + 0.04124903306365013, + 0.035456378012895584, + 0.025568421930074692, + 0.08003929257392883 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.807788074016571, + "style_target": 1, + "support_probability": 0.9358105659484863, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1796386663520457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15368597209453583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5499953627586365, + 0.885040819644928, + 0.37613803148269653, + 0.4402555525302887, + 0.0748896598815918, + 0.11449594795703888, + 0.08410567790269852, + 0.06286453455686569, + 0.0774131715297699 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.1417868435382843, + "style_target": 0, + "support_probability": 0.9148972630500793, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9006156641488237, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05107298493385315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.362587034702301, + 0.38637447357177734, + 0.9702200293540955, + 0.4812798798084259, + 0.03247881308197975, + 0.11091866344213486, + 0.0629362091422081, + 0.0588388592004776, + 0.022205142304301262, + 0.030805757269263268, + 0.09804680943489075 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 1, + "style_probability": 0.835561990737915, + "style_target": 1, + "support_probability": 0.954806387424469, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7992288325749063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10023646801710129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31333020329475403, + 0.33493560552597046, + 0.939349889755249, + 0.45140373706817627, + 0.05529932677745819, + 0.055205442011356354, + 0.05010630190372467, + 0.052519869059324265, + 0.11470082402229309, + 0.16144321858882904 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 0, + "style_probability": 0.612054705619812, + "style_target": 0, + "support_probability": 0.9181450009346008, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38679225031206926, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3105110824108124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6727808117866516, + 0.5028663277626038, + 0.4308730959892273, + 0.35752516984939575, + 0.04588066413998604 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 1, + "style_probability": 0.519442081451416, + "style_target": 1, + "support_probability": 0.6436129808425903, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5927066864670303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32130762934684753, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6772624850273132, + 0.5913442373275757, + 0.5078325271606445, + 0.3622705042362213, + 0.04205453023314476 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 0, + "style_probability": 0.9034464955329895, + "style_target": 0, + "support_probability": 0.7337722778320312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06959118125458306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6878162622451782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19263328611850739, + 0.3854922652244568, + 0.17944349348545074, + 0.8577722907066345, + 0.07822995632886887, + 0.06938210874795914 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 0, + "style_probability": 0.49897700548171997, + "style_target": 1, + "support_probability": 0.05781667307019234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03166047624471945, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4506305158138275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16044870018959045, + 0.37715277075767517, + 0.2532927989959717, + 0.937525749206543, + 0.053143210709095, + 0.11227382719516754 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 1, + "style_probability": 0.18173450231552124, + "style_target": 0, + "support_probability": 0.36679327487945557, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.648742487831465, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05803700163960457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32847100496292114, + 0.4406380355358124, + 0.9068321585655212, + 0.46236640214920044, + 0.05954728648066521, + 0.051350854337215424, + 0.1130886822938919, + 0.10283149778842926, + 0.18457239866256714 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 0, + "style_probability": 0.4072970747947693, + "style_target": 1, + "support_probability": 0.7764122486114502, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7335814795845552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09756860136985779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2994670569896698, + 0.44464272260665894, + 0.9060319066047668, + 0.4761597514152527, + 0.05252021551132202, + 0.11366899311542511, + 0.14975202083587646, + 0.15257428586483002, + 0.06077776849269867 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 0, + "style_probability": 0.5116568207740784, + "style_target": 0, + "support_probability": 0.79788738489151, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8687861960707316, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0418737456202507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.334253191947937, + 0.97698974609375, + 0.24510681629180908, + 0.19441957771778107, + 0.16409993171691895, + 0.45247548818588257, + 0.059877365827560425, + 0.08089236170053482, + 0.06465794146060944, + 0.12832513451576233, + 0.09426993876695633 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.3799833357334137, + "style_target": 1, + "support_probability": 0.967073380947113, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9458073330461055, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03346701338887215, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1009:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30132314562797546, + 0.9793078303337097, + 0.24094471335411072, + 0.2166927307844162, + 0.037214245647192, + 0.07075207680463791, + 0.0626482143998146, + 0.035541873425245285, + 0.04340272396802902, + 0.03653842583298683, + 0.09432338178157806 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.9065504670143127, + "style_target": 0, + "support_probability": 0.9680055975914001, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.534669750982939, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22781510651111603, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1396118700504303, + 0.29947832226753235, + 0.8516526818275452, + 0.2690175473690033, + 0.1483767181634903 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 0, + "style_probability": 0.34839242696762085, + "style_target": 1, + "support_probability": 0.8113387227058411, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4128348742252318, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4199853241443634, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12739136815071106, + 0.24376897513866425, + 0.8279432058334351, + 0.23885470628738403, + 0.08818082511425018 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 1, + "style_probability": 0.2231651097536087, + "style_target": 0, + "support_probability": 0.6666855216026306, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08128389675811327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.43017488718032837, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1990608274936676, + 0.25456055998802185, + 0.1990608274936676, + 0.853675901889801, + 0.0757395476102829, + 0.37595877051353455 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 0, + "style_probability": 0.4879779815673828, + "style_target": 1, + "support_probability": 0.10511383414268494, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005580832090355399, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.714520275592804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1829448938369751, + 0.2639683783054352, + 0.1829448938369751, + 0.895169734954834, + 0.1066666916012764, + 0.11095567792654037 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 1, + "style_probability": 0.10703767091035843, + "style_target": 0, + "support_probability": 0.09592483192682266, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7943774421426696, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07752679288387299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6864480376243591, + 0.16144859790802002, + 0.6869505047798157, + 0.9368811249732971, + 0.09120889008045197, + 0.18577255308628082, + 0.17600414156913757, + 0.17567339539527893, + 0.226701557636261 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 1, + "style_probability": 0.5058267116546631, + "style_target": 1, + "support_probability": 0.9153362512588501, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8530912962124134, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08136240392923355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7201303243637085, + 0.15558448433876038, + 0.7281238436698914, + 0.9511600732803345, + 0.09646806865930557, + 0.10107932239770889, + 0.13267704844474792, + 0.08852532505989075, + 0.08223985880613327 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 0, + "style_probability": 0.733697772026062, + "style_target": 0, + "support_probability": 0.9013814330101013, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5787401322289586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09883835166692734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26712459325790405, + 0.2569471597671509, + 0.9132078886032104, + 0.3154955506324768, + 0.08272472769021988, + 0.09960407018661499, + 0.05401737987995148, + 0.06196577847003937, + 0.07789043337106705, + 0.22295156121253967, + 0.1616826355457306 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 0, + "style_probability": 0.46491509675979614, + "style_target": 1, + "support_probability": 0.7183325290679932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3856367346271604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19064933061599731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2897473871707916, + 0.25392448902130127, + 0.9144101738929749, + 0.3240484297275543, + 0.11484754830598831, + 0.08341489732265472, + 0.23554465174674988, + 0.09723738580942154, + 0.1653018742799759, + 0.16715732216835022, + 0.09205220639705658 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 1, + "style_probability": 0.37153446674346924, + "style_target": 0, + "support_probability": 0.5374516844749451, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6357967946588059, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0810418650507927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.970084547996521, + 0.23443344235420227, + 0.26119327545166016, + 0.5029594898223877, + 0.05536855012178421 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 1, + "style_probability": 0.508678674697876, + "style_target": 1, + "support_probability": 0.8464392423629761, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.628831998184455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09928996115922928, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9524655342102051, + 0.24296768009662628, + 0.3057384192943573, + 0.48303845524787903, + 0.17817485332489014 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 1, + "style_probability": 0.41265222430229187, + "style_target": 0, + "support_probability": 0.8245111107826233, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12301333562152131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5948975086212158, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5270598530769348, + 0.8528904318809509, + 0.3016203045845032, + 0.3530218303203583, + 0.11574455350637436, + 0.10428924113512039 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.815944492816925, + "style_target": 1, + "support_probability": 0.12853272259235382, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018577664680371097, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8056451082229614, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5134521126747131, + 0.8377528786659241, + 0.3048627972602844, + 0.33026549220085144, + 0.06269191205501556, + 0.11816275119781494 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.32423561811447144, + "style_target": 0, + "support_probability": 0.02731507085263729, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9069170647592399, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12785372138023376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1050:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29340803623199463, + 0.5105980634689331, + 0.9966827034950256, + 0.27167433500289917, + 0.0917765200138092, + 0.07287709414958954, + 0.06513100117444992, + 0.1573440581560135, + 0.18265996873378754 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.6839491724967957, + "style_target": 1, + "support_probability": 0.9454768300056458, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3684754455778083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1778315305709839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1050:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3395560383796692, + 0.6015865206718445, + 0.999065101146698, + 0.31723299622535706, + 0.146616593003273, + 0.09567161649465561, + 0.16600705683231354, + 0.08759547024965286, + 0.09596029669046402 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.10159703344106674, + "style_target": 0, + "support_probability": 0.9345170259475708, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.529504685476973, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6144388318061829, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2602621018886566, + 0.7004765272140503, + 0.8660345673561096, + 0.23678813874721527, + 0.04567050561308861, + 0.10198736935853958, + 0.04035124555230141, + 0.05907867103815079, + 0.09665105491876602, + 0.12355522811412811 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 1, + "style_probability": 0.8633900284767151, + "style_target": 1, + "support_probability": 0.48273950815200806, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2099993849279692, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4793815612792969, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19136376678943634, + 0.6142170429229736, + 0.8184048533439636, + 0.18361368775367737, + 0.14293153584003448, + 0.07488662004470825, + 0.05684533715248108, + 0.05244554579257965, + 0.08697319030761719, + 0.07965736836194992 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 1, + "style_probability": 0.31190910935401917, + "style_target": 0, + "support_probability": 0.5408247113227844, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07401264954779171, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8098586201667786, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27407705783843994, + 0.5768064260482788, + 0.424860417842865, + 0.4004572033882141, + 0.17826876044273376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 1, + "style_probability": 0.6705085039138794, + "style_target": 1, + "support_probability": 0.07803228497505188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11298483768258917, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7989028692245483, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2719253599643707, + 0.5996100902557373, + 0.4744817018508911, + 0.4134855568408966, + 0.0671960860490799 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 0, + "style_probability": 0.8567741513252258, + "style_target": 0, + "support_probability": 0.11383328586816788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3224102711559468, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5139123201370239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37780457735061646, + 0.13942702114582062, + 0.2380864918231964, + 0.8842570781707764, + 0.059398576617240906, + 0.12961265444755554 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.565169095993042, + "style_target": 1, + "support_probability": 0.3227590024471283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1222244405229186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7229758501052856, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.412553071975708, + 0.18484875559806824, + 0.27149757742881775, + 0.890302836894989, + 0.09405604749917984, + 0.16155561804771423 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.372444212436676, + "style_target": 0, + "support_probability": 0.14894376695156097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5165226036900856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33173060417175293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8227062821388245, + 0.40885618329048157, + 0.15719088912010193, + 0.40887823700904846, + 0.0661306232213974, + 0.048134323209524155, + 0.0969780907034874, + 0.11852116882801056, + 0.15933917462825775 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 0, + "style_probability": 0.46874740719795227, + "style_target": 1, + "support_probability": 0.7528244256973267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18213836467848735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3256474733352661, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.866814911365509, + 0.41710197925567627, + 0.19910703599452972, + 0.41239944100379944, + 0.13702493906021118, + 0.18031156063079834, + 0.06437308341264725, + 0.06869502365589142, + 0.0825941264629364 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 1, + "style_probability": 0.19405809044837952, + "style_target": 0, + "support_probability": 0.8346964120864868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32474670484529516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5571085810661316, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5388835072517395, + 0.4052622318267822, + 0.6673601865768433, + 0.5663557648658752, + 0.05398901179432869, + 0.07013806700706482, + 0.06115420162677765, + 0.03723999112844467, + 0.07536710053682327, + 0.11550243198871613, + 0.1286012977361679 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 1, + "style_probability": 0.5385898351669312, + "style_target": 1, + "support_probability": 0.6024796366691589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5133321276168039, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5540140867233276, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6493473649024963, + 0.4911012649536133, + 0.7244546413421631, + 0.6213235259056091, + 0.1380140334367752, + 0.08201860636472702, + 0.04922657832503319, + 0.08326224982738495, + 0.14016389846801758, + 0.15239907801151276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 0, + "style_probability": 0.8458006978034973, + "style_target": 0, + "support_probability": 0.48007479310035706, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4560960535819609, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4855395257472992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30479100346565247, + 0.7614935636520386, + 0.3331771492958069, + 0.30980995297431946, + 0.10110173374414444 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 0, + "style_probability": 0.4956934154033661, + "style_target": 1, + "support_probability": 0.5289695858955383, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40659606507448015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.406328022480011, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3138793706893921, + 0.7624582052230835, + 0.3606303930282593, + 0.28572162985801697, + 0.102862648665905 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 1, + "style_probability": 0.39173921942710876, + "style_target": 0, + "support_probability": 0.5017660856246948, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5960011453190517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06482056528329849, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11236938089132309, + 0.2767038941383362, + 0.9853894114494324, + 0.40512949228286743, + 0.16871848702430725, + 0.05674077570438385 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.7052903175354004, + "style_target": 1, + "support_probability": 0.526188313961029, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06529458899481841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15630663931369781, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10720358043909073, + 0.31806445121765137, + 0.9968670010566711, + 0.44001758098602295, + 0.08273579925298691, + 0.14711500704288483 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.2132744938135147, + "style_target": 0, + "support_probability": 0.3767038881778717, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.17139695158623994, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6420252323150635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24852262437343597, + 0.17906323075294495, + 0.543798565864563, + 0.9656026363372803, + 0.13805146515369415, + 0.13624334335327148, + 0.0756879672408104, + 0.19030551612377167, + 0.06927435845136642 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 1, + "style_probability": 0.6524495482444763, + "style_target": 1, + "support_probability": 0.09442009776830673, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.502303428749251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4016672968864441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22424739599227905, + 0.19566361606121063, + 0.6143513917922974, + 0.9830741286277771, + 0.14271734654903412, + 0.16045695543289185, + 0.09186920523643494, + 0.09920237958431244, + 0.160386323928833 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 0, + "style_probability": 0.8139982223510742, + "style_target": 0, + "support_probability": 0.3674300014972687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19297654148592214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46440598368644714, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8027992248535156, + 0.2576070725917816, + 0.12105410546064377, + 0.32863184809684753, + 0.02693156525492668, + 0.04210423678159714, + 0.13830122351646423, + 0.0751430094242096, + 0.08480097353458405, + 0.1317397505044937, + 0.16014637053012848 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 0, + "style_probability": 0.3228643536567688, + "style_target": 1, + "support_probability": 0.29289743304252625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17713971960820424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.493060439825058, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8680800199508667, + 0.24845995008945465, + 0.12966887652873993, + 0.2953372597694397, + 0.07323212176561356, + 0.06759821623563766, + 0.4136890470981598, + 0.06332175433635712, + 0.17681507766246796, + 0.08490017801523209, + 0.09934816509485245 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 1, + "style_probability": 0.1882350891828537, + "style_target": 0, + "support_probability": 0.3887936770915985, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4640315867218282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5486326217651367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39521533250808716, + 0.2760573923587799, + 0.2891392707824707, + 0.7509124279022217, + 0.10806923359632492 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 1, + "style_probability": 0.8641228079795837, + "style_target": 1, + "support_probability": 0.2986776828765869, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12834681819429555, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7535041570663452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41717374324798584, + 0.281602144241333, + 0.3388323485851288, + 0.6687368154525757, + 0.10988016426563263 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 1, + "style_probability": 0.4895267188549042, + "style_target": 0, + "support_probability": 0.17792607843875885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05529337980591934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7915908098220825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.727525532245636, + 0.19101151823997498, + 0.14730608463287354, + 0.3347542881965637, + 0.1401800811290741, + 0.07668611407279968 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.6251563429832458, + "style_target": 1, + "support_probability": 0.04080253839492798, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026941675446520173, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8336570858955383, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7032227516174316, + 0.20991206169128418, + 0.170559823513031, + 0.31953638792037964, + 0.0802120566368103, + 0.19729067385196686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.2178744226694107, + "style_target": 0, + "support_probability": 0.03404585272073746, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7041537839623103, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32673701643943787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4161953628063202, + 0.9067920446395874, + 0.18637850880622864, + 0.14475798606872559, + 0.06298227608203888, + 0.07425487041473389, + 0.10287840664386749, + 0.13684800267219543, + 0.07591745257377625 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.5669836401939392, + "style_target": 1, + "support_probability": 0.6204420328140259, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16025942984061123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3106555640697479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38998931646347046, + 0.9327893257141113, + 0.2055826187133789, + 0.13044694066047668, + 0.08967418968677521, + 0.21786876022815704, + 0.09181416034698486, + 0.15003085136413574, + 0.08916337043046951 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.07426796108484268, + "style_target": 0, + "support_probability": 0.6559138298034668, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.79454942933675, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07229427993297577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.85099858045578, + 0.18018534779548645, + 0.45960095524787903, + 0.28029847145080566, + 0.08347619324922562, + 0.023803070187568665, + 0.022019444033503532, + 0.0462920144200325, + 0.056964896619319916, + 0.09590829163789749, + 0.06365366280078888 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.8423217535018921, + "style_target": 1, + "support_probability": 0.943504273891449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7278226417242328, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12465595453977585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8474485874176025, + 0.19319410622119904, + 0.4007483124732971, + 0.24704977869987488, + 0.059565525501966476, + 0.08255475759506226, + 0.0379825085401535, + 0.05828485265374184, + 0.050678253173828125, + 0.12225042283535004, + 0.06611883640289307 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.4992011487483978, + "style_target": 0, + "support_probability": 0.8877536654472351, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17029874786311647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8075783848762512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16965287923812866, + 0.7867668271064758, + 0.2084815949201584, + 0.31783467531204224, + 0.13421395421028137 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.42619287967681885, + "style_target": 1, + "support_probability": 0.25488778948783875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2573459343699852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8524316549301147, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1702495515346527, + 0.8069690465927124, + 0.19941860437393188, + 0.3818870782852173, + 0.07112733274698257 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.7791023254394531, + "style_target": 0, + "support_probability": 0.2348787933588028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6667011513732267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18804959952831268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23643779754638672, + 0.23394539952278137, + 0.9017332196235657, + 0.3590437173843384, + 0.12693721055984497, + 0.09217754751443863 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.5965926647186279, + "style_target": 1, + "support_probability": 0.7719520330429077, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5354853120574983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22984495759010315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22786302864551544, + 0.24684864282608032, + 0.9028939604759216, + 0.36256077885627747, + 0.46184876561164856, + 0.1199943870306015 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.48442479968070984, + "style_target": 0, + "support_probability": 0.6362176537513733, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7879915026485229, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2480594366788864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20676206052303314, + 0.928215742111206, + 0.29852572083473206, + 0.20676206052303314, + 0.07377681136131287, + 0.1492440402507782, + 0.08125055581331253, + 0.23115894198417664, + 0.09372314810752869 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.7303573489189148, + "style_target": 1, + "support_probability": 0.8479995131492615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39275694991248306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16122284531593323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18760022521018982, + 0.9780479669570923, + 0.3036070764064789, + 0.18760022521018982, + 0.07401968538761139, + 0.09245173633098602, + 0.1430055797100067, + 0.1061578020453453, + 0.1900985687971115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.2712462246417999, + "style_target": 0, + "support_probability": 0.9005646109580994, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1704292668659618, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7446230053901672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.299617737531662, + 0.6112536787986755, + 0.3914766311645508, + 0.4107581377029419, + 0.044030532240867615, + 0.07739868760108948, + 0.08512984961271286, + 0.06857766956090927, + 0.0790935754776001, + 0.2146253138780594, + 0.053994320333004 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 0, + "style_probability": 0.45213064551353455, + "style_target": 1, + "support_probability": 0.13438758254051208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3489389926553903, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7850615978240967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2999624013900757, + 0.6616744995117188, + 0.47841134667396545, + 0.4940668046474457, + 0.10741834342479706, + 0.06715340912342072, + 0.060981761664152145, + 0.0733371302485466, + 0.06552004814147949, + 0.06166605278849602, + 0.030963128432631493 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 0, + "style_probability": 0.7131577134132385, + "style_target": 0, + "support_probability": 0.17956379055976868, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7372797068739652, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21103480458259583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9749780893325806, + 0.3327459692955017, + 0.41486209630966187, + 0.14464682340621948, + 0.056434959173202515 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 0, + "style_probability": 0.4294288456439972, + "style_target": 1, + "support_probability": 0.7032372951507568, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5802903024117951, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.226579487323761, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9714325070381165, + 0.41318461298942566, + 0.4955986738204956, + 0.1579780876636505, + 0.23816224932670593 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 1, + "style_probability": 0.37517616152763367, + "style_target": 0, + "support_probability": 0.624392569065094, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01840119558595175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8007171750068665, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6903725862503052, + 0.20611056685447693, + 0.2304457277059555, + 0.32491204142570496, + 0.0927494540810585, + 0.08477932959794998 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 0, + "style_probability": 0.3576437830924988, + "style_target": 1, + "support_probability": 0.0219911839812994, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00755114908081811, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6667065024375916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7414801120758057, + 0.22817884385585785, + 0.1931721270084381, + 0.39975592494010925, + 0.11208238452672958, + 0.09293597936630249 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 1, + "style_probability": 0.19865640997886658, + "style_target": 0, + "support_probability": 0.035872217267751694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04782383423663395, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7301557064056396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2647453844547272, + 0.7233639359474182, + 0.34940171241760254, + 0.3875640630722046, + 0.07589909434318542, + 0.11308954656124115, + 0.4055456519126892, + 0.10904651135206223, + 0.11331754922866821 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 1, + "style_probability": 0.5559409260749817, + "style_target": 1, + "support_probability": 0.06853116303682327, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07089856948056476, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8879343271255493, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19989252090454102, + 0.72844398021698, + 0.3580171465873718, + 0.3112364113330841, + 0.04381253570318222, + 0.1051344946026802, + 0.06508193910121918, + 0.056368496268987656, + 0.09178273379802704 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 0, + "style_probability": 0.8185454607009888, + "style_target": 0, + "support_probability": 0.048514895141124725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024483933162050633, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9313574433326721, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9016207456588745, + 0.22672820091247559, + 0.38191887736320496, + 0.22672820091247559, + 0.08964988589286804, + 0.06785155832767487, + 0.05420534312725067, + 0.14389696717262268, + 0.061659540981054306, + 0.12945246696472168, + 0.07505428045988083 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 0, + "style_probability": 0.45459288358688354, + "style_target": 1, + "support_probability": 0.031085999682545662, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023485009668840127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9254274368286133, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.91317218542099, + 0.18844598531723022, + 0.31584346294403076, + 0.18844598531723022, + 0.09287431836128235, + 0.10948986560106277, + 0.12613710761070251, + 0.19781719148159027, + 0.30245092511177063, + 0.0867401510477066, + 0.33129024505615234 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 1, + "style_probability": 0.35073402523994446, + "style_target": 0, + "support_probability": 0.017655054107308388, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13011649137061423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8111416697502136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20452247560024261, + 0.9351866245269775, + 0.35355252027511597, + 0.14381423592567444, + 0.19766998291015625 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 1, + "style_probability": 0.753363311290741, + "style_target": 1, + "support_probability": 0.07652576267719269, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045360764917655935, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7961443066596985, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21307621896266937, + 0.9282979369163513, + 0.3424082100391388, + 0.1671242117881775, + 0.12116114050149918 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 1, + "style_probability": 0.3873582184314728, + "style_target": 0, + "support_probability": 0.06586737185716629, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6650874310288568, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04788676276803017, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16307425498962402, + 0.16053758561611176, + 0.5268152356147766, + 0.9977600574493408, + 0.07637707889080048, + 0.13823577761650085 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 1, + "style_probability": 0.5215060114860535, + "style_target": 1, + "support_probability": 0.8649583458900452, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4845419421442284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07299549877643585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1843896210193634, + 0.19123508036136627, + 0.5580019354820251, + 0.9924015998840332, + 0.07004349678754807, + 0.1663634330034256 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 1, + "style_probability": 0.2653891444206238, + "style_target": 0, + "support_probability": 0.8030238747596741, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.678433390778265, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3130776882171631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3091922700405121, + 0.8646863102912903, + 0.061656538397073746, + 0.30181780457496643, + 0.062491536140441895, + 0.06633670628070831, + 0.08359450101852417, + 0.03249964863061905, + 0.04414128139615059 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.8114507794380188, + "style_target": 1, + "support_probability": 0.6636612415313721, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.040633701038157305, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4579751491546631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2997607886791229, + 0.8889377117156982, + 0.12353596836328506, + 0.3598964810371399, + 0.09098131209611893, + 0.0637332871556282, + 0.07466911524534225, + 0.08841138333082199, + 0.14187578856945038 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.08944925665855408, + "style_target": 0, + "support_probability": 0.6145229339599609, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4223270548747978, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1642824411392212, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4077708423137665, + 0.42956262826919556, + 0.1585950404405594, + 0.9236104488372803, + 0.029853682965040207, + 0.029019320383667946, + 0.09192809462547302, + 0.07980670034885406, + 0.05662517994642258, + 0.02917998842895031, + 0.056976232677698135 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 1, + "style_probability": 0.791694700717926, + "style_target": 1, + "support_probability": 0.35818400979042053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08622401993087106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2944151759147644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40667763352394104, + 0.4548813998699188, + 0.2081153392791748, + 0.8838410377502441, + 0.0653948113322258, + 0.058427147567272186, + 0.03623608127236366, + 0.060729291290044785, + 0.06611612439155579, + 0.04081616550683975 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 1, + "style_probability": 0.3604481816291809, + "style_target": 0, + "support_probability": 0.19433918595314026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045598473897826446, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4006568491458893, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.790534496307373, + 0.23529312014579773, + 0.38559168577194214, + 0.34863531589508057, + 0.10255573689937592 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.23469075560569763, + "style_target": 1, + "support_probability": 0.05145317688584328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0823535834220237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2740235924720764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8335919380187988, + 0.23384417593479156, + 0.36367616057395935, + 0.3529331386089325, + 0.05707881227135658 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.6433241367340088, + "style_target": 0, + "support_probability": 0.04644178971648216, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4408506346302387, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061077509075403214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2942645847797394, + 0.919577956199646, + 0.1909164935350418, + 0.14082376658916473, + 0.11047118157148361, + 0.2573833167552948 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 0, + "style_probability": 0.37227120995521545, + "style_target": 1, + "support_probability": 0.6816291809082031, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.057996936260328785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0987151488661766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26759687066078186, + 0.9722598195075989, + 0.2685549557209015, + 0.21933093667030334, + 0.14698542654514313, + 0.12547089159488678 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 1, + "style_probability": 0.0656280368566513, + "style_target": 0, + "support_probability": 0.7346075773239136, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009746012995759619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9735804796218872, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5753129124641418, + 0.29503610730171204, + 0.7990490794181824, + 0.2430424988269806, + 0.11399969458580017, + 0.12788709998130798, + 0.09953009337186813, + 0.09773752093315125, + 0.13988207280635834 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.36926716566085815, + "style_target": 1, + "support_probability": 0.009730713441967964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03913887218886763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9202658534049988, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5452473759651184, + 0.3186657428741455, + 0.809343159198761, + 0.22968582808971405, + 0.08305377513170242, + 0.2053651362657547, + 0.33693623542785645, + 0.11828915774822235, + 0.11904462426900864 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.5738356113433838, + "style_target": 0, + "support_probability": 0.030633561313152313, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6813128204628988, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.043547745794057846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19856008887290955, + 0.6059806942939758, + 0.42312362790107727, + 0.9726304411888123, + 0.07619822025299072, + 0.12870997190475464, + 0.08982796967029572, + 0.10428258776664734, + 0.10219917446374893, + 0.06368571519851685, + 0.11626046150922775 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.3235571086406708, + "style_target": 1, + "support_probability": 0.8602063059806824, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8880023290776142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06253937631845474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1195:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2111615687608719, + 0.6495380997657776, + 0.4590113162994385, + 0.9771239161491394, + 0.11061281710863113, + 0.07900485396385193, + 0.05867959186434746, + 0.07551977783441544, + 0.08607291430234909, + 0.10257039219141006, + 0.0632285326719284 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.8440665602684021, + "style_target": 0, + "support_probability": 0.8465222716331482, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2929701879269462, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5225507020950317, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0801486000418663, + 0.26999810338020325, + 0.2302207350730896, + 0.6528877019882202, + 0.06221556290984154 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.7797170877456665, + "style_target": 1, + "support_probability": 0.37214136123657227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01207966275962612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7033135294914246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09758350998163223, + 0.2288554608821869, + 0.19079093635082245, + 0.7096307277679443, + 0.08588922023773193 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.13405075669288635, + "style_target": 0, + "support_probability": 0.2158246785402298, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5673253088181542, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4075053930282593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23664425313472748, + 0.8711816668510437, + 0.07804695516824722, + 0.35998469591140747, + 0.059418655931949615, + 0.07315570116043091 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 1, + "style_probability": 0.632414698600769, + "style_target": 1, + "support_probability": 0.6363946199417114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4672589184390356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46747589111328125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2896310091018677, + 0.8271844387054443, + 0.0977812111377716, + 0.3402855098247528, + 0.07804737985134125, + 0.05861073359847069 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 1, + "style_probability": 0.4290415346622467, + "style_target": 0, + "support_probability": 0.6016770601272583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1688811638542502, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7846410274505615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6096101403236389, + 0.18521688878536224, + 0.6096101403236389, + 0.6876789927482605, + 0.06623717397451401, + 0.05131414905190468, + 0.10706835985183716, + 0.05391644686460495, + 0.12066888064146042 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.5610675811767578, + "style_target": 1, + "support_probability": 0.2580828368663788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08287978741226051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7577903270721436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5075904130935669, + 0.19961567223072052, + 0.5075904130935669, + 0.6909831166267395, + 0.09145164489746094, + 0.1209431067109108, + 0.14739267528057098, + 0.0872405469417572, + 0.11819349974393845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.22581346333026886, + "style_target": 0, + "support_probability": 0.18691593408584595, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016887539206008263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9813920855522156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6342356204986572, + 0.5127374529838562, + 0.2695229649543762, + 0.5127374529838562, + 0.08074189722537994, + 0.16147515177726746, + 0.07057341933250427, + 0.11270489543676376, + 0.2075648456811905, + 0.15193839371204376, + 0.13094864785671234 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.7152653932571411, + "style_target": 1, + "support_probability": 0.013959490694105625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033233852513713507, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9856016039848328, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6834454536437988, + 0.5583258271217346, + 0.24349021911621094, + 0.5583258271217346, + 0.09942822903394699, + 0.13096845149993896, + 0.26062577962875366, + 0.15081308782100677, + 0.15347178280353546, + 0.13601955771446228 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.19371679425239563, + "style_target": 0, + "support_probability": 0.01641734130680561, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6170929176042922, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3304575979709625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8990561366081238, + 0.259008526802063, + 0.20358803868293762, + 0.259008526802063, + 0.08750563114881516 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.7145108580589294, + "style_target": 1, + "support_probability": 0.42235881090164185, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07372881198393709, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40415191650390625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9637283682823181, + 0.25266510248184204, + 0.22031913697719574, + 0.25266510248184204, + 0.1403820812702179 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.1860886514186859, + "style_target": 0, + "support_probability": 0.33997753262519836, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012247264680051578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9657430052757263, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30471327900886536, + 0.6862573623657227, + 0.23421061038970947, + 0.4999990463256836, + 0.1383867710828781, + 0.10553063452243805 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 0, + "style_probability": 0.37342387437820435, + "style_target": 1, + "support_probability": 0.011304018087685108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014845820489723325, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9694942235946655, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28826355934143066, + 0.6691422462463379, + 0.23637306690216064, + 0.5087919235229492, + 0.06914117187261581, + 0.13734130561351776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 0, + "style_probability": 0.7227960228919983, + "style_target": 0, + "support_probability": 0.00640434306114912, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.602109036951884, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19591118395328522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2628823518753052, + 0.700556218624115, + 0.4243049919605255, + 0.4906100630760193, + 0.06188099831342697, + 0.15401431918144226, + 0.17397551238536835, + 0.08621435612440109, + 0.12563028931617737 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 0, + "style_probability": 0.49158668518066406, + "style_target": 1, + "support_probability": 0.834916889667511, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6526381181851967, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09233919531106949, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24203774333000183, + 0.7533413171768188, + 0.42988306283950806, + 0.47693905234336853, + 0.10369501262903214, + 0.08051152527332306, + 0.0686199888586998, + 0.03463217243552208, + 0.1168917566537857 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 1, + "style_probability": 0.3756798505783081, + "style_target": 0, + "support_probability": 0.908784031867981, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.057087310674673514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9375392198562622, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7507293820381165, + 0.26234373450279236, + 0.1696949303150177, + 0.46882957220077515, + 0.048352062702178955, + 0.056224897503852844, + 0.05804295837879181, + 0.18350335955619812, + 0.09274578094482422, + 0.07170511037111282, + 0.067695252597332 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 0, + "style_probability": 0.34475892782211304, + "style_target": 1, + "support_probability": 0.1205233708024025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047660452082643426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7825347185134888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8152250647544861, + 0.28670623898506165, + 0.18325552344322205, + 0.4829404354095459, + 0.18717943131923676, + 0.10859423130750656, + 0.6807866096496582, + 0.06737092137336731, + 0.07230854779481888, + 0.11621733009815216 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 1, + "style_probability": 0.19707049429416656, + "style_target": 0, + "support_probability": 0.21727371215820312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05114000091403513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7369439601898193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4892989695072174, + 0.2395639568567276, + 0.1661272495985031, + 0.2395639568567276, + 0.0872846469283104 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 0, + "style_probability": 0.42729929089546204, + "style_target": 1, + "support_probability": 0.11637098342180252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11460063854593357, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7246952652931213, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.49754786491394043, + 0.2349284440279007, + 0.1726710945367813, + 0.2349284440279007, + 0.035575151443481445 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 0, + "style_probability": 0.824438214302063, + "style_target": 0, + "support_probability": 0.1084868460893631, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08169778460458214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5169104337692261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5919921398162842, + 0.17836038768291473, + 0.5853732824325562, + 0.2794022262096405, + 0.11991873383522034, + 0.11301103234291077 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 1, + "style_probability": 0.5713790059089661, + "style_target": 1, + "support_probability": 0.0745806023478508, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10122967579011513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28793802857398987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6418536305427551, + 0.14884251356124878, + 0.5697124600410461, + 0.24580664932727814, + 0.10988662391901016, + 0.3641766309738159 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 1, + "style_probability": 0.4028502106666565, + "style_target": 0, + "support_probability": 0.13748110830783844, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3923647100337377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5464891791343689, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22769282758235931, + 0.6688205599784851, + 0.1059112548828125, + 0.22409003973007202, + 0.032522838562726974, + 0.07913751155138016, + 0.07529944181442261, + 0.06789194047451019, + 0.13737823069095612 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.6634507775306702, + "style_target": 1, + "support_probability": 0.5129916667938232, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02982482367843422, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6028929352760315, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31946665048599243, + 0.6597720980644226, + 0.10957334190607071, + 0.3075518012046814, + 0.07763450592756271, + 0.07394286245107651, + 0.06559883058071136, + 0.14306384325027466, + 0.10650552064180374 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.22804851830005646, + "style_target": 0, + "support_probability": 0.4375672936439514, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8725623685747027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01580481231212616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30937063694000244, + 0.30937063694000244, + 0.3450579345226288, + 0.9981332421302795, + 0.02898526005446911, + 0.04010278359055519, + 0.0835675373673439, + 0.10560865700244904, + 0.18397818505764008, + 0.057395584881305695, + 0.05885864794254303 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 1, + "style_probability": 0.6251386404037476, + "style_target": 1, + "support_probability": 0.9141671657562256, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.847781008862587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.011884409934282303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3408450782299042, + 0.3408450782299042, + 0.32297274470329285, + 0.9997640252113342, + 0.07273587584495544, + 0.07046709209680557, + 0.16270938515663147, + 0.08538147062063217, + 0.04874236136674881, + 0.1800895631313324, + 0.10170801728963852 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 0, + "style_probability": 0.7396215796470642, + "style_target": 0, + "support_probability": 0.8807416558265686, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7658437802136003, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20384962856769562, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41998323798179626, + 0.9030459523200989, + 0.4403719902038574, + 0.8685951828956604, + 0.18855497241020203 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.9394772052764893, + "style_target": 1, + "support_probability": 0.7312524914741516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10714850587116212, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25053009390830994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5240262746810913, + 0.9400348663330078, + 0.49801212549209595, + 0.8728169798851013, + 0.15672826766967773 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.24464178085327148, + "style_target": 0, + "support_probability": 0.6421349048614502, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7022171114579521, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08140555769205093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9277248978614807, + 0.17549444735050201, + 0.3584462106227875, + 0.434829980134964, + 0.06816791743040085, + 0.16929271817207336 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 1, + "style_probability": 0.8104878664016724, + "style_target": 1, + "support_probability": 0.734016478061676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5449674503267076, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07259775698184967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9174941778182983, + 0.1755048781633377, + 0.3594875633716583, + 0.35137301683425903, + 0.05931820720434189, + 0.06339193135499954 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 1, + "style_probability": 0.3685910403728485, + "style_target": 0, + "support_probability": 0.810907781124115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04703355113792274, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8270978331565857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47061705589294434, + 0.2249438762664795, + 0.2503283619880676, + 0.7073115110397339, + 0.10803037881851196, + 0.13818271458148956, + 0.053989555686712265, + 0.08853419125080109, + 0.1172219067811966 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 1, + "style_probability": 0.8559176921844482, + "style_target": 1, + "support_probability": 0.018967045471072197, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019435611497566726, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.839966356754303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4765462875366211, + 0.19741226732730865, + 0.2662041485309601, + 0.6791792511940002, + 0.1234213262796402, + 0.09579793363809586, + 0.07490473985671997, + 0.09008722752332687, + 0.05421636626124382 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 1, + "style_probability": 0.42138081789016724, + "style_target": 0, + "support_probability": 0.016318069770932198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3002354920839604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44317373633384705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35635244846343994, + 0.27561643719673157, + 0.26767128705978394, + 0.9052921533584595, + 0.11377140879631042, + 0.09174474328756332, + 0.1430942714214325, + 0.05721757933497429, + 0.05691692605614662, + 0.08255370706319809, + 0.13911837339401245 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.7844502329826355, + "style_target": 1, + "support_probability": 0.25035810470581055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040224843571377115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4769136607646942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33078011870384216, + 0.26093751192092896, + 0.24438366293907166, + 0.9352334141731262, + 0.07954610139131546, + 0.07353214919567108, + 0.1950453370809555, + 0.10694196075201035, + 0.16460368037223816, + 0.09341651201248169 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.21699188649654388, + "style_target": 0, + "support_probability": 0.3720030188560486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22370066076440617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4857681691646576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7934274673461914, + 0.38774603605270386, + 0.44987794756889343, + 0.35408833622932434, + 0.18183432519435883 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 0, + "style_probability": 0.3140597343444824, + "style_target": 1, + "support_probability": 0.31815314292907715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3384047515471742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4594469368457794, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7936687469482422, + 0.44170862436294556, + 0.4188411831855774, + 0.30381450057029724, + 0.1519242525100708 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 0, + "style_probability": 0.5705634355545044, + "style_target": 0, + "support_probability": 0.3159275949001312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4060843516447611, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1027355045080185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2062496691942215, + 0.1713028997182846, + 0.6846456527709961, + 0.23388642072677612, + 0.07303471118211746, + 0.22922168672084808 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.28923889994621277, + "style_target": 1, + "support_probability": 0.6681567430496216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5857146478896426, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1331745684146881, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21263277530670166, + 0.17568165063858032, + 0.6847395300865173, + 0.2295229434967041, + 0.05020226538181305, + 0.06389956921339035 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.8200526237487793, + "style_target": 0, + "support_probability": 0.6453813314437866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31566350735825743, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3833908140659332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4323657155036926, + 0.8544955849647522, + 0.47273799777030945, + 0.4323657155036926, + 0.07923703640699387, + 0.07261088490486145, + 0.10177935659885406, + 0.06313910335302353, + 0.12288094311952591 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 0, + "style_probability": 0.4964025914669037, + "style_target": 1, + "support_probability": 0.29149189591407776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20304019620559025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22313697636127472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4168577194213867, + 0.8894387483596802, + 0.5714789032936096, + 0.4168577194213867, + 0.09535380452871323, + 0.16511140763759613, + 0.06723123043775558, + 0.0736301988363266, + 0.06746357679367065 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 1, + "style_probability": 0.34070974588394165, + "style_target": 0, + "support_probability": 0.5058192014694214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15564088927383476, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8382837176322937, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35876545310020447, + 0.4249788522720337, + 0.26865482330322266, + 0.6974731087684631, + 0.08319821953773499, + 0.09576212614774704, + 0.08178956061601639, + 0.09562788903713226, + 0.09831849485635757, + 0.1076580360531807, + 0.21542860567569733 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 0, + "style_probability": 0.3425162136554718, + "style_target": 1, + "support_probability": 0.2364213764667511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13142322320855654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8939070701599121, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33616387844085693, + 0.36814936995506287, + 0.2879451513290405, + 0.71856689453125, + 0.08074888586997986, + 0.069735586643219, + 0.07695448398590088, + 0.1446344405412674, + 0.09626918286085129, + 0.15563108026981354 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 0, + "style_probability": 0.5132901668548584, + "style_target": 0, + "support_probability": 0.129133403301239, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8414411192811247, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3720984160900116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2646274268627167, + 0.28454217314720154, + 0.1845519095659256, + 0.9315203428268433, + 0.11703596264123917 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.9434865713119507, + "style_target": 1, + "support_probability": 0.5783191919326782, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7238132144120324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2637603282928467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25393053889274597, + 0.28421327471733093, + 0.1964869499206543, + 0.9360267519950867, + 0.12235879898071289 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.49062395095825195, + "style_target": 0, + "support_probability": 0.724800169467926, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7799601585620377, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1834971308708191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2006518393754959, + 0.25868237018585205, + 0.9559416770935059, + 0.2178829312324524, + 0.09447523206472397, + 0.12500742077827454 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.8634227514266968, + "style_target": 1, + "support_probability": 0.6798532605171204, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12719191140724076, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24924394488334656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16192135214805603, + 0.24563200771808624, + 0.9840553998947144, + 0.2233634740114212, + 0.04850905388593674, + 0.1370757669210434 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.13551539182662964, + "style_target": 0, + "support_probability": 0.5969017148017883, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5341310808341548, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13605788350105286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6394376158714294, + 0.3861598074436188, + 0.7583905458450317, + 0.06170283630490303, + 0.043391600251197815, + 0.07923823595046997, + 0.046550553292036057, + 0.027255672961473465, + 0.21892091631889343 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 1, + "style_probability": 0.7218077182769775, + "style_target": 1, + "support_probability": 0.8072772026062012, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4460697110100808, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11587084829807281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7162204384803772, + 0.3681958019733429, + 0.7325683832168579, + 0.06838231533765793, + 0.07112245261669159, + 0.053095314651727676, + 0.10677272081375122, + 0.07554453611373901, + 0.0512666329741478 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 1, + "style_probability": 0.3551030457019806, + "style_target": 0, + "support_probability": 0.8512058258056641, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6019484687386014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22231055796146393, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27122724056243896, + 0.09297787398099899, + 0.5670692920684814, + 0.8422378301620483, + 0.10039785504341125, + 0.03319280594587326, + 0.11832612007856369, + 0.07772267609834671, + 0.04026927053928375, + 0.0718630999326706 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.7632476687431335, + "style_target": 1, + "support_probability": 0.6914879679679871, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5137935697501138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3238537907600403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2713620960712433, + 0.0973551943898201, + 0.5505183935165405, + 0.8116756677627563, + 0.15283909440040588, + 0.15412160754203796, + 0.05157334730029106, + 0.17983588576316833, + 0.13934412598609924, + 0.10070717334747314, + 0.05802082270383835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.4951455295085907, + "style_target": 0, + "support_probability": 0.6196693181991577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4008906034350538, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.594779908657074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1752099245786667, + 0.2875775694847107, + 0.5122275352478027, + 0.4224551022052765, + 0.028182683512568474 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.8789074420928955, + "style_target": 1, + "support_probability": 0.481320858001709, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03154831570147383, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7932754158973694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2872750163078308, + 0.3970402777194977, + 0.5379511117935181, + 0.41917967796325684, + 0.09796465188264847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.1771378368139267, + "style_target": 0, + "support_probability": 0.4107285439968109, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7274120615691915, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11256027966737747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5582027435302734, + 0.28402799367904663, + 0.8923202157020569, + 0.22640883922576904, + 0.09632498025894165, + 0.042907677590847015 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 1, + "style_probability": 0.7519777417182922, + "style_target": 1, + "support_probability": 0.7581579089164734, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4497126957166422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17312391102313995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5307140946388245, + 0.2840328812599182, + 0.8574140667915344, + 0.30250558257102966, + 0.09120863676071167, + 0.07623108476400375 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 0, + "style_probability": 0.5089079141616821, + "style_target": 0, + "support_probability": 0.6801238656044006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018173205017206852, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6382573246955872, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3727359175682068, + 0.2719390392303467, + 0.42970526218414307, + 0.2096126228570938, + 0.14845873415470123, + 0.06715209037065506, + 0.05885070189833641, + 0.061073802411556244, + 0.03592391312122345 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.5423579216003418, + "style_target": 1, + "support_probability": 0.058394063264131546, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008850972101486693, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.757559061050415, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3669615685939789, + 0.28305816650390625, + 0.3897925317287445, + 0.20338043570518494, + 0.0515914261341095, + 0.12429969012737274, + 0.05231838300824165, + 0.10204305499792099, + 0.13576945662498474 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.3053601384162903, + "style_target": 0, + "support_probability": 0.024592170491814613, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.01714877142526065, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8983049392700195, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22375129163265228, + 0.6364864706993103, + 0.41008296608924866, + 0.47003644704818726, + 0.08296439051628113, + 0.06037392467260361, + 0.09645707905292511, + 0.06983812153339386, + 0.16945894062519073, + 0.07915827631950378, + 0.032577842473983765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.5871022343635559, + "style_target": 1, + "support_probability": 0.016767824068665504, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004787514220557723, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9211509823799133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2505892515182495, + 0.6125337481498718, + 0.40398114919662476, + 0.5254626870155334, + 0.053811948746442795, + 0.06040245667099953, + 0.16488155722618103, + 0.05830695107579231, + 0.08794903010129929, + 0.0646795704960823 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.1974417269229889, + "style_target": 0, + "support_probability": 0.017924318090081215, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.19242915218244927, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4728340804576874, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2084479182958603, + 0.7833208441734314, + 0.16231881082057953, + 0.20985843241214752, + 0.12703853845596313 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 0, + "style_probability": 0.34015101194381714, + "style_target": 1, + "support_probability": 0.3074939548969269, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07157070064653309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.530194103717804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24421681463718414, + 0.9206096529960632, + 0.27637743949890137, + 0.27117589116096497, + 0.2789272665977478 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 1, + "style_probability": 0.316806823015213, + "style_target": 0, + "support_probability": 0.21472565829753876, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029440963301765346, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.74611896276474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41100355982780457, + 0.28441333770751953, + 0.36540037393569946, + 0.41100355982780457, + 0.064867302775383, + 0.06604201346635818 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.40836358070373535, + "style_target": 1, + "support_probability": 0.0981375053524971, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11108910281579182, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.736073911190033, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36479800939559937, + 0.3018595278263092, + 0.3987727761268616, + 0.36479800939559937, + 0.06208204850554466, + 0.048159193247556686 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.8454950451850891, + "style_target": 0, + "support_probability": 0.12418621778488159, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5813139971320689, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36430954933166504, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8979508876800537, + 0.10972528904676437, + 0.15532810986042023, + 0.6171109080314636, + 0.03891308978199959, + 0.03260546550154686, + 0.03703121468424797, + 0.0486738346517086, + 0.10748691111803055 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.5136920213699341, + "style_target": 1, + "support_probability": 0.613364577293396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5468729165553577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40698739886283875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9052232503890991, + 0.1048671305179596, + 0.19572052359580994, + 0.568378746509552, + 0.07459327578544617, + 0.06928572058677673, + 0.11136553436517715, + 0.08741965144872665, + 0.05987995117902756 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.4804098904132843, + "style_target": 0, + "support_probability": 0.5725451707839966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047183098026814564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9277212619781494, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17643648386001587, + 0.7530214786529541, + 0.2827392816543579, + 0.17643648386001587, + 0.10576270520687103, + 0.32105258107185364, + 0.07871649414300919, + 0.0872063860297203, + 0.09698706865310669, + 0.10919182747602463, + 0.12242420017719269 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 0, + "style_probability": 0.4716499149799347, + "style_target": 1, + "support_probability": 0.04015523940324783, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025003073912069373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9893787503242493, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15752743184566498, + 0.6858938932418823, + 0.45032086968421936, + 0.15752743184566498, + 0.13932809233665466, + 0.08025158941745758, + 0.22394803166389465, + 0.16142399609088898, + 0.09854020923376083, + 0.09182364493608475 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 1, + "style_probability": 0.19236093759536743, + "style_target": 0, + "support_probability": 0.009928935207426548, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2524619569038181, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5238946080207825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20127417147159576, + 0.4373572766780853, + 0.48242709040641785, + 0.38998982310295105, + 0.06884776055812836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.8308610320091248, + "style_target": 1, + "support_probability": 0.3147214949131012, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02002070102989747, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8240275382995605, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16544871032238007, + 0.43863070011138916, + 0.40952634811401367, + 0.3550799489021301, + 0.0512021966278553 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.36253055930137634, + "style_target": 0, + "support_probability": 0.029910746961832047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4364885098831839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4482898712158203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5779473185539246, + 0.542081356048584, + 0.24633799493312836, + 0.7249758243560791, + 0.12109175324440002, + 0.09523401409387589 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.7644208669662476, + "style_target": 1, + "support_probability": 0.47269877791404724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03801984553336923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6407387256622314, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6335932016372681, + 0.5631015300750732, + 0.285660982131958, + 0.7919671535491943, + 0.18854744732379913, + 0.12475039809942245 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.23174722492694855, + "style_target": 0, + "support_probability": 0.24324218928813934, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4902237733626157, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2497110664844513, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34270623326301575, + 0.960820734500885, + 0.4087408483028412, + 0.22866486012935638, + 0.08005646616220474, + 0.2054414004087448, + 0.10888687521219254, + 0.06670254468917847, + 0.12147045880556107 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.7111204266548157, + "style_target": 1, + "support_probability": 0.33792799711227417, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2913781573706752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2563529312610626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3339117467403412, + 0.9386194348335266, + 0.4128248989582062, + 0.23547059297561646, + 0.06966336071491241, + 0.1247859075665474, + 0.08316569030284882, + 0.28537923097610474, + 0.09642044454813004 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.44842100143432617, + "style_target": 0, + "support_probability": 0.36020681262016296, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7297349127817787, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06830847263336182, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.414117693901062, + 0.3392053544521332, + 0.5476632118225098, + 0.96200031042099, + 0.06810732930898666, + 0.08364179730415344, + 0.11442840099334717, + 0.12789106369018555, + 0.06325791031122208, + 0.08906275033950806, + 0.055099066346883774 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.5655204653739929, + "style_target": 1, + "support_probability": 0.8174484968185425, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5230749523014344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10922355949878693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.4240818917751312, + 0.36544719338417053, + 0.564172089099884, + 0.9431423544883728, + 0.053241975605487823, + 0.10956685245037079, + 0.07389674335718155, + 0.12103602290153503, + 0.10720483958721161, + 0.16587118804454803, + 0.08290573954582214, + 0.050315238535404205 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.4089873135089874, + "style_target": 0, + "support_probability": 0.6473733186721802, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2271291804925865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.41692841053009033, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1667288839817047, + 0.34608522057533264, + 0.33890318870544434, + 0.7812595367431641, + 0.14989472925662994 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 1, + "style_probability": 0.782230019569397, + "style_target": 1, + "support_probability": 0.15409712493419647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14630106536720913, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.29940688610076904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17020879685878754, + 0.4203287363052368, + 0.4549115300178528, + 0.908958375453949, + 0.1405780017375946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 0, + "style_probability": 0.5042128562927246, + "style_target": 0, + "support_probability": 0.35075831413269043, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3647403283070498, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8745532631874084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3046603500843048, + 0.334953635931015, + 0.62044757604599, + 0.5722678899765015, + 0.054924122989177704, + 0.05887492001056671 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.8517905473709106, + "style_target": 1, + "support_probability": 0.3677825927734375, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10251019957813937, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9240004420280457, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23036229610443115, + 0.30411359667778015, + 0.5546743273735046, + 0.4792526364326477, + 0.08208278566598892, + 0.12378241121768951 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.3749699890613556, + "style_target": 0, + "support_probability": 0.3112283945083618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.760150295678613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12874656915664673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9476174116134644, + 0.5128792524337769, + 0.2308279275894165, + 0.17564308643341064, + 0.18217311799526215, + 0.032973866909742355, + 0.060337938368320465, + 0.06065420061349869, + 0.1492030769586563 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.863121747970581, + "style_target": 1, + "support_probability": 0.6799211502075195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4135225152710254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09799187630414963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9489777088165283, + 0.5265029072761536, + 0.22796161472797394, + 0.1689278781414032, + 0.0816279798746109, + 0.10275902599096298, + 0.15679582953453064, + 0.07344478368759155, + 0.14271856844425201 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.29535406827926636, + "style_target": 0, + "support_probability": 0.7049838304519653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33800480222437457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5799902081489563, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.217474564909935, + 0.2636173367500305, + 0.6324917674064636, + 0.8111290335655212, + 0.09304802864789963, + 0.2865576148033142, + 0.17318549752235413, + 0.1428668349981308, + 0.10177429765462875, + 0.07723934948444366, + 0.1964224874973297 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.7497758865356445, + "style_target": 1, + "support_probability": 0.27589917182922363, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01700873382301706, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.712949812412262, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22857531905174255, + 0.37455227971076965, + 0.7770839929580688, + 0.7746323347091675, + 0.08533253520727158, + 0.08973383903503418, + 0.16608664393424988, + 0.16214020550251007, + 0.08705088496208191, + 0.11574117094278336 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.10200139135122299, + "style_target": 0, + "support_probability": 0.16074955463409424, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8378385801934775, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06916215270757675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20327228307724, + 0.9467269778251648, + 0.3641660213470459, + 0.2299838662147522, + 0.0434943325817585 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 1, + "style_probability": 0.7569003701210022, + "style_target": 1, + "support_probability": 0.8963721990585327, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6680083599971584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17662231624126434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16754937171936035, + 0.8998810052871704, + 0.3296145498752594, + 0.19607087969779968, + 0.1399536281824112 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 0, + "style_probability": 0.5191127061843872, + "style_target": 0, + "support_probability": 0.7816401124000549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34940559723276227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19990701973438263, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22556473314762115, + 0.2574649751186371, + 0.9362791180610657, + 0.47344914078712463, + 0.08997948467731476, + 0.08062190562486649 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.7503141164779663, + "style_target": 1, + "support_probability": 0.14120951294898987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16396310458384314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1669263392686844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22017019987106323, + 0.24093618988990784, + 0.9340874552726746, + 0.4236758351325989, + 0.10174675285816193, + 0.08813455700874329 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.2190038561820984, + "style_target": 0, + "support_probability": 0.1851472705602646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5685501725826767, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17765775322914124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3619774281978607, + 0.2074487954378128, + 0.9057765603065491, + 0.2645622193813324, + 0.10664381831884384, + 0.09798643738031387, + 0.13069744408130646, + 0.04601434990763664, + 0.053272951394319534 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.7844377756118774, + "style_target": 1, + "support_probability": 0.6351003646850586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1914361305947137, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14430978894233704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35556942224502563, + 0.21563822031021118, + 0.941872239112854, + 0.2815532684326172, + 0.08090878278017044, + 0.14127404987812042, + 0.07538112252950668, + 0.166624054312706, + 0.19534030556678772 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.2894163429737091, + "style_target": 0, + "support_probability": 0.7800617814064026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2497533253975705, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2930183708667755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.237031489610672, + 0.823050320148468, + 0.2654684782028198, + 0.44437041878700256, + 0.04489404708147049, + 0.10045118629932404, + 0.052819687873125076, + 0.08736705780029297, + 0.05231350287795067, + 0.04884286969900131, + 0.032558735460042953 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 1, + "style_probability": 0.7774287462234497, + "style_target": 1, + "support_probability": 0.133946493268013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.103698452931936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38661861419677734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2249649465084076, + 0.7557570934295654, + 0.20986178517341614, + 0.4063088297843933, + 0.04763728007674217, + 0.05582534149289131, + 0.07180973142385483, + 0.17354443669319153, + 0.056464266031980515, + 0.13550812005996704, + 0.11894240230321884 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 0, + "style_probability": 0.5382852554321289, + "style_target": 0, + "support_probability": 0.06316591799259186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5392015371891108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6915072202682495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18155300617218018, + 0.2820703387260437, + 0.8328769207000732, + 0.6737024784088135, + 0.0695020779967308 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.762336015701294, + "style_target": 1, + "support_probability": 0.46829354763031006, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34437544496792255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6551356315612793, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18208077549934387, + 0.28326043486595154, + 0.8301427364349365, + 0.6344096064567566, + 0.06395133584737778 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.41521838307380676, + "style_target": 0, + "support_probability": 0.4550037682056427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5336812317457474, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07232259213924408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3282180726528168, + 0.2806164026260376, + 0.9481353163719177, + 0.3282180726528168, + 0.07416386157274246, + 0.08759567141532898 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 1, + "style_probability": 0.5909696817398071, + "style_target": 1, + "support_probability": 0.5968784689903259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2146240669892296, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09558217227458954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3457021415233612, + 0.2909358739852905, + 0.9604917168617249, + 0.3457021415233612, + 0.062354590743780136, + 0.11037220060825348 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 1, + "style_probability": 0.27572011947631836, + "style_target": 0, + "support_probability": 0.6207324266433716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3314633335671277, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.827512800693512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8630814552307129, + 0.08646173030138016, + 0.08646173030138016, + 0.6870257258415222, + 0.1285572648048401, + 0.11768309026956558, + 0.11182210594415665, + 0.028422381728887558, + 0.07092424482107162 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.83600252866745, + "style_target": 1, + "support_probability": 0.1921553760766983, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1959833189837439, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.660703718662262, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8630186915397644, + 0.07598292827606201, + 0.07598292827606201, + 0.7005642652511597, + 0.06952998787164688, + 0.08503953367471695, + 0.1130199208855629, + 0.04312247410416603, + 0.05937890335917473 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.394588440656662, + "style_target": 0, + "support_probability": 0.3275623023509979, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8177031450732842, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08010188490152359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5928048491477966, + 0.9318903088569641, + 0.04735846072435379, + 0.5928048491477966, + 0.03718939796090126, + 0.05988505855202675, + 0.12982797622680664, + 0.08803807199001312, + 0.04284569248557091, + 0.04959830269217491, + 0.10973420739173889 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.7909775376319885, + "style_target": 1, + "support_probability": 0.7649961113929749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5835719411249631, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06567224860191345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5485212802886963, + 0.9234501123428345, + 0.047123685479164124, + 0.5485212802886963, + 0.035819027572870255, + 0.07309448719024658, + 0.057057566940784454, + 0.0768556073307991, + 0.09366363286972046, + 0.046070657670497894, + 0.03467899188399315 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.3425975441932678, + "style_target": 0, + "support_probability": 0.6786144375801086, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005438437109403812, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9903944134712219, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2829136252403259, + 0.618747889995575, + 0.31853675842285156, + 0.3314807116985321, + 0.17673359811306 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.7257193922996521, + "style_target": 1, + "support_probability": 0.004148043692111969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015903695774096601, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9891617298126221, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3470055162906647, + 0.6368680596351624, + 0.3014813959598541, + 0.3586651384830475, + 0.17041084170341492 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.29639437794685364, + "style_target": 0, + "support_probability": 0.003532343078404665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24740967039514258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1413893848657608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3048495650291443, + 0.12857423722743988, + 0.788589358329773, + 0.3048495650291443, + 0.35090526938438416, + 0.14945797622203827 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 1, + "style_probability": 0.6111596822738647, + "style_target": 1, + "support_probability": 0.37096184492111206, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19039542746359403, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2010817974805832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29073911905288696, + 0.1429051160812378, + 0.7572007179260254, + 0.29073911905288696, + 0.11309193819761276, + 0.17085681855678558 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 1, + "style_probability": 0.4736483693122864, + "style_target": 0, + "support_probability": 0.2736521065235138, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.48547594851995607, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3781041204929352, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24174384772777557, + 0.24174384772777557, + 0.2071208506822586, + 0.86916184425354, + 0.09273254126310349, + 0.1936485320329666, + 0.1946285516023636, + 0.057445213198661804, + 0.11376674473285675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.7823900580406189, + "style_target": 1, + "support_probability": 0.40560317039489746, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28002080406990976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31166866421699524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22239147126674652, + 0.22239147126674652, + 0.21577613055706024, + 0.8676316142082214, + 0.06914285570383072, + 0.10513857007026672, + 0.09068245440721512, + 0.14191432297229767, + 0.06490228325128555 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.40101292729377747, + "style_target": 0, + "support_probability": 0.3797813653945923, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.609876103231471, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.396809846162796, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9471747875213623, + 0.3473897874355316, + 0.3976231813430786, + 0.2707175016403198, + 0.09104417264461517, + 0.1047748476266861, + 0.06080373004078865, + 0.10154704749584198, + 0.09768286347389221, + 0.030258195474743843, + 0.05323795974254608 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.8080551028251648, + "style_target": 1, + "support_probability": 0.3900511562824249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14474041011103983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45695072412490845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9572099447250366, + 0.47078049182891846, + 0.4968390166759491, + 0.24109824001789093, + 0.05984412133693695, + 0.04862603172659874, + 0.36744028329849243, + 0.05472024157643318, + 0.0555780753493309, + 0.04976053908467293 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.31022191047668457, + "style_target": 0, + "support_probability": 0.401052862405777, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9032039231521773, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1235959604382515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1626:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12576943635940552, + 0.21261464059352875, + 0.969975471496582, + 0.17181824147701263, + 0.20367152988910675 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 1, + "style_probability": 0.8585793375968933, + "style_target": 1, + "support_probability": 0.9115179777145386, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6529793042631865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22991250455379486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1626:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18498416244983673, + 0.25472158193588257, + 0.9043260216712952, + 0.15323910117149353, + 0.09842915087938309 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 0, + "style_probability": 0.5429961681365967, + "style_target": 0, + "support_probability": 0.7610682845115662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7739379753304916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05505923181772232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39857324957847595, + 0.886792004108429, + 0.12419480830430984, + 0.2605448365211487, + 0.0601067952811718, + 0.22322514653205872 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.6539088487625122, + "style_target": 1, + "support_probability": 0.9078090786933899, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7027746783171831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05326475203037262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.375804603099823, + 0.8536401987075806, + 0.12563028931617737, + 0.2603417932987213, + 0.061074405908584595, + 0.06601691246032715 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.4466072916984558, + "style_target": 0, + "support_probability": 0.8826019167900085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1915495934182556, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7626767158508301, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30369865894317627, + 0.26897189021110535, + 0.6624951362609863, + 0.43846046924591064, + 0.1656150221824646, + 0.12946172058582306, + 0.22773705422878265, + 0.10244038701057434, + 0.1979168951511383 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.7514528632164001, + "style_target": 1, + "support_probability": 0.16180585324764252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04566224782291879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6520543694496155, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2840263545513153, + 0.27150824666023254, + 0.7088822722434998, + 0.44769614934921265, + 0.06380371749401093, + 0.14999446272850037, + 0.07133658230304718, + 0.05213344469666481, + 0.13562875986099243 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.31360864639282227, + "style_target": 0, + "support_probability": 0.23345573246479034, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8606202288790727, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2947189211845398, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1636:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9918276071548462, + 0.3809601366519928, + 0.1601562649011612, + 0.4799615144729614, + 0.07118897885084152, + 0.06959594786167145, + 0.10997388511896133, + 0.07579472661018372, + 0.07891537994146347, + 0.052259959280490875, + 0.19122326374053955 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 1, + "style_probability": 0.5877595543861389, + "style_target": 1, + "support_probability": 0.8551802635192871, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8189705930617454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2513161599636078, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1636:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.97974693775177, + 0.42925766110420227, + 0.14296744763851166, + 0.5015131235122681, + 0.06271811574697495, + 0.04250679910182953, + 0.031686391681432724, + 0.04192038252949715, + 0.067540742456913, + 0.12716162204742432, + 0.10409192740917206 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 1, + "style_probability": 0.4035647511482239, + "style_target": 0, + "support_probability": 0.8725079298019409, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3214462270536897, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33609169721603394, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41828522086143494, + 0.28865230083465576, + 0.9523292183876038, + 0.1335529237985611, + 0.09112747013568878 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 0, + "style_probability": 0.421195387840271, + "style_target": 1, + "support_probability": 0.4957609176635742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5526672430009125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4147714376449585, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44354739785194397, + 0.34285131096839905, + 0.9641655087471008, + 0.15416112542152405, + 0.11957047134637833 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 0, + "style_probability": 0.787415623664856, + "style_target": 0, + "support_probability": 0.432415634393692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2005770966202808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6563178896903992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3638460636138916, + 0.23347434401512146, + 0.3192691504955292, + 0.6926323771476746, + 0.0707482248544693, + 0.10384306311607361 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.5063726902008057, + "style_target": 1, + "support_probability": 0.2306760847568512, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032873832416009344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9011270403862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35670435428619385, + 0.2676331400871277, + 0.3331346809864044, + 0.7158860564231873, + 0.1702767163515091, + 0.15079103410243988 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.32717934250831604, + "style_target": 0, + "support_probability": 0.046463288366794586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6149591092524371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.125743567943573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8900454640388489, + 0.2549588680267334, + 0.2692112326622009, + 0.297336220741272, + 0.08185748010873795, + 0.2631267309188843, + 0.09274335205554962, + 0.042629655450582504, + 0.06575225293636322 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.5428422093391418, + "style_target": 1, + "support_probability": 0.6715037822723389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15848525202615696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19219617545604706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8872031569480896, + 0.26966843008995056, + 0.3259507715702057, + 0.3069620430469513, + 0.09870140254497528, + 0.08272112905979156, + 0.11260246485471725, + 0.08335254341363907, + 0.14182792603969574 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.14887496829032898, + "style_target": 0, + "support_probability": 0.6603941917419434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008508408752252068, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9507242441177368, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4958486557006836, + 0.28019246459007263, + 0.3926631808280945, + 0.35452765226364136, + 0.06221672147512436, + 0.0818256288766861, + 0.13360737264156342, + 0.24428129196166992, + 0.07379011809825897, + 0.07705310732126236, + 0.05655766278505325 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 0, + "style_probability": 0.47666311264038086, + "style_target": 1, + "support_probability": 0.015094751492142677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026093821043107375, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9585370421409607, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4450629949569702, + 0.24992144107818604, + 0.3896617889404297, + 0.45255985856056213, + 0.10840953141450882, + 0.1149468794465065, + 0.15084044635295868, + 0.0636816918849945, + 0.04684973508119583, + 0.07100159674882889 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 0, + "style_probability": 0.8251931071281433, + "style_target": 0, + "support_probability": 0.02001021057367325, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7879498441421231, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2654545307159424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5753703713417053, + 0.18469692766666412, + 0.27375519275665283, + 0.9545062780380249, + 0.08315331488847733 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.6917774081230164, + "style_target": 1, + "support_probability": 0.6401833295822144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14538408189391655, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3363344669342041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6954084634780884, + 0.252001017332077, + 0.3144175410270691, + 0.9860588908195496, + 0.15427935123443604 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.061020199209451675, + "style_target": 0, + "support_probability": 0.7250040173530579, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4570331452762394, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5311558246612549, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7022536396980286, + 0.613609790802002, + 0.145279198884964, + 0.8944408297538757, + 0.11725077033042908, + 0.09488420188426971 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 1, + "style_probability": 0.7404769659042358, + "style_target": 1, + "support_probability": 0.31750816106796265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2489981411783413, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6577205061912537, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5705673098564148, + 0.5629178285598755, + 0.20847506821155548, + 0.8788187503814697, + 0.07877718657255173, + 0.08576378971338272 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 0, + "style_probability": 0.6063756942749023, + "style_target": 0, + "support_probability": 0.14841200411319733, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6712108085429223, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3983103930950165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5187401175498962, + 0.38484853506088257, + 0.5187401175498962, + 0.7732612490653992, + 0.08061844110488892, + 0.039449095726013184, + 0.08570581674575806, + 0.07478495687246323, + 0.05656042695045471 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.9231775403022766, + "style_target": 1, + "support_probability": 0.5557775497436523, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1441548720145627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6545537710189819, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44386640191078186, + 0.3519149422645569, + 0.44386640191078186, + 0.740307092666626, + 0.08247210830450058, + 0.13086198270320892, + 0.06230980157852173, + 0.1037917509675026, + 0.1210993230342865 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.3804837763309479, + "style_target": 0, + "support_probability": 0.19251509010791779, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6518170709619081, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3499417304992676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25016549229621887, + 0.37882524728775024, + 0.6317470669746399, + 0.908481776714325, + 0.05412144213914871, + 0.10198457539081573, + 0.05284363403916359, + 0.047906357795000076, + 0.06003371626138687, + 0.07436704635620117, + 0.04922066256403923 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.8638189435005188, + "style_target": 1, + "support_probability": 0.5315502285957336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06488377834451453, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4396822154521942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35422030091285706, + 0.3414131700992584, + 0.5973939895629883, + 0.9288910627365112, + 0.10385311394929886, + 0.10449452698230743, + 0.24496686458587646, + 0.1543358862400055, + 0.09917645156383514, + 0.08210345357656479 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.13977354764938354, + "style_target": 0, + "support_probability": 0.564952552318573, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.49997884257470415, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43076759576797485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21103046834468842, + 0.5442002415657043, + 0.2961634695529938, + 0.5168745517730713, + 0.11608052998781204 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 1, + "style_probability": 0.8631541728973389, + "style_target": 1, + "support_probability": 0.7958154678344727, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3253351518276387, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5580841302871704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2056896686553955, + 0.519990861415863, + 0.3115185499191284, + 0.5065522789955139, + 0.0772067978978157 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 0, + "style_probability": 0.5006175637245178, + "style_target": 0, + "support_probability": 0.7663425207138062, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6558707754324823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09053120017051697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8993998765945435, + 0.3814980387687683, + 0.44012555480003357, + 0.46444034576416016, + 0.059147052466869354, + 0.2225174754858017 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 1, + "style_probability": 0.7128183245658875, + "style_target": 1, + "support_probability": 0.6706728339195251, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5071658078911635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09202294051647186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8745835423469543, + 0.3870723843574524, + 0.41217488050460815, + 0.46513813734054565, + 0.11745794862508774, + 0.05798155441880226 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 1, + "style_probability": 0.4589468240737915, + "style_target": 0, + "support_probability": 0.600553035736084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5073221505964045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18665668368339539, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3651406764984131, + 0.3613433837890625, + 0.9388015866279602, + 0.42786747217178345, + 0.04986013099551201, + 0.12152105569839478, + 0.10711494088172913, + 0.058190666139125824, + 0.17891044914722443 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 1, + "style_probability": 0.5738065242767334, + "style_target": 1, + "support_probability": 0.614798367023468, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1932243844480129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22960957884788513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3457013964653015, + 0.3588491380214691, + 0.9471703171730042, + 0.43964478373527527, + 0.06041281297802925, + 0.16992120444774628, + 0.08632367849349976, + 0.13804256916046143, + 0.10689999163150787 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 1, + "style_probability": 0.19793401658535004, + "style_target": 0, + "support_probability": 0.6624019742012024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16714753757407888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1286603808403015, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3780549466609955, + 0.8139538168907166, + 0.3444504737854004, + 0.18914860486984253, + 0.1379317045211792, + 0.10432586818933487, + 0.09372986108064651, + 0.1083931252360344, + 0.12324587255716324, + 0.09214314073324203, + 0.3375268280506134 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 1, + "style_probability": 0.6922516822814941, + "style_target": 1, + "support_probability": 0.15517130494117737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049847833069042176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2588133215904236, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42369258403778076, + 0.7695755958557129, + 0.4359152317047119, + 0.18511910736560822, + 0.11902865022420883, + 0.06493125855922699, + 0.08601934462785721, + 0.08470196276903152, + 0.06389684975147247, + 0.14073312282562256, + 0.07213082909584045 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 0, + "style_probability": 0.5163500308990479, + "style_target": 0, + "support_probability": 0.04648668318986893, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.34684292004433986, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17153827846050262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9095627665519714, + 0.2507096529006958, + 0.310354620218277, + 0.2836555242538452, + 0.0725000724196434 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 0, + "style_probability": 0.21558476984500885, + "style_target": 1, + "support_probability": 0.6940699815750122, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6933591022131795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09361215680837631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9582599997520447, + 0.3059302270412445, + 0.44247549772262573, + 0.29904136061668396, + 0.17597085237503052 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 1, + "style_probability": 0.48072752356529236, + "style_target": 0, + "support_probability": 0.8240537643432617, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4238730192992501, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21558275818824768, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10509797930717468, + 0.913251519203186, + 0.12704749405384064, + 0.2572891116142273, + 0.09593988209962845, + 0.15419286489486694 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 1, + "style_probability": 0.5934496521949768, + "style_target": 1, + "support_probability": 0.45022594928741455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12021548424383101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5692203044891357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11002861708402634, + 0.9079822897911072, + 0.1735163778066635, + 0.217486172914505, + 0.19068536162376404, + 0.14985373616218567 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 1, + "style_probability": 0.461410254240036, + "style_target": 0, + "support_probability": 0.10482732951641083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.451698529829411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14583918452262878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14854839444160461, + 0.24476061761379242, + 0.8751839995384216, + 0.3120165467262268, + 0.063471220433712, + 0.1772010773420334, + 0.07246368378400803, + 0.10632964968681335, + 0.0683864876627922 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 1, + "style_probability": 0.5129209756851196, + "style_target": 1, + "support_probability": 0.6251927614212036, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3014769694384496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15616662800312042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1253604143857956, + 0.3115150034427643, + 0.9701361060142517, + 0.40085428953170776, + 0.19133363664150238, + 0.17858488857746124, + 0.15008725225925446, + 0.055359404534101486, + 0.10851024836301804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 1, + "style_probability": 0.2837386131286621, + "style_target": 0, + "support_probability": 0.7996008992195129, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1294128369641617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6541991233825684, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30013981461524963, + 0.24226869642734528, + 0.560950756072998, + 0.26118311285972595, + 0.0649184063076973, + 0.08181848376989365, + 0.09151867032051086, + 0.12035999447107315, + 0.10876088589429855, + 0.13266783952713013, + 0.07920943200588226 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 1, + "style_probability": 0.5489010810852051, + "style_target": 1, + "support_probability": 0.09753663837909698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10436268411792785, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7894424796104431, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25768300890922546, + 0.2620146870613098, + 0.5453319549560547, + 0.2401563972234726, + 0.0989430770277977, + 0.07920773327350616, + 0.06423976272344589, + 0.08185156434774399, + 0.1205635741353035, + 0.08665251731872559, + 0.05335761234164238 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 0, + "style_probability": 0.7959595918655396, + "style_target": 0, + "support_probability": 0.034992970526218414, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8822962781425082, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05794548988342285, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9755684733390808, + 0.2169269621372223, + 0.4917042851448059, + 0.6284074783325195, + 0.21164220571517944 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 1, + "style_probability": 0.5416096448898315, + "style_target": 1, + "support_probability": 0.9707335829734802, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8433214619240224, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06959013640880585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9837107062339783, + 0.15444952249526978, + 0.5422988533973694, + 0.6365953683853149, + 0.2745351493358612 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 1, + "style_probability": 0.4054962992668152, + "style_target": 0, + "support_probability": 0.9592731595039368, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03589937503160153, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29654476046562195, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5513862371444702, + 0.4564273953437805, + 0.29588666558265686, + 0.5513862371444702, + 0.071811243891716, + 0.17448802292346954 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 1, + "style_probability": 0.7140211462974548, + "style_target": 1, + "support_probability": 0.07351133227348328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01589643295937549, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35372841358184814, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6024051308631897, + 0.57797771692276, + 0.3186849057674408, + 0.6024051308631897, + 0.11170937120914459, + 0.06644406914710999 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 0, + "style_probability": 0.7160052061080933, + "style_target": 0, + "support_probability": 0.049666061997413635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01804593837212949, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9578793048858643, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6509172916412354, + 0.5059983134269714, + 0.1948470026254654, + 0.32528093457221985, + 0.12374064326286316, + 0.22187440097332, + 0.12972308695316315, + 0.10188563913106918, + 0.23610493540763855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 0, + "style_probability": 0.42437994480133057, + "style_target": 1, + "support_probability": 0.031347256153821945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011328901473951951, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9889494180679321, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6008244156837463, + 0.43206074833869934, + 0.22954536974430084, + 0.30657052993774414, + 0.08199168741703033, + 0.28048253059387207, + 0.13402846455574036, + 0.10698877274990082, + 0.0944630354642868 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 0, + "style_probability": 0.8500251173973083, + "style_target": 0, + "support_probability": 0.00816353503614664, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005716464553177768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9933248162269592, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7408815622329712, + 0.34793931245803833, + 0.20883949100971222, + 0.34793931245803833, + 0.1282954216003418, + 0.07597171515226364, + 0.07745818048715591, + 0.06166651099920273, + 0.03336401656270027, + 0.12335477024316788 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 0, + "style_probability": 0.4472655653953552, + "style_target": 1, + "support_probability": 0.007959268055856228, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02239910106912936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9755565524101257, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7986181378364563, + 0.3293462097644806, + 0.17919033765792847, + 0.3293462097644806, + 0.09455254673957825, + 0.0753859281539917, + 0.15110336244106293, + 0.07447168976068497, + 0.11262714117765427, + 0.1518305540084839 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 1, + "style_probability": 0.44740214943885803, + "style_target": 0, + "support_probability": 0.03319172561168671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38297665883431437, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.717524528503418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6581090688705444, + 0.2976630628108978, + 0.3556009829044342, + 0.20387090742588043, + 0.11234207451343536 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 1, + "style_probability": 0.7569695115089417, + "style_target": 1, + "support_probability": 0.18587233126163483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24104578098637913, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6709277033805847, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.608527421951294, + 0.29487353563308716, + 0.3437516689300537, + 0.23128466308116913, + 0.12643681466579437 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 0, + "style_probability": 0.5067546367645264, + "style_target": 0, + "support_probability": 0.2345375269651413, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.13617057016717848, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7449770569801331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11505216360092163, + 0.27177461981773376, + 0.4132726490497589, + 0.600330114364624, + 0.0242465790361166, + 0.04425978660583496 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.6984478831291199, + "style_target": 1, + "support_probability": 0.20676782727241516, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05740471133796807, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.887386679649353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1384708732366562, + 0.27688732743263245, + 0.37073439359664917, + 0.517066478729248, + 0.09147881716489792, + 0.050179965794086456 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.34926947951316833, + "style_target": 0, + "support_probability": 0.09422178566455841, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7522601114856126, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06494464725255966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9559893608093262, + 0.20330126583576202, + 0.26180413365364075, + 0.3992231488227844, + 0.08823078870773315, + 0.13368989527225494, + 0.09058840572834015, + 0.05437599495053291, + 0.04353976622223854 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.7868643403053284, + "style_target": 1, + "support_probability": 0.6592346429824829, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14231081642103915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04304579272866249, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9903663396835327, + 0.1949770599603653, + 0.25083866715431213, + 0.5091433525085449, + 0.16841843724250793, + 0.12072890996932983, + 0.13934847712516785, + 0.1249341145157814, + 0.15578381717205048 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.027207888662815094, + "style_target": 0, + "support_probability": 0.8733299970626831, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020262255094319784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9383300542831421, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5467658042907715, + 0.5467658042907715, + 0.8216167688369751, + 0.2912837564945221, + 0.059735238552093506, + 0.23949222266674042, + 0.06672006100416183, + 0.09213466197252274, + 0.12581545114517212, + 0.061070289462804794, + 0.12524500489234924 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 1, + "style_probability": 0.8323063254356384, + "style_target": 1, + "support_probability": 0.00763155659660697, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020627555373780755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8622626066207886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.53907710313797, + 0.53907710313797, + 0.7981185913085938, + 0.2876492142677307, + 0.07150599360466003, + 0.0688929557800293, + 0.09269900619983673, + 0.09677152335643768, + 0.062289901077747345, + 0.0735466256737709, + 0.08444944769144058 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 0, + "style_probability": 0.552890419960022, + "style_target": 0, + "support_probability": 0.01703062281012535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014181892918674293, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.770585298538208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7777743935585022, + 0.28726592659950256, + 0.5281088948249817, + 0.21944637596607208, + 0.07005397975444794 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 1, + "style_probability": 0.5038944482803345, + "style_target": 1, + "support_probability": 0.019434163346886635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03250845631287455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7782998085021973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8074043393135071, + 0.292041540145874, + 0.5586528182029724, + 0.1999724954366684, + 0.05435042828321457 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 0, + "style_probability": 0.849046528339386, + "style_target": 0, + "support_probability": 0.032486189156770706, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6001472108106896, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22439220547676086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7923858761787415, + 0.3358250558376312, + 0.18188761174678802, + 0.2126850038766861, + 0.0373251847922802, + 0.033320970833301544 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 1, + "style_probability": 0.6440178155899048, + "style_target": 1, + "support_probability": 0.591829776763916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.605744892162778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1563083827495575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7974219918251038, + 0.37630659341812134, + 0.1895911544561386, + 0.23485812544822693, + 0.0716594010591507, + 0.04593200981616974 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 0, + "style_probability": 0.545050323009491, + "style_target": 0, + "support_probability": 0.6621372103691101, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8218579960494334, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.032172899693250656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20915670692920685, + 0.2887912094593048, + 0.9180176854133606, + 0.3395939767360687, + 0.38359305262565613, + 0.08670570701360703, + 0.05377990007400513, + 0.05051054432988167, + 0.09010951220989227 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 0, + "style_probability": 0.47215771675109863, + "style_target": 1, + "support_probability": 0.9194485545158386, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.557962679999564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03130415081977844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14418958127498627, + 0.34987539052963257, + 0.9613651633262634, + 0.34460216760635376, + 0.1393187791109085, + 0.22425302863121033, + 0.14537173509597778, + 0.04632152244448662, + 0.20778459310531616 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 1, + "style_probability": 0.16010235249996185, + "style_target": 0, + "support_probability": 0.9420093894004822, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7607847328968627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03615541011095047, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3475213050842285, + 0.2267635315656662, + 0.3475213050842285, + 0.9477999806404114, + 0.07148239016532898, + 0.33292269706726074, + 0.07062821835279465, + 0.06993894279003143, + 0.06464280933141708, + 0.06529621034860611 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 0, + "style_probability": 0.46158167719841003, + "style_target": 1, + "support_probability": 0.8293068408966064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6398938262634376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08760306984186172, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31158921122550964, + 0.19978947937488556, + 0.31158921122550964, + 0.9509671926498413, + 0.04696817323565483, + 0.08449546992778778, + 0.09333953261375427, + 0.08840233832597733, + 0.03100871667265892, + 0.03893182426691055, + 0.04927441477775574 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 0, + "style_probability": 0.6115191578865051, + "style_target": 0, + "support_probability": 0.73517245054245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8060881963666995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.034383077174425125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9628873467445374, + 0.2583793103694916, + 0.3892197608947754, + 0.193587988615036, + 0.07930534332990646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 1, + "style_probability": 0.8032289743423462, + "style_target": 1, + "support_probability": 0.7850514054298401, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7521609299846154, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03509370610117912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9749723076820374, + 0.22937791049480438, + 0.36874088644981384, + 0.2028985321521759, + 0.04711503908038139 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 0, + "style_probability": 0.6702578067779541, + "style_target": 0, + "support_probability": 0.7401158213615417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06154062370763047, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7439124584197998, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1488010138273239, + 0.5631173849105835, + 0.1963089406490326, + 0.2886328399181366, + 0.04899713397026062, + 0.06930331885814667 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.5662364959716797, + "style_target": 1, + "support_probability": 0.13853612542152405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04548479561152179, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5091637969017029, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12049298733472824, + 0.7372975945472717, + 0.3262329697608948, + 0.2570343017578125, + 0.09103424847126007, + 0.21757614612579346 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.21494422852993011, + "style_target": 0, + "support_probability": 0.5024701356887817, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021136395576084488, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8921676278114319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6757131218910217, + 0.13458099961280823, + 0.3821253478527069, + 0.2333548367023468, + 0.0830555185675621, + 0.05222165212035179, + 0.06825800985097885, + 0.07373517751693726, + 0.13223546743392944 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 0, + "style_probability": 0.39530351758003235, + "style_target": 1, + "support_probability": 0.0614907406270504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16834847020589172, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7726725935935974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6623497605323792, + 0.14461925625801086, + 0.3945262134075165, + 0.20607705414295197, + 0.0234752856194973, + 0.033556241542100906, + 0.0296463705599308, + 0.06531327962875366, + 0.16965624690055847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 0, + "style_probability": 0.7782419919967651, + "style_target": 0, + "support_probability": 0.10513168573379517, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.45607061590020825, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30575698614120483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4932098686695099, + 0.25666606426239014, + 0.3912668526172638, + 0.7490999102592468, + 0.30850526690483093, + 0.08795224130153656, + 0.08493542671203613, + 0.11887802928686142, + 0.0700218454003334, + 0.09155013412237167, + 0.18151071667671204 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 1, + "style_probability": 0.5065828561782837, + "style_target": 1, + "support_probability": 0.5637016892433167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3688661915813816, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34939420223236084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45380735397338867, + 0.2336338311433792, + 0.34919214248657227, + 0.7383121252059937, + 0.142963707447052, + 0.15375877916812897, + 0.09262121468782425, + 0.08175990730524063, + 0.10351258516311646, + 0.20194382965564728, + 0.10770275443792343 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 1, + "style_probability": 0.3751126229763031, + "style_target": 0, + "support_probability": 0.4307677745819092, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5585049764609161, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1050676703453064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.310823529958725, + 0.8646628856658936, + 0.680357813835144, + 0.32811272144317627, + 0.02769390493631363 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 1, + "style_probability": 0.7934740781784058, + "style_target": 1, + "support_probability": 0.585749089717865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33914677036125396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14622750878334045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2559952139854431, + 0.8192394971847534, + 0.5973020792007446, + 0.29642120003700256, + 0.04518870636820793 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 1, + "style_probability": 0.40683305263519287, + "style_target": 0, + "support_probability": 0.5734250545501709, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8100225035945348, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05254356935620308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3737025260925293, + 0.3814360499382019, + 0.9145041108131409, + 0.19447460770606995, + 0.055290013551712036, + 0.03644712641835213 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.728480875492096, + "style_target": 1, + "support_probability": 0.8602795004844666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7223370671666371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06767875701189041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34801843762397766, + 0.37911170721054077, + 0.9089298248291016, + 0.18849104642868042, + 0.03105752170085907, + 0.09193053096532822 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.36785125732421875, + "style_target": 0, + "support_probability": 0.8632977604866028, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.864458355538514, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.013889812864363194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4629760682582855, + 0.5554866194725037, + 0.9983999133110046, + 0.4907345175743103, + 0.04692158102989197, + 0.061816778033971786, + 0.04499339684844017, + 0.07406746596097946, + 0.05475235357880592 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 1, + "style_probability": 0.6919757723808289, + "style_target": 1, + "support_probability": 0.9006761908531189, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6256957203525987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01420850120484829, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49385225772857666, + 0.5319437384605408, + 0.9990156888961792, + 0.5011332035064697, + 0.07825198769569397, + 0.07232268154621124, + 0.07646121829748154, + 0.08355579525232315, + 0.1375030279159546 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 1, + "style_probability": 0.36557236313819885, + "style_target": 0, + "support_probability": 0.9080861210823059, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15442805970704626, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5452350974082947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21664492785930634, + 0.7582356333732605, + 0.17359685897827148, + 0.15178123116493225, + 0.05183940753340721, + 0.052639078348875046, + 0.04146935045719147, + 0.07753746956586838, + 0.04340355843305588, + 0.12233861535787582, + 0.04423939809203148 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 1, + "style_probability": 0.5904987454414368, + "style_target": 1, + "support_probability": 0.2508244216442108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12127677736021054, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6517235040664673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23995834589004517, + 0.6394951939582825, + 0.16973930597305298, + 0.16967278718948364, + 0.0876346305012703, + 0.07034822553396225, + 0.095334492623806, + 0.08009382337331772, + 0.059280507266521454, + 0.0755777657032013, + 0.06261745095252991 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 0, + "style_probability": 0.5204073786735535, + "style_target": 0, + "support_probability": 0.18237660825252533, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7696964874552713, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1015455573797226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9681942462921143, + 0.16524891555309296, + 0.563869297504425, + 0.479910671710968, + 0.05875639244914055 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.4137258529663086, + "style_target": 1, + "support_probability": 0.8791399002075195, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8587455466356906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09397906810045242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9806991219520569, + 0.16689449548721313, + 0.5903603434562683, + 0.5138460993766785, + 0.052152909338474274 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.699403703212738, + "style_target": 0, + "support_probability": 0.8588960766792297, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7164746618231898, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06573858112096786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8634703755378723, + 0.232253760099411, + 0.3319697380065918, + 0.5229465365409851, + 0.05397753044962883, + 0.13572834432125092 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 0, + "style_probability": 0.4656892716884613, + "style_target": 1, + "support_probability": 0.915443480014801, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26940973156516484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06415669620037079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9357702136039734, + 0.23314553499221802, + 0.4629586935043335, + 0.47399693727493286, + 0.06853573024272919, + 0.12383414059877396 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 1, + "style_probability": 0.12807337939739227, + "style_target": 0, + "support_probability": 0.8999489545822144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46352527018370704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42941972613334656, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38337814807891846, + 0.9801335334777832, + 0.33018630743026733, + 0.38337814807891846, + 0.06271106004714966, + 0.05531967431306839, + 0.14454491436481476, + 0.08091238141059875, + 0.08578632026910782 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 0, + "style_probability": 0.3699466288089752, + "style_target": 1, + "support_probability": 0.4091207981109619, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5866005064421744, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3493628203868866, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35752713680267334, + 0.9745556712150574, + 0.3470515012741089, + 0.35752713680267334, + 0.14888805150985718, + 0.1743151992559433, + 0.14012527465820312, + 0.14083516597747803, + 0.11517725139856339 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 0, + "style_probability": 0.5232457518577576, + "style_target": 0, + "support_probability": 0.4953227639198303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10137958848480687, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2858150005340576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21051129698753357, + 0.7172474265098572, + 0.7674140334129333, + 0.6026004552841187, + 0.06456972658634186, + 0.04304712265729904, + 0.05342445150017738, + 0.07375548779964447, + 0.09947660565376282, + 0.040051914751529694, + 0.045153841376304626 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.28727591037750244, + "style_target": 1, + "support_probability": 0.2116355448961258, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24759298750798575, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2175925225019455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2054411619901657, + 0.7158504128456116, + 0.7918886542320251, + 0.5518933534622192, + 0.10929940640926361, + 0.055932316929101944, + 0.044282495975494385, + 0.029487255960702896, + 0.04941653087735176, + 0.14527127146720886, + 0.062326688319444656 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.7164252400398254, + "style_target": 0, + "support_probability": 0.2723684012889862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1693515565971413, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6345115900039673, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3901028335094452, + 0.18083570897579193, + 0.576945424079895, + 0.5618497133255005, + 0.12949591875076294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 1, + "style_probability": 0.5148393511772156, + "style_target": 1, + "support_probability": 0.2983826994895935, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020089647657541, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6575042605400085, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35756370425224304, + 0.22634264826774597, + 0.7090941667556763, + 0.7064678072929382, + 0.09173651784658432 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 1, + "style_probability": 0.17573405802249908, + "style_target": 0, + "support_probability": 0.2670823037624359, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10875552883888008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7446293234825134, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46316277980804443, + 0.869994580745697, + 0.41825419664382935, + 0.38325703144073486, + 0.20650885999202728, + 0.07290660589933395 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 0, + "style_probability": 0.4405378997325897, + "style_target": 1, + "support_probability": 0.2108970284461975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05372423213918554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.782146155834198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47065550088882446, + 0.8423308730125427, + 0.4574527144432068, + 0.3837910294532776, + 0.06484461575746536, + 0.11252539604902267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 1, + "style_probability": 0.4691254794597626, + "style_target": 0, + "support_probability": 0.099418506026268, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1474907595704315, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6761065721511841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6276949644088745, + 0.42595452070236206, + 0.887186586856842, + 0.4086182117462158, + 0.05573917180299759, + 0.05666080862283707, + 0.07942935824394226, + 0.3320504426956177, + 0.060597214847803116 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.42443275451660156, + "style_target": 1, + "support_probability": 0.275602251291275, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2340478291769453, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6648094058036804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6295432448387146, + 0.42399924993515015, + 0.9001547694206238, + 0.400547057390213, + 0.1068674698472023, + 0.06988838315010071, + 0.05985255911946297, + 0.08107051253318787, + 0.14010928571224213 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.7812473177909851, + "style_target": 0, + "support_probability": 0.22364048659801483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04129430858797203, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8682066798210144, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34163761138916016, + 0.17284244298934937, + 0.2894948124885559, + 0.6082425713539124, + 0.07736226171255112, + 0.16523927450180054, + 0.09927324205636978, + 0.22936241328716278, + 0.07143519073724747, + 0.14878778159618378, + 0.19343118369579315 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 0, + "style_probability": 0.44326379895210266, + "style_target": 1, + "support_probability": 0.032512079924345016, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021388981543043663, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9042009711265564, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3726951777935028, + 0.2101646512746811, + 0.28913575410842896, + 0.7029842734336853, + 0.11268962174654007, + 0.08450614660978317, + 0.0663454532623291, + 0.17778891324996948, + 0.08611617982387543, + 0.1561044454574585 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 0, + "style_probability": 0.5012462735176086, + "style_target": 0, + "support_probability": 0.0320269949734211, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47568719623479383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16462768614292145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8904664516448975, + 0.5284373164176941, + 0.2137688547372818, + 0.5556419491767883, + 0.13044479489326477 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 1, + "style_probability": 0.7861648797988892, + "style_target": 1, + "support_probability": 0.4166203737258911, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2746697698064117, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24728092551231384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8652040362358093, + 0.48482128977775574, + 0.21496345102787018, + 0.5066115856170654, + 0.11889681965112686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 0, + "style_probability": 0.5175992846488953, + "style_target": 0, + "support_probability": 0.31251823902130127, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.93725267809188, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04919302091002464, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1930:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5535251498222351, + 0.3526066541671753, + 0.9856383204460144, + 0.326638400554657, + 0.0750865787267685, + 0.04597172886133194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 1, + "style_probability": 0.928344190120697, + "style_target": 1, + "support_probability": 0.9465700387954712, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6714532542390098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08484674990177155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1930:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5299347043037415, + 0.3353075385093689, + 0.9720099568367004, + 0.26073282957077026, + 0.03168937563896179, + 0.05907503888010979 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 0, + "style_probability": 0.5104802846908569, + "style_target": 0, + "support_probability": 0.897849977016449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4956664873904139, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19557109475135803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3300400972366333, + 0.7055923342704773, + 0.3300400972366333, + 0.9252204895019531, + 0.05126717686653137, + 0.054429877549409866, + 0.15123219788074493, + 0.08625095337629318, + 0.08237557858228683 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.8932710289955139, + "style_target": 1, + "support_probability": 0.4354432225227356, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13553295844935934, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15468674898147583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4325059950351715, + 0.7764813303947449, + 0.4325059950351715, + 0.9701932668685913, + 0.05448893830180168, + 0.2376933991909027, + 0.0762486681342125, + 0.07742945104837418, + 0.09661027789115906 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.3944056034088135, + "style_target": 0, + "support_probability": 0.6275243163108826, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7339330695112182, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1271163672208786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47031643986701965, + 0.8958399891853333, + 0.4794754981994629, + 0.3396027684211731, + 0.20089417695999146, + 0.09276953339576721, + 0.049862686544656754, + 0.04324820265173912, + 0.12157589197158813, + 0.058461207896471024, + 0.03932463750243187 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 1, + "style_probability": 0.7968243360519409, + "style_target": 1, + "support_probability": 0.7907068133354187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42301638212050746, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33721837401390076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5513273477554321, + 0.8131527304649353, + 0.45939841866493225, + 0.34096601605415344, + 0.05254276096820831, + 0.13283850252628326, + 0.12682925164699554, + 0.053774550557136536, + 0.116840660572052, + 0.06381502747535706, + 0.06470014899969101 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 0, + "style_probability": 0.5111080408096313, + "style_target": 0, + "support_probability": 0.6864928603172302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01319163083035171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9834344983100891, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16612999141216278, + 0.28317171335220337, + 0.20094700157642365, + 0.741154134273529, + 0.17758499085903168 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.2602818012237549, + "style_target": 1, + "support_probability": 0.013461612164974213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013382538193689538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9871111512184143, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16771374642848969, + 0.26682430505752563, + 0.195878803730011, + 0.7360029220581055, + 0.08243443071842194 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.7861302495002747, + "style_target": 0, + "support_probability": 0.005046230275183916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4395435288000105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4011397063732147, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5146751999855042, + 0.13322271406650543, + 0.7907139658927917, + 0.305027037858963, + 0.16088594496250153, + 0.06130937859416008 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 0, + "style_probability": 0.31790465116500854, + "style_target": 1, + "support_probability": 0.6111151576042175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05452576753607028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5274450778961182, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6057178378105164, + 0.19253498315811157, + 0.8992380499839783, + 0.2987931966781616, + 0.07097016274929047, + 0.08135993033647537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 1, + "style_probability": 0.10354074835777283, + "style_target": 0, + "support_probability": 0.5528271198272705, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5784658883388538, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45990321040153503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8142989277839661, + 0.38211068511009216, + 0.38211068511009216, + 0.12931804358959198, + 0.09180086106061935, + 0.1280289888381958, + 0.04820675775408745, + 0.1613062024116516, + 0.09548769146203995 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 0, + "style_probability": 0.3680776357650757, + "style_target": 1, + "support_probability": 0.5762370228767395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6346591531071182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49394774436950684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8159075379371643, + 0.43700161576271057, + 0.43700161576271057, + 0.14008590579032898, + 0.038005948066711426, + 0.12163334339857101, + 0.05675836279988289, + 0.0596277117729187, + 0.044496290385723114 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 1, + "style_probability": 0.43138769268989563, + "style_target": 0, + "support_probability": 0.6999613642692566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.003737748147559855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.943061351776123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13205237686634064, + 0.4833245575428009, + 0.3282738924026489, + 0.2413448989391327, + 0.03884255141019821, + 0.053854621946811676, + 0.09056300669908524, + 0.1257908046245575, + 0.07041555643081665, + 0.11416086554527283, + 0.12564970552921295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.35863152146339417, + "style_target": 1, + "support_probability": 0.0029259820003062487, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005622520232921386, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9124990105628967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12947185337543488, + 0.4897273778915405, + 0.35246342420578003, + 0.25816142559051514, + 0.0593000091612339, + 0.031206147745251656, + 0.028963658958673477, + 0.05624952167272568, + 0.033974789083004, + 0.03741707652807236 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.8102447390556335, + "style_target": 0, + "support_probability": 0.003722638823091984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6536568589468208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22740572690963745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9905357956886292, + 0.27585217356681824, + 0.2889336347579956, + 0.14541807770729065, + 0.34081539511680603 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.7806805968284607, + "style_target": 1, + "support_probability": 0.5292794108390808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3109476948624594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19576427340507507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9963400363922119, + 0.34974417090415955, + 0.3420509994029999, + 0.18060550093650818, + 0.07278583943843842 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.30415868759155273, + "style_target": 0, + "support_probability": 0.51667320728302, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7171193936490158, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17216543853282928, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12873224914073944, + 0.8516152501106262, + 0.2431803196668625, + 0.23566706478595734, + 0.04539079964160919, + 0.09567762166261673 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 1, + "style_probability": 0.8322051763534546, + "style_target": 1, + "support_probability": 0.8710696697235107, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5746981143547482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3073090612888336, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15784220397472382, + 0.714664101600647, + 0.2189609408378601, + 0.20984786748886108, + 0.08035566657781601, + 0.08336377143859863 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 0, + "style_probability": 0.5935381054878235, + "style_target": 0, + "support_probability": 0.8225023746490479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07732697173374839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.475400447845459, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9489669799804688, + 0.6465902924537659, + 0.293125182390213, + 0.216878280043602, + 0.06868476420640945, + 0.07329204678535461, + 0.04206440597772598, + 0.03679562732577324, + 0.05266651511192322 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 1, + "style_probability": 0.8430210947990417, + "style_target": 1, + "support_probability": 0.02069811336696148, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07024590952183998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3977367579936981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9307113885879517, + 0.5870276689529419, + 0.28866761922836304, + 0.23508058488368988, + 0.14273202419281006, + 0.14671601355075836, + 0.10147123783826828, + 0.0954618975520134, + 0.06301309168338776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 0, + "style_probability": 0.6506716012954712, + "style_target": 0, + "support_probability": 0.02803897112607956, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.869167128487579, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0194961316883564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9488832354545593, + 0.6545623540878296, + 0.6822948455810547, + 0.38681942224502563, + 0.10095840692520142, + 0.06422174721956253, + 0.07365773618221283, + 0.10094466060400009, + 0.06222621724009514, + 0.07827208936214447, + 0.07272995263338089 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.8510331511497498, + "style_target": 1, + "support_probability": 0.8957927227020264, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.376502557514937, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.018885597586631775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9553157091140747, + 0.6821767687797546, + 0.726485013961792, + 0.37381216883659363, + 0.20457623898983002, + 0.11142732203006744, + 0.21908921003341675, + 0.11962016671895981, + 0.153396338224411, + 0.16856546700000763 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.17858803272247314, + "style_target": 0, + "support_probability": 0.9475401639938354, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7163840854073169, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2748306095600128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19804219901561737, + 0.9255974292755127, + 0.1679968684911728, + 0.5166218280792236, + 0.07065912336111069 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 1, + "style_probability": 0.6422890424728394, + "style_target": 1, + "support_probability": 0.8159990310668945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8041340277692611, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2512662410736084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19369715452194214, + 0.9154014587402344, + 0.17238934338092804, + 0.5088053941726685, + 0.060404222458601 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 0, + "style_probability": 0.7592740654945374, + "style_target": 0, + "support_probability": 0.8852309584617615, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6977855036629167, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14955241978168488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2017:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4206383526325226, + 0.9607471227645874, + 0.2727185785770416, + 0.24978087842464447, + 0.05181988701224327, + 0.06635314971208572 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 1, + "style_probability": 0.5155701041221619, + "style_target": 1, + "support_probability": 0.44492435455322266, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8505841455852826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14771141111850739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2017:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4399670958518982, + 0.9701639413833618, + 0.2928318977355957, + 0.23214012384414673, + 0.059930458664894104, + 0.08927751332521439 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 0, + "style_probability": 0.8395700454711914, + "style_target": 0, + "support_probability": 0.4492090940475464, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05959963785865953, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8046277761459351, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3218472898006439, + 0.4489326477050781, + 0.5207899808883667, + 0.284919798374176, + 0.0753474310040474, + 0.07620541751384735, + 0.059771910309791565, + 0.05827467516064644, + 0.07880433648824692 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 0, + "style_probability": 0.22866752743721008, + "style_target": 1, + "support_probability": 0.1701064556837082, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012888224525032847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8210204243659973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38545939326286316, + 0.5258725881576538, + 0.689565896987915, + 0.36791250109672546, + 0.05475674569606781, + 0.1460050642490387, + 0.17784996330738068, + 0.07898471504449844, + 0.1650308072566986 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 1, + "style_probability": 0.07357989996671677, + "style_target": 0, + "support_probability": 0.1834057867527008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04285309784081102, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.836518406867981, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30876243114471436, + 0.335105836391449, + 0.5680428743362427, + 0.48240798711776733, + 0.09710577875375748, + 0.05900164321064949, + 0.05480080470442772, + 0.04796982556581497, + 0.10943122208118439, + 0.14477211236953735 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.42205536365509033, + "style_target": 1, + "support_probability": 0.049540165811777115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08432038336667035, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.783511221408844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33710578083992004, + 0.3424050807952881, + 0.5680099725723267, + 0.43612954020500183, + 0.08971528708934784, + 0.11458825320005417, + 0.14878176152706146, + 0.07903213053941727, + 0.07824233919382095, + 0.10369144380092621 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.6426627039909363, + "style_target": 0, + "support_probability": 0.06278187036514282, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7708403185739855, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21625150740146637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8893553018569946, + 0.3857885003089905, + 0.3426559567451477, + 0.3857885003089905, + 0.1597764790058136 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.7849713563919067, + "style_target": 1, + "support_probability": 0.852391242980957, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4996013277656661, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22601915895938873, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8667494058609009, + 0.3438379466533661, + 0.42385631799697876, + 0.3438379466533661, + 0.03497349098324776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.25690436363220215, + "style_target": 0, + "support_probability": 0.8682548999786377, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7615845940012015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23752422630786896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4783518612384796, + 0.512512743473053, + 0.4592675566673279, + 0.9650794863700867, + 0.07954499870538712, + 0.10030461102724075 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.7823315262794495, + "style_target": 1, + "support_probability": 0.48158347606658936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1978243427598313, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3074108362197876, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5686047673225403, + 0.514336347579956, + 0.49870166182518005, + 0.979332447052002, + 0.08024398237466812, + 0.11259914934635162 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.20883552730083466, + "style_target": 0, + "support_probability": 0.47716063261032104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33799022887390234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2238660454750061, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4356148838996887, + 0.23862527310848236, + 0.20621292293071747, + 0.8703153729438782, + 0.08850808441638947, + 0.07238682359457016, + 0.14242230355739594, + 0.13158434629440308, + 0.0691431537270546 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 1, + "style_probability": 0.694055438041687, + "style_target": 1, + "support_probability": 0.37285953760147095, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24394063052439224, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3701481819152832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39911383390426636, + 0.2223651260137558, + 0.1642562299966812, + 0.7581081986427307, + 0.24412250518798828, + 0.09920376539230347, + 0.12684522569179535, + 0.2818249762058258, + 0.14645566046237946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 0, + "style_probability": 0.6367673873901367, + "style_target": 0, + "support_probability": 0.20950525999069214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10136042337613338, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7388712763786316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4138423800468445, + 0.1947503387928009, + 0.47426608204841614, + 0.4138423800468445, + 0.11623603850603104, + 0.0691654235124588, + 0.047107674181461334, + 0.06340385228395462, + 0.1114569902420044, + 0.10000409185886383 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.9154430031776428, + "style_target": 1, + "support_probability": 0.056149061769247055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044648067980013, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.722790002822876, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3567277193069458, + 0.20456092059612274, + 0.46866539120674133, + 0.3567277193069458, + 0.1035294383764267, + 0.03921981528401375, + 0.04310688376426697, + 0.09076599776744843, + 0.09391005337238312, + 0.07291223108768463 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.3066759407520294, + "style_target": 0, + "support_probability": 0.060534361749887466, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0770039340749193, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.43938833475112915, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1921055018901825, + 0.5494612455368042, + 0.6641301512718201, + 0.5962597727775574, + 0.0837741419672966 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 0, + "style_probability": 0.45898890495300293, + "style_target": 1, + "support_probability": 0.1460266411304474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18233644286960002, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.505618155002594, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15470479428768158, + 0.519189715385437, + 0.6632441878318787, + 0.5459423661231995, + 0.07741425186395645 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 0, + "style_probability": 0.5250349640846252, + "style_target": 0, + "support_probability": 0.1962359994649887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021549591577153956, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6297244429588318, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17905117571353912, + 0.2382151186466217, + 0.21499821543693542, + 0.6717624664306641, + 0.2832685708999634, + 0.08152876794338226 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 0, + "style_probability": 0.3813455104827881, + "style_target": 1, + "support_probability": 0.02547379955649376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005239408060514732, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6365343928337097, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14319460093975067, + 0.23676195740699768, + 0.18057283759117126, + 0.695746660232544, + 0.10941103845834732, + 0.08935824781656265 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 1, + "style_probability": 0.06889462471008301, + "style_target": 0, + "support_probability": 0.07218198478221893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42318682578910854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05076100677251816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2926064133644104, + 0.3348952531814575, + 0.9852102398872375, + 0.4054484963417053, + 0.10287376493215561, + 0.09428022056818008, + 0.2250959873199463, + 0.1555231213569641, + 0.0921347364783287 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 0, + "style_probability": 0.3488764762878418, + "style_target": 1, + "support_probability": 0.4334200322628021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5199278168701513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07267706096172333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3166678845882416, + 0.35348621010780334, + 0.9894030094146729, + 0.395854651927948, + 0.05040445923805237, + 0.08359072357416153, + 0.11850053071975708, + 0.42498865723609924, + 0.1006883978843689 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 0, + "style_probability": 0.7924723029136658, + "style_target": 0, + "support_probability": 0.302249938249588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10862627371392103, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7091348767280579, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5019571781158447, + 0.8979347348213196, + 0.3940824568271637, + 0.45485353469848633, + 0.10695777088403702, + 0.06301426142454147, + 0.07646315544843674, + 0.10533278435468674, + 0.09281415492296219, + 0.0910157561302185 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 1, + "style_probability": 0.5739246606826782, + "style_target": 1, + "support_probability": 0.10164599120616913, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0493241693840372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8120933771133423, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5234150290489197, + 0.9127202033996582, + 0.3678133189678192, + 0.42936521768569946, + 0.17191565036773682, + 0.10756029188632965, + 0.14060629904270172, + 0.11525216698646545, + 0.07533323019742966, + 0.16575996577739716 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 0, + "style_probability": 0.5023902654647827, + "style_target": 0, + "support_probability": 0.06197188049554825, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8929357683306208, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1975390464067459, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12143649905920029, + 0.9814671874046326, + 0.624561607837677, + 0.12143649905920029, + 0.08679526299238205 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 1, + "style_probability": 0.8041101098060608, + "style_target": 1, + "support_probability": 0.7892621755599976, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7926878052722778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21640051901340485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15624740719795227, + 0.9454833269119263, + 0.5501587986946106, + 0.15624740719795227, + 0.17985442280769348 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 0, + "style_probability": 0.6108265519142151, + "style_target": 0, + "support_probability": 0.7702562808990479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7397583438987202, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06687090545892715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7872994542121887, + 0.283132940530777, + 0.14517593383789062, + 0.945329487323761, + 0.028343018144369125, + 0.03490549325942993 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.7696490287780762, + "style_target": 1, + "support_probability": 0.8123944401741028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5417698724702695, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10443548113107681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7297570109367371, + 0.3480600118637085, + 0.12570194900035858, + 0.911831259727478, + 0.03384850174188614, + 0.07314417511224747 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.3911949694156647, + "style_target": 0, + "support_probability": 0.7482951879501343, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7055631371798425, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24003788828849792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5717145204544067, + 0.4181928038597107, + 0.26006120443344116, + 0.8213791251182556, + 0.11208175122737885, + 0.0959276407957077, + 0.04316450282931328, + 0.04778364673256874, + 0.08078456670045853 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.8821664452552795, + "style_target": 1, + "support_probability": 0.7358546853065491, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06635427778386867, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27261894941329956, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.523429274559021, + 0.500190794467926, + 0.39991581439971924, + 0.9152518510818481, + 0.16466116905212402, + 0.14955690503120422, + 0.12380174547433853, + 0.057776033878326416, + 0.10750270634889603 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.1389162391424179, + "style_target": 0, + "support_probability": 0.772930920124054, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37339175295985705, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24361968040466309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.41502881050109863, + 0.427409827709198, + 0.8145886659622192, + 0.32989218831062317, + 0.06914760172367096, + 0.08330991119146347, + 0.0612531341612339, + 0.06497253477573395, + 0.05047624185681343, + 0.0579148605465889 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 1, + "style_probability": 0.5109785795211792, + "style_target": 1, + "support_probability": 0.6243913173675537, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4968024148433461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27959969639778137, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3601433038711548, + 0.3912140130996704, + 0.7839828133583069, + 0.2870332896709442, + 0.0868474692106247, + 0.04202362522482872, + 0.03908831253647804, + 0.06523068994283676, + 0.08808278292417526, + 0.13861629366874695 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 1, + "style_probability": 0.4456765949726105, + "style_target": 0, + "support_probability": 0.7049643993377686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025141948173506866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7025501728057861, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3746265470981598, + 0.27515798807144165, + 0.300721138715744, + 0.741693913936615, + 0.4112878143787384 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 0, + "style_probability": 0.41639137268066406, + "style_target": 1, + "support_probability": 0.05120769515633583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08544654356537176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6840541362762451, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44841551780700684, + 0.31481871008872986, + 0.29556989669799805, + 0.7938063144683838, + 0.15867860615253448 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 0, + "style_probability": 0.8203892707824707, + "style_target": 0, + "support_probability": 0.07305280864238739, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007457038471145356, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8333092331886292, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15677320957183838, + 0.3810602128505707, + 0.5305636525154114, + 0.4096367359161377, + 0.0705086886882782, + 0.07114747911691666 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 0, + "style_probability": 0.21131505072116852, + "style_target": 1, + "support_probability": 0.029121924191713333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007893582920189018, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8450498580932617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2079758495092392, + 0.3402007818222046, + 0.5760378837585449, + 0.46161845326423645, + 0.03670114651322365, + 0.09229330718517303 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 1, + "style_probability": 0.13286718726158142, + "style_target": 0, + "support_probability": 0.023066096007823944, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8102185684293439, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0633377656340599, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32029762864112854, + 0.18893800675868988, + 0.9769749641418457, + 0.42230331897735596, + 0.04885012283921242, + 0.0953822061419487, + 0.08211911469697952, + 0.12409930676221848, + 0.07674858719110489 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 1, + "style_probability": 0.6720168590545654, + "style_target": 1, + "support_probability": 0.8563830852508545, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6959518475329645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04269785434007645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40499988198280334, + 0.2684515118598938, + 0.9929070472717285, + 0.5541014075279236, + 0.06656071543693542, + 0.2024068981409073, + 0.11864723265171051, + 0.05380720645189285, + 0.0706750899553299 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 0, + "style_probability": 0.5787350535392761, + "style_target": 0, + "support_probability": 0.8706070780754089, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03207634677101926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.849268913269043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.840206503868103, + 0.21731600165367126, + 0.2995396852493286, + 0.21731600165367126, + 0.0948343575000763, + 0.07843948155641556, + 0.1598455309867859, + 0.15612955391407013, + 0.09164170175790787, + 0.17220501601696014, + 0.2908962070941925 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 0, + "style_probability": 0.3757379353046417, + "style_target": 1, + "support_probability": 0.06748087704181671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1465089049095737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7799737453460693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8233001828193665, + 0.20489239692687988, + 0.23061369359493256, + 0.20489239692687988, + 0.10305428504943848, + 0.18082353472709656, + 0.0742875188589096, + 0.11059550195932388, + 0.13891196250915527, + 0.1723533421754837 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 0, + "style_probability": 0.8449000716209412, + "style_target": 0, + "support_probability": 0.11425323784351349, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8011509911116399, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03888462856411934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6927380561828613, + 0.4447379410266876, + 0.9887996912002563, + 0.37340837717056274, + 0.07706675678491592 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.5602670907974243, + "style_target": 1, + "support_probability": 0.8019133806228638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.435479316792897, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05195794999599457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7041603922843933, + 0.5105461478233337, + 0.9900319576263428, + 0.41637471318244934, + 0.11397034674882889 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.2503323256969452, + "style_target": 0, + "support_probability": 0.7823905348777771, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6544423526557988, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16885460913181305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32108086347579956, + 0.42088836431503296, + 0.8054524064064026, + 0.1775180548429489, + 0.11283508688211441, + 0.03934113308787346 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 1, + "style_probability": 0.6694785952568054, + "style_target": 1, + "support_probability": 0.887812077999115, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5989777061604461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19691626727581024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3232539892196655, + 0.42807167768478394, + 0.738959550857544, + 0.17757393419742584, + 0.04056442901492119, + 0.3145151734352112 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 0, + "style_probability": 0.5424106121063232, + "style_target": 0, + "support_probability": 0.8431298136711121, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.557013659525844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2422567456960678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27381670475006104, + 0.23264053463935852, + 0.3564092516899109, + 0.8536549210548401, + 0.07378683239221573, + 0.09946286678314209, + 0.0806276798248291, + 0.07432988286018372, + 0.06907398998737335 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.8164052367210388, + "style_target": 1, + "support_probability": 0.3908546268939972, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13577460273214742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43183138966560364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2790527939796448, + 0.22397741675376892, + 0.3495439887046814, + 0.8158032894134521, + 0.10229115933179855, + 0.20944666862487793, + 0.11896231770515442, + 0.31847330927848816, + 0.17494051158428192 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.22541417181491852, + "style_target": 0, + "support_probability": 0.18024179339408875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02527914763494649, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7443272471427917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2305956780910492, + 0.5981643795967102, + 0.5072137117385864, + 0.2305956780910492, + 0.1624898612499237, + 0.05708242207765579, + 0.045853015035390854, + 0.08767387270927429, + 0.05865912139415741, + 0.06916321069002151, + 0.06347265839576721 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.7093911170959473, + "style_target": 1, + "support_probability": 0.0098474295809865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005459628744201745, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7222145199775696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26772212982177734, + 0.6367976665496826, + 0.5502851009368896, + 0.26772212982177734, + 0.04460025951266289, + 0.043058015406131744, + 0.0686541497707367, + 0.10243166983127594, + 0.06522922217845917, + 0.09050820022821426 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.306832492351532, + "style_target": 0, + "support_probability": 0.018985966220498085, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8154015540773413, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1110410988330841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19479969143867493, + 0.1419389396905899, + 0.27153781056404114, + 0.9556845426559448, + 0.04351477324962616 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 1, + "style_probability": 0.8161828517913818, + "style_target": 1, + "support_probability": 0.7721696496009827, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.745853767290099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12857012450695038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23465688526630402, + 0.18535733222961426, + 0.26642370223999023, + 0.9287776350975037, + 0.06849195063114166 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 0, + "style_probability": 0.6426101922988892, + "style_target": 0, + "support_probability": 0.7560296058654785, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7186177487281394, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3181726634502411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32462844252586365, + 0.8209964632987976, + 0.20509445667266846, + 0.5004842281341553, + 0.04799174144864082, + 0.05338461324572563 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.8723976612091064, + "style_target": 1, + "support_probability": 0.7492921352386475, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6105728283721716, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24834924936294556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3483104705810547, + 0.8376007080078125, + 0.23925136029720306, + 0.47598910331726074, + 0.1196136400103569, + 0.10803894698619843 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.39500942826271057, + "style_target": 0, + "support_probability": 0.8250581622123718, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2658036799186036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.489810585975647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8304449319839478, + 0.5251358151435852, + 0.32550519704818726, + 0.2328747808933258, + 0.0720517560839653, + 0.07529141753911972, + 0.05941930413246155, + 0.06801555305719376, + 0.05756969377398491 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.7023167014122009, + "style_target": 1, + "support_probability": 0.47018539905548096, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.052285656719983864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5495320558547974, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8765749335289001, + 0.5467572212219238, + 0.3316262364387512, + 0.27924785017967224, + 0.13557350635528564, + 0.22687861323356628, + 0.09135890007019043, + 0.06456899642944336, + 0.12574583292007446 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.25819912552833557, + "style_target": 0, + "support_probability": 0.5119438171386719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29480610174433663, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29162901639938354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7771005034446716, + 0.3823916018009186, + 0.69602370262146, + 0.20572242140769958, + 0.09860691428184509, + 0.05755987763404846, + 0.07241692394018173, + 0.09659307450056076, + 0.05214231461286545, + 0.07543399930000305, + 0.048525188118219376 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.8318457007408142, + "style_target": 1, + "support_probability": 0.2835890054702759, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02822525836456184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5501994490623474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6702387928962708, + 0.332180380821228, + 0.6460301280021667, + 0.13696536421775818, + 0.08546756207942963, + 0.10656438767910004, + 0.06389093399047852, + 0.19256757199764252, + 0.0639670342206955, + 0.08161640167236328, + 0.07857660204172134 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.3513433039188385, + "style_target": 0, + "support_probability": 0.09348244220018387, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3145241840591934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15958808362483978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31984809041023254, + 0.29714521765708923, + 0.9208536148071289, + 0.35275694727897644, + 0.13449034094810486 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.7755792140960693, + "style_target": 1, + "support_probability": 0.29045480489730835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1127024348241956, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19221438467502594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2767914831638336, + 0.28641024231910706, + 0.8964499831199646, + 0.3111107647418976, + 0.07989861816167831 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.4106726348400116, + "style_target": 0, + "support_probability": 0.20477283000946045, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8088607538781462, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2845766544342041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26972463726997375, + 0.5000396966934204, + 0.9824284911155701, + 0.39985373616218567, + 0.07151491940021515, + 0.17475642263889313 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.8470512628555298, + "style_target": 1, + "support_probability": 0.8443525433540344, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44877270265786506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14560087025165558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26364725828170776, + 0.6011738777160645, + 0.99216228723526, + 0.5429741740226746, + 0.07444234937429428, + 0.1314874142408371 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.3695342540740967, + "style_target": 0, + "support_probability": 0.9421390295028687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6298780378643606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14908698201179504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3645750880241394, + 0.16531938314437866, + 0.2756311595439911, + 0.9894323348999023, + 0.08513107150793076, + 0.21000254154205322, + 0.0998581051826477, + 0.10085957497358322, + 0.15453463792800903 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 1, + "style_probability": 0.6714538335800171, + "style_target": 1, + "support_probability": 0.8651401400566101, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.519945694962102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2220613658428192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3346947431564331, + 0.18874908983707428, + 0.24879762530326843, + 0.9828345775604248, + 0.1414768397808075, + 0.0610506609082222, + 0.11812366545200348, + 0.10773185640573502, + 0.16954292356967926 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 1, + "style_probability": 0.3963870108127594, + "style_target": 0, + "support_probability": 0.88802570104599, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8636097575786792, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23036730289459229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16259141266345978, + 0.2703288793563843, + 0.4176665246486664, + 0.9650880694389343, + 0.03959684818983078, + 0.1069069504737854, + 0.07691285014152527, + 0.17845121026039124, + 0.10365039110183716, + 0.07603607326745987, + 0.17352816462516785 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.8199739456176758, + "style_target": 1, + "support_probability": 0.8102599382400513, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7397188291448271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25231704115867615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17250119149684906, + 0.2534518539905548, + 0.39485055208206177, + 0.9623389840126038, + 0.05019547790288925, + 0.1255679726600647, + 0.04546040669083595, + 0.046267442405223846, + 0.05638599768280983, + 0.042851902544498444, + 0.06776101142168045 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.43563926219940186, + "style_target": 0, + "support_probability": 0.8357367515563965, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8712770989297794, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0961192175745964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3408065438270569, + 0.17701976001262665, + 0.27387920022010803, + 0.9448744058609009, + 0.04138479381799698 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.8628085851669312, + "style_target": 1, + "support_probability": 0.8997818827629089, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06799570774845287, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12123961746692657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4112307131290436, + 0.16549499332904816, + 0.3072633445262909, + 0.9765102863311768, + 0.11596118658781052 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.05383923649787903, + "style_target": 0, + "support_probability": 0.7295102477073669, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8348723017883658, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07952777296304703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43008938431739807, + 0.9351570010185242, + 0.3854987621307373, + 0.43008938431739807, + 0.05890990421175957, + 0.030989306047558784 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 1, + "style_probability": 0.7134976983070374, + "style_target": 1, + "support_probability": 0.8714318871498108, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7943065164583206, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07030226290225983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43861040472984314, + 0.8929426074028015, + 0.3885287046432495, + 0.43861040472984314, + 0.05481971427798271, + 0.07971838861703873 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 0, + "style_probability": 0.5864604115486145, + "style_target": 0, + "support_probability": 0.900200366973877, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9457130028936435, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1896430402994156, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:2209:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23404274880886078, + 0.9947210550308228, + 0.5680429339408875, + 0.15364280343055725, + 0.0934041440486908, + 0.06351214647293091, + 0.07147277146577835, + 0.031848687678575516, + 0.045297086238861084 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.9122285842895508, + "style_target": 1, + "support_probability": 0.9261545538902283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5512073006024577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25760236382484436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2209:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26562583446502686, + 0.986781895160675, + 0.4971334636211395, + 0.15831688046455383, + 0.09670308232307434, + 0.06235850602388382, + 0.09695222228765488, + 0.05145450308918953, + 0.1483563482761383 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.33887824416160583, + "style_target": 0, + "support_probability": 0.7885934710502625, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8805180753360974, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09951221197843552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9813778400421143, + 0.3824901282787323, + 0.342469722032547, + 0.42102962732315063, + 0.04261147975921631, + 0.10750830173492432, + 0.057445842772722244, + 0.04476775974035263, + 0.07246600836515427, + 0.07088623940944672, + 0.06709421426057816 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.5553829669952393, + "style_target": 1, + "support_probability": 0.9147319793701172, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.513587690394911, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10549712181091309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9937366247177124, + 0.5183691382408142, + 0.4329955577850342, + 0.5141399502754211, + 0.12386998534202576, + 0.11351940035820007, + 0.12973622977733612, + 0.08410178124904633, + 0.10725177824497223, + 0.0978483334183693 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.10324173420667648, + "style_target": 0, + "support_probability": 0.9163281917572021, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8922032589202666, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07099940627813339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28708338737487793, + 0.9851888418197632, + 0.14154021441936493, + 0.13497143983840942, + 0.042997170239686966 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 1, + "style_probability": 0.5995544791221619, + "style_target": 1, + "support_probability": 0.927666187286377, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7780356017536058, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10955875366926193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25214919447898865, + 0.9523246884346008, + 0.15643654763698578, + 0.11274613440036774, + 0.12228687852621078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 1, + "style_probability": 0.33289510011672974, + "style_target": 0, + "support_probability": 0.8980435729026794, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8630376796848126, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04233578220009804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9652431607246399, + 0.33544471859931946, + 0.6910972595214844, + 0.7439854145050049, + 0.03816014900803566, + 0.06951501220464706 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.740635097026825, + "style_target": 1, + "support_probability": 0.8194668292999268, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7065706268653634, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06210976094007492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9321797490119934, + 0.31788066029548645, + 0.6619278788566589, + 0.7055845856666565, + 0.10063174366950989, + 0.09375203400850296 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.3987988531589508, + "style_target": 0, + "support_probability": 0.749579906463623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009455580302907237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.974428117275238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19957707822322845, + 0.7048379182815552, + 0.4425186216831207, + 0.2508144676685333, + 0.07383053004741669, + 0.1858244687318802, + 0.09331753849983215, + 0.1252347230911255, + 0.0835360586643219 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.6863284111022949, + "style_target": 1, + "support_probability": 0.0062255095690488815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0024184903605484484, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.95674729347229, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2780660390853882, + 0.6689829230308533, + 0.46802017092704773, + 0.2755274772644043, + 0.08779897540807724, + 0.22338925302028656, + 0.12373761087656021, + 0.13731753826141357, + 0.2990385890007019 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.10460072010755539, + "style_target": 0, + "support_probability": 0.01700950600206852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4870060974744526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2970975935459137, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11221276223659515, + 0.3781512379646301, + 0.9143969416618347, + 0.23205840587615967, + 0.0678696259856224, + 0.09298398345708847, + 0.06811069697141647, + 0.0744309201836586, + 0.046662356704473495, + 0.058037374168634415, + 0.07784939557313919 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 1, + "style_probability": 0.7313684225082397, + "style_target": 1, + "support_probability": 0.4568292200565338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32396843023819477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.428475946187973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13770532608032227, + 0.3576424717903137, + 0.8462199568748474, + 0.23688393831253052, + 0.05022694915533066, + 0.03663548827171326, + 0.11417790502309799, + 0.08902797847986221, + 0.08072950690984726, + 0.08242534101009369, + 0.04296335577964783 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 0, + "style_probability": 0.7106925249099731, + "style_target": 0, + "support_probability": 0.2957908511161804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16712389103536437, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.58249431848526, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8655369281768799, + 0.7430289387702942, + 0.15963132679462433, + 0.34347912669181824, + 0.048514269292354584 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.4055160880088806, + "style_target": 1, + "support_probability": 0.23619890213012695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3786794029566008, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.627088725566864, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8991808295249939, + 0.7848338484764099, + 0.17304310202598572, + 0.3576284646987915, + 0.35919541120529175 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.8277183175086975, + "style_target": 0, + "support_probability": 0.2673835754394531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022477378315015707, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9642154574394226, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5880737900733948, + 0.4053645730018616, + 0.5880737900733948, + 0.3996022045612335, + 0.0471469983458519, + 0.08860365301370621 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 0, + "style_probability": 0.26963379979133606, + "style_target": 1, + "support_probability": 0.03815361112356186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002101571851994059, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.973899781703949, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5620554685592651, + 0.3488004207611084, + 0.5620554685592651, + 0.40567123889923096, + 0.13773581385612488, + 0.11417985707521439 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 1, + "style_probability": 0.068717822432518, + "style_target": 0, + "support_probability": 0.024193376302719116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1792574176823886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.285804808139801, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31291693449020386, + 0.964500367641449, + 0.12486746162176132, + 0.16748841106891632, + 0.0920877754688263, + 0.10195699334144592, + 0.10387659072875977, + 0.1978185623884201, + 0.26591184735298157 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 0, + "style_probability": 0.4106430113315582, + "style_target": 1, + "support_probability": 0.1444864720106125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5740849531428022, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0950600728392601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26613926887512207, + 0.9642457365989685, + 0.12948904931545258, + 0.2078125774860382, + 0.1104213073849678, + 0.13924825191497803, + 0.07951465249061584, + 0.10450137406587601, + 0.1965135782957077 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 1, + "style_probability": 0.47433292865753174, + "style_target": 0, + "support_probability": 0.5614991784095764, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016972436903930112, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7794052958488464, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2731812298297882, + 0.31430932879447937, + 0.8741858005523682, + 0.44164785742759705, + 0.10203336924314499, + 0.09850147366523743, + 0.10883668065071106, + 0.1403685361146927, + 0.12174674868583679, + 0.22323693335056305, + 0.24210000038146973 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.13752929866313934, + "style_target": 1, + "support_probability": 0.03897862508893013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11288470711555301, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6012265086174011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2530520260334015, + 0.29217490553855896, + 0.9055448770523071, + 0.45985567569732666, + 0.1673223227262497, + 0.10134051740169525, + 0.18492452800273895, + 0.19419725239276886, + 0.12081275880336761, + 0.16453245282173157, + 0.24430696666240692 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.7444112300872803, + "style_target": 0, + "support_probability": 0.06544049084186554, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9220769949220147, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05173759534955025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27494823932647705, + 0.07886876165866852, + 0.2070501148700714, + 0.993721604347229, + 0.12152311950922012 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.5156055688858032, + "style_target": 1, + "support_probability": 0.9567530751228333, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7737283865702674, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13702884316444397, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.259594202041626, + 0.08493255823850632, + 0.22088249027729034, + 0.9931005239486694, + 0.17702166736125946 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.3856778144836426, + "style_target": 0, + "support_probability": 0.8884276747703552, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5985027804768971, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1964655965566635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4237617552280426, + 0.10347515344619751, + 0.890072762966156, + 0.3080565929412842, + 0.1127069890499115, + 0.14317673444747925 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 0, + "style_probability": 0.49488934874534607, + "style_target": 1, + "support_probability": 0.7791032195091248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1759129541952356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22647997736930847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3803058862686157, + 0.12534743547439575, + 0.9793190360069275, + 0.41824325919151306, + 0.094364233314991, + 0.09088771045207977 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 1, + "style_probability": 0.1741081327199936, + "style_target": 0, + "support_probability": 0.7861998677253723, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028467217727173963, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8907656073570251, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2868232727050781, + 0.17657499015331268, + 0.5028076171875, + 0.2868232727050781, + 0.09348888695240021, + 0.08433836698532104, + 0.05391982942819595, + 0.13053558766841888, + 0.14326944947242737 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 1, + "style_probability": 0.5389494895935059, + "style_target": 1, + "support_probability": 0.027524910867214203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09455063247331842, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8811624050140381, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22214503586292267, + 0.16078691184520721, + 0.5339438915252686, + 0.22214503586292267, + 0.07184193283319473, + 0.053533803671598434, + 0.11749850958585739, + 0.10448874533176422, + 0.11432409286499023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 0, + "style_probability": 0.810339629650116, + "style_target": 0, + "support_probability": 0.05844452977180481, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4162847614658802, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.520264208316803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16642265021800995, + 0.20319409668445587, + 0.7527267336845398, + 0.13739991188049316, + 0.0807185173034668, + 0.10719370096921921, + 0.09922903776168823, + 0.05866701528429985, + 0.06921813637018204, + 0.09537144750356674, + 0.17884305119514465 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 0, + "style_probability": 0.46347978711128235, + "style_target": 1, + "support_probability": 0.6680694818496704, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17030313120241025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.644184947013855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19403403997421265, + 0.1985500007867813, + 0.7284973859786987, + 0.16043062508106232, + 0.06864102184772491, + 0.08296499401330948, + 0.05221789702773094, + 0.06826023757457733, + 0.06481106579303741, + 0.04702882096171379, + 0.23455484211444855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 1, + "style_probability": 0.27065929770469666, + "style_target": 0, + "support_probability": 0.34410080313682556, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0872817060505663, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8850423097610474, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7943660616874695, + 0.5183635950088501, + 0.5233992338180542, + 0.4975915849208832, + 0.1505233347415924 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 1, + "style_probability": 0.814709484577179, + "style_target": 1, + "support_probability": 0.053132008761167526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10132253319455835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8417048454284668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6983552575111389, + 0.5122213363647461, + 0.5110393762588501, + 0.45825469493865967, + 0.12972694635391235 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 0, + "style_probability": 0.6961057782173157, + "style_target": 0, + "support_probability": 0.12931185960769653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6417599349152859, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04235396906733513, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9948728680610657, + 0.3244403898715973, + 0.297124981880188, + 0.3244403898715973, + 0.17620566487312317, + 0.08167705684900284 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.6402779817581177, + "style_target": 1, + "support_probability": 0.6825945377349854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5697439563352132, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0875006765127182, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9850338101387024, + 0.28510549664497375, + 0.2861754298210144, + 0.28510549664497375, + 0.06776729226112366, + 0.14096949994564056 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.3503173291683197, + "style_target": 0, + "support_probability": 0.7289570569992065, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.87667370562982, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31946465373039246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40561389923095703, + 0.3751142621040344, + 0.40561389923095703, + 0.9686832427978516, + 0.03398970887064934, + 0.061233047395944595, + 0.11290355771780014, + 0.27872708439826965, + 0.0722217708826065 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.9234088659286499, + "style_target": 1, + "support_probability": 0.8219152688980103, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1978825868881046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2408810406923294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4363677501678467, + 0.45143240690231323, + 0.4363677501678467, + 0.9878787398338318, + 0.07908803969621658, + 0.5199568867683411, + 0.10081473737955093, + 0.097501240670681, + 0.13117235898971558 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.10201713442802429, + "style_target": 0, + "support_probability": 0.8807019591331482, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2607415302277079, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49532464146614075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5101991295814514, + 0.29622548818588257, + 0.3810127377510071, + 0.8059666156768799, + 0.08749237656593323, + 0.09195642918348312, + 0.07331311702728271, + 0.04810744896531105, + 0.0849456787109375, + 0.12586089968681335, + 0.0865560844540596 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 1, + "style_probability": 0.7826105356216431, + "style_target": 1, + "support_probability": 0.14373476803302765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1313465744516384, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4935557544231415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5067503452301025, + 0.26167526841163635, + 0.4026210904121399, + 0.7859678268432617, + 0.0942113846540451, + 0.10166345536708832, + 0.061068419367074966, + 0.1187763586640358, + 0.1401025354862213, + 0.19174453616142273, + 0.23367434740066528 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 0, + "style_probability": 0.5397801399230957, + "style_target": 0, + "support_probability": 0.10801190137863159, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04198266721316387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7755216360092163, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28422486782073975, + 0.24084199965000153, + 0.28422486782073975, + 0.8651561737060547, + 0.15850387513637543 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 0, + "style_probability": 0.3922075033187866, + "style_target": 1, + "support_probability": 0.03938913345336914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2078574381515635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.43424367904663086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.283760666847229, + 0.23260432481765747, + 0.283760666847229, + 0.9086642265319824, + 0.041647352278232574 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 0, + "style_probability": 0.7179105281829834, + "style_target": 0, + "support_probability": 0.10564546287059784, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7038025858745253, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06497449427843094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9448012709617615, + 0.5260681509971619, + 0.1233534961938858, + 0.1934908926486969, + 0.12124259024858475, + 0.08620619028806686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 0, + "style_probability": 0.4381203055381775, + "style_target": 1, + "support_probability": 0.832411527633667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5870029886328101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13920605182647705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9266313314437866, + 0.6159646511077881, + 0.11285523325204849, + 0.1947057545185089, + 0.08409814536571503, + 0.08446679264307022 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 1, + "style_probability": 0.32551801204681396, + "style_target": 0, + "support_probability": 0.7703316807746887, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7390824545984999, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06860298663377762, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11538345366716385, + 0.49407950043678284, + 0.9356303215026855, + 0.14268113672733307, + 0.05835666507482529, + 0.06489099562168121, + 0.09060510247945786, + 0.036818839609622955, + 0.0345681793987751 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.6965917348861694, + "style_target": 1, + "support_probability": 0.8064563870429993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26346402695545756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05829933285713196, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1468856781721115, + 0.5443791747093201, + 0.9800400733947754, + 0.1602904200553894, + 0.06695400923490524, + 0.12824493646621704, + 0.10180533677339554, + 0.12214560061693192, + 0.09310896694660187 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.1939593106508255, + "style_target": 0, + "support_probability": 0.8419604897499084, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8731550026557752, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.008562896400690079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15789397060871124, + 0.13129107654094696, + 0.15789397060871124, + 0.9821304678916931, + 0.056839942932128906, + 0.1062874123454094, + 0.06529557704925537, + 0.10189549624919891, + 0.1559043973684311, + 0.05946343019604683, + 0.058688681572675705 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 1, + "style_probability": 0.6313973665237427, + "style_target": 1, + "support_probability": 0.9302576780319214, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.895672670675534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.006413334049284458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16402730345726013, + 0.11745550483465195, + 0.16402730345726013, + 0.9947794675827026, + 0.2092970907688141, + 0.0654681846499443, + 0.10116087645292282, + 0.0711250975728035, + 0.038879744708538055, + 0.026234881952404976 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 0, + "style_probability": 0.8280184864997864, + "style_target": 0, + "support_probability": 0.9129626154899597, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5537788134071896, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13601873815059662, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6205254197120667, + 0.1543572098016739, + 0.7697046399116516, + 0.43586254119873047, + 0.12005510926246643 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 0, + "style_probability": 0.35094285011291504, + "style_target": 1, + "support_probability": 0.9185828566551208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12860075789964567, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15147729218006134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7308319211006165, + 0.13580572605133057, + 0.875453531742096, + 0.3815355598926544, + 0.09592729061841965 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 1, + "style_probability": 0.08465410023927689, + "style_target": 0, + "support_probability": 0.8813195824623108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05226606710849052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6581598520278931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09794556349515915, + 0.1820688247680664, + 0.3142263889312744, + 0.7729855179786682, + 0.11039559543132782, + 0.21798117458820343 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 0, + "style_probability": 0.3488894999027252, + "style_target": 1, + "support_probability": 0.06619955599308014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06744712237323738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3265116512775421, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10299407690763474, + 0.17437145113945007, + 0.2747538685798645, + 0.7470695972442627, + 0.047353532165288925, + 0.09964407980442047 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 0, + "style_probability": 0.5060538649559021, + "style_target": 0, + "support_probability": 0.10931085795164108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036333437380894514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9472755193710327, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7654983401298523, + 0.47300150990486145, + 0.47300150990486145, + 0.15436972677707672, + 0.17266587913036346, + 0.13858447968959808, + 0.07046772539615631, + 0.11658526211977005, + 0.1917337030172348 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.3014375865459442, + "style_target": 1, + "support_probability": 0.06624622642993927, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05991371692237429, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9255543351173401, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7492170333862305, + 0.4818994104862213, + 0.4818994104862213, + 0.146256223320961, + 0.05004124343395233, + 0.12595324218273163, + 0.17485125362873077, + 0.05399586632847786, + 0.04841325804591179 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.6874316930770874, + "style_target": 0, + "support_probability": 0.0804462656378746, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8147617096017877, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060679949820041656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08659164607524872, + 0.9579785466194153, + 0.3735102713108063, + 0.34016191959381104, + 0.0369269959628582, + 0.05907547473907471, + 0.10746213048696518, + 0.08431994169950485, + 0.03677946329116821, + 0.07989758253097534, + 0.14730428159236908 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 0, + "style_probability": 0.4134342074394226, + "style_target": 1, + "support_probability": 0.8883990049362183, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.348392355026268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09960907697677612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.06456095725297928, + 0.9749525785446167, + 0.3510596454143524, + 0.3898976147174835, + 0.09642202407121658, + 0.09927066415548325, + 0.19595417380332947, + 0.12300007045269012, + 0.0330972783267498, + 0.07333788275718689 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 1, + "style_probability": 0.23151709139347076, + "style_target": 0, + "support_probability": 0.8109472393989563, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4107467342024407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18908001482486725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.07825601100921631, + 0.9565624594688416, + 0.5286933183670044, + 0.07825601100921631, + 0.09083110839128494 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 1, + "style_probability": 0.6351242661476135, + "style_target": 1, + "support_probability": 0.44306111335754395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38540148449298767, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18651176989078522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.09808628261089325, + 0.906338632106781, + 0.5145547986030579, + 0.09808628261089325, + 0.04702122136950493 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 1, + "style_probability": 0.4895278513431549, + "style_target": 0, + "support_probability": 0.4937707483768463, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6815802033186179, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06579470634460449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7322478294372559, + 0.18644095957279205, + 0.7488725185394287, + 0.2926703691482544, + 0.03728235512971878, + 0.04500555992126465 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.7473590970039368, + "style_target": 1, + "support_probability": 0.9496450424194336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44884384651971154, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10824184864759445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6536786556243896, + 0.1782170981168747, + 0.6909000277519226, + 0.293874055147171, + 0.02762720175087452, + 0.0475962795317173 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.2679154574871063, + "style_target": 0, + "support_probability": 0.8639910221099854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0202706349625523, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9812256693840027, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2617413401603699, + 0.6410993337631226, + 0.44919922947883606, + 0.2617413401603699, + 0.15669046342372894, + 0.10271018743515015, + 0.1294628232717514, + 0.05195212736725807, + 0.1586570292711258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.7647565603256226, + "style_target": 1, + "support_probability": 0.016407517716288567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004720453690163472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9668236970901489, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24722321331501007, + 0.7191548943519592, + 0.4355679452419281, + 0.24722321331501007, + 0.09341682493686676, + 0.39278486371040344, + 0.10250671952962875, + 0.10629939287900925, + 0.1531817466020584 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.19488345086574554, + "style_target": 0, + "support_probability": 0.02736450545489788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8903641825527753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13060277700424194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33284440636634827, + 0.29738664627075195, + 0.20106078684329987, + 0.9904775619506836, + 0.08618719130754471, + 0.06334714591503143, + 0.10256209969520569, + 0.13773711025714874, + 0.06235705316066742, + 0.07015475630760193, + 0.06218856945633888 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 1, + "style_probability": 0.8216284513473511, + "style_target": 1, + "support_probability": 0.7424951791763306, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7702144458553817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10117886960506439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2816079556941986, + 0.2782573699951172, + 0.1944873034954071, + 0.9844949841499329, + 0.0701647698879242, + 0.08865489810705185, + 0.03683087229728699, + 0.12809158861637115, + 0.06803327053785324, + 0.09142467379570007, + 0.0718119889497757 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 0, + "style_probability": 0.633825421333313, + "style_target": 0, + "support_probability": 0.7686189413070679, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42429099424812355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6754719018936157, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19149017333984375, + 0.916272759437561, + 0.15936294198036194, + 0.5393018126487732, + 0.047908879816532135 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.8583841323852539, + "style_target": 1, + "support_probability": 0.1531936675310135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26382459267577485, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.531275749206543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20469902455806732, + 0.9202255010604858, + 0.15789343416690826, + 0.5923578143119812, + 0.054625965654850006 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.39938050508499146, + "style_target": 0, + "support_probability": 0.24442656338214874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00542002985579263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9647821187973022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17145800590515137, + 0.5115290880203247, + 0.3052418529987335, + 0.3518659770488739, + 0.05876467004418373, + 0.044863972812891006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.560644268989563, + "style_target": 1, + "support_probability": 0.015284578315913677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015525906683869798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9795179963111877, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21539218723773956, + 0.6882535815238953, + 0.30604878067970276, + 0.4303775727748871, + 0.10910143703222275, + 0.1456703394651413 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.11133717745542526, + "style_target": 0, + "support_probability": 0.014813439920544624, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11933111233943852, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7859290242195129, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3115953803062439, + 0.3115953803062439, + 0.39277252554893494, + 0.2139691710472107, + 0.1495400369167328, + 0.055623896420001984, + 0.034325916320085526, + 0.10583140701055527, + 0.06507385522127151 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.8416874408721924, + "style_target": 1, + "support_probability": 0.09543151408433914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014499562124163893, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.880431592464447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28586244583129883, + 0.28586244583129883, + 0.352039635181427, + 0.23510849475860596, + 0.14742816984653473, + 0.09549587219953537, + 0.12840668857097626, + 0.11637373268604279, + 0.18472599983215332 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.28802740573883057, + "style_target": 0, + "support_probability": 0.034178223460912704, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7641372310246415, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13139532506465912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2204337865114212, + 0.1712437868118286, + 0.2204337865114212, + 0.9760002493858337, + 0.06191578507423401, + 0.049199849367141724, + 0.11327110230922699, + 0.05012477934360504, + 0.052240040153265, + 0.1046108826994896, + 0.08227572590112686 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 1, + "style_probability": 0.755439281463623, + "style_target": 1, + "support_probability": 0.8448559045791626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5939637285946954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13497509062290192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.185003399848938, + 0.14685606956481934, + 0.185003399848938, + 0.9630817770957947, + 0.25917282700538635, + 0.11311184614896774, + 0.1291181892156601, + 0.11599895358085632, + 0.14032244682312012, + 0.12628941237926483 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 0, + "style_probability": 0.5491186380386353, + "style_target": 0, + "support_probability": 0.734305202960968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03455447187375263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.32747942209243774, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.311295747756958, + 0.8354631662368774, + 0.23877111077308655, + 0.396528422832489, + 0.07777919620275497 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 0, + "style_probability": 0.22551442682743073, + "style_target": 1, + "support_probability": 0.09008783102035522, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023013552180908094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3684740960597992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36173585057258606, + 0.8368502855300903, + 0.28181031346321106, + 0.41104447841644287, + 0.1423356980085373 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 1, + "style_probability": 0.15959960222244263, + "style_target": 0, + "support_probability": 0.06588900089263916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06678804267866598, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24839043617248535, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3810635805130005, + 0.34117212891578674, + 0.7501522302627563, + 0.13895481824874878, + 0.10362152755260468, + 0.0782841145992279 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 0, + "style_probability": 0.29303082823753357, + "style_target": 1, + "support_probability": 0.2964058816432953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011131749379214089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4686710834503174, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3942001760005951, + 0.4586509168148041, + 0.852875828742981, + 0.1718890368938446, + 0.17676305770874023, + 0.1289415806531906 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 1, + "style_probability": 0.11942357569932938, + "style_target": 0, + "support_probability": 0.1835097372531891, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008917308792583489, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8398663997650146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3425222933292389, + 0.27228865027427673, + 0.17760951817035675, + 0.24771684408187866, + 0.12505091726779938, + 0.12050608545541763, + 0.1057305559515953, + 0.13923625648021698, + 0.0883045643568039 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 0, + "style_probability": 0.4906943440437317, + "style_target": 1, + "support_probability": 0.01827779971063137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031794146217398116, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8332856297492981, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3912922143936157, + 0.3318546414375305, + 0.23399749398231506, + 0.3169997036457062, + 0.08423477411270142, + 0.149858757853508, + 0.11831850558519363, + 0.1770220398902893, + 0.22732342779636383 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 0, + "style_probability": 0.7301162481307983, + "style_target": 0, + "support_probability": 0.020921895280480385, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23073703112328384, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5380274057388306, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7678140997886658, + 0.11374369263648987, + 0.43840187788009644, + 0.4976870119571686, + 0.017304830253124237, + 0.03510737791657448, + 0.04341345652937889, + 0.03366396948695183, + 0.06646130234003067, + 0.032371122390031815 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 1, + "style_probability": 0.5898289084434509, + "style_target": 1, + "support_probability": 0.18991291522979736, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2085749002799123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6497493386268616, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7959238290786743, + 0.13446809351444244, + 0.3911515474319458, + 0.519257128238678, + 0.059406477957963943, + 0.07227129489183426, + 0.11044829338788986, + 0.10019083321094513, + 0.024712033569812775, + 0.030672568827867508, + 0.08712183684110641 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 0, + "style_probability": 0.5179357528686523, + "style_target": 0, + "support_probability": 0.10589339584112167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47685627894601, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7513731122016907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3233553469181061, + 0.27166748046875, + 0.31485065817832947, + 0.82328861951828, + 0.11766646802425385 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 1, + "style_probability": 0.8797951936721802, + "style_target": 1, + "support_probability": 0.38231775164604187, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1110626703091163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8057423830032349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.279327929019928, + 0.19956541061401367, + 0.3013004958629608, + 0.771878719329834, + 0.19668011367321014 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 1, + "style_probability": 0.44385287165641785, + "style_target": 0, + "support_probability": 0.30257415771484375, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6942116144216475, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4332844614982605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2512153089046478, + 0.2512153089046478, + 0.8800324201583862, + 0.3141828179359436, + 0.048089295625686646, + 0.0925004780292511 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 1, + "style_probability": 0.853385329246521, + "style_target": 1, + "support_probability": 0.5060442686080933, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47552278092694844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.47837984561920166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25206953287124634, + 0.25206953287124634, + 0.8498661518096924, + 0.32488295435905457, + 0.05670502036809921, + 0.09964671730995178 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 0, + "style_probability": 0.5048567652702332, + "style_target": 0, + "support_probability": 0.5217142105102539, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3967220282831545, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4651763141155243, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27472928166389465, + 0.20176251232624054, + 0.049822840839624405, + 0.8026623129844666, + 0.056835420429706573, + 0.06582547724246979, + 0.020594431087374687, + 0.03722452372312546, + 0.13558164238929749 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.7107193470001221, + "style_target": 1, + "support_probability": 0.46718305349349976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024067901807146196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4941554069519043, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37951594591140747, + 0.19302143156528473, + 0.053664688020944595, + 0.8702500462532043, + 0.0843871608376503, + 0.35091322660446167, + 0.10697159171104431, + 0.05751737952232361, + 0.12249451875686646 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.09145307540893555, + "style_target": 0, + "support_probability": 0.41256392002105713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31658010790544544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.66202312707901, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8286464810371399, + 0.20319128036499023, + 0.4917094111442566, + 0.20329000055789948, + 0.06261415779590607, + 0.045393772423267365, + 0.05301075056195259, + 0.07634750008583069, + 0.04198405519127846, + 0.07711184024810791, + 0.1238938421010971 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 1, + "style_probability": 0.7423674464225769, + "style_target": 1, + "support_probability": 0.23909707367420197, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16619287165928626, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5958704352378845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7750426530838013, + 0.2290976196527481, + 0.43719029426574707, + 0.22069859504699707, + 0.0600907988846302, + 0.045655202120542526, + 0.12601885199546814, + 0.05606527253985405, + 0.09124284982681274, + 0.13641059398651123 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 1, + "style_probability": 0.4447261691093445, + "style_target": 0, + "support_probability": 0.2911621630191803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6626383443228978, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14868147671222687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5838620662689209, + 0.4642467498779297, + 0.7134642601013184, + 0.6015055775642395, + 0.05954575538635254 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.8577282428741455, + "style_target": 1, + "support_probability": 0.9056127667427063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5266559290824001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3136514127254486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5482539534568787, + 0.3818749189376831, + 0.69430011510849, + 0.5377505421638489, + 0.1643436849117279 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.36229267716407776, + "style_target": 0, + "support_probability": 0.7701712250709534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10310215648738758, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7709689736366272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2707793712615967, + 0.18195219337940216, + 0.33446988463401794, + 0.8684987425804138, + 0.0658532902598381, + 0.09731946140527725 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.6713745594024658, + "style_target": 1, + "support_probability": 0.07832121104001999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01525811530737671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8667125105857849, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33542755246162415, + 0.2253962904214859, + 0.296962708234787, + 0.9327311515808105, + 0.2153782695531845, + 0.13901318609714508 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.13107310235500336, + "style_target": 0, + "support_probability": 0.05831420049071312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1278459307193644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9181053042411804, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2455364614725113, + 0.4433116912841797, + 0.2455364614725113, + 0.6339566111564636, + 0.06987440586090088, + 0.06427023559808731, + 0.050334084779024124, + 0.09742888063192368, + 0.05675111338496208 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 1, + "style_probability": 0.874062716960907, + "style_target": 1, + "support_probability": 0.06471167504787445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1112578779051846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8233159184455872, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2846211791038513, + 0.3684620261192322, + 0.2846211791038513, + 0.6601777076721191, + 0.0562758632004261, + 0.057703811675310135, + 0.021885046735405922, + 0.04967761039733887, + 0.06862489879131317 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 0, + "style_probability": 0.6266763210296631, + "style_target": 0, + "support_probability": 0.17753079533576965, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3601678916963864, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8775361180305481, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18164049088954926, + 0.3986108601093292, + 0.2627600431442261, + 0.6036720275878906, + 0.050536368042230606, + 0.0456843376159668, + 0.0426362119615078, + 0.04700180143117905, + 0.03491370379924774, + 0.06434983760118484, + 0.0866241455078125 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.9091629981994629, + "style_target": 1, + "support_probability": 0.16354413330554962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030942345999743832, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9390472769737244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2296430468559265, + 0.3174075782299042, + 0.25740960240364075, + 0.547740638256073, + 0.1611616015434265, + 0.04144733399152756, + 0.06702163070440292, + 0.06634122133255005, + 0.03334280848503113, + 0.07804863154888153, + 0.0941716879606247 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.24352125823497772, + "style_target": 0, + "support_probability": 0.05105648934841156, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047977838068014166, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8718962669372559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26950570940971375, + 0.48081278800964355, + 0.2851130962371826, + 0.36368459463119507, + 0.15226112306118011 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.7942522764205933, + "style_target": 1, + "support_probability": 0.04995071515440941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008206540759297805, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.919095516204834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35287874937057495, + 0.5739428997039795, + 0.30944785475730896, + 0.3781757354736328, + 0.1737751066684723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.20039045810699463, + "style_target": 0, + "support_probability": 0.06315924227237701, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14510119191675744, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.47465837001800537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23767279088497162, + 0.21170467138290405, + 0.5989739298820496, + 0.2813083827495575, + 0.0484497956931591, + 0.07267475873231888 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.7986708283424377, + "style_target": 1, + "support_probability": 0.1823297142982483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028617911923700994, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7266685366630554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1973397433757782, + 0.20154085755348206, + 0.5147045254707336, + 0.23331132531166077, + 0.1601024568080902, + 0.12377599626779556 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.4877423346042633, + "style_target": 0, + "support_probability": 0.06573594361543655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4016814771350851, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7511950135231018, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6543376445770264, + 0.05827878415584564, + 0.504273533821106, + 0.6543376445770264, + 0.05928844213485718, + 0.1667591780424118, + 0.07621818035840988, + 0.05610502138733864, + 0.08265291899442673 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.8032649755477905, + "style_target": 1, + "support_probability": 0.29385966062545776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14892715950065613, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7846407294273376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6535239219665527, + 0.06820008903741837, + 0.4639297425746918, + 0.6535239219665527, + 0.03382663056254387, + 0.01862242817878723, + 0.17182420194149017, + 0.0880180150270462, + 0.1014765202999115 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.3195417821407318, + "style_target": 0, + "support_probability": 0.27576932311058044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0049156116022687915, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.834523618221283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5193727016448975, + 0.47493264079093933, + 0.20940089225769043, + 0.12168153375387192, + 0.15033011138439178, + 0.11211196333169937, + 0.11883553117513657, + 0.22675703465938568, + 0.23630744218826294, + 0.06696033477783203, + 0.07154670357704163 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.5383957624435425, + "style_target": 1, + "support_probability": 0.005184690933674574, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002276869416118303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8397419452667236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5551962852478027, + 0.4792163670063019, + 0.23315058648586273, + 0.15454134345054626, + 0.1405762881040573, + 0.08915281295776367, + 0.3391401469707489, + 0.11506001651287079, + 0.22246558964252472, + 0.12245196104049683 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.17870180308818817, + "style_target": 0, + "support_probability": 0.006752749904990196, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5060918809627882, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6615678668022156, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34596481919288635, + 0.45030608773231506, + 0.43889251351356506, + 0.9341922998428345, + 0.07322309166193008 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 1, + "style_probability": 0.7370750904083252, + "style_target": 1, + "support_probability": 0.38918885588645935, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28270878164858276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7573615908622742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29552021622657776, + 0.40153926610946655, + 0.365957111120224, + 0.9016008377075195, + 0.13038431107997894 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 1, + "style_probability": 0.39460286498069763, + "style_target": 0, + "support_probability": 0.38048020005226135, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7956245997338519, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029448483139276505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.553352415561676, + 0.9313725233078003, + 0.4209255278110504, + 0.6839967966079712, + 0.06533897668123245, + 0.027554133906960487 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.7791768908500671, + "style_target": 1, + "support_probability": 0.9017664194107056, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5301352777370916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029261382296681404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4535757601261139, + 0.904461681842804, + 0.34571048617362976, + 0.6410656571388245, + 0.05166918411850929, + 0.05339060351252556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.33266541361808777, + "style_target": 0, + "support_probability": 0.8427351117134094, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32286165138300227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.168343186378479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.742710292339325, + 0.195829838514328, + 0.3458005487918854, + 0.27280348539352417, + 0.03637827932834625, + 0.08042704313993454, + 0.17977574467658997, + 0.15315434336662292, + 0.05071873590350151 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.6243681311607361, + "style_target": 1, + "support_probability": 0.6299974918365479, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03348874353473182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2042597085237503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7953081130981445, + 0.1755867302417755, + 0.3525516986846924, + 0.3344132602214813, + 0.06002718582749367, + 0.23769859969615936, + 0.08280614018440247, + 0.07714749872684479, + 0.06908730417490005 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.11401938647031784, + "style_target": 0, + "support_probability": 0.5783915519714355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7134702543056232, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13191647827625275, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.796263575553894, + 0.1373961865901947, + 0.5743735432624817, + 0.7783672213554382, + 0.07280384749174118, + 0.037735339254140854, + 0.03822213411331177, + 0.16261303424835205, + 0.1054295152425766, + 0.16634075343608856, + 0.03953302279114723 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.6332073211669922, + "style_target": 1, + "support_probability": 0.9482178092002869, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5109273473266109, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3542889356613159, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7282019853591919, + 0.14213065803050995, + 0.5000010132789612, + 0.6622250080108643, + 0.1664060652256012, + 0.08969200402498245, + 0.0701243057847023, + 0.0884891226887703, + 0.03492072597146034, + 0.04972727224230766, + 0.07036174833774567 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.31210729479789734, + "style_target": 0, + "support_probability": 0.8962550163269043, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09834453715740878, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6765482425689697, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2923307418823242, + 0.8484887480735779, + 0.4127260744571686, + 0.38907065987586975, + 0.08682504296302795 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.5995675325393677, + "style_target": 1, + "support_probability": 0.15175604820251465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021019663334884798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7814093828201294, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29001766443252563, + 0.8228065371513367, + 0.38016974925994873, + 0.36789315938949585, + 0.13861487805843353 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.2361643761396408, + "style_target": 0, + "support_probability": 0.044576697051525116, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7895274644686433, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10535223037004471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3888368010520935, + 0.9868298172950745, + 0.1276136338710785, + 0.1281694769859314, + 0.03674905374646187, + 0.08753430098295212 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.7036265134811401, + "style_target": 1, + "support_probability": 0.75515216588974, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15828472082049366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15706755220890045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4792584478855133, + 0.9938643574714661, + 0.1907244324684143, + 0.19230201840400696, + 0.05319777876138687, + 0.10232073813676834 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.14245285093784332, + "style_target": 0, + "support_probability": 0.6631385087966919, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041239462555195054, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.916374921798706, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19047531485557556, + 0.5606256723403931, + 0.4261413514614105, + 0.4568760395050049, + 0.2550845146179199, + 0.10665226727724075, + 0.11152329295873642, + 0.03458261862397194, + 0.09997133165597916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 1, + "style_probability": 0.6139532923698425, + "style_target": 1, + "support_probability": 0.09504906088113785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06393393885997292, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8605427742004395, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1777031272649765, + 0.5374956727027893, + 0.38155266642570496, + 0.47393539547920227, + 0.06442996859550476, + 0.2411198616027832, + 0.06007785722613335, + 0.19645167887210846, + 0.1477680206298828 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 0, + "style_probability": 0.5029801726341248, + "style_target": 0, + "support_probability": 0.1212504431605339, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4521635179615753, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48308223485946655, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5222618579864502, + 0.30855199694633484, + 0.3178155720233917, + 0.7010471224784851, + 0.07858636230230331, + 0.04953170195221901, + 0.26319101452827454, + 0.1139879822731018, + 0.07596749812364578, + 0.03718995675444603, + 0.07753702253103256 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.6607182025909424, + "style_target": 1, + "support_probability": 0.5706101059913635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2385389137113056, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7101144790649414, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:negative", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.4906337261199951, + 0.29439330101013184, + 0.3197802007198334, + 0.6426112055778503, + 0.14136023819446564, + 0.06931404769420624, + 0.08896932750940323, + 0.05268055573105812, + 0.10936978459358215, + 0.08995109796524048, + 0.07372903823852539, + 0.10420822352170944, + 0.1257181018590927, + 0.06329576671123505 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.3872305452823639, + "style_target": 0, + "support_probability": 0.29146459698677063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3314471270481763, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6489530801773071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.630165696144104, + 0.2648317813873291, + 0.5905006527900696, + 0.12578129768371582, + 0.12480753660202026 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.8194984793663025, + "style_target": 1, + "support_probability": 0.32945048809051514, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04993719325900616, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7259622812271118, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8491184115409851, + 0.34877046942710876, + 0.7155288457870483, + 0.20092859864234924, + 0.14807943999767303 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.1876196563243866, + "style_target": 0, + "support_probability": 0.328620046377182, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3018789159367936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27376314997673035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3478420376777649, + 0.4512287676334381, + 0.19590450823307037, + 0.9941917061805725, + 0.10739242285490036, + 0.07608446478843689 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 1, + "style_probability": 0.872179388999939, + "style_target": 1, + "support_probability": 0.14667503535747528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26971765421214866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1694072186946869, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32188865542411804, + 0.48716673254966736, + 0.23093236982822418, + 0.9872238636016846, + 0.16672638058662415, + 0.08464076370000839 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 0, + "style_probability": 0.683039665222168, + "style_target": 0, + "support_probability": 0.2336510419845581, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6436700332153587, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19824302196502686, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1644502878189087, + 0.8037548065185547, + 0.2120659053325653, + 0.7619352340698242, + 0.07334767282009125, + 0.15228593349456787, + 0.08701169490814209, + 0.11134455353021622, + 0.08265405148267746 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 1, + "style_probability": 0.7635775804519653, + "style_target": 1, + "support_probability": 0.7319201231002808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4681940248294925, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1981508731842041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19711615145206451, + 0.78947913646698, + 0.18713857233524323, + 0.7096109986305237, + 0.05408184975385666, + 0.12423320859670639 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 1, + "style_probability": 0.400749146938324, + "style_target": 0, + "support_probability": 0.7019113302230835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0945292795128781, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9349769949913025, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42624130845069885, + 0.13333918154239655, + 0.40952304005622864, + 0.13459491729736328, + 0.06825802475214005, + 0.059318218380212784, + 0.04431728646159172, + 0.10607340186834335, + 0.08455350250005722, + 0.10345368087291718, + 0.16559287905693054 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.8218749165534973, + "style_target": 1, + "support_probability": 0.07826478034257889, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004556439423182028, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9666487574577332, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40374383330345154, + 0.20327068865299225, + 0.4696353077888489, + 0.23190352320671082, + 0.161713108420372, + 0.06475529819726944, + 0.3064388930797577, + 0.09936190396547318, + 0.05620407313108444, + 0.09314049780368805 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.128830224275589, + "style_target": 0, + "support_probability": 0.0717424601316452, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6247696050534375, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36068058013916016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8607606887817383, + 0.5385708808898926, + 0.4394058883190155, + 0.2384047508239746, + 0.049537237733602524 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.7581803202629089, + "style_target": 1, + "support_probability": 0.6688247919082642, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43232027803544054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33482876420021057, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8206226229667664, + 0.40918561816215515, + 0.42875540256500244, + 0.2329747974872589, + 0.19087976217269897 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.32438117265701294, + "style_target": 0, + "support_probability": 0.6934599876403809, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6170793272183432, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0752524882555008, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.220892533659935, + 0.30819568037986755, + 0.9896886348724365, + 0.2532220780849457, + 0.11143998056650162, + 0.05372817814350128 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.5427409410476685, + "style_target": 1, + "support_probability": 0.579391360282898, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6774169348828936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05983753502368927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20846450328826904, + 0.3051935136318207, + 0.9833574295043945, + 0.2704838216304779, + 0.09905274957418442 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.3752201795578003, + "style_target": 0, + "support_probability": 0.6795235276222229, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44659528645966423, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41209402680397034, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13216158747673035, + 0.4624043107032776, + 0.4589550495147705, + 0.5912243127822876, + 0.051144301891326904, + 0.04741276800632477, + 0.11190146207809448, + 0.06487423181533813, + 0.031812720000743866 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.7565426826477051, + "style_target": 1, + "support_probability": 0.48930346965789795, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012993841949861763, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5654255151748657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25566595792770386, + 0.6238780617713928, + 0.5529651641845703, + 0.6749217510223389, + 0.08951056748628616, + 0.10028795897960663, + 0.10156740248203278, + 0.10401718318462372, + 0.10862095654010773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.04315616190433502, + "style_target": 0, + "support_probability": 0.3888706862926483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038575592952498135, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9657834768295288, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5812997817993164, + 0.7017664909362793, + 0.36242491006851196, + 0.4478690028190613, + 0.05633942037820816, + 0.09212839603424072, + 0.11067212373018265, + 0.16871868073940277, + 0.06445500254631042, + 0.0915353074669838 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.7663114666938782, + "style_target": 1, + "support_probability": 0.02217158116400242, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.048651828362625604, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9199318885803223, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5645383596420288, + 0.6870018839836121, + 0.315965473651886, + 0.4486684203147888, + 0.06333236396312714, + 0.07776229083538055, + 0.08744419366121292, + 0.14115481078624725, + 0.09713389724493027, + 0.07326800376176834, + 0.09165177494287491 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.49651646614074707, + "style_target": 0, + "support_probability": 0.04626459255814552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21564699130301487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4391568899154663, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3171927034854889, + 0.49651575088500977, + 0.21498645842075348, + 0.9226120114326477, + 0.13971421122550964 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 1, + "style_probability": 0.6900899410247803, + "style_target": 1, + "support_probability": 0.14610503613948822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1783097996339236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6030537486076355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33507296442985535, + 0.4311850666999817, + 0.22863632440567017, + 0.895045280456543, + 0.09630163013935089, + 0.06856045126914978, + 0.25118985772132874, + 0.11451036483049393, + 0.15976694226264954, + 0.08209923654794693, + 0.07326837629079819 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 1, + "style_probability": 0.4272857904434204, + "style_target": 0, + "support_probability": 0.10718046873807907, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7015613571402858, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1928659826517105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8947708010673523, + 0.3461020290851593, + 0.20484162867069244, + 0.47802165150642395, + 0.032538700848817825, + 0.07210336625576019 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.6987844109535217, + "style_target": 1, + "support_probability": 0.5835601091384888, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24639628768356295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20992320775985718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9665049910545349, + 0.4429818391799927, + 0.23954907059669495, + 0.49853160977363586, + 0.10073261708021164, + 0.10102169215679169 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.17672914266586304, + "style_target": 0, + "support_probability": 0.606942355632782, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5068495766076225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.723886251449585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1668820083141327, + 0.1486092507839203, + 0.24693770706653595, + 0.8106205463409424, + 0.03918706253170967, + 0.15965403616428375, + 0.10722199827432632, + 0.060197360813617706, + 0.06973689049482346 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.8896393775939941, + "style_target": 1, + "support_probability": 0.1911531388759613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17857434277801448, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5302925109863281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15441884100437164, + 0.16435973346233368, + 0.21966075897216797, + 0.8102749586105347, + 0.07813777029514313, + 0.1371617615222931, + 0.1218784898519516, + 0.04864927753806114, + 0.0675923079252243 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.4203243851661682, + "style_target": 0, + "support_probability": 0.2189255803823471, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7836661843089757, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1623826026916504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13765855133533478, + 0.22354084253311157, + 0.25085312128067017, + 0.8766918182373047, + 0.018943382427096367, + 0.03140184283256531, + 0.02041291445493698, + 0.0518643744289875, + 0.04266640916466713, + 0.020301038399338722, + 0.030320435762405396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.8272360563278198, + "style_target": 1, + "support_probability": 0.9400824904441833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5613279143185477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25172728300094604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16463489830493927, + 0.18506036698818207, + 0.24523429572582245, + 0.8471677899360657, + 0.07498405128717422, + 0.046560220420360565, + 0.06235058605670929, + 0.06344813853502274, + 0.03139769658446312 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.3622870147228241, + "style_target": 0, + "support_probability": 0.8904299736022949, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2973531976740554, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6611583828926086, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24002379179000854, + 0.401108056306839, + 0.15845006704330444, + 0.5227053761482239, + 0.11142493784427643 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.903140664100647, + "style_target": 1, + "support_probability": 0.2667955756187439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01515609715616834, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8923054337501526, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23735006153583527, + 0.4753674566745758, + 0.20571672916412354, + 0.6077924370765686, + 0.1269463151693344 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.24000020325183868, + "style_target": 0, + "support_probability": 0.17296527326107025, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8047436270070587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13774022459983826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8968766331672668, + 0.6185325980186462, + 0.23966443538665771, + 0.378287136554718, + 0.05632231757044792, + 0.07079866528511047 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 1, + "style_probability": 0.9325489401817322, + "style_target": 1, + "support_probability": 0.7623639702796936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6698303079356885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04752398654818535, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9295077919960022, + 0.6326814293861389, + 0.18174214661121368, + 0.3791162073612213, + 0.12459439784288406, + 0.2127314954996109 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 1, + "style_probability": 0.49597883224487305, + "style_target": 0, + "support_probability": 0.8191335797309875, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5505324731478348, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19698084890842438, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9800381064414978, + 0.23410646617412567, + 0.21445155143737793, + 0.29993367195129395, + 0.04811515659093857, + 0.06341797113418579, + 0.15233556926250458, + 0.1212419718503952, + 0.041900210082530975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.6535316705703735, + "style_target": 1, + "support_probability": 0.443509042263031, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41765422278801445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17977802455425262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9721196293830872, + 0.22193865478038788, + 0.19216446578502655, + 0.2878459393978119, + 0.08241447061300278, + 0.03389504924416542 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.30127373337745667, + "style_target": 0, + "support_probability": 0.49376389384269714, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8143833408079768, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05271204188466072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19132371246814728, + 0.37891629338264465, + 0.9195669889450073, + 0.3069269061088562, + 0.09029607474803925, + 0.0896705910563469, + 0.04969801753759384, + 0.07354103773832321, + 0.0790691003203392, + 0.037122856825590134 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.7277678847312927, + "style_target": 1, + "support_probability": 0.970192551612854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17398484194150685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.089628204703331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23987534642219543, + 0.4767850637435913, + 0.951467752456665, + 0.5374671220779419, + 0.08243491500616074, + 0.10691951960325241, + 0.08417057245969772, + 0.0552508719265461, + 0.08833582699298859, + 0.07492417097091675 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.0885973572731018, + "style_target": 0, + "support_probability": 0.9495998024940491, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6256150133701291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1002749428153038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3290444314479828, + 0.0694885104894638, + 0.317134827375412, + 0.9675734043121338, + 0.08259705454111099 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 1, + "style_probability": 0.8194558620452881, + "style_target": 1, + "support_probability": 0.4534422755241394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4655802369151126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1305055469274521, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36373695731163025, + 0.07966144382953644, + 0.33893951773643494, + 0.9546914100646973, + 0.07887361198663712 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 0, + "style_probability": 0.6470246315002441, + "style_target": 0, + "support_probability": 0.45572325587272644, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.675289041603924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5088390111923218, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5155301094055176, + 0.921413242816925, + 0.41910320520401, + 0.21038588881492615, + 0.040090568363666534, + 0.05191480368375778 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.6615248918533325, + "style_target": 1, + "support_probability": 0.589464008808136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3230108015033075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5722506642341614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.495976060628891, + 0.8862548470497131, + 0.36413219571113586, + 0.21532119810581207, + 0.08677693456411362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.25048133730888367, + "style_target": 0, + "support_probability": 0.4334646761417389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19675503528533198, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8029637932777405, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5734478831291199, + 0.2746794521808624, + 0.29346439242362976, + 0.2555611729621887, + 0.05935161933302879, + 0.07231774181127548, + 0.061295200139284134, + 0.06242785602807999, + 0.04925765097141266 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.8237141966819763, + "style_target": 1, + "support_probability": 0.09395254403352737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011506084516832151, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.721385657787323, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6461862921714783, + 0.35239431262016296, + 0.37632912397384644, + 0.18159474432468414, + 0.12725387513637543, + 0.2793220281600952, + 0.06808141618967056, + 0.167887344956398, + 0.13531725108623505 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.12026166170835495, + "style_target": 0, + "support_probability": 0.15370984375476837, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.584686750387768, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1956591010093689, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23926737904548645, + 0.9242916703224182, + 0.3599027693271637, + 0.6558075547218323, + 0.05490673705935478, + 0.16483305394649506, + 0.043574996292591095, + 0.054717086255550385, + 0.06557570397853851, + 0.09316866099834442, + 0.046152129769325256 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 1, + "style_probability": 0.6423314809799194, + "style_target": 1, + "support_probability": 0.6116328239440918, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5185098546811883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2217010259628296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2438652068376541, + 0.8873304128646851, + 0.3951868712902069, + 0.5859114527702332, + 0.09365521371364594, + 0.09855411946773529, + 0.06920502334833145, + 0.08937516808509827, + 0.12298576533794403, + 0.07591116428375244, + 0.08778950572013855 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 0, + "style_probability": 0.5820093750953674, + "style_target": 0, + "support_probability": 0.5343013405799866, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08337052800931721, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7614248991012573, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6372784972190857, + 0.4572945833206177, + 0.19117598235607147, + 0.18735258281230927, + 0.09440523386001587 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 1, + "style_probability": 0.635520339012146, + "style_target": 1, + "support_probability": 0.09096233546733856, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3619843080778447, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7149838209152222, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.646733820438385, + 0.4429458677768707, + 0.15677882730960846, + 0.21105289459228516, + 0.0636182650923729 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 0, + "style_probability": 0.9126158356666565, + "style_target": 0, + "support_probability": 0.17574530839920044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43855489959370003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20384001731872559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8527395725250244, + 0.5657007098197937, + 0.607705295085907, + 0.37027090787887573, + 0.16356244683265686, + 0.16438040137290955 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 0, + "style_probability": 0.38104185461997986, + "style_target": 1, + "support_probability": 0.5945510864257812, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4532602983971678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19675756990909576, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8429844379425049, + 0.519371747970581, + 0.613696277141571, + 0.40841296315193176, + 0.09804865717887878 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 1, + "style_probability": 0.2669645845890045, + "style_target": 0, + "support_probability": 0.6273048520088196, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6272361453239333, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2136605679988861, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25828447937965393, + 0.25151440501213074, + 0.8720571398735046, + 0.18224623799324036, + 0.11901213228702545, + 0.06419535726308823, + 0.05125897750258446, + 0.213825985789299, + 0.16018067300319672 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 0, + "style_probability": 0.43251752853393555, + "style_target": 1, + "support_probability": 0.851176917552948, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16458395800457912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2934426963329315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30138352513313293, + 0.2098330408334732, + 0.9526808261871338, + 0.27371251583099365, + 0.19107678532600403, + 0.3012308180332184, + 0.1074678897857666, + 0.11417354643344879, + 0.1009160578250885 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 1, + "style_probability": 0.1786857396364212, + "style_target": 0, + "support_probability": 0.7596983313560486, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13895744330036194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7953469753265381, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25004929304122925, + 0.26802003383636475, + 0.2837541401386261, + 0.8895105719566345, + 0.12346797436475754, + 0.24907486140727997, + 0.24341094493865967, + 0.2501949667930603, + 0.1010890081524849, + 0.3507181704044342, + 0.22935841977596283 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.39626625180244446, + "style_target": 1, + "support_probability": 0.24676360189914703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1584694187355318, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8969155550003052, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30289173126220703, + 0.32739144563674927, + 0.2817986309528351, + 0.8776479959487915, + 0.2553662061691284, + 0.1355242133140564, + 0.0916689783334732, + 0.21631570160388947, + 0.14783872663974762, + 0.10224192589521408, + 0.11716600507497787 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.7622557878494263, + "style_target": 0, + "support_probability": 0.1539199948310852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7419561629361656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13198897242546082, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.773749589920044, + 0.6343244910240173, + 0.8640430569648743, + 0.5225187540054321, + 0.20905959606170654 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.8160035014152527, + "style_target": 1, + "support_probability": 0.841061532497406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2950582098237391, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22082269191741943, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8385238647460938, + 0.736552894115448, + 0.9508834481239319, + 0.6829349994659424, + 0.17883816361427307 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.18985708057880402, + "style_target": 0, + "support_probability": 0.8001865744590759, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24832786457456857, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8231953978538513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3519136607646942, + 0.6886798143386841, + 0.6862766146659851, + 0.35839441418647766, + 0.2206689864397049, + 0.1541069746017456 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 1, + "style_probability": 0.6844199299812317, + "style_target": 1, + "support_probability": 0.3543807864189148, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0860439932641645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.886940062046051, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35726645588874817, + 0.5773993730545044, + 0.6374053955078125, + 0.4083412289619446, + 0.038231756538152695, + 0.09779387712478638 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 1, + "style_probability": 0.43452268838882446, + "style_target": 0, + "support_probability": 0.2534460425376892, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8671794242281491, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045543868094682693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16146306693553925, + 0.5980086326599121, + 0.14409559965133667, + 0.9946374297142029, + 0.09575550258159637, + 0.03139859437942505, + 0.11657606810331345, + 0.08295770734548569, + 0.11559765040874481 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.80154949426651, + "style_target": 1, + "support_probability": 0.8947834968566895, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6171320943522165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08279422670602798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18820370733737946, + 0.5846541523933411, + 0.17289869487285614, + 0.9883913397789001, + 0.04833067208528519, + 0.07152777165174484 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.4080195724964142, + "style_target": 0, + "support_probability": 0.7581446766853333, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6233577901458911, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2983234226703644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31046003103256226, + 0.5883460640907288, + 0.31046003103256226, + 0.9439353346824646, + 0.05406760424375534, + 0.14880408346652985, + 0.09021085500717163, + 0.09461890906095505, + 0.060546934604644775, + 0.30258631706237793 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.7392480373382568, + "style_target": 1, + "support_probability": 0.5300738215446472, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24707973492044275, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41262513399124146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3214055001735687, + 0.5586045980453491, + 0.3214055001735687, + 0.9418866038322449, + 0.1186675876379013, + 0.13354766368865967, + 0.14190778136253357, + 0.08726532757282257, + 0.05979219079017639, + 0.11522772908210754 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.3296586573123932, + "style_target": 0, + "support_probability": 0.5662034153938293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24152226243664954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4373798072338104, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2697373628616333, + 0.29875051975250244, + 0.2697373628616333, + 0.7737892866134644, + 0.12540514767169952 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.8194206357002258, + "style_target": 1, + "support_probability": 0.29875367879867554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11111921810262282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.35845303535461426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26242953538894653, + 0.3324010968208313, + 0.26242953538894653, + 0.7932919859886169, + 0.1885906159877777 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.411240816116333, + "style_target": 0, + "support_probability": 0.2696276605129242, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37155475516781394, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39907121658325195, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8870396614074707, + 0.7523389458656311, + 0.4132104814052582, + 0.4954528212547302, + 0.05857003107666969, + 0.1407441347837448 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 1, + "style_probability": 0.8823587894439697, + "style_target": 1, + "support_probability": 0.26157599687576294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27690093314896114, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38478708267211914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8638055920600891, + 0.7100635766983032, + 0.4303132891654968, + 0.521489679813385, + 0.09371662884950638 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 0, + "style_probability": 0.6138758659362793, + "style_target": 0, + "support_probability": 0.3525797426700592, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8681561748441026, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30938974022865295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5747347474098206, + 0.9743624925613403, + 0.7131946682929993, + 0.2408730685710907, + 0.0632273480296135, + 0.11584603786468506, + 0.04432272911071777, + 0.08038896322250366, + 0.12592671811580658 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.877317488193512, + "style_target": 1, + "support_probability": 0.8220685124397278, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17765959731202988, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3908109962940216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.644804835319519, + 0.9928370118141174, + 0.6807746291160583, + 0.23768405616283417, + 0.08488573879003525, + 0.16747744381427765, + 0.1460115909576416, + 0.0663764700293541, + 0.1827978938817978 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.14844489097595215, + "style_target": 0, + "support_probability": 0.744537353515625, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7838089519944985, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09516303986310959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2579215168952942, + 0.3177143931388855, + 0.866990327835083, + 0.4182390868663788, + 0.05505239963531494, + 0.05100139230489731, + 0.08880900591611862, + 0.050395797938108444, + 0.04421444609761238, + 0.0981898084282875 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 1, + "style_probability": 0.8484721183776855, + "style_target": 1, + "support_probability": 0.9628222584724426, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5920168154167245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2516133487224579, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21321521699428558, + 0.3477971851825714, + 0.7685540914535522, + 0.4523079991340637, + 0.09742741286754608, + 0.03956751525402069, + 0.04663565382361412, + 0.048023950308561325, + 0.08504406362771988, + 0.2001856416463852 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 0, + "style_probability": 0.5525095462799072, + "style_target": 0, + "support_probability": 0.9476135969161987, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30633424766116574, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17183667421340942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16384606063365936, + 0.8428903818130493, + 0.15221580862998962, + 0.25720909237861633, + 0.13929955661296844 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.24968799948692322, + "style_target": 1, + "support_probability": 0.5457415580749512, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3671194414767046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15634164214134216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16839444637298584, + 0.863426923751831, + 0.1650378704071045, + 0.26060575246810913, + 0.035629838705062866 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.5061943531036377, + "style_target": 0, + "support_probability": 0.5123692750930786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22606963129796614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3187328577041626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.406049519777298, + 0.8228856921195984, + 0.40179166197776794, + 0.178677037358284, + 0.18986406922340393, + 0.15277665853500366 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 0, + "style_probability": 0.45263218879699707, + "style_target": 1, + "support_probability": 0.2904731035232544, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36110714130458216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07544410973787308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.371481329202652, + 0.940559983253479, + 0.27181535959243774, + 0.15599071979522705, + 0.051995232701301575, + 0.10115591436624527 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 1, + "style_probability": 0.3801835775375366, + "style_target": 0, + "support_probability": 0.7959561944007874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43143088825165965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1618780642747879, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42887333035469055, + 0.25808948278427124, + 0.8490343689918518, + 0.4712778329849243, + 0.11350268870592117, + 0.13422714173793793, + 0.10347455739974976, + 0.22717638313770294, + 0.11061771959066391 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.418300986289978, + "style_target": 1, + "support_probability": 0.5689720511436462, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39509922234336026, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21890772879123688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4853745400905609, + 0.27601465582847595, + 0.8378210663795471, + 0.48603126406669617, + 0.17577512562274933, + 0.12125947326421738, + 0.2249809205532074, + 0.08592267334461212, + 0.2208823263645172 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.6293143630027771, + "style_target": 0, + "support_probability": 0.4316192865371704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047098852566577164, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6358731389045715, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.453972727060318, + 0.3337658941745758, + 0.15539668500423431, + 0.3659871220588684, + 0.11343557387590408, + 0.1434314101934433, + 0.06607235968112946, + 0.07446505129337311, + 0.08949078619480133, + 0.15212640166282654, + 0.11382788419723511 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.2662689685821533, + "style_target": 1, + "support_probability": 0.17919225990772247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10698792346526567, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5768789649009705, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44662705063819885, + 0.35566866397857666, + 0.13560955226421356, + 0.3832724392414093, + 0.06930036097764969, + 0.037268731743097305, + 0.08109772950410843, + 0.05749490484595299, + 0.12939806282520294, + 0.05437096953392029 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.5950387716293335, + "style_target": 0, + "support_probability": 0.2838272750377655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11010523437918067, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3527030944824219, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27489328384399414, + 0.5895319581031799, + 0.4221043586730957, + 0.402080237865448, + 0.0963326171040535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.6371544003486633, + "style_target": 1, + "support_probability": 0.1750039905309677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015651824554526555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42752593755722046, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34752383828163147, + 0.6980298757553101, + 0.525963306427002, + 0.462027907371521, + 0.16438239812850952 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.14742256700992584, + "style_target": 0, + "support_probability": 0.23342019319534302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38069254234240796, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5945028066635132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41962161660194397, + 0.6476477980613708, + 0.6272876262664795, + 0.7683107852935791, + 0.08424290269613266, + 0.06462646275758743 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 1, + "style_probability": 0.8735135793685913, + "style_target": 1, + "support_probability": 0.3117905557155609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040444453593804865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7917497754096985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3844929039478302, + 0.6589465141296387, + 0.5903192758560181, + 0.6296514272689819, + 0.16019068658351898, + 0.06759443134069443 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 1, + "style_probability": 0.488016813993454, + "style_target": 0, + "support_probability": 0.08440948277711868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13873111905449975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9579494595527649, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35232922434806824, + 0.5231106877326965, + 0.6375503540039062, + 0.3833014667034149, + 0.06442302465438843, + 0.12946651875972748, + 0.06063822656869888, + 0.06160782277584076, + 0.05917008966207504 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.8580511212348938, + "style_target": 1, + "support_probability": 0.06172298640012741, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022300644875476925, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.976967453956604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3236599266529083, + 0.521161675453186, + 0.5971760153770447, + 0.35898539423942566, + 0.1449897438287735, + 0.17785318195819855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.30774256587028503, + "style_target": 0, + "support_probability": 0.017610155045986176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014226308160989883, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9895221590995789, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21941488981246948, + 0.3846770226955414, + 0.3158738911151886, + 0.4846422076225281, + 0.09532620757818222, + 0.13829605281352997, + 0.11347409337759018, + 0.09732932597398758, + 0.11176974326372147, + 0.08543583750724792, + 0.09572234749794006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.7075672745704651, + "style_target": 1, + "support_probability": 0.01594381593167782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016476212723657716, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9905732274055481, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23751196265220642, + 0.32398372888565063, + 0.38703247904777527, + 0.5092479586601257, + 0.10099426656961441, + 0.12245859950780869, + 0.19982028007507324, + 0.15642467141151428, + 0.17324085533618927, + 0.08874440938234329 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.10572155565023422, + "style_target": 0, + "support_probability": 0.00952291302382946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8220562757755139, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12529116868972778, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5617526173591614, + 0.9951325058937073, + 0.23550328612327576, + 0.24859489500522614, + 0.11536824703216553 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 1, + "style_probability": 0.7689536809921265, + "style_target": 1, + "support_probability": 0.5601574182510376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4702182458147914, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22882132232189178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5585207939147949, + 0.9778574705123901, + 0.2298898994922638, + 0.30491164326667786, + 0.10136678814888 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 0, + "style_probability": 0.5771504640579224, + "style_target": 0, + "support_probability": 0.3614138066768646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37475681569442193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28678280115127563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9554396271705627, + 0.36205407977104187, + 0.2980895936489105, + 0.3371661305427551, + 0.32023027539253235, + 0.1382974535226822 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.8182827830314636, + "style_target": 1, + "support_probability": 0.18497136235237122, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18459628743271098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20144003629684448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9480750560760498, + 0.4009089767932892, + 0.245518296957016, + 0.3111332654953003, + 0.22896267473697662 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.2953617572784424, + "style_target": 0, + "support_probability": 0.2420486956834793, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6948528803115452, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4001723527908325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8490993976593018, + 0.256311297416687, + 0.1423739492893219, + 0.23272132873535156, + 0.06537169963121414, + 0.16180028021335602, + 0.031746022403240204, + 0.2838010787963867, + 0.08352991938591003 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.9494571685791016, + "style_target": 1, + "support_probability": 0.5246300101280212, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18730411844033368, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32282352447509766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9293013215065002, + 0.37970414757728577, + 0.20414230227470398, + 0.3197021484375, + 0.08115723729133606, + 0.3165997564792633, + 0.0844033733010292, + 0.08053536713123322, + 0.070699542760849 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.39291447401046753, + "style_target": 0, + "support_probability": 0.5650600790977478, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.06870111392574901, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7255273461341858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29650983214378357, + 0.21995379030704498, + 0.8047813177108765, + 0.5527195334434509, + 0.14811545610427856, + 0.06696518510580063, + 0.03506520390510559, + 0.0889008641242981, + 0.1556731015443802, + 0.3487339913845062, + 0.14398238062858582 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 1, + "style_probability": 0.7001122832298279, + "style_target": 1, + "support_probability": 0.04559415206313133, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04979955832973766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6266283988952637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31420108675956726, + 0.2589096128940582, + 0.7255790829658508, + 0.4407346248626709, + 0.0713641494512558, + 0.07805617898702621, + 0.08679637312889099, + 0.12719295918941498, + 0.1386466920375824, + 0.07030191272497177, + 0.10380128771066666 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 1, + "style_probability": 0.4321036636829376, + "style_target": 0, + "support_probability": 0.06680340319871902, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2260080301676665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.33893021941185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19218620657920837, + 0.24569928646087646, + 0.35038939118385315, + 0.8260583281517029, + 0.1355089396238327 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 0, + "style_probability": 0.4505792260169983, + "style_target": 1, + "support_probability": 0.43007567524909973, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3055626477179648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2975553274154663, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17764784395694733, + 0.2284407615661621, + 0.44710031151771545, + 0.8825154900550842, + 0.1419808268547058 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 0, + "style_probability": 0.5052904486656189, + "style_target": 0, + "support_probability": 0.45323893427848816, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5052221630366991, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16275513172149658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9951706528663635, + 0.17561089992523193, + 0.4924995005130768, + 0.4309040307998657, + 0.050755295902490616, + 0.11653666943311691 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.5485908389091492, + "style_target": 1, + "support_probability": 0.5747976303100586, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49125880596805027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15618890523910522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9984847903251648, + 0.16688290238380432, + 0.4585379660129547, + 0.39069798588752747, + 0.10698919743299484 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.3135218024253845, + "style_target": 0, + "support_probability": 0.6942852735519409, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6112912918554559, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17429831624031067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46290841698646545, + 0.25817689299583435, + 0.8943098783493042, + 0.368387371301651, + 0.18409961462020874, + 0.13385486602783203, + 0.08785998821258545, + 0.050409380346536636, + 0.1272861808538437 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.5945981740951538, + "style_target": 1, + "support_probability": 0.5394914746284485, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14243756673409375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23302289843559265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5257694721221924, + 0.3403978645801544, + 0.9672653675079346, + 0.4072054326534271, + 0.09631666541099548, + 0.1923588067293167, + 0.11452289670705795, + 0.13533183932304382, + 0.29538971185684204 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.2649320960044861, + "style_target": 0, + "support_probability": 0.5441827178001404, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.16250749589119978, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47351816296577454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2960905432701111, + 0.3217296004295349, + 0.4420674741268158, + 0.5497542023658752, + 0.06823716312646866, + 0.07438430190086365, + 0.10111119598150253, + 0.11304160952568054, + 0.07939665764570236, + 0.050166741013526917, + 0.08264844864606857 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 0, + "style_probability": 0.48915529251098633, + "style_target": 1, + "support_probability": 0.20296765863895416, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3131429759741914, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4152556359767914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30104324221611023, + 0.3459610044956207, + 0.5240882635116577, + 0.6202858686447144, + 0.04017395153641701, + 0.05348169803619385, + 0.05248785763978958, + 0.04544597491621971, + 0.059163451194763184, + 0.08018375933170319, + 0.1257997304201126 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 0, + "style_probability": 0.8055714964866638, + "style_target": 0, + "support_probability": 0.26099830865859985, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1159133083903632, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6405574083328247, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30498388409614563, + 0.38767462968826294, + 0.5637955665588379, + 0.2624589204788208, + 0.10162139683961868 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 0, + "style_probability": 0.48169973492622375, + "style_target": 1, + "support_probability": 0.3433465361595154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020801500420109302, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9130887985229492, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.362464964389801, + 0.4791080951690674, + 0.5396087765693665, + 0.24634496867656708, + 0.0962815061211586, + 0.2244795560836792, + 0.06570567935705185, + 0.11304440349340439, + 0.17705346643924713, + 0.1029513031244278, + 0.08173215389251709 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 1, + "style_probability": 0.3587334156036377, + "style_target": 0, + "support_probability": 0.03948300704360008, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7313982062030177, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.34681522846221924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5179539322853088, + 0.5635589957237244, + 0.3717057406902313, + 0.9041416049003601, + 0.0839633047580719, + 0.06195799633860588 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.5778656601905823, + "style_target": 1, + "support_probability": 0.7178287506103516, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11439760260351133, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.42719191312789917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5993890762329102, + 0.6979656219482422, + 0.5532292723655701, + 0.9742446541786194, + 0.11497378349304199, + 0.1780087798833847 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.07854029536247253, + "style_target": 0, + "support_probability": 0.6511717438697815, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24488880113383635, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5835726261138916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6164169907569885, + 0.226237952709198, + 0.3933367133140564, + 0.18435557186603546, + 0.08699460327625275, + 0.09911521524190903, + 0.22416090965270996, + 0.10031156241893768, + 0.11733607947826385 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 1, + "style_probability": 0.6466757655143738, + "style_target": 1, + "support_probability": 0.311460018157959, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29511073192373144, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5671730637550354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6417471170425415, + 0.25219109654426575, + 0.41961386799812317, + 0.1251799613237381, + 0.0409562923014164, + 0.15652254223823547, + 0.1088620126247406, + 0.046433333307504654, + 0.07342622429132462 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 0, + "style_probability": 0.7220401763916016, + "style_target": 0, + "support_probability": 0.3060150742530823, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8116393542627234, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07797204703092575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24568034708499908, + 0.37151309847831726, + 0.8978697061538696, + 0.4230365455150604, + 0.06446617841720581, + 0.06673579663038254, + 0.17272524535655975, + 0.08495032787322998, + 0.1051580011844635, + 0.05248411372303963, + 0.12313693016767502 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 1, + "style_probability": 0.5251951217651367, + "style_target": 1, + "support_probability": 0.971015989780426, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8030034375453984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08911419659852982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21321113407611847, + 0.4844191074371338, + 0.9058959484100342, + 0.4066142439842224, + 0.04404253512620926, + 0.05266380310058594, + 0.09167644381523132, + 0.09792829304933548, + 0.07811439782381058 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 1, + "style_probability": 0.44253039360046387, + "style_target": 0, + "support_probability": 0.9478197693824768, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6777883072830766, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.38747403025627136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3044396638870239, + 0.7967643737792969, + 0.11053991317749023, + 0.3044396638870239, + 0.14335781335830688 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 1, + "style_probability": 0.9097436666488647, + "style_target": 1, + "support_probability": 0.45478305220603943, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47950331364343035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4780307710170746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2849346697330475, + 0.7666827440261841, + 0.13884814083576202, + 0.2849346697330475, + 0.04702598601579666 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 0, + "style_probability": 0.7327859401702881, + "style_target": 0, + "support_probability": 0.46309512853622437, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009484106939624615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6476794481277466, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42421194911003113, + 0.4125652611255646, + 0.18519431352615356, + 0.1361476182937622, + 0.08366726338863373, + 0.10634857416152954 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 0, + "style_probability": 0.47815021872520447, + "style_target": 1, + "support_probability": 0.026554105803370476, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009336922378967263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7318477630615234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42285552620887756, + 0.3925858438014984, + 0.19435778260231018, + 0.13699820637702942, + 0.10066745430231094 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 1, + "style_probability": 0.22750386595726013, + "style_target": 0, + "support_probability": 0.024450309574604034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4350143862096919, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2269020825624466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09931262582540512, + 0.2437373250722885, + 0.9199633002281189, + 0.09931262582540512, + 0.08542943000793457, + 0.0359329879283905, + 0.04184364154934883, + 0.04014122858643532, + 0.08426594734191895 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.7452301979064941, + "style_target": 1, + "support_probability": 0.26023027300834656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05819006619239882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.259722501039505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11165998131036758, + 0.3123328387737274, + 0.9566811323165894, + 0.11165998131036758, + 0.05975451320409775, + 0.046087589114904404, + 0.07236145436763763, + 0.0766056478023529, + 0.10163486748933792 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.2430819422006607, + "style_target": 0, + "support_probability": 0.34961944818496704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01432519156260573, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9713267683982849, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48466113209724426, + 0.295820415019989, + 0.29065608978271484, + 0.16835567355155945, + 0.07336084544658661, + 0.09725605696439743, + 0.07243281602859497, + 0.04162922874093056, + 0.05203015357255936, + 0.060089804232120514, + 0.08831986784934998 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 1, + "style_probability": 0.6598251461982727, + "style_target": 1, + "support_probability": 0.01764967106282711, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0204551369998336, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9303815960884094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48720425367355347, + 0.3023660480976105, + 0.3301401436328888, + 0.19939936697483063, + 0.1166151836514473, + 0.09170128405094147, + 0.07511869072914124, + 0.0553947389125824, + 0.09344230592250824, + 0.14709259569644928, + 0.09778117388486862 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 0, + "style_probability": 0.5685119032859802, + "style_target": 0, + "support_probability": 0.02681743912398815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002115599385667492, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9668543338775635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3653514087200165, + 0.21008166670799255, + 0.21008166670799255, + 0.23418329656124115, + 0.07627099007368088 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.30314815044403076, + "style_target": 1, + "support_probability": 0.003664620453491807, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018525369727203762, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.969187319278717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4045063555240631, + 0.18823394179344177, + 0.18823394179344177, + 0.1824924349784851, + 0.05601870268583298 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.8804993629455566, + "style_target": 0, + "support_probability": 0.00852483045309782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03177938855267624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5321954488754272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4003274142742157, + 0.8369078040122986, + 0.297383576631546, + 0.18111838400363922, + 0.06241879612207413, + 0.10212237387895584 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 0, + "style_probability": 0.3896864950656891, + "style_target": 1, + "support_probability": 0.05524781346321106, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050239297468800535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4220142364501953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3641303479671478, + 0.8147748112678528, + 0.25144538283348083, + 0.1544443666934967, + 0.10756111890077591 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 1, + "style_probability": 0.1923053115606308, + "style_target": 0, + "support_probability": 0.11381153017282486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007921893216914028, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.792076051235199, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.293375700712204, + 0.29943785071372986, + 0.20605382323265076, + 0.2514384090900421, + 0.1258978247642517, + 0.20979498326778412, + 0.1349395364522934, + 0.17871399223804474, + 0.20504890382289886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 0, + "style_probability": 0.403361052274704, + "style_target": 1, + "support_probability": 0.03725524619221687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009005505649267603, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9493635892868042, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3308045268058777, + 0.38171038031578064, + 0.30058184266090393, + 0.36009377241134644, + 0.2264053076505661, + 0.21924477815628052, + 0.19270013272762299, + 0.2033763974905014, + 0.15476559102535248 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 1, + "style_probability": 0.10218944400548935, + "style_target": 0, + "support_probability": 0.019280822947621346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11666080845354916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4728407859802246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13560247421264648, + 0.5509237051010132, + 0.4687529504299164, + 0.162980318069458, + 0.12313403934240341, + 0.2600727081298828, + 0.13719162344932556, + 0.16721513867378235, + 0.05697325989603996, + 0.050751473754644394, + 0.059055913239717484 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 0, + "style_probability": 0.4800153374671936, + "style_target": 1, + "support_probability": 0.17545413970947266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2870846313892713, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33596065640449524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1389952301979065, + 0.5955150127410889, + 0.39306169748306274, + 0.15346938371658325, + 0.08068365603685379, + 0.0910044014453888, + 0.11768751591444016, + 0.08845893293619156, + 0.10834149271249771, + 0.046109389513731, + 0.04465186595916748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 0, + "style_probability": 0.5049372315406799, + "style_target": 0, + "support_probability": 0.4344848096370697, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9133026871985797, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05951738730072975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3011:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19734251499176025, + 0.8642938733100891, + 0.19734251499176025, + 0.9617906212806702, + 0.07641065865755081 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8207666873931885, + "style_target": 1, + "support_probability": 0.9791456460952759, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5250609375420703, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10286255925893784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3011:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2431456297636032, + 0.9100079536437988, + 0.2431456297636032, + 0.9803553819656372, + 0.12863928079605103 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.15960125625133514, + "style_target": 0, + "support_probability": 0.9627685546875, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3154492553549784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20042957365512848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39628249406814575, + 0.2672450542449951, + 0.5142723917961121, + 0.9617462158203125, + 0.07545939832925797, + 0.03145306557416916 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 1, + "style_probability": 0.64082932472229, + "style_target": 1, + "support_probability": 0.22701714932918549, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2881351569562001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23514609038829803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34873121976852417, + 0.2923435568809509, + 0.4683683216571808, + 0.933912992477417, + 0.049029987305402756, + 0.0679052546620369 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 0, + "style_probability": 0.60989910364151, + "style_target": 0, + "support_probability": 0.18983691930770874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7352533688803351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32835012674331665, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13028326630592346, + 0.22118303179740906, + 0.1959179937839508, + 0.9765806198120117, + 0.12234620749950409, + 0.19312584400177002, + 0.07753576338291168, + 0.10760495066642761, + 0.03667336329817772 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.7429472208023071, + "style_target": 1, + "support_probability": 0.7240519523620605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5927181316376487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2696211338043213, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13265755772590637, + 0.22184881567955017, + 0.20920313894748688, + 0.9685229659080505, + 0.04372984915971756, + 0.07182992994785309 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.3755740821361542, + "style_target": 0, + "support_probability": 0.7535032629966736, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011166770435427537, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9750295281410217, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24393734335899353, + 0.19882157444953918, + 0.3353361487388611, + 0.2896325886249542, + 0.08885201811790466, + 0.14416436851024628, + 0.28442877531051636, + 0.06313589960336685, + 0.11816912889480591, + 0.0802779346704483 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.743478536605835, + "style_target": 1, + "support_probability": 0.011635839007794857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031960539273737307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9454215168952942, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25303229689598083, + 0.2505865693092346, + 0.40438395738601685, + 0.42075181007385254, + 0.16698332130908966, + 0.13384731113910675, + 0.23548762500286102, + 0.0857330858707428, + 0.07312943786382675, + 0.09185128659009933 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.12074676156044006, + "style_target": 0, + "support_probability": 0.038573745638132095, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.18873073999091608, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36036622524261475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42000943422317505, + 0.4899251461029053, + 0.4152669608592987, + 0.16648653149604797, + 0.12244612723588943 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 0, + "style_probability": 0.4656960070133209, + "style_target": 1, + "support_probability": 0.5426081418991089, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03146494319536536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.40096813440322876, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42763110995292664, + 0.7305427193641663, + 0.43302613496780396, + 0.30120399594306946, + 0.14721670746803284 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 1, + "style_probability": 0.1254061907529831, + "style_target": 0, + "support_probability": 0.4158834218978882, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7762710956572468, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12832969427108765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9495511054992676, + 0.26873716711997986, + 0.33488765358924866, + 0.4090534448623657, + 0.15609510242938995, + 0.19506771862506866 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 0, + "style_probability": 0.4342734217643738, + "style_target": 1, + "support_probability": 0.8180089592933655, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8354856401225915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11316836625337601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9788151383399963, + 0.2894733250141144, + 0.3057687282562256, + 0.45249345898628235, + 0.0979372188448906, + 0.20898640155792236 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 0, + "style_probability": 0.6498273015022278, + "style_target": 0, + "support_probability": 0.8292926549911499, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7448857824593837, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038226403295993805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36741432547569275, + 0.9342241883277893, + 0.25344061851501465, + 0.2648066282272339, + 0.036527086049318314, + 0.04175224155187607, + 0.04080317169427872, + 0.09567972272634506, + 0.15164320170879364 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 0, + "style_probability": 0.38520166277885437, + "style_target": 1, + "support_probability": 0.8851262331008911, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.804925709203957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01736612431704998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3448992967605591, + 0.9475144743919373, + 0.2105485051870346, + 0.22815200686454773, + 0.03723134100437164, + 0.04954342171549797 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 1, + "style_probability": 0.2798512876033783, + "style_target": 0, + "support_probability": 0.9333882927894592, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01866108385233521, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8915004134178162, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36419451236724854, + 0.2039055973291397, + 0.3892452120780945, + 0.4673845171928406, + 0.07741155475378036, + 0.20968492329120636, + 0.20578600466251373, + 0.08311676234006882, + 0.08890125155448914, + 0.19131772220134735 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 1, + "style_probability": 0.5213922262191772, + "style_target": 1, + "support_probability": 0.025230126455426216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004743830337404391, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9019502401351929, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.46380648016929626, + 0.21527104079723358, + 0.4586747884750366, + 0.499475359916687, + 0.18093112111091614, + 0.14561115205287933, + 0.20859000086784363, + 0.17625342309474945, + 0.07956015318632126, + 0.1597730815410614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 1, + "style_probability": 0.24323974549770355, + "style_target": 0, + "support_probability": 0.04159882292151451, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7866287534609339, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04550272226333618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.716066300868988, + 0.25981760025024414, + 0.8782558441162109, + 0.1841975748538971, + 0.04505418613553047 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 1, + "style_probability": 0.8725582361221313, + "style_target": 1, + "support_probability": 0.776889979839325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5830048929909415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1294425129890442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6715031266212463, + 0.3094460964202881, + 0.8506825566291809, + 0.20135965943336487, + 0.1199764683842659, + 0.09060317277908325, + 0.1311608999967575, + 0.12424583733081818, + 0.0707148090004921, + 0.06558053940534592, + 0.0840788260102272 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 1, + "style_probability": 0.4913918972015381, + "style_target": 0, + "support_probability": 0.6319791674613953, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1443601271637328, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.627631425857544, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9315400719642639, + 0.11206784099340439, + 0.43211981654167175, + 0.33974361419677734, + 0.2202148288488388, + 0.062966488301754 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.7914772033691406, + "style_target": 1, + "support_probability": 0.11227784305810928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011057074490267969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6619027853012085, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9474380612373352, + 0.16134899854660034, + 0.551047682762146, + 0.4071787893772125, + 0.07450469583272934, + 0.0813201516866684 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.22274181246757507, + "style_target": 0, + "support_probability": 0.09360754489898682, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6372757304465088, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2310551553964615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3831985890865326, + 0.2587997317314148, + 0.9668986797332764, + 0.32035765051841736, + 0.2639772295951843, + 0.053723353892564774, + 0.0594230480492115, + 0.29148122668266296, + 0.06474119424819946 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 1, + "style_probability": 0.6874496936798096, + "style_target": 1, + "support_probability": 0.5369396209716797, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5634035383569337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2937546372413635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42060986161231995, + 0.2354414165019989, + 0.955915093421936, + 0.30257999897003174, + 0.13909007608890533, + 0.12120161205530167, + 0.2701832056045532, + 0.09700130671262741, + 0.0778898149728775 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 1, + "style_probability": 0.47899582982063293, + "style_target": 0, + "support_probability": 0.6385116577148438, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.047162240704150725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.779662013053894, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17506402730941772, + 0.503441333770752, + 0.923323392868042, + 0.17506402730941772, + 0.09096737205982208, + 0.0784183219075203, + 0.11493296921253204, + 0.08564118295907974, + 0.03691159933805466, + 0.16288931667804718, + 0.07815857231616974 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6284491419792175, + "style_target": 1, + "support_probability": 0.0423123762011528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013010523513414538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9411013722419739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17585332691669464, + 0.47693806886672974, + 0.8912898302078247, + 0.17585332691669464, + 0.05872604623436928, + 0.059500012546777725, + 0.0670386552810669, + 0.057598765939474106, + 0.04772387444972992 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 1, + "style_probability": 0.4003944396972656, + "style_target": 0, + "support_probability": 0.006158981006592512, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15982487331953002, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2076680064201355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10745366662740707, + 0.8998283743858337, + 0.14950767159461975, + 0.13124994933605194, + 0.1111956536769867 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 0, + "style_probability": 0.32831206917762756, + "style_target": 1, + "support_probability": 0.17460860311985016, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23538914337206712, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2227734923362732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10011770576238632, + 0.9267261028289795, + 0.16616058349609375, + 0.12381469458341599, + 0.06954239308834076, + 0.05977943539619446, + 0.11436425149440765, + 0.06136530637741089, + 0.04471981152892113, + 0.05677715688943863, + 0.04540728032588959 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 1, + "style_probability": 0.23037341237068176, + "style_target": 0, + "support_probability": 0.22080327570438385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07045309346336283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6306384801864624, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6419564485549927, + 0.16692472994327545, + 0.7586309313774109, + 0.6419564485549927, + 0.0957353338599205, + 0.12985920906066895 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 0, + "style_probability": 0.378092497587204, + "style_target": 1, + "support_probability": 0.1983535885810852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006485000580747179, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8842957615852356, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6383535861968994, + 0.13029161095619202, + 0.9231929779052734, + 0.6383535861968994, + 0.11935663968324661, + 0.10681893676519394 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 1, + "style_probability": 0.13879220187664032, + "style_target": 0, + "support_probability": 0.07296255975961685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05051400284221226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5969395041465759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6291005611419678, + 0.3693447709083557, + 0.8134655952453613, + 0.5386927723884583, + 0.07039985805749893, + 0.04525135084986687, + 0.2381500005722046, + 0.1233804002404213, + 0.04522867500782013 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 0, + "style_probability": 0.39853787422180176, + "style_target": 1, + "support_probability": 0.07091761380434036, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3412871053169141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2789583206176758, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6212930083274841, + 0.37968769669532776, + 0.867642879486084, + 0.5265429019927979, + 0.10404173284769058, + 0.09513813257217407, + 0.032114554196596146, + 0.08063285052776337, + 0.045269619673490524 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 0, + "style_probability": 0.8066663146018982, + "style_target": 0, + "support_probability": 0.23365333676338196, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.31941476865005713, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5797947645187378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7764608860015869, + 0.2970377504825592, + 0.3576085567474365, + 0.5430758595466614, + 0.10267903655767441, + 0.14252009987831116, + 0.11552079021930695, + 0.06720315665006638, + 0.14948108792304993, + 0.17371392250061035, + 0.05342547222971916 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 1, + "style_probability": 0.691339910030365, + "style_target": 1, + "support_probability": 0.18504944443702698, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33864713789492484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5161445140838623, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7999420166015625, + 0.32026222348213196, + 0.39157092571258545, + 0.5520480275154114, + 0.06442967802286148, + 0.055090490728616714, + 0.10227324813604355, + 0.04675716906785965, + 0.12618325650691986 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 0, + "style_probability": 0.5413171648979187, + "style_target": 0, + "support_probability": 0.21070094406604767, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14687969234735568, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6576613187789917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3899769186973572, + 0.1998276263475418, + 0.49908900260925293, + 0.6950633525848389, + 0.03583119437098503 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 1, + "style_probability": 0.7913433313369751, + "style_target": 1, + "support_probability": 0.11609572172164917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011086328426838602, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9172651171684265, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4004732072353363, + 0.2639773190021515, + 0.5584543943405151, + 0.587801992893219, + 0.07764599472284317 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 0, + "style_probability": 0.5079842209815979, + "style_target": 0, + "support_probability": 0.0083515839651227, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7582464931343746, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16274762153625488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9725481271743774, + 0.5471057891845703, + 0.38569802045822144, + 0.35670891404151917, + 0.06640861183404922, + 0.1083172857761383 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.8269573450088501, + "style_target": 1, + "support_probability": 0.6405848860740662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5233672060408908, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24011291563510895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9490194916725159, + 0.5447406768798828, + 0.4042356014251709, + 0.3611242473125458, + 0.19543275237083435 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.4853009283542633, + "style_target": 0, + "support_probability": 0.5531206727027893, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7984668637882695, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14646120369434357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6309566497802734, + 0.30409717559814453, + 0.957610547542572, + 0.25433349609375, + 0.05493362620472908, + 0.09652011841535568, + 0.06415532529354095, + 0.07926715165376663, + 0.08008942753076553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.8139135241508484, + "style_target": 1, + "support_probability": 0.6762510538101196, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2502554906029289, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19382335245609283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6402780413627625, + 0.3326181471347809, + 0.9796872138977051, + 0.27232247591018677, + 0.08864998072385788, + 0.10362162441015244, + 0.1308811753988266, + 0.10230779647827148, + 0.15911225974559784 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.22264792025089264, + "style_target": 0, + "support_probability": 0.6802048683166504, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9093772801446084, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06401568651199341, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5073047876358032, + 0.20545750856399536, + 0.9587459564208984, + 0.09744855016469955, + 0.0390642024576664, + 0.07290536910295486, + 0.056230831891298294, + 0.0356583446264267, + 0.04298811033368111, + 0.053804002702236176, + 0.04453810304403305 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 1, + "style_probability": 0.8745212554931641, + "style_target": 1, + "support_probability": 0.9742072224617004, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.832997981441515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09295438975095749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5023627281188965, + 0.20826351642608643, + 0.9369049072265625, + 0.14549250900745392, + 0.11830874532461166, + 0.0432218499481678, + 0.056026674807071686, + 0.10165097564458847, + 0.05535159260034561, + 0.06016487255692482, + 0.04728928953409195 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 0, + "style_probability": 0.5403991341590881, + "style_target": 0, + "support_probability": 0.9523652195930481, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33820743030052114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20138052105903625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22269299626350403, + 0.8073151111602783, + 0.3472205698490143, + 0.2311030924320221, + 0.05655281990766525 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 0, + "style_probability": 0.3648267984390259, + "style_target": 1, + "support_probability": 0.5211191177368164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6030596508769932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15037983655929565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23131676018238068, + 0.8757116198539734, + 0.3746594786643982, + 0.15871399641036987, + 0.07328255474567413 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 0, + "style_probability": 0.6385411024093628, + "style_target": 0, + "support_probability": 0.6127748489379883, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.218768749855964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45416590571403503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33041250705718994, + 0.4521705210208893, + 0.6981912851333618, + 0.39476948976516724, + 0.2004900872707367, + 0.08980707824230194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 1, + "style_probability": 0.6424520611763, + "style_target": 1, + "support_probability": 0.170556902885437, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13893027515965176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5810092687606812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3414582312107086, + 0.481426477432251, + 0.697529137134552, + 0.3575601577758789, + 0.17397160828113556 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 1, + "style_probability": 0.40197494626045227, + "style_target": 0, + "support_probability": 0.1744958609342575, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.19835134757519046, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7347033619880676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34393325448036194, + 0.22297172248363495, + 0.4690593183040619, + 0.9088636636734009, + 0.13447849452495575, + 0.24048727750778198, + 0.19644978642463684, + 0.11419279128313065, + 0.1402702033519745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 0, + "style_probability": 0.4281388223171234, + "style_target": 1, + "support_probability": 0.33576616644859314, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0771599022806842, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49991780519485474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3869912028312683, + 0.17528384923934937, + 0.4933539628982544, + 0.9471020698547363, + 0.12340100854635239, + 0.21772296726703644, + 0.20847295224666595, + 0.1940026730298996, + 0.20280282199382782 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 1, + "style_probability": 0.06481646001338959, + "style_target": 0, + "support_probability": 0.7594843506813049, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10167439276349789, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6221070885658264, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4530864655971527, + 0.5763111114501953, + 0.6448201537132263, + 0.4530864655971527, + 0.12468177825212479, + 0.1305134892463684, + 0.11687402427196503, + 0.27358123660087585, + 0.18789689242839813, + 0.2422323226928711, + 0.11259850114583969 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 0, + "style_probability": 0.4819253087043762, + "style_target": 1, + "support_probability": 0.09145008772611618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3039334637668967, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30521759390830994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45648252964019775, + 0.607374370098114, + 0.654076099395752, + 0.45648252964019775, + 0.15090662240982056, + 0.05962969362735748, + 0.0608942024409771, + 0.1308959722518921, + 0.24953743815422058, + 0.12737716734409332, + 0.0702671930193901 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 0, + "style_probability": 0.6941351890563965, + "style_target": 0, + "support_probability": 0.31384947896003723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13960473162671846, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8257612586021423, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4630935490131378, + 0.6183953881263733, + 0.9013259410858154, + 0.2803589701652527, + 0.1076134592294693 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.8134891986846924, + "style_target": 1, + "support_probability": 0.07511036098003387, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015405664848457441, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8994253873825073, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.512872040271759, + 0.6987899541854858, + 0.9168288707733154, + 0.30495485663414, + 0.14339697360992432 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.3302942216396332, + "style_target": 0, + "support_probability": 0.04419712722301483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1016292240072735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6245840191841125, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33943626284599304, + 0.7954373359680176, + 0.5898751020431519, + 0.24035492539405823, + 0.04076174274086952, + 0.1365123838186264 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 1, + "style_probability": 0.7758297324180603, + "style_target": 1, + "support_probability": 0.05083693936467171, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13465986051698997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6640149354934692, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38564836978912354, + 0.7749373316764832, + 0.5885220766067505, + 0.24043059349060059, + 0.08160314708948135, + 0.061561424285173416 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 0, + "style_probability": 0.6934126615524292, + "style_target": 0, + "support_probability": 0.07939647138118744, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14600163418707177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44340983033180237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2596583962440491, + 0.5425744652748108, + 0.6045613288879395, + 0.9205898642539978, + 0.023058559745550156, + 0.02705259807407856, + 0.03017704375088215, + 0.07476387917995453, + 0.14092758297920227 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.6673969030380249, + "style_target": 1, + "support_probability": 0.11023437231779099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19014984232861284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3322910666465759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2876170873641968, + 0.4461599290370941, + 0.5828791856765747, + 0.9140744209289551, + 0.04759087786078453, + 0.06640360504388809 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.4544987976551056, + "style_target": 0, + "support_probability": 0.1712566614151001, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28821865582561407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31543105840682983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6143904328346252, + 0.563474178314209, + 0.8639901280403137, + 0.4508398175239563, + 0.20252066850662231, + 0.17726460099220276, + 0.08677677810192108, + 0.09498033672571182, + 0.06270044296979904, + 0.08960835635662079, + 0.1870253086090088 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 0, + "style_probability": 0.4576011896133423, + "style_target": 1, + "support_probability": 0.15993720293045044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0624431689566762, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3479245901107788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7010512948036194, + 0.6207979321479797, + 0.9122324585914612, + 0.5365626811981201, + 0.17837172746658325, + 0.09989741444587708, + 0.2676147520542145, + 0.2175394892692566, + 0.2612828314304352, + 0.18588857352733612 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 1, + "style_probability": 0.09430436789989471, + "style_target": 0, + "support_probability": 0.17440561950206757, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03330276362910334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4944717288017273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18507252633571625, + 0.6481744647026062, + 0.5423026084899902, + 0.12859046459197998, + 0.17629463970661163 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 0, + "style_probability": 0.31306886672973633, + "style_target": 1, + "support_probability": 0.025119932368397713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021082015686807863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.34292879700660706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2079097479581833, + 0.6934329867362976, + 0.5674484372138977, + 0.12534405291080475, + 0.11797960102558136 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 1, + "style_probability": 0.29570332169532776, + "style_target": 0, + "support_probability": 0.034455619752407074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.340600970509087, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.457381933927536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22484537959098816, + 0.25701096653938293, + 0.4311515986919403, + 0.9438802003860474, + 0.27727970480918884, + 0.1262032687664032 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 0, + "style_probability": 0.43326714634895325, + "style_target": 1, + "support_probability": 0.38923415541648865, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3943495509432756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.502080500125885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25621119141578674, + 0.262469619512558, + 0.4688783586025238, + 0.9566206336021423, + 0.11398419737815857, + 0.2920989990234375 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 0, + "style_probability": 0.737903892993927, + "style_target": 0, + "support_probability": 0.35154473781585693, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.034691781345367094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38642042875289917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2001897096633911, + 0.21349504590034485, + 0.1300889253616333, + 0.8154146671295166, + 0.07927712053060532, + 0.06422556191682816, + 0.07990441471338272, + 0.1850394457578659, + 0.13186015188694 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 0, + "style_probability": 0.34066471457481384, + "style_target": 1, + "support_probability": 0.024357609450817108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012752216049963612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4345528483390808, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2315051406621933, + 0.33283987641334534, + 0.1892254501581192, + 0.8363111615180969, + 0.09658196568489075, + 0.13812977075576782, + 0.0710153728723526, + 0.09636061638593674, + 0.09400401264429092 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 1, + "style_probability": 0.25377708673477173, + "style_target": 0, + "support_probability": 0.04195183143019676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018041320723575498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8325524926185608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1851787269115448, + 0.17335490882396698, + 0.8027306795120239, + 0.308726042509079, + 0.06351173669099808, + 0.09426809102296829, + 0.10176313668489456, + 0.04048103094100952, + 0.11510138213634491, + 0.11946124583482742, + 0.12002942711114883 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 0, + "style_probability": 0.2193048745393753, + "style_target": 1, + "support_probability": 0.03591512143611908, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014022851865838295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8527886271476746, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18214239180088043, + 0.20551151037216187, + 0.7606873512268066, + 0.3127416968345642, + 0.08267875015735626, + 0.09221892803907394, + 0.21338985860347748, + 0.10255241394042969, + 0.21115213632583618, + 0.11326257884502411, + 0.06443910300731659 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 1, + "style_probability": 0.3683449923992157, + "style_target": 0, + "support_probability": 0.030137598514556885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48595567070825063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.140577495098114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4162280559539795, + 0.2188127487897873, + 0.9430894255638123, + 0.20262682437896729, + 0.06944600492715836 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.8129891157150269, + "style_target": 1, + "support_probability": 0.5682147145271301, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23869466633471975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21676339209079742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3912726938724518, + 0.22498057782649994, + 0.9129179120063782, + 0.20349445939064026, + 0.04069758951663971, + 0.06055084243416786, + 0.12148899585008621, + 0.26227712631225586, + 0.11934509128332138, + 0.07797657698392868, + 0.06402191519737244 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.46867385506629944, + "style_target": 0, + "support_probability": 0.3963159918785095, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7440191736382001, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4386974573135376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8780573606491089, + 0.5087279677391052, + 0.6550999283790588, + 0.7877551913261414, + 0.06552761048078537, + 0.044147782027721405 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.6923524737358093, + "style_target": 1, + "support_probability": 0.8480716943740845, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0667882070597335, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5626267790794373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8986449241638184, + 0.5213221311569214, + 0.6048826575279236, + 0.8056620955467224, + 0.1070760041475296, + 0.17032809555530548 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.05583301559090614, + "style_target": 0, + "support_probability": 0.6076395511627197, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5800548223165833, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45305144786834717, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6458447575569153, + 0.3693181276321411, + 0.6618514060974121, + 0.42947012186050415, + 0.11277426034212112, + 0.12608222663402557, + 0.04234921187162399, + 0.04354923591017723, + 0.04114394262433052 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 1, + "style_probability": 0.9192254543304443, + "style_target": 1, + "support_probability": 0.5063760280609131, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2895492090538588, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.507594883441925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5394588112831116, + 0.36341556906700134, + 0.6405322551727295, + 0.397390753030777, + 0.12605373561382294, + 0.05442686006426811, + 0.14087311923503876, + 0.17979124188423157, + 0.09689010679721832 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 0, + "style_probability": 0.5473747849464417, + "style_target": 0, + "support_probability": 0.41315746307373047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5630688983978036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32043221592903137, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3704375922679901, + 0.8140613436698914, + 0.23124182224273682, + 0.31705567240715027, + 0.08007655292749405, + 0.09719975292682648, + 0.06667986512184143, + 0.05549495294690132, + 0.1049288883805275, + 0.06166151538491249, + 0.04272586479783058 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.6904271245002747, + "style_target": 1, + "support_probability": 0.4944956302642822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.380276599378093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3152232766151428, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3661094307899475, + 0.7650275826454163, + 0.2487224042415619, + 0.3096245229244232, + 0.09691158682107925, + 0.16497191786766052, + 0.20679284632205963, + 0.06876200437545776, + 0.18853074312210083 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.3457685708999634, + "style_target": 0, + "support_probability": 0.5441169142723083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17667574851654808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8750708103179932, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30067744851112366, + 0.6933926939964294, + 0.3660487234592438, + 0.6017976403236389, + 0.04922347515821457 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.7076022624969482, + "style_target": 1, + "support_probability": 0.1678183376789093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01765266999787001, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.901759922504425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2930701673030853, + 0.6719704866409302, + 0.3410758078098297, + 0.5848190784454346, + 0.08908487856388092 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.17693312466144562, + "style_target": 0, + "support_probability": 0.12333360314369202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5958740805906643, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.32421860098838806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7195190787315369, + 0.6548261046409607, + 0.9341307282447815, + 0.2338438630104065, + 0.11807897686958313, + 0.05334967374801636 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 1, + "style_probability": 0.7154791355133057, + "style_target": 1, + "support_probability": 0.6177409291267395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6340591868862699, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2265038937330246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6888853907585144, + 0.6580460071563721, + 0.9038923382759094, + 0.23338449001312256, + 0.0876983031630516, + 0.18876835703849792 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 0, + "style_probability": 0.7676796913146973, + "style_target": 0, + "support_probability": 0.5835295915603638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06981004950048698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9263617992401123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8612959980964661, + 0.26793545484542847, + 0.31288841366767883, + 0.3219585418701172, + 0.045840561389923096, + 0.08470994979143143, + 0.06685875356197357, + 0.109330914914608, + 0.04524506255984306 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.7818966507911682, + "style_target": 1, + "support_probability": 0.04080725833773613, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04824711853652488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9161859750747681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8088732957839966, + 0.24814361333847046, + 0.2989106774330139, + 0.33060118556022644, + 0.13964419066905975, + 0.11242473870515823 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.3615424931049347, + "style_target": 0, + "support_probability": 0.058363739401102066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13157394246393359, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46373531222343445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8167014122009277, + 0.23467081785202026, + 0.30440032482147217, + 0.29583320021629333, + 0.22719277441501617, + 0.09961311519145966, + 0.08282041549682617, + 0.08683998137712479, + 0.09800498187541962, + 0.2190839797258377, + 0.08798166364431381 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.6844505667686462, + "style_target": 1, + "support_probability": 0.13080236315727234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007386524972565427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5844607353210449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8270076513290405, + 0.30102676153182983, + 0.37236422300338745, + 0.36471596360206604, + 0.12248361855745316, + 0.13074800372123718, + 0.107088103890419, + 0.099337138235569, + 0.1220819428563118, + 0.1770305633544922 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.10175333172082901, + "style_target": 0, + "support_probability": 0.10404740273952484, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8223750764708916, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.21038417518138885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14860722422599792, + 0.4047836661338806, + 0.29904788732528687, + 0.9155519008636475, + 0.10240531712770462 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 1, + "style_probability": 0.8713875412940979, + "style_target": 1, + "support_probability": 0.7108197808265686, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6170985044312141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15917228162288666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12600521743297577, + 0.34250712394714355, + 0.2829991579055786, + 0.9057515263557434, + 0.15960198640823364 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 1, + "style_probability": 0.4302893579006195, + "style_target": 0, + "support_probability": 0.688629150390625, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7399826883064982, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3743129074573517, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8600286841392517, + 0.49554890394210815, + 0.14574216306209564, + 0.8290873765945435, + 0.11222109943628311, + 0.10697369277477264 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.9008975028991699, + "style_target": 1, + "support_probability": 0.7188466191291809, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3096156146397189, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6191413998603821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8448304533958435, + 0.5163387656211853, + 0.1483154147863388, + 0.8167285323143005, + 0.17147588729858398 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.47335681319236755, + "style_target": 0, + "support_probability": 0.4631412923336029, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1269892103143272, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6717002391815186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6983399987220764, + 0.3798062801361084, + 0.9211664795875549, + 0.26445338129997253, + 0.08952080458402634, + 0.2510293126106262, + 0.25468459725379944, + 0.13027629256248474, + 0.14741270244121552 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.8139992952346802, + "style_target": 1, + "support_probability": 0.07973070442676544, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010461083452791228, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8654898405075073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7259225249290466, + 0.34679943323135376, + 0.9291884899139404, + 0.2553655207157135, + 0.18464510142803192, + 0.33013781905174255, + 0.176753968000412, + 0.15179045498371124, + 0.17208276689052582 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.1950734257698059, + "style_target": 0, + "support_probability": 0.06432268023490906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7857415637210061, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07315264642238617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18970972299575806, + 0.46652543544769287, + 0.9774637818336487, + 0.13647586107254028, + 0.06805092841386795, + 0.07213485985994339, + 0.056956853717565536, + 0.09246141463518143, + 0.09197699278593063, + 0.052497297525405884, + 0.04539669677615166 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 1, + "style_probability": 0.807273805141449, + "style_target": 1, + "support_probability": 0.7163432240486145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7712995263999456, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14164774119853973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19599983096122742, + 0.5129387974739075, + 0.9409728050231934, + 0.14030396938323975, + 0.0774029791355133, + 0.058099567890167236, + 0.04185468703508377, + 0.07410422712564468, + 0.07921683043241501, + 0.06792019307613373, + 0.09631748497486115 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 0, + "style_probability": 0.727655291557312, + "style_target": 0, + "support_probability": 0.7767652273178101, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5434536486128589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5600489377975464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7892818450927734, + 0.20536592602729797, + 0.36287206411361694, + 0.1670028120279312, + 0.07335073500871658 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.7906411290168762, + "style_target": 1, + "support_probability": 0.43291419744491577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09060816045431355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7872961759567261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7532169222831726, + 0.21051020920276642, + 0.389138400554657, + 0.18086306750774384, + 0.12497039884328842, + 0.09821276366710663, + 0.1735953837633133, + 0.12760314345359802, + 0.10877127945423126, + 0.059762950986623764, + 0.18428181111812592 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.23473872244358063, + "style_target": 0, + "support_probability": 0.12389258295297623, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17511371646826382, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8385759592056274, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33963489532470703, + 0.3817863464355469, + 0.08961852639913559, + 0.7446238994598389, + 0.13447749614715576, + 0.1095021590590477 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.7486430406570435, + "style_target": 1, + "support_probability": 0.14449456334114075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004498531732066736, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9761813879013062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3420434892177582, + 0.46267107129096985, + 0.14943400025367737, + 0.6794149279594421, + 0.17513658106327057, + 0.09315069019794464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.13418236374855042, + "style_target": 0, + "support_probability": 0.03824716433882713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2709689487965736, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35900217294692993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6833503246307373, + 0.1767263412475586, + 0.2891140878200531, + 0.2561247646808624, + 0.06341098994016647, + 0.08497639000415802, + 0.05571388080716133, + 0.07111542671918869, + 0.07228583842515945 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 1, + "style_probability": 0.7014220952987671, + "style_target": 1, + "support_probability": 0.2411389797925949, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15551034391301405, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4359750747680664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6724636554718018, + 0.1611977219581604, + 0.3179672956466675, + 0.23863065242767334, + 0.12562064826488495, + 0.03879348933696747, + 0.09485011547803879, + 0.1692587435245514, + 0.12879852950572968 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 1, + "style_probability": 0.4949508309364319, + "style_target": 0, + "support_probability": 0.15206584334373474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34162153196142103, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8127230405807495, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20966164767742157, + 0.22397133708000183, + 0.49025946855545044, + 0.5453870296478271, + 0.035945404320955276, + 0.048483606427907944, + 0.05456198751926422, + 0.05724503844976425, + 0.06747560948133469, + 0.04984080046415329, + 0.07470983266830444 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 1, + "style_probability": 0.9086188077926636, + "style_target": 1, + "support_probability": 0.18429110944271088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2003114559556877, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7406947612762451, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16342227160930634, + 0.21408890187740326, + 0.4146879315376282, + 0.49510863423347473, + 0.04856611043214798, + 0.03162555769085884, + 0.06683612614870071, + 0.051524724811315536, + 0.03980305790901184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 0, + "style_probability": 0.6744072437286377, + "style_target": 0, + "support_probability": 0.23246504366397858, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5252852195435708, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2433103770017624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3035131096839905, + 0.8719519972801208, + 0.13817015290260315, + 0.716842770576477, + 0.174762561917305 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7222427129745483, + "style_target": 1, + "support_probability": 0.5216118693351746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10237619582502565, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24764658510684967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36653193831443787, + 0.9165976047515869, + 0.1854581981897354, + 0.7627202868461609, + 0.1552726924419403 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.20934553444385529, + "style_target": 0, + "support_probability": 0.5241495370864868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6103443107122135, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.056371767073869705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9979531764984131, + 0.39766594767570496, + 0.5997743010520935, + 0.45374366641044617, + 0.05676216632127762, + 0.10144086182117462 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 1, + "style_probability": 0.6469933390617371, + "style_target": 1, + "support_probability": 0.7769046425819397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6090904103035456, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05406906083226204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9921797513961792, + 0.39708882570266724, + 0.5305353999137878, + 0.4047098159790039, + 0.10264814645051956, + 0.07381482422351837 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 0, + "style_probability": 0.6095630526542664, + "style_target": 0, + "support_probability": 0.7705424427986145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010912816196151155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.976677417755127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1905539482831955, + 0.34634363651275635, + 0.8632676005363464, + 0.18481336534023285, + 0.03658467158675194, + 0.039568234235048294, + 0.05184483528137207, + 0.05643638223409653, + 0.09699849039316177 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.6354987025260925, + "style_target": 1, + "support_probability": 0.0075464448891580105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0077845647320728874, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9891996383666992, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21000471711158752, + 0.34180113673210144, + 0.8284920454025269, + 0.18855339288711548, + 0.06227032467722893, + 0.1277938336133957 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.25523215532302856, + "style_target": 0, + "support_probability": 0.007001119200140238, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2559703157150911, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6196236610412598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.58852618932724, + 0.5569487810134888, + 0.6749086976051331, + 0.16459201276302338, + 0.11558245122432709, + 0.061704374849796295, + 0.05319599807262421, + 0.06974967569112778, + 0.06114177033305168, + 0.08985298126935959 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.8678992986679077, + "style_target": 1, + "support_probability": 0.1740243285894394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02801657487116348, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5238876938819885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6868996620178223, + 0.5338212251663208, + 0.7641341090202332, + 0.18999895453453064, + 0.07305839657783508, + 0.05882619693875313, + 0.1663486659526825, + 0.11297576874494553, + 0.07466506212949753, + 0.12187430262565613 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.11798400431871414, + "style_target": 0, + "support_probability": 0.413636177778244, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.24559331410160085, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.517855167388916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33942916989326477, + 0.4616059362888336, + 0.2413831204175949, + 0.921595573425293, + 0.1369665116071701 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 0, + "style_probability": 0.3041302561759949, + "style_target": 1, + "support_probability": 0.2878464460372925, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10290834020598122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5535469651222229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36385515332221985, + 0.479257732629776, + 0.23497633635997772, + 0.9861051440238953, + 0.1446828991174698 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 1, + "style_probability": 0.14241716265678406, + "style_target": 0, + "support_probability": 0.49577775597572327, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1256401322412789, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9088935256004333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3230639398097992, + 0.8169033527374268, + 0.29214292764663696, + 0.3230639398097992, + 0.12204170972108841, + 0.1301404982805252 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 0, + "style_probability": 0.46981680393218994, + "style_target": 1, + "support_probability": 0.11836867034435272, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13965152775580575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9327751994132996, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29344460368156433, + 0.8896464705467224, + 0.2844555079936981, + 0.29344460368156433, + 0.06440331041812897, + 0.1282324194908142 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 0, + "style_probability": 0.7582813501358032, + "style_target": 0, + "support_probability": 0.09742733091115952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3059486932037972, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3055575489997864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7383908629417419, + 0.4658947288990021, + 0.4239257872104645, + 0.3501583933830261, + 0.09683048725128174, + 0.0867132619023323, + 0.09043177217245102, + 0.10978040844202042, + 0.15861894190311432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 1, + "style_probability": 0.5251055359840393, + "style_target": 1, + "support_probability": 0.4864300489425659, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20530747233624555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3296522796154022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7107276916503906, + 0.4698864221572876, + 0.42833414673805237, + 0.37733978033065796, + 0.08362890779972076, + 0.12550926208496094 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 1, + "style_probability": 0.4095088243484497, + "style_target": 0, + "support_probability": 0.4725264012813568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31366875247807435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4316173791885376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30058106780052185, + 0.6900478601455688, + 0.5159052014350891, + 0.3114890456199646, + 0.1937595158815384, + 0.07828997820615768, + 0.06863319873809814, + 0.06328824907541275, + 0.09920061379671097, + 0.07270949333906174 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 0, + "style_probability": 0.39561933279037476, + "style_target": 1, + "support_probability": 0.588038980960846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06777477643705404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2637573480606079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5512943863868713, + 0.7494947910308838, + 0.6278151273727417, + 0.35843518376350403, + 0.12949666380882263, + 0.13558851182460785, + 0.09472314268350601, + 0.08386682718992233, + 0.12012077122926712, + 0.10184022784233093 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 1, + "style_probability": 0.12130787968635559, + "style_target": 0, + "support_probability": 0.775662362575531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21443921314364894, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5821153521537781, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26072564721107483, + 0.21896471083164215, + 0.32723468542099, + 0.6849596500396729, + 0.04431429132819176 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.8700698614120483, + "style_target": 1, + "support_probability": 0.1473589539527893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03674300726191598, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6402130722999573, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21638819575309753, + 0.2070150226354599, + 0.3288739323616028, + 0.7209251523017883, + 0.07351319491863251, + 0.052317745983600616, + 0.04608951136469841, + 0.06766896694898605, + 0.1395120471715927, + 0.07737771421670914, + 0.23211005330085754 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.4166143238544464, + "style_target": 0, + "support_probability": 0.03747761994600296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7438316129095263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27369192242622375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5643531084060669, + 0.32729899883270264, + 0.83067786693573, + 0.22408324480056763, + 0.07288727164268494, + 0.03053252398967743 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.876690149307251, + "style_target": 1, + "support_probability": 0.8682613372802734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24702111177261502, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.317007839679718, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6186078190803528, + 0.3765833377838135, + 0.8836163282394409, + 0.2489023059606552, + 0.07616528123617172, + 0.1536562144756317 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.2615252137184143, + "style_target": 0, + "support_probability": 0.8599708676338196, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.782822579419933, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14271330833435059, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9775010347366333, + 0.4313243329524994, + 0.3465486466884613, + 0.28194841742515564, + 0.08875949680805206, + 0.09766002744436264, + 0.15346884727478027, + 0.17896753549575806, + 0.048243604600429535 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.8305078148841858, + "style_target": 1, + "support_probability": 0.7167914509773254, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7509031718929413, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10096719115972519, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.950790524482727, + 0.4360467493534088, + 0.2699258029460907, + 0.3353163003921509, + 0.09232596307992935, + 0.1000744104385376, + 0.08516784012317657, + 0.164739727973938, + 0.07267188280820847 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.4889451563358307, + "style_target": 0, + "support_probability": 0.8112829923629761, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5392566310923054, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4809698164463043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3445398807525635, + 0.33839714527130127, + 0.7956336140632629, + 0.23356595635414124, + 0.13508553802967072, + 0.10123974829912186, + 0.04896511510014534, + 0.11915604025125504, + 0.08533966541290283, + 0.11964205652475357, + 0.07018999010324478 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.89067143201828, + "style_target": 1, + "support_probability": 0.2890078127384186, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19080207450496256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44574135541915894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3302682042121887, + 0.3389586806297302, + 0.7789429426193237, + 0.21259061992168427, + 0.6413329243659973, + 0.2897045910358429, + 0.09869606047868729, + 0.1072913110256195, + 0.14589840173721313 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.2803298830986023, + "style_target": 0, + "support_probability": 0.42990291118621826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02346409599385897, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8866512775421143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.532943844795227, + 0.3629834055900574, + 0.24025188386440277, + 0.6593528985977173, + 0.10502927005290985 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 0, + "style_probability": 0.2343960702419281, + "style_target": 1, + "support_probability": 0.04429304227232933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11260384350023868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3330747187137604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6108025312423706, + 0.46353501081466675, + 0.2642454206943512, + 0.8406041860580444, + 0.12468917667865753 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 1, + "style_probability": 0.4131399989128113, + "style_target": 0, + "support_probability": 0.3399312496185303, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5218060208893979, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42780110239982605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6007564067840576, + 0.7795586585998535, + 0.4391945004463196, + 0.4707566201686859, + 0.058501534163951874, + 0.07422984391450882 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 0, + "style_probability": 0.4189811646938324, + "style_target": 1, + "support_probability": 0.7641800045967102, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5377120285775794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42904311418533325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5555453300476074, + 0.7339230179786682, + 0.5141802430152893, + 0.4247344434261322, + 0.04515484347939491, + 0.052853845059871674 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 0, + "style_probability": 0.6363420486450195, + "style_target": 0, + "support_probability": 0.7474618554115295, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10682746295048891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8277196884155273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49271470308303833, + 0.4109708070755005, + 0.5922269225120544, + 0.8730177879333496, + 0.056924231350421906, + 0.09925363212823868, + 0.13793359696865082, + 0.05714164301753044, + 0.1409716010093689 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 0, + "style_probability": 0.44655001163482666, + "style_target": 1, + "support_probability": 0.10155197978019714, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18904620694780366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7284336686134338, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4903421103954315, + 0.44589412212371826, + 0.6065604090690613, + 0.8993148803710938, + 0.04869534447789192, + 0.03223751112818718, + 0.024125853553414345, + 0.02844076231122017, + 0.11161193251609802 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 0, + "style_probability": 0.7199387550354004, + "style_target": 0, + "support_probability": 0.28739315271377563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41813427588656205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09944894164800644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4670632779598236, + 0.5274975299835205, + 0.6794490218162537, + 0.8556956052780151, + 0.06822006404399872, + 0.08456812798976898, + 0.06976204365491867, + 0.11324925720691681, + 0.13677026331424713, + 0.05117902532219887, + 0.07993195950984955 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 1, + "style_probability": 0.7735832929611206, + "style_target": 1, + "support_probability": 0.2531580924987793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13454777933110199, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1364506632089615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5317621827125549, + 0.5037932395935059, + 0.738332986831665, + 0.9009577035903931, + 0.13510292768478394, + 0.09870254993438721, + 0.23753555119037628, + 0.1482846885919571, + 0.08263534307479858, + 0.06094285845756531 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 0, + "style_probability": 0.6084318161010742, + "style_target": 0, + "support_probability": 0.20884622633457184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24955385112972167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.19820532202720642, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1825108379125595, + 0.21260125935077667, + 0.1825108379125595, + 0.9233912825584412, + 0.10725074261426926 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 0, + "style_probability": 0.3909794092178345, + "style_target": 1, + "support_probability": 0.2237480729818344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10745012590097083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.19874480366706848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21310116350650787, + 0.22478029131889343, + 0.21310116350650787, + 0.9566834568977356, + 0.1156960129737854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 1, + "style_probability": 0.36000892519950867, + "style_target": 0, + "support_probability": 0.1835382580757141, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.39811190046240835, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19589512050151825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3146715462207794, + 0.8171416521072388, + 0.42338961362838745, + 0.31947359442710876, + 0.16772963106632233, + 0.1158350333571434 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 0, + "style_probability": 0.26599982380867004, + "style_target": 1, + "support_probability": 0.6225705742835999, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7727560055610231, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16251778602600098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3102571666240692, + 0.8568605184555054, + 0.48855769634246826, + 0.3089368939399719, + 0.06508955359458923, + 0.06390700489282608 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 0, + "style_probability": 0.8153032064437866, + "style_target": 0, + "support_probability": 0.8179908990859985, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38508757650667286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2605910003185272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3149109184741974, + 0.21020834147930145, + 0.48332053422927856, + 0.9459683299064636, + 0.08893764019012451, + 0.0707477405667305, + 0.11664068698883057, + 0.07881511747837067, + 0.061878252774477005 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 1, + "style_probability": 0.608855128288269, + "style_target": 1, + "support_probability": 0.2660219073295593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25992261103228137, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3373570740222931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32044628262519836, + 0.21123644709587097, + 0.46690991520881653, + 0.9441622495651245, + 0.09770846366882324, + 0.11141487956047058 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 1, + "style_probability": 0.4763147234916687, + "style_target": 0, + "support_probability": 0.15006482601165771, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22277931209863452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6641469597816467, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23472905158996582, + 0.8935938477516174, + 0.37394773960113525, + 0.23472905158996582, + 0.14409637451171875, + 0.1451960653066635, + 0.278055876493454, + 0.213423952460289, + 0.15308499336242676, + 0.07846732437610626, + 0.13847070932388306 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 1, + "style_probability": 0.6316277980804443, + "style_target": 1, + "support_probability": 0.2282262146472931, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2276660874760806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4243614077568054, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28615447878837585, + 0.9397420883178711, + 0.46583953499794006, + 0.28615447878837585, + 0.15423554182052612, + 0.09560921788215637, + 0.30916666984558105, + 0.12627683579921722, + 0.16587886214256287, + 0.152645543217659 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 1, + "style_probability": 0.21657605469226837, + "style_target": 0, + "support_probability": 0.8063141703605652, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45455372839474506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1850506067276001, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3798665404319763, + 0.29910808801651, + 0.9775565266609192, + 0.36934396624565125, + 0.07007540017366409 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.5574297308921814, + "style_target": 1, + "support_probability": 0.41181641817092896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5214826478374279, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16773635149002075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.397930383682251, + 0.3020817041397095, + 0.9641330242156982, + 0.3624614477157593, + 0.11917999386787415, + 0.04090762138366699, + 0.10985589772462845, + 0.13975487649440765, + 0.07710563391447067, + 0.06071237102150917, + 0.1434507519006729 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.30146053433418274, + "style_target": 0, + "support_probability": 0.5906562805175781, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5953487830645577, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09595928341150284, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7047733664512634, + 0.27342504262924194, + 0.7047733664512634, + 0.8816918134689331, + 0.10662174224853516, + 0.09274519979953766 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.712507963180542, + "style_target": 1, + "support_probability": 0.4943627417087555, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2813739274988673, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09254737943410873, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7836793065071106, + 0.3464566469192505, + 0.7836793065071106, + 0.9440584778785706, + 0.05732675641775131, + 0.13641974329948425 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.31075936555862427, + "style_target": 0, + "support_probability": 0.5854400396347046, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6022156157804288, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08503389358520508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4387796223163605, + 0.48057836294174194, + 0.639371931552887, + 0.6379899978637695, + 0.04147549346089363, + 0.03950638696551323, + 0.05541782081127167, + 0.03828169032931328, + 0.06365394592285156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 1, + "style_probability": 0.8469217419624329, + "style_target": 1, + "support_probability": 0.8866777420043945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5647675556137091, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11714483052492142, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4822458028793335, + 0.3673964738845825, + 0.6233395338058472, + 0.5744825601577759, + 0.04049992561340332, + 0.06887666881084442, + 0.05258653685450554, + 0.06344135105609894, + 0.06110472232103348 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 0, + "style_probability": 0.6017475128173828, + "style_target": 0, + "support_probability": 0.8939763903617859, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44373688742842027, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5065387487411499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3819042444229126, + 0.32829543948173523, + 0.3819042444229126, + 0.6068223118782043, + 0.1593358814716339, + 0.11684788763523102, + 0.05599402263760567, + 0.06239788606762886, + 0.062095608562231064, + 0.03422132506966591, + 0.07774271070957184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.8321670293807983, + "style_target": 1, + "support_probability": 0.4700656831264496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27303440048263294, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45960375666618347, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4268385171890259, + 0.33816421031951904, + 0.4268385171890259, + 0.597992479801178, + 0.30994290113449097, + 0.06319761276245117, + 0.06728513538837433, + 0.3158089518547058, + 0.13310909271240234 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.39273229241371155, + "style_target": 0, + "support_probability": 0.5813114643096924, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.21932207715464722, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5803520679473877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24893730878829956, + 0.9202038645744324, + 0.3057495951652527, + 0.4920494854450226, + 0.09201222658157349 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 1, + "style_probability": 0.534587025642395, + "style_target": 1, + "support_probability": 0.22993236780166626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029216206465303013, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.550719678401947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24371285736560822, + 0.9308871030807495, + 0.27435439825057983, + 0.4782661199569702, + 0.1542278677225113 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 1, + "style_probability": 0.22217650711536407, + "style_target": 0, + "support_probability": 0.2121397703886032, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.427272847534632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33413293957710266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4851958751678467, + 0.29256466031074524, + 0.7973300814628601, + 0.505463719367981, + 0.05120642110705376, + 0.09779011458158493 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 0, + "style_probability": 0.38037005066871643, + "style_target": 1, + "support_probability": 0.8106072545051575, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5060437296154033, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.282206654548645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5762386322021484, + 0.23138327896595, + 0.8121346831321716, + 0.535448431968689, + 0.032730527222156525, + 0.08967471867799759 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 1, + "style_probability": 0.43943026661872864, + "style_target": 0, + "support_probability": 0.89427250623703, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17649956829895394, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6170802116394043, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4004664719104767, + 0.5186152458190918, + 0.2702395021915436, + 0.9406397938728333, + 0.07611629366874695, + 0.07804923504590988, + 0.08927885442972183, + 0.07485012710094452, + 0.16966569423675537 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.1706305593252182, + "style_target": 1, + "support_probability": 0.4717079997062683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3814538126197653, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5706726908683777, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41104811429977417, + 0.5339746475219727, + 0.2652667164802551, + 0.9624542593955994, + 0.10394265502691269, + 0.06845810264348984, + 0.05478653311729431, + 0.1260426640510559, + 0.07910723239183426 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.5845724940299988, + "style_target": 0, + "support_probability": 0.611163854598999, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.04396604857660027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9174767136573792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.300073504447937, + 0.4037522077560425, + 0.33423784375190735, + 0.7803930044174194, + 0.20089101791381836, + 0.21168328821659088, + 0.17062948644161224, + 0.25385570526123047, + 0.1538778841495514, + 0.17198055982589722, + 0.26460322737693787 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 0, + "style_probability": 0.3569968044757843, + "style_target": 1, + "support_probability": 0.053294189274311066, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17263300509148172, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5934432148933411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32117483019828796, + 0.46895474195480347, + 0.3490861654281616, + 0.9122721552848816, + 0.21875019371509552, + 0.09708235412836075, + 0.31555628776550293, + 0.11567456275224686, + 0.10107813775539398, + 0.09714186191558838 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 1, + "style_probability": 0.39445653557777405, + "style_target": 0, + "support_probability": 0.6472540497779846, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8085339428053068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0214659683406353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.46978631615638733, + 0.3915654420852661, + 0.9278210401535034, + 0.46978631615638733, + 0.16056935489177704 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 1, + "style_probability": 0.8244613409042358, + "style_target": 1, + "support_probability": 0.9205760955810547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36360134610158906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05978260934352875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4388280510902405, + 0.3704623281955719, + 0.8507896661758423, + 0.4388280510902405, + 0.24291102588176727 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 0, + "style_probability": 0.5513701438903809, + "style_target": 0, + "support_probability": 0.6546790599822998, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9195391070831268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04259388893842697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2137177437543869, + 0.2577608525753021, + 0.9987620115280151, + 0.13870328664779663, + 0.11064717918634415, + 0.057912442833185196 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.7780658602714539, + "style_target": 1, + "support_probability": 0.9164109826087952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7740907251304547, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04726192727684975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22958263754844666, + 0.26513296365737915, + 0.9962413311004639, + 0.11003800481557846, + 0.07311659306287766 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.3614879846572876, + "style_target": 0, + "support_probability": 0.9268561601638794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7462307217596654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04041363671422005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1589631289243698, + 0.9695572257041931, + 0.2832396924495697, + 0.18312008678913116, + 0.04080945998430252, + 0.050297811627388, + 0.055990900844335556, + 0.06948645412921906, + 0.10009623318910599 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.6689842343330383, + "style_target": 1, + "support_probability": 0.8628138303756714, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30318704486506903, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05894135311245918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1892438381910324, + 0.9808931350708008, + 0.31342560052871704, + 0.21223831176757812, + 0.0938134491443634, + 0.12300848960876465, + 0.052365437150001526, + 0.12573261559009552, + 0.07166633009910583 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.17720355093479156, + "style_target": 0, + "support_probability": 0.8795581459999084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03836611690408317, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8698095679283142, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33430835604667664, + 0.7710943818092346, + 0.2956755757331848, + 0.32554078102111816, + 0.05931691825389862, + 0.082724928855896, + 0.1447540670633316, + 0.05887917801737785, + 0.029408155009150505, + 0.2861151695251465, + 0.13754689693450928 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 1, + "style_probability": 0.7774109244346619, + "style_target": 1, + "support_probability": 0.021381480619311333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03887441260174038, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8462142944335938, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3810131251811981, + 0.7757185697555542, + 0.3171360492706299, + 0.37590348720550537, + 0.031556423753499985, + 0.06859457492828369, + 0.19311580061912537, + 0.10047180205583572, + 0.059291962534189224, + 0.09662415832281113, + 0.2703707218170166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 0, + "style_probability": 0.5030922293663025, + "style_target": 0, + "support_probability": 0.03431329131126404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25528834057495803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6251063942909241, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3891165852546692, + 0.7584395408630371, + 0.5364682078361511, + 0.4061339795589447, + 0.039089396595954895 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.8899912238121033, + "style_target": 1, + "support_probability": 0.11559426039457321, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02101375793851029, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8562275767326355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36369413137435913, + 0.7366845607757568, + 0.5192928314208984, + 0.3889707922935486, + 0.03362170234322548, + 0.046533774584531784, + 0.03742510452866554, + 0.047432925552129745, + 0.05372360721230507, + 0.05599178001284599 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.4262133240699768, + "style_target": 0, + "support_probability": 0.015481531620025635, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.762536108165507, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2098802924156189, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.279080867767334, + 0.39071640372276306, + 0.9475551247596741, + 0.3026966154575348, + 0.24384739995002747, + 0.06145424023270607 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.6742578148841858, + "style_target": 1, + "support_probability": 0.6652964949607849, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3735240159200843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29546189308166504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29461929202079773, + 0.4418463110923767, + 0.9687469005584717, + 0.33161917328834534, + 0.06322479248046875, + 0.06535971164703369 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.25814053416252136, + "style_target": 0, + "support_probability": 0.6625351905822754, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6343651318804112, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3208605945110321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5049889087677002, + 0.897866427898407, + 0.24446508288383484, + 0.2811942994594574, + 0.042611002922058105, + 0.04943598061800003, + 0.07422459870576859, + 0.24504445493221283, + 0.044382039457559586 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 1, + "style_probability": 0.8382514119148254, + "style_target": 1, + "support_probability": 0.5731849670410156, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6789379029717739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2869059443473816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4345044791698456, + 0.8651587963104248, + 0.2681352496147156, + 0.3113265633583069, + 0.12845800817012787, + 0.06502074003219604, + 0.08171595633029938, + 0.11247842013835907, + 0.23491047322750092 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 0, + "style_probability": 0.7106528878211975, + "style_target": 0, + "support_probability": 0.6574373841285706, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4068419397679556, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5635737180709839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:positive", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.43724310398101807, + 0.5178372263908386, + 0.546389639377594, + 0.31346121430397034, + 0.07237622886896133, + 0.08907032757997513, + 0.04941222444176674, + 0.02162756398320198, + 0.06034643203020096, + 0.046355053782463074, + 0.33494672179222107, + 0.04019991308450699 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.9036458730697632, + "style_target": 1, + "support_probability": 0.31814470887184143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08375038994843445, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7885075211524963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47241586446762085, + 0.44635477662086487, + 0.5140059590339661, + 0.27891919016838074, + 0.17077858746051788, + 0.04120165854692459, + 0.0898679718375206, + 0.11882061511278152, + 0.08496486395597458 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.33946794271469116, + "style_target": 0, + "support_probability": 0.16567614674568176, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5339891942905393, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16360406577587128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47778746485710144, + 0.6966130137443542, + 0.437490850687027, + 0.5873017311096191, + 0.05180392414331436 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 0, + "style_probability": 0.35229459404945374, + "style_target": 1, + "support_probability": 0.9152568578720093, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04925618582850732, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37400397658348083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6324010491371155, + 0.7437915802001953, + 0.48590266704559326, + 0.5185743570327759, + 0.07098910957574844 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 1, + "style_probability": 0.045145291835069656, + "style_target": 0, + "support_probability": 0.837456226348877, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5246988467584414, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20033270120620728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5259548425674438, + 0.9583595991134644, + 0.2544577717781067, + 0.21450898051261902, + 0.0831887423992157, + 0.09451133757829666 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 1, + "style_probability": 0.6318874359130859, + "style_target": 1, + "support_probability": 0.3930894434452057, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.586769641811536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17104576528072357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4991133511066437, + 0.9500525593757629, + 0.2259007692337036, + 0.22884434461593628, + 0.19895969331264496, + 0.07490658015012741 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 0, + "style_probability": 0.6740256547927856, + "style_target": 0, + "support_probability": 0.5196632146835327, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0679243010983803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9270089864730835, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11700581014156342, + 0.503730058670044, + 0.8230912685394287, + 0.24707838892936707, + 0.09410574287176132, + 0.16273248195648193, + 0.14827117323875427, + 0.10980594903230667, + 0.11861813068389893 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 0, + "style_probability": 0.37069952487945557, + "style_target": 1, + "support_probability": 0.06226017326116562, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07699851125342906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.930046796798706, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10126741975545883, + 0.4918375313282013, + 0.7928075790405273, + 0.20769086480140686, + 0.08862178772687912, + 0.09967246651649475, + 0.07848250865936279, + 0.10542149841785431, + 0.0737140104174614 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 0, + "style_probability": 0.7504605650901794, + "style_target": 0, + "support_probability": 0.04511883482336998, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5081304274495789, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37392327189445496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19235759973526, + 0.12302719056606293, + 0.27780914306640625, + 0.9688214063644409, + 0.09791413694620132, + 0.14738091826438904, + 0.07636243849992752, + 0.17324723303318024, + 0.08782019466161728, + 0.0777435153722763, + 0.15447114408016205 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 1, + "style_probability": 0.5243289470672607, + "style_target": 1, + "support_probability": 0.5204092860221863, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.036475145161047856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2804182171821594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19341818988323212, + 0.12557777762413025, + 0.26337286829948425, + 0.9841991662979126, + 0.08309261500835419, + 0.12880632281303406, + 0.10516636818647385, + 0.07064640522003174, + 0.14559926092624664, + 0.09264034777879715 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 1, + "style_probability": 0.08048991113901138, + "style_target": 0, + "support_probability": 0.5321893095970154, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8554350432606096, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1043919175863266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3635876178741455, + 0.2750513255596161, + 0.9909279942512512, + 0.28176215291023254, + 0.058938365429639816 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 1, + "style_probability": 0.9063575863838196, + "style_target": 1, + "support_probability": 0.8485420942306519, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7744867257365995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.050676874816417694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37591156363487244, + 0.29212790727615356, + 0.9899119734764099, + 0.2748691737651825, + 0.16161730885505676 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 0, + "style_probability": 0.5698735117912292, + "style_target": 0, + "support_probability": 0.9347108006477356, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12469830653238292, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8105072379112244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10016295313835144, + 0.10016295313835144, + 0.47584691643714905, + 0.5265614986419678, + 0.0885271355509758, + 0.08911214023828506 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.8502843976020813, + "style_target": 1, + "support_probability": 0.11669713258743286, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12717032988886426, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7448030114173889, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10524066537618637, + 0.10524066537618637, + 0.4320220351219177, + 0.49504202604293823, + 0.033360742032527924 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.4603906571865082, + "style_target": 0, + "support_probability": 0.23535585403442383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19082400173074496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5217648148536682, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5669606924057007, + 0.7873969078063965, + 0.09215456247329712, + 0.6388040781021118, + 0.057508472353219986, + 0.0877847671508789, + 0.04595756158232689, + 0.06836673617362976, + 0.07515684515237808 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.614633321762085, + "style_target": 1, + "support_probability": 0.286902517080307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024393531832410754, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5573971271514893, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6828072667121887, + 0.8524870276451111, + 0.1441745012998581, + 0.7710685729980469, + 0.06932321190834045, + 0.25515124201774597, + 0.1764776110649109, + 0.10509146004915237, + 0.10841439664363861 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.1065046489238739, + "style_target": 0, + "support_probability": 0.33004939556121826, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8419112345617694, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.036670003086328506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4366942346096039, + 0.7252574563026428, + 0.9052311778068542, + 0.3750194013118744, + 0.13030868768692017, + 0.06569597870111465, + 0.08584890514612198, + 0.027155794203281403, + 0.09640615433454514, + 0.06741432100534439 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.8602322340011597, + "style_target": 1, + "support_probability": 0.9767289161682129, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7126496754497299, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05117448419332504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4532702565193176, + 0.6410629749298096, + 0.8590909242630005, + 0.387726753950119, + 0.05911567062139511, + 0.0376308336853981, + 0.09296578913927078, + 0.056364502757787704, + 0.0685347393155098, + 0.04158040136098862, + 0.039183955639600754 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.44979217648506165, + "style_target": 0, + "support_probability": 0.9706704616546631, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11713014816925639, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.916775107383728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5610141754150391, + 0.4426126778125763, + 0.49019184708595276, + 0.5610141754150391, + 0.03942030295729637 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 1, + "style_probability": 0.5866143703460693, + "style_target": 1, + "support_probability": 0.13388174772262573, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08136532082152802, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9684241414070129, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5711247324943542, + 0.5212936401367188, + 0.513303279876709, + 0.5711247324943542, + 0.06418118625879288 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 0, + "style_probability": 0.8420331478118896, + "style_target": 0, + "support_probability": 0.050290580838918686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47319859604997677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.462851881980896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3778761625289917, + 0.3271510601043701, + 0.7450065016746521, + 0.3778761625289917, + 0.12918417155742645, + 0.044518519192934036 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 1, + "style_probability": 0.5335311889648438, + "style_target": 1, + "support_probability": 0.40947386622428894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5266055701659835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3154374361038208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3777693212032318, + 0.31294840574264526, + 0.788938045501709, + 0.3777693212032318, + 0.06424208730459213 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 1, + "style_probability": 0.41188475489616394, + "style_target": 0, + "support_probability": 0.5302614569664001, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6670247292401619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2646149694919586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8445043563842773, + 0.43945375084877014, + 0.32847198843955994, + 0.43945375084877014, + 0.0855172649025917, + 0.03814605996012688, + 0.0739230215549469, + 0.23084241151809692, + 0.10863206535577774 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.6594140529632568, + "style_target": 1, + "support_probability": 0.8322903513908386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3816525863117464, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26435115933418274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.923004150390625, + 0.4276307225227356, + 0.2948933243751526, + 0.4276307225227356, + 0.08720433712005615, + 0.0877520963549614, + 0.08818770945072174, + 0.07740318030118942, + 0.142610564827919 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.30282697081565857, + "style_target": 0, + "support_probability": 0.8622696995735168, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.471066623906545, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29323822259902954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3566:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3282911479473114, + 0.4487893879413605, + 0.9496537446975708, + 0.3282911479473114, + 0.04986744374036789, + 0.04215161129832268, + 0.06346610188484192, + 0.02224332094192505, + 0.05133717134594917, + 0.07680840790271759, + 0.046667180955410004 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.2850327491760254, + "style_target": 1, + "support_probability": 0.7658706903457642, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8467960764110667, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2241934835910797, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3566:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4158138930797577, + 0.4767252504825592, + 0.9618197679519653, + 0.4158138930797577, + 0.034325435757637024, + 0.044024378061294556, + 0.05916740000247955, + 0.053467411547899246, + 0.06070392206311226, + 0.060989491641521454 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.8164540529251099, + "style_target": 0, + "support_probability": 0.817983865737915, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3993026438267923, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2616245746612549, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22770750522613525, + 0.41370779275894165, + 0.6790574193000793, + 0.3651060461997986, + 0.10743726789951324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.8376819491386414, + "style_target": 1, + "support_probability": 0.3530244529247284, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019367373341206258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.36004874110221863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2813306748867035, + 0.5261178016662598, + 0.7436072826385498, + 0.43312162160873413, + 0.11945644021034241 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.13169153034687042, + "style_target": 0, + "support_probability": 0.24113346636295319, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21292494967960324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2739782929420471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40902841091156006, + 0.9174051880836487, + 0.47204893827438354, + 0.4623758792877197, + 0.07983028888702393, + 0.11390218883752823 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6844444870948792, + "style_target": 1, + "support_probability": 0.4785982668399811, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20203548847768546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17564386129379272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4840158224105835, + 0.866803765296936, + 0.4640759825706482, + 0.47116413712501526, + 0.06487277150154114, + 0.1699722409248352 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 1, + "style_probability": 0.4734915792942047, + "style_target": 0, + "support_probability": 0.5417033433914185, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7875469916614075, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05097911134362221, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43737199902534485, + 0.6272861957550049, + 0.908629834651947, + 0.15850374102592468, + 0.08481428027153015, + 0.019036704674363136, + 0.05666792392730713, + 0.04693209379911423, + 0.07144399732351303 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.7266431450843811, + "style_target": 1, + "support_probability": 0.7970678210258484, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6234869450099885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04683253541588783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41462013125419617, + 0.6722478270530701, + 0.8830094933509827, + 0.175290048122406, + 0.05258578807115555, + 0.05627278983592987 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.3456844389438629, + "style_target": 0, + "support_probability": 0.7938408255577087, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1442529171131879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8905689120292664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8080419898033142, + 0.3073508143424988, + 0.27759066224098206, + 0.2957100570201874, + 0.11042953282594681, + 0.060653794556856155, + 0.042214829474687576, + 0.08318279683589935, + 0.08314476907253265, + 0.11777786910533905 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.8393906354904175, + "style_target": 1, + "support_probability": 0.05747271329164505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033926202207450074, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6418973803520203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8918761610984802, + 0.3208451271057129, + 0.31440573930740356, + 0.300617516040802, + 0.1237635463476181, + 0.08309824764728546, + 0.3746914863586426, + 0.14494353532791138, + 0.07846516370773315, + 0.0986870750784874 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.20649246871471405, + "style_target": 0, + "support_probability": 0.18154007196426392, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13745364863446513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5892698764801025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3031955659389496, + 0.5622317790985107, + 0.20383207499980927, + 0.1506700962781906, + 0.13452591001987457 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 0, + "style_probability": 0.3738732635974884, + "style_target": 1, + "support_probability": 0.30095845460891724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12672516341334308, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5788800120353699, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30500125885009766, + 0.5418554544448853, + 0.25205159187316895, + 0.14063453674316406, + 0.08514468371868134 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 1, + "style_probability": 0.4051693379878998, + "style_target": 0, + "support_probability": 0.3813556134700775, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28847357765214326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6634833216667175, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5292584896087646, + 0.7609136700630188, + 0.24043257534503937, + 0.5936883091926575, + 0.044991832226514816, + 0.09812197834253311 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 0, + "style_probability": 0.4029519557952881, + "style_target": 1, + "support_probability": 0.3541439175605774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26202913167208325, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7728354334831238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5400243401527405, + 0.7817147970199585, + 0.1940537542104721, + 0.6244134902954102, + 0.11384814232587814, + 0.27382078766822815 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 0, + "style_probability": 0.6713582277297974, + "style_target": 0, + "support_probability": 0.28177496790885925, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.09676732368255099, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7671812772750854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6019583344459534, + 0.5917466878890991, + 0.5748052000999451, + 0.29972952604293823, + 0.052177708595991135, + 0.12179069966077805, + 0.04598907008767128, + 0.06234103813767433, + 0.06049463525414467 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 0, + "style_probability": 0.28591322898864746, + "style_target": 1, + "support_probability": 0.22498661279678345, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022879943105534598, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8640955090522766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6515256762504578, + 0.8565050363540649, + 0.766857922077179, + 0.47649094462394714, + 0.07066348940134048, + 0.23058602213859558, + 0.0809030532836914, + 0.09066648781299591, + 0.1294788271188736 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 1, + "style_probability": 0.10581613332033157, + "style_target": 0, + "support_probability": 0.20395101606845856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38153533023380604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12410350888967514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4942243993282318, + 0.41497737169265747, + 0.4942243993282318, + 0.9370563626289368, + 0.08243415504693985, + 0.09622626006603241, + 0.1258639544248581, + 0.08298217505216599, + 0.1250677853822708, + 0.10919660329818726 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 0, + "style_probability": 0.38664594292640686, + "style_target": 1, + "support_probability": 0.501501739025116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3169222097483965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18627630174160004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5082786083221436, + 0.42303037643432617, + 0.5082786083221436, + 0.9180845022201538, + 0.06000214442610741, + 0.16371433436870575, + 0.052444349974393845, + 0.08384007215499878, + 0.108550064265728, + 0.08901029080152512, + 0.11939918994903564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 0, + "style_probability": 0.5574268698692322, + "style_target": 0, + "support_probability": 0.3323315382003784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19341292741839414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6730062365531921, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3280116021633148, + 0.690923810005188, + 0.3094892203807831, + 0.1504519134759903, + 0.07737919688224792 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.7219365835189819, + "style_target": 1, + "support_probability": 0.09577907621860504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09445374204042742, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6886489391326904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34419727325439453, + 0.6315350532531738, + 0.29530030488967896, + 0.16730952262878418, + 0.05200670287013054, + 0.127424418926239, + 0.1783953160047531, + 0.12522664666175842, + 0.14956454932689667, + 0.09606034308671951, + 0.15519607067108154 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.35946738719940186, + "style_target": 0, + "support_probability": 0.07117287069559097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12125339684749369, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4166075587272644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34077292680740356, + 0.3556211590766907, + 0.31981831789016724, + 0.6415016651153564, + 0.03861796110868454, + 0.08175303786993027 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.7548536062240601, + "style_target": 1, + "support_probability": 0.1494446098804474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009462394389304728, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.46407845616340637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42573070526123047, + 0.38511765003204346, + 0.337631493806839, + 0.6751412749290466, + 0.07786455005407333, + 0.19162103533744812 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.19060690701007843, + "style_target": 0, + "support_probability": 0.14974671602249146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01916711737328591, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.954097330570221, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6513561010360718, + 0.27861812710762024, + 0.46292978525161743, + 0.6659910082817078, + 0.041960906237363815, + 0.09928660839796066, + 0.11251293867826462, + 0.04933741316199303, + 0.1093834862112999 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 1, + "style_probability": 0.6519742012023926, + "style_target": 1, + "support_probability": 0.023498408496379852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013952334738139898, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.948650598526001, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6460906267166138, + 0.25013646483421326, + 0.5292807817459106, + 0.6263432502746582, + 0.2476055920124054, + 0.1358436942100525, + 0.15422721207141876, + 0.07973556965589523, + 0.11115651577711105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 0, + "style_probability": 0.5691421031951904, + "style_target": 0, + "support_probability": 0.01395469345152378, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4305881544621302, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3123425841331482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2613604664802551, + 0.27003729343414307, + 0.43746161460876465, + 0.2613604664802551, + 0.28613826632499695, + 0.05624130740761757, + 0.04361196607351303, + 0.029438713565468788, + 0.0695151537656784, + 0.025526339188218117, + 0.06112469360232353 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.8683319687843323, + "style_target": 1, + "support_probability": 0.7704646587371826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2595562309277467, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.290813148021698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24602283537387848, + 0.24249140918254852, + 0.47940030694007874, + 0.24602283537387848, + 0.04899439215660095, + 0.09395276755094528, + 0.059841059148311615, + 0.03445613384246826, + 0.17715796828269958 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.18303368985652924, + "style_target": 0, + "support_probability": 0.7342294454574585, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7046877951524612, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10587393492460251, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19804009795188904, + 0.8308695554733276, + 0.25214576721191406, + 0.16515079140663147, + 0.07795778661966324 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 1, + "style_probability": 0.5163064002990723, + "style_target": 1, + "support_probability": 0.8523369431495667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7023068858000271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07491632550954819, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2071157693862915, + 0.8301316499710083, + 0.26524800062179565, + 0.2138635367155075, + 0.0428253635764122, + 0.08101387321949005, + 0.08506922423839569, + 0.17774894833564758, + 0.08819397538900375, + 0.05125263333320618, + 0.23899394273757935 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 1, + "style_probability": 0.42997968196868896, + "style_target": 0, + "support_probability": 0.8677352070808411, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37387728233248313, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4470910131931305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2813345789909363, + 0.36192911863327026, + 0.311714231967926, + 0.7481505274772644, + 0.2497962862253189, + 0.10366452485322952 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.6130000948905945, + "style_target": 1, + "support_probability": 0.3741114139556885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08569136669150465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3732845187187195, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29584363102912903, + 0.42584824562072754, + 0.21169231832027435, + 0.8587813377380371, + 0.05684308707714081, + 0.14446242153644562 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.3437429964542389, + "style_target": 0, + "support_probability": 0.34812089800834656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05280154371659959, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7036777138710022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12341026216745377, + 0.6942853331565857, + 0.21316640079021454, + 0.6594561338424683, + 0.07332442700862885, + 0.10439006984233856, + 0.1565704345703125, + 0.23014786839485168, + 0.06725381314754486 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 0, + "style_probability": 0.41748160123825073, + "style_target": 1, + "support_probability": 0.15508463978767395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4125200128307407, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4564270079135895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14194348454475403, + 0.7742393612861633, + 0.1716194897890091, + 0.7666481137275696, + 0.11280456930398941, + 0.08428386598825455, + 0.07058054953813553, + 0.10168864578008652, + 0.29624778032302856 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 0, + "style_probability": 0.7363309860229492, + "style_target": 0, + "support_probability": 0.4773055613040924, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2989457723166252, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48901715874671936, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19140981137752533, + 0.17901748418807983, + 0.8287808895111084, + 0.14841817319393158, + 0.06153753772377968, + 0.11592618376016617, + 0.06401810050010681, + 0.09030815958976746, + 0.05855630710721016, + 0.15109750628471375, + 0.09200111031532288 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 0, + "style_probability": 0.48889848589897156, + "style_target": 1, + "support_probability": 0.227980837225914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1590504544639492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6008481383323669, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21881328523159027, + 0.18329226970672607, + 0.8435521125793457, + 0.1448032259941101, + 0.05947661027312279, + 0.05735776573419571, + 0.10622109472751617, + 0.09135985374450684, + 0.08037688583135605 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 1, + "style_probability": 0.2658579647541046, + "style_target": 0, + "support_probability": 0.12278955429792404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07790027772559203, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.777061402797699, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4182175397872925, + 0.5342345833778381, + 0.2330903857946396, + 0.6030093431472778, + 0.07560649514198303 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 0, + "style_probability": 0.4162387549877167, + "style_target": 1, + "support_probability": 0.12507189810276031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09286711674678379, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8102608323097229, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41038575768470764, + 0.48283180594444275, + 0.2526125907897949, + 0.5817908644676208, + 0.1153184100985527 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 0, + "style_probability": 0.5906785726547241, + "style_target": 0, + "support_probability": 0.11171102523803711, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05397411430296895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3639312982559204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27128854393959045, + 0.7800478339195251, + 0.3836887776851654, + 0.342159628868103, + 0.07073410600423813, + 0.15705294907093048 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.37710773944854736, + "style_target": 1, + "support_probability": 0.04520208016037941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.039036435083725035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3587248623371124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26956072449684143, + 0.7975093126296997, + 0.39303937554359436, + 0.328105628490448, + 0.03693300858139992, + 0.2257004827260971 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.5353570580482483, + "style_target": 0, + "support_probability": 0.030917180702090263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04628959975589201, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9219278693199158, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6314307451248169, + 0.3040337860584259, + 0.3467870354652405, + 0.28853046894073486, + 0.07155723124742508, + 0.04442489147186279, + 0.05420977994799614, + 0.05972650647163391, + 0.048281922936439514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 0, + "style_probability": 0.44328105449676514, + "style_target": 1, + "support_probability": 0.0190989151597023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036066879332439426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7424423098564148, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7541996836662292, + 0.35992053151130676, + 0.5332625508308411, + 0.3742945194244385, + 0.06026912480592728, + 0.17636726796627045, + 0.053300075232982635, + 0.04835127294063568, + 0.10523109883069992 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 1, + "style_probability": 0.345496267080307, + "style_target": 0, + "support_probability": 0.12955237925052643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15708956603312169, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7565929889678955, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.46064746379852295, + 0.46064746379852295, + 0.538098931312561, + 0.5564367771148682, + 0.06962568312883377, + 0.08219412714242935, + 0.05669405311346054, + 0.08384764939546585, + 0.08664644509553909, + 0.05616240203380585, + 0.08726751804351807 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 0, + "style_probability": 0.20908740162849426, + "style_target": 1, + "support_probability": 0.43130430579185486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2042602730513843, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.672196626663208, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45622652769088745, + 0.45622652769088745, + 0.5075289011001587, + 0.6017523407936096, + 0.08102481067180634, + 0.06996720284223557, + 0.061152134090662, + 0.066941037774086, + 0.052447691559791565, + 0.04119723290205002, + 0.11071567982435226 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 1, + "style_probability": 0.2739526629447937, + "style_target": 0, + "support_probability": 0.5016420483589172, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8313684982435042, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07432425767183304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9205504655838013, + 0.6334484815597534, + 0.19468577206134796, + 0.4067244231700897, + 0.111061692237854 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.8065416812896729, + "style_target": 1, + "support_probability": 0.8416280746459961, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6846430749740823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1145457774400711, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8829360604286194, + 0.5691633820533752, + 0.20124858617782593, + 0.4029924273490906, + 0.0973362997174263, + 0.10210337489843369, + 0.03965207189321518, + 0.060149699449539185, + 0.04119788110256195, + 0.35512250661849976, + 0.10339750349521637 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.41530317068099976, + "style_target": 0, + "support_probability": 0.7945729494094849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006620448602443383, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8181895017623901, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5044069290161133, + 0.41540414094924927, + 0.3059764802455902, + 0.5515250563621521, + 0.10258444398641586, + 0.21396476030349731 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.7866038084030151, + "style_target": 1, + "support_probability": 0.003875895170494914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014484024229360858, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7524237036705017, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5783377289772034, + 0.44611796736717224, + 0.29953983426094055, + 0.5258236527442932, + 0.13830861449241638, + 0.12321369349956512 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.16718891263008118, + "style_target": 0, + "support_probability": 0.005452022422105074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5219016479694836, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3724818825721741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8629961013793945, + 0.358562171459198, + 0.32118797302246094, + 0.6000171303749084, + 0.1384577751159668, + 0.14315271377563477, + 0.1162729263305664, + 0.07563379406929016, + 0.1621018648147583 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 1, + "style_probability": 0.8293216228485107, + "style_target": 1, + "support_probability": 0.3508254289627075, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3119077347217954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44825366139411926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8158572912216187, + 0.3104364573955536, + 0.3210109770298004, + 0.6518124938011169, + 0.06805182248353958, + 0.05586624518036842, + 0.07463643699884415, + 0.06387609988451004, + 0.0673861876130104 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 0, + "style_probability": 0.5945417881011963, + "style_target": 0, + "support_probability": 0.24767647683620453, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31253491195043576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5767911672592163, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44932758808135986, + 0.28214821219444275, + 0.8345431089401245, + 0.3428983986377716, + 0.0829668790102005, + 0.11110746115446091, + 0.11258473992347717, + 0.15817658603191376, + 0.11114457994699478, + 0.16681329905986786 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.856857180595398, + "style_target": 1, + "support_probability": 0.12167666107416153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11888936861662583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6608067154884338, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5053390264511108, + 0.2729314863681793, + 0.8338766098022461, + 0.3602757453918457, + 0.06757001578807831, + 0.10470739752054214, + 0.09366553276777267, + 0.15368424355983734, + 0.19342486560344696 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.3733077049255371, + "style_target": 0, + "support_probability": 0.0879589170217514, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.532367127711895, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29467281699180603, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5898115634918213, + 0.3191385269165039, + 0.8649036288261414, + 0.30948716402053833, + 0.17947430908679962 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.5113087296485901, + "style_target": 1, + "support_probability": 0.6385560035705566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.056805118098410066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4570206105709076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6704645752906799, + 0.2871750295162201, + 0.9133455753326416, + 0.30803290009498596, + 0.12062244862318039 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.06578757613897324, + "style_target": 0, + "support_probability": 0.538457453250885, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24215113940785418, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33834442496299744, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30189260840415955, + 0.42317354679107666, + 0.5761306881904602, + 0.392081618309021, + 0.11918292194604874, + 0.09943969547748566 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 1, + "style_probability": 0.7870315909385681, + "style_target": 1, + "support_probability": 0.197265163064003, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0705820487453003, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5279737710952759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2993815541267395, + 0.4315430223941803, + 0.6130978465080261, + 0.3809930086135864, + 0.06835167109966278, + 0.0836261436343193 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 1, + "style_probability": 0.48170071840286255, + "style_target": 0, + "support_probability": 0.06908506155014038, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42295651344718316, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5529744029045105, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1806374341249466, + 0.547631025314331, + 0.5580947399139404, + 0.7873767614364624, + 0.06254925578832626, + 0.11201437562704086, + 0.15038344264030457, + 0.17424499988555908, + 0.11672786623239517 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.8470461368560791, + "style_target": 1, + "support_probability": 0.33682122826576233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10676538499934898, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7506343722343445, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15420785546302795, + 0.5936812162399292, + 0.5437949895858765, + 0.7607633471488953, + 0.08340071886777878, + 0.13880427181720734 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.3200611472129822, + "style_target": 0, + "support_probability": 0.17464618384838104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7855056355209376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06799469143152237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9684218764305115, + 0.3979053199291229, + 0.12669959664344788, + 0.2810092568397522, + 0.11930310726165771, + 0.06724236905574799, + 0.06526608765125275, + 0.06066618859767914, + 0.10050994902849197, + 0.3641858696937561, + 0.22510108351707458 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.8481901288032532, + "style_target": 1, + "support_probability": 0.6572337746620178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2638987301845584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06910227984189987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9833536148071289, + 0.3952523469924927, + 0.14323045313358307, + 0.34430620074272156, + 0.09419138729572296, + 0.09176824241876602, + 0.11874239891767502, + 0.14410951733589172, + 0.07826139032840729, + 0.12657670676708221 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.20895056426525116, + "style_target": 0, + "support_probability": 0.8035505414009094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.653081329366992, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10372433066368103, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35369873046875, + 0.24677027761936188, + 0.23251770436763763, + 0.8105354905128479, + 0.1068490520119667 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.7623803019523621, + "style_target": 1, + "support_probability": 0.7625394463539124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.501380914439066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.09347230195999146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.313389390707016, + 0.2693108916282654, + 0.2278410792350769, + 0.7386381030082703, + 0.05767003819346428 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.4413679838180542, + "style_target": 0, + "support_probability": 0.7914193272590637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7320083169829021, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5431788563728333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1299292892217636, + 0.20301681756973267, + 0.6193815469741821, + 0.8872619867324829, + 0.036410361528396606, + 0.14738652110099792 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 1, + "style_probability": 0.938412606716156, + "style_target": 1, + "support_probability": 0.5151817202568054, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4469532018429291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5942447185516357, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13617603480815887, + 0.21309739351272583, + 0.6509475111961365, + 0.8674266338348389, + 0.06931091099977493 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 0, + "style_probability": 0.611811637878418, + "style_target": 0, + "support_probability": 0.5036957859992981, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9718894354213582, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.01892080344259739, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9930800795555115, + 0.7429183721542358, + 0.11435442417860031, + 0.09975853562355042, + 0.04941853880882263, + 0.143345445394516, + 0.08914565294981003, + 0.05014276131987572, + 0.08287408947944641 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.9287530183792114, + "style_target": 1, + "support_probability": 0.9794757962226868, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8264323907734225, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015105040743947029, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9995803236961365, + 0.8621670603752136, + 0.19434963166713715, + 0.16974152624607086, + 0.10985974222421646, + 0.0982012078166008, + 0.1174740120768547, + 0.06836767494678497, + 0.11688851565122604 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.17201779782772064, + "style_target": 0, + "support_probability": 0.9933812618255615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.031862176090832355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9271596670150757, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2526870369911194, + 0.250232070684433, + 0.2526870369911194, + 0.9116178154945374, + 0.16928884387016296, + 0.06844257563352585, + 0.11227762699127197, + 0.14749675989151, + 0.2077915072441101, + 0.052940934896469116, + 0.1345282346010208 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 1, + "style_probability": 0.6996467709541321, + "style_target": 1, + "support_probability": 0.02797497808933258, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04215391510421285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.838748574256897, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2588833272457123, + 0.22732792794704437, + 0.2588833272457123, + 0.8877773284912109, + 0.2055175006389618, + 0.16006876528263092, + 0.09331385791301727, + 0.10669806599617004, + 0.2545010447502136, + 0.25372380018234253, + 0.13148410618305206 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 1, + "style_probability": 0.4770369529724121, + "style_target": 0, + "support_probability": 0.057668693363666534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15442307541342304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16269557178020477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3268371522426605, + 0.8518972992897034, + 0.3624860346317291, + 0.4162661135196686, + 0.05092618241906166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 1, + "style_probability": 0.5139185786247253, + "style_target": 1, + "support_probability": 0.11357925832271576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28661220634647705, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15325325727462769, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31235334277153015, + 0.8861863613128662, + 0.3540991246700287, + 0.4504299759864807, + 0.03993320092558861 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 0, + "style_probability": 0.8947772979736328, + "style_target": 0, + "support_probability": 0.0782386064529419, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08278467541146894, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.830299973487854, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19976116716861725, + 0.3029586374759674, + 0.49840331077575684, + 0.19976116716861725, + 0.11799889802932739, + 0.15300336480140686 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 0, + "style_probability": 0.3544210195541382, + "style_target": 1, + "support_probability": 0.20288778841495514, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09339517986136077, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7333672046661377, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3816065490245819, + 0.3990248143672943, + 0.8178511261940002, + 0.3816065490245819, + 0.10423502326011658, + 0.21222810447216034 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 1, + "style_probability": 0.3758033514022827, + "style_target": 0, + "support_probability": 0.2780856490135193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17393154026675509, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4970325827598572, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3040863573551178, + 0.43532228469848633, + 0.8223036527633667, + 0.33910468220710754, + 0.10465501993894577, + 0.08877155929803848, + 0.09179672598838806, + 0.10313327610492706, + 0.12161054462194443 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 1, + "style_probability": 0.5075136423110962, + "style_target": 1, + "support_probability": 0.18329304456710815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2759166858230768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3095911145210266, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30449825525283813, + 0.4549656808376312, + 0.8213104605674744, + 0.33675268292427063, + 0.09453068673610687, + 0.20364534854888916, + 0.16422605514526367, + 0.11843196302652359, + 0.04421650990843773 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 0, + "style_probability": 0.688859760761261, + "style_target": 0, + "support_probability": 0.2626679539680481, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1578564173456681, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9142364859580994, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45718470215797424, + 0.3773197829723358, + 0.7675618529319763, + 0.45718470215797424, + 0.07486701756715775, + 0.07626549899578094, + 0.0914069339632988, + 0.07500346750020981, + 0.0592363141477108, + 0.035613320767879486, + 0.14603640139102936 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.45734986662864685, + "style_target": 1, + "support_probability": 0.10572460293769836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2702161368269853, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8910112380981445, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47254636883735657, + 0.4308093786239624, + 0.8067858815193176, + 0.47254636883735657, + 0.0676681399345398, + 0.10832930356264114, + 0.0648636594414711, + 0.08396172523498535, + 0.2357785850763321, + 0.06448225677013397, + 0.06482186168432236 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.7590057849884033, + "style_target": 0, + "support_probability": 0.14166516065597534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5770547508292841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1785307675600052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8868817090988159, + 0.28001338243484497, + 0.16243499517440796, + 0.4516148567199707, + 0.08434164524078369 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.709595799446106, + "style_target": 1, + "support_probability": 0.7043668031692505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.060908792966163534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20770876109600067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9006602168083191, + 0.38168594241142273, + 0.20536455512046814, + 0.5739122033119202, + 0.13989807665348053 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.13532279431819916, + "style_target": 0, + "support_probability": 0.5570428967475891, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5492995775275755, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36548370122909546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4051021933555603, + 0.21738477051258087, + 0.8804277181625366, + 0.262141615152359, + 0.0387239046394825, + 0.12755614519119263 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 1, + "style_probability": 0.7063116431236267, + "style_target": 1, + "support_probability": 0.6349722146987915, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.141470095863647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5162715911865234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41240987181663513, + 0.2450176328420639, + 0.8398789167404175, + 0.2558751106262207, + 0.12570689618587494, + 0.15864714980125427 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 0, + "style_probability": 0.5740928649902344, + "style_target": 0, + "support_probability": 0.1750498116016388, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29607107238255753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5674828886985779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27939730882644653, + 0.4784238934516907, + 0.31735631823539734, + 0.930252730846405, + 0.15916678309440613, + 0.07991912215948105, + 0.10293018817901611, + 0.08157049864530563, + 0.21718674898147583 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.7956593036651611, + "style_target": 1, + "support_probability": 0.2465067207813263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03374885348367253, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7824452519416809, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3256664276123047, + 0.43377625942230225, + 0.27002397179603577, + 0.8746277689933777, + 0.15381626784801483, + 0.09736024588346481 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.29504263401031494, + "style_target": 0, + "support_probability": 0.08241697400808334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07495881603077206, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8795981407165527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.623083770275116, + 0.33439114689826965, + 0.6891762018203735, + 0.4441019892692566, + 0.11887379735708237, + 0.12867529690265656, + 0.08631887286901474, + 0.08144329488277435, + 0.03205864876508713, + 0.03477631136775017, + 0.15214383602142334 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8184681534767151, + "style_target": 1, + "support_probability": 0.029678480699658394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008021928417560225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7942788600921631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7842457890510559, + 0.4404574930667877, + 0.6789706349372864, + 0.5078045129776001, + 0.07777822762727737, + 0.06143297627568245, + 0.17129871249198914, + 0.052277520298957825, + 0.12715306878089905, + 0.09609764069318771 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.13399028778076172, + "style_target": 0, + "support_probability": 0.07841022312641144, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5509241667307954, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18647797405719757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6176971793174744, + 0.9675572514533997, + 0.378704309463501, + 0.4522857069969177, + 0.1765364557504654 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.3879563510417938, + "style_target": 1, + "support_probability": 0.5949109792709351, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.742604785382724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12306321412324905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6604486703872681, + 0.9604580402374268, + 0.3534729778766632, + 0.47800663113594055, + 0.0671219751238823 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.6332233548164368, + "style_target": 0, + "support_probability": 0.6841427683830261, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01052412342178001, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9557991027832031, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3092113137245178, + 0.39133384823799133, + 0.3916258215904236, + 0.5097860097885132, + 0.045907486230134964, + 0.06823038309812546 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.4722493588924408, + "style_target": 1, + "support_probability": 0.00947581697255373, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03159979010555791, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8521620035171509, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28258442878723145, + 0.3962607979774475, + 0.36995193362236023, + 0.5243539810180664, + 0.23424170911312103, + 0.07464919984340668 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.7737205028533936, + "style_target": 0, + "support_probability": 0.023774487897753716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0692479220791731, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8966257572174072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5548783540725708, + 0.3525044620037079, + 0.17242100834846497, + 0.6615683436393738, + 0.047375358641147614, + 0.05624210461974144, + 0.06284521520137787, + 0.033967554569244385, + 0.09852923452854156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 0, + "style_probability": 0.3502129912376404, + "style_target": 1, + "support_probability": 0.10563825815916061, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05397632443291061, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8361541628837585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6127966046333313, + 0.2984766960144043, + 0.19572891294956207, + 0.7032545804977417, + 0.04162496700882912, + 0.30598127841949463, + 0.1071443185210228, + 0.05955560505390167, + 0.08371499925851822 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 1, + "style_probability": 0.3524666428565979, + "style_target": 0, + "support_probability": 0.2056075781583786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30792337730992736, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8000153303146362, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15043452382087708, + 0.27872219681739807, + 0.761201798915863, + 0.1860920786857605, + 0.07024644315242767, + 0.04469030722975731, + 0.08314844220876694, + 0.0962897464632988, + 0.04904115945100784, + 0.10093462467193604 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 1, + "style_probability": 0.6581320762634277, + "style_target": 1, + "support_probability": 0.25328928232192993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19178397074591064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8531813025474548, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2081671506166458, + 0.30146458745002747, + 0.6991730332374573, + 0.18919774889945984, + 0.04288839176297188, + 0.04337839409708977, + 0.04479451850056648, + 0.056900687515735626, + 0.12824465334415436, + 0.028079090639948845, + 0.07624020427465439 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 0, + "style_probability": 0.7183889150619507, + "style_target": 0, + "support_probability": 0.13047802448272705, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.28460524234923446, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4074842631816864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28624752163887024, + 0.4013172686100006, + 0.2666371464729309, + 0.858982264995575, + 0.18312819302082062 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 0, + "style_probability": 0.47717007994651794, + "style_target": 1, + "support_probability": 0.5905380845069885, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6278530187752897, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.37245339155197144, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.287585973739624, + 0.4276377856731415, + 0.24888591468334198, + 0.9236190915107727, + 0.2958315312862396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 0, + "style_probability": 0.7378625273704529, + "style_target": 0, + "support_probability": 0.6589585542678833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8051320514156981, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015365787781774998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9704256653785706, + 0.3084971308708191, + 0.6907663345336914, + 0.4331217110157013, + 0.05448993667960167, + 0.18445813655853271 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 1, + "style_probability": 0.5410387516021729, + "style_target": 1, + "support_probability": 0.8609495162963867, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5717547129587779, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019977597519755363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9736896753311157, + 0.3040330708026886, + 0.6784235835075378, + 0.4064429998397827, + 0.0705064982175827 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 1, + "style_probability": 0.33188048005104065, + "style_target": 0, + "support_probability": 0.7613900899887085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007688508758277502, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9889004230499268, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3495958745479584, + 0.3088989853858948, + 0.2088446170091629, + 0.1732129454612732, + 0.17123979330062866, + 0.0985649824142456, + 0.1104799136519432, + 0.12221451848745346, + 0.18724578619003296 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 0, + "style_probability": 0.3104022443294525, + "style_target": 1, + "support_probability": 0.020202994346618652, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006766957361306546, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9552831649780273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3133147060871124, + 0.3859984576702118, + 0.1987675428390503, + 0.16762685775756836, + 0.1590036004781723, + 0.4063034653663635, + 0.1822042167186737, + 0.0884433463215828, + 0.14417840540409088 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 1, + "style_probability": 0.10191459208726883, + "style_target": 0, + "support_probability": 0.10420473664999008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05253775660158389, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9159259796142578, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3220366835594177, + 0.3472002446651459, + 0.6067428588867188, + 0.7249048948287964, + 0.05784579738974571, + 0.06570836156606674, + 0.17582805454730988, + 0.05707274749875069, + 0.059334639459848404, + 0.09260161966085434, + 0.10670779645442963 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.2561630308628082, + "style_target": 1, + "support_probability": 0.07241452485322952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20085886440264997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9232955574989319, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2619456946849823, + 0.3103397786617279, + 0.562639594078064, + 0.7628496885299683, + 0.06123393401503563, + 0.07536359876394272, + 0.06308352947235107, + 0.0809144377708435, + 0.10530052334070206, + 0.21198278665542603 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.7706187963485718, + "style_target": 0, + "support_probability": 0.10135934501886368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8242564216487932, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04934694990515709, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9195711016654968, + 0.36279523372650146, + 0.7398524284362793, + 0.22472137212753296, + 0.2654769718647003 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.900862991809845, + "style_target": 1, + "support_probability": 0.8986884951591492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4445025516772885, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0498218797147274, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9256390929222107, + 0.34478631615638733, + 0.731025218963623, + 0.20522427558898926, + 0.1486252397298813 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.36191701889038086, + "style_target": 0, + "support_probability": 0.9012517333030701, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18277944759042142, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7764568328857422, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.284579873085022, + 0.2005769908428192, + 0.5196630954742432, + 0.284579873085022, + 0.06216117739677429, + 0.10948140174150467 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 1, + "style_probability": 0.6826296448707581, + "style_target": 1, + "support_probability": 0.2177393138408661, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07613467027698738, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9213364720344543, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29033100605010986, + 0.2317969650030136, + 0.43952569365501404, + 0.29033100605010986, + 0.09122998267412186, + 0.08507359772920609 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 1, + "style_probability": 0.41401243209838867, + "style_target": 0, + "support_probability": 0.0987362191081047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15361157923814966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7161276936531067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43176302313804626, + 0.7201884984970093, + 0.37735962867736816, + 0.6209264397621155, + 0.07449225336313248, + 0.19089780747890472, + 0.13111527264118195, + 0.09966427087783813, + 0.22118456661701202 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.7917808890342712, + "style_target": 1, + "support_probability": 0.2238830029964447, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04862122031030273, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7988260388374329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40310585498809814, + 0.7068139314651489, + 0.3468044400215149, + 0.5881071090698242, + 0.14514879882335663, + 0.11979032307863235 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.2944469153881073, + "style_target": 0, + "support_probability": 0.09605736285448074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005260934267427338, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9768632650375366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14313696324825287, + 0.2086598426103592, + 0.6475498676300049, + 0.15251776576042175, + 0.057904791086912155, + 0.17897479236125946, + 0.09354808181524277, + 0.13611537218093872, + 0.11868029087781906, + 0.11504345387220383, + 0.21219561994075775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.6208983063697815, + "style_target": 1, + "support_probability": 0.006749693304300308, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017751305710343088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9815669059753418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1658361554145813, + 0.2542169392108917, + 0.7005575299263, + 0.21157127618789673, + 0.10649683326482773, + 0.07921865582466125, + 0.3091701567173004, + 0.1295195370912552, + 0.10624430328607559, + 0.08693943917751312 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.1505676656961441, + "style_target": 0, + "support_probability": 0.010020450688898563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7721014546298423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13542166352272034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23387376964092255, + 0.2889181673526764, + 0.6657426953315735, + 0.9041857123374939, + 0.10024911910295486 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.5686516165733337, + "style_target": 1, + "support_probability": 0.8711270689964294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10290061624276309, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2704627513885498, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34469297528266907, + 0.3588572144508362, + 0.7072102427482605, + 0.9634754657745361, + 0.17544519901275635 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.11836009472608566, + "style_target": 0, + "support_probability": 0.8077760934829712, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.670256348146697, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.177159383893013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18068067729473114, + 0.10845055431127548, + 0.9036415219306946, + 0.1230521947145462, + 0.11851554363965988, + 0.04774785041809082 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 1, + "style_probability": 0.5622493624687195, + "style_target": 1, + "support_probability": 0.8101791739463806, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8165279315538442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17882323265075684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1686142086982727, + 0.09962329268455505, + 0.9346740245819092, + 0.08101674914360046, + 0.04634051397442818, + 0.05487844720482826 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 0, + "style_probability": 0.7899500727653503, + "style_target": 0, + "support_probability": 0.8149795532226562, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6956169438216738, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10124102979898453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4670872092247009, + 0.9152823686599731, + 0.2015298455953598, + 0.23200903832912445, + 0.045638490468263626, + 0.0669373869895935, + 0.08447589725255966, + 0.09026642888784409, + 0.11393772810697556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 0, + "style_probability": 0.38829970359802246, + "style_target": 1, + "support_probability": 0.8943316340446472, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.615752799083786, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09728176146745682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47534096240997314, + 0.9154354333877563, + 0.20797626674175262, + 0.2093627154827118, + 0.1269548237323761, + 0.11488957703113556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 1, + "style_probability": 0.24273064732551575, + "style_target": 0, + "support_probability": 0.8839972615242004, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010936998678699851, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9078745245933533, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16035005450248718, + 0.5409948825836182, + 0.14556115865707397, + 0.21399442851543427, + 0.10347016155719757, + 0.11902376264333725, + 0.1237979456782341, + 0.11211397498846054, + 0.10716917365789413, + 0.10760286450386047, + 0.1468842476606369 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 0, + "style_probability": 0.36586418747901917, + "style_target": 1, + "support_probability": 0.024358853697776794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01184658695258093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7678088545799255, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2425355464220047, + 0.847025990486145, + 0.18672239780426025, + 0.3129294216632843, + 0.1091574877500534, + 0.12826207280158997, + 0.2231365442276001, + 0.16078677773475647, + 0.15160056948661804, + 0.1353072077035904 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 1, + "style_probability": 0.1361505389213562, + "style_target": 0, + "support_probability": 0.08068273961544037, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27619744020536247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.37099894881248474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21011555194854736, + 0.2889364957809448, + 0.7118874788284302, + 0.21011555194854736, + 0.11720333248376846 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 1, + "style_probability": 0.5124848484992981, + "style_target": 1, + "support_probability": 0.2237946093082428, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26501251033493783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5425286293029785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21955624222755432, + 0.26839450001716614, + 0.7070078253746033, + 0.21955624222755432, + 0.12739180028438568 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 0, + "style_probability": 0.8889716863632202, + "style_target": 0, + "support_probability": 0.11678530275821686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08870382248804076, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8285580277442932, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5552778244018555, + 0.2700457274913788, + 0.5133823752403259, + 0.3742833435535431, + 0.11073938757181168, + 0.06624513864517212 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.6269614100456238, + "style_target": 1, + "support_probability": 0.06892887502908707, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06174939656210128, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.792961061000824, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5245265960693359, + 0.3081061840057373, + 0.4660857617855072, + 0.3758181929588318, + 0.1458916813135147 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.4507454037666321, + "style_target": 0, + "support_probability": 0.06836573034524918, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.25534635117104, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5120947957038879, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8727333545684814, + 0.29355019330978394, + 0.34785956144332886, + 0.29355019330978394, + 0.06462572515010834, + 0.07418837398290634, + 0.14430935680866241, + 0.09927862137556076, + 0.10366097837686539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 0, + "style_probability": 0.46681448817253113, + "style_target": 1, + "support_probability": 0.22289755940437317, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30215631141069244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09337592124938965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9588200449943542, + 0.27225351333618164, + 0.30313313007354736, + 0.27225351333618164, + 0.10202385485172272, + 0.36961638927459717, + 0.13927753269672394, + 0.054866716265678406, + 0.16234084963798523 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 1, + "style_probability": 0.1578942984342575, + "style_target": 0, + "support_probability": 0.8391032814979553, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022665767516587575, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.928318202495575, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.13596603274345398, + 0.13596603274345398, + 0.6764516830444336, + 0.3195043206214905, + 0.12238264828920364, + 0.06404755264520645, + 0.05902504920959473, + 0.16513006389141083, + 0.09011884778738022, + 0.07672937959432602 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.38761621713638306, + "style_target": 1, + "support_probability": 0.038632020354270935, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019056815776671263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9506934285163879, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14467059075832367, + 0.14467059075832367, + 0.7149229049682617, + 0.3003365695476532, + 0.07428011298179626, + 0.039778053760528564, + 0.05527472496032715, + 0.052547331899404526, + 0.11965969949960709, + 0.0844060406088829, + 0.04858782887458801 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.6437968611717224, + "style_target": 0, + "support_probability": 0.034261304885149, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6260865676210964, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.40963295102119446, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2934257686138153, + 0.07015150785446167, + 0.35870426893234253, + 0.9616921544075012, + 0.0901440754532814 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.8287597298622131, + "style_target": 1, + "support_probability": 0.46655169129371643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10216264086437594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4553558826446533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31307733058929443, + 0.10153742879629135, + 0.46401193737983704, + 0.9804471731185913, + 0.18358029425144196 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.2716222107410431, + "style_target": 0, + "support_probability": 0.4305630028247833, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6441047161498488, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33725568652153015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15846340358257294, + 0.4198038578033447, + 0.9592333436012268, + 0.15846340358257294, + 0.07916062325239182, + 0.0463574081659317 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 1, + "style_probability": 0.7042967081069946, + "style_target": 1, + "support_probability": 0.6207153797149658, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6623899611134089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2344568520784378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18541347980499268, + 0.37795785069465637, + 0.9224199652671814, + 0.18541347980499268, + 0.04222088307142258, + 0.0708659365773201 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 0, + "style_probability": 0.5851342678070068, + "style_target": 0, + "support_probability": 0.655523955821991, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006580678862271318, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9910767078399658, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19873030483722687, + 0.377275675535202, + 0.1731073558330536, + 0.6610464453697205, + 0.09778283536434174, + 0.06689808517694473, + 0.15430302917957306, + 0.09798026829957962, + 0.08305493742227554 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.6458179354667664, + "style_target": 1, + "support_probability": 0.009054246358573437, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010231873465292588, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9866358637809753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2009998857975006, + 0.39864295721054077, + 0.18617160618305206, + 0.6354106664657593, + 0.0931820347905159, + 0.13566607236862183 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.23916403949260712, + "style_target": 0, + "support_probability": 0.014387772418558598, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09952024328200688, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9381988048553467, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3195731043815613, + 0.6662232279777527, + 0.6329655647277832, + 0.2750936448574066, + 0.08263909071683884, + 0.10914262384176254, + 0.04287227615714073, + 0.13558217883110046, + 0.16694393754005432, + 0.09703809767961502 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.8993877172470093, + "style_target": 1, + "support_probability": 0.03780253231525421, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01595491035992694, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8318371772766113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.390188068151474, + 0.7544816136360168, + 0.6871455311775208, + 0.3109700381755829, + 0.1621878445148468, + 0.1001100018620491, + 0.5094767212867737, + 0.18211425840854645, + 0.1029953733086586, + 0.13170325756072998 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.19721470773220062, + "style_target": 0, + "support_probability": 0.0606975257396698, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7683531109908063, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09602757543325424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16633906960487366, + 0.9715318083763123, + 0.19113990664482117, + 0.10160790383815765, + 0.03917934000492096 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.30882778763771057, + "style_target": 1, + "support_probability": 0.9266034960746765, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8468510690901455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07894126325845718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17578153312206268, + 0.9673293828964233, + 0.19142381846904755, + 0.1226201057434082, + 0.1647319793701172 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.5078853964805603, + "style_target": 0, + "support_probability": 0.9388759732246399, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13977408259137555, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8240640163421631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37547746300697327, + 0.3237575888633728, + 0.5145407915115356, + 0.4174600839614868, + 0.08312079310417175, + 0.030861711129546165 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.4618759751319885, + "style_target": 1, + "support_probability": 0.3291827440261841, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34757347105455927, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7076589465141296, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4147840440273285, + 0.32581499218940735, + 0.5496148467063904, + 0.41690728068351746, + 0.04005887359380722, + 0.05103759095072746 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.8221160173416138, + "style_target": 0, + "support_probability": 0.48646003007888794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01879907241486046, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8297167420387268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5294359922409058, + 0.25705626606941223, + 0.3041841387748718, + 0.4912763237953186, + 0.08086322993040085, + 0.20755192637443542, + 0.15244503319263458, + 0.17082349956035614, + 0.14748424291610718 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 0, + "style_probability": 0.2814037799835205, + "style_target": 1, + "support_probability": 0.030326591804623604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02402112316944882, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4379904270172119, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6928443908691406, + 0.35916730761528015, + 0.3758261203765869, + 0.6571386456489563, + 0.14933361113071442, + 0.14607377350330353, + 0.17951835691928864, + 0.158237487077713, + 0.17212431132793427 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 1, + "style_probability": 0.23705723881721497, + "style_target": 0, + "support_probability": 0.23700539767742157, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.49899132699377313, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29972976446151733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8001192808151245, + 0.5059681534767151, + 0.45272693037986755, + 0.4693198502063751, + 0.09894803166389465, + 0.09583539515733719, + 0.05550563707947731, + 0.1139184832572937, + 0.13413715362548828, + 0.10686764121055603, + 0.057408034801483154 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 0, + "style_probability": 0.32013383507728577, + "style_target": 1, + "support_probability": 0.792906641960144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6105832538753054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22825667262077332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8034511804580688, + 0.5559095740318298, + 0.5277327299118042, + 0.4889181852340698, + 0.14500324428081512, + 0.10288450121879578, + 0.11455581337213516, + 0.061002571135759354, + 0.05148172751069069, + 0.08190495520830154, + 0.08309295028448105 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 1, + "style_probability": 0.4878656566143036, + "style_target": 0, + "support_probability": 0.8624764680862427, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3850783255689109, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14910881221294403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35542675852775574, + 0.3167347013950348, + 0.9455084800720215, + 0.23300741612911224, + 0.0801115334033966 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6583011150360107, + "style_target": 1, + "support_probability": 0.6273215413093567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24502109169345979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22793054580688477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3593813180923462, + 0.3182497024536133, + 0.9228987693786621, + 0.24498692154884338, + 0.12655296921730042, + 0.13029266893863678, + 0.10493643581867218, + 0.09913090616464615, + 0.146961972117424, + 0.12759961187839508, + 0.15614798665046692 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.39419180154800415, + "style_target": 0, + "support_probability": 0.4768541157245636, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8342155777740506, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10330715775489807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2581867575645447, + 0.38074779510498047, + 0.9551101326942444, + 0.32625627517700195, + 0.044355228543281555, + 0.08459728956222534 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.8291457891464233, + "style_target": 1, + "support_probability": 0.8187707662582397, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2638459170634938, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0733937919139862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2645065486431122, + 0.42820999026298523, + 0.9767200350761414, + 0.408204048871994, + 0.08244941383600235, + 0.10159358382225037 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.19398605823516846, + "style_target": 0, + "support_probability": 0.8420655131340027, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39107702495955654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27304768562316895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4603641927242279, + 0.23765107989311218, + 0.1188168153166771, + 0.8340815305709839, + 0.07068862020969391, + 0.042960479855537415, + 0.042054396122694016, + 0.05652519315481186, + 0.0479690320789814 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 1, + "style_probability": 0.7481006383895874, + "style_target": 1, + "support_probability": 0.42178258299827576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3125507781340378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32248833775520325, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41980665922164917, + 0.23080526292324066, + 0.14064094424247742, + 0.8147162199020386, + 0.058750275522470474, + 0.06986042857170105, + 0.11718165129423141, + 0.07882344722747803, + 0.053672872483730316 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 0, + "style_probability": 0.517507016658783, + "style_target": 0, + "support_probability": 0.5924116969108582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25905447441258306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8129146695137024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5203990340232849, + 0.2707560956478119, + 0.8253244757652283, + 0.5203990340232849, + 0.08186213672161102, + 0.0705239549279213, + 0.05894685536623001, + 0.03888898715376854, + 0.09462198615074158, + 0.04476325213909149, + 0.03948909416794777 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.8871418833732605, + "style_target": 1, + "support_probability": 0.11484042555093765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11405258837681237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7655466794967651, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5156278014183044, + 0.2636055648326874, + 0.8099398016929626, + 0.5156278014183044, + 0.1572568416595459, + 0.09191551804542542, + 0.12824410200119019, + 0.0636320561170578, + 0.08987647294998169 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.31753528118133545, + "style_target": 0, + "support_probability": 0.1647556722164154, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.27129764877479173, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7816308736801147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7451761960983276, + 0.10157527774572372, + 0.5137979388237, + 0.439791738986969, + 0.04536484554409981 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.8254342675209045, + "style_target": 1, + "support_probability": 0.36681675910949707, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015014909009108129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8370625972747803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8398021459579468, + 0.12538573145866394, + 0.5008236765861511, + 0.5065661072731018, + 0.10981932282447815 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.13393627107143402, + "style_target": 0, + "support_probability": 0.36429381370544434, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8532624469839618, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13601234555244446, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4101:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24789409339427948, + 0.9630626440048218, + 0.36058419942855835, + 0.24789409339427948, + 0.085018090903759, + 0.0816856101155281 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 1, + "style_probability": 0.7673695683479309, + "style_target": 1, + "support_probability": 0.8695225119590759, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6459288806024778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21278025209903717, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4101:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2760290801525116, + 0.9407984614372253, + 0.3579029440879822, + 0.2760290801525116, + 0.1408277153968811, + 0.2073569893836975 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 0, + "style_probability": 0.5984465479850769, + "style_target": 0, + "support_probability": 0.6689453125, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9188400231987766, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08460886776447296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9841566681861877, + 0.40420955419540405, + 0.3422623574733734, + 0.4056908190250397, + 0.07529354095458984, + 0.09199845790863037, + 0.05037865415215492, + 0.04153410717844963, + 0.06512179970741272 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.8396737575531006, + "style_target": 1, + "support_probability": 0.7833102345466614, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7051893245041247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16179008781909943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9792903661727905, + 0.39019975066185, + 0.3582485020160675, + 0.4003002643585205, + 0.08391314744949341, + 0.04970903322100639 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.42787274718284607, + "style_target": 0, + "support_probability": 0.6305402517318726, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6469785531160896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3204326033592224, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3065038025379181, + 0.8952068090438843, + 0.46976181864738464, + 0.2256745845079422, + 0.07393984496593475, + 0.11478958278894424, + 0.10809260606765747, + 0.05286350101232529, + 0.06260591745376587, + 0.08107319474220276 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 1, + "style_probability": 0.8431510925292969, + "style_target": 1, + "support_probability": 0.40113380551338196, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3559340976568295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25801363587379456, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3668135106563568, + 0.9416502118110657, + 0.5570025444030762, + 0.24837149679660797, + 0.13649307191371918, + 0.12103194743394852, + 0.11268393695354462, + 0.18638980388641357, + 0.1516447812318802, + 0.11749915778636932 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 0, + "style_probability": 0.5929252505302429, + "style_target": 0, + "support_probability": 0.4215957224369049, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7509209718063437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1927204579114914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32741087675094604, + 0.3884373605251312, + 0.4967826306819916, + 0.9318253993988037, + 0.07364549487829208 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 1, + "style_probability": 0.8217191100120544, + "style_target": 1, + "support_probability": 0.7764520049095154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5331848665251855, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20033419132232666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35573717951774597, + 0.36111029982566833, + 0.5475592613220215, + 0.9004520773887634, + 0.06617957353591919 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 0, + "style_probability": 0.5377459526062012, + "style_target": 0, + "support_probability": 0.7084696888923645, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.836904798278459, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14128515124320984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34395846724510193, + 0.9391862750053406, + 0.2561245560646057, + 0.23449638485908508, + 0.03640949726104736, + 0.06864984333515167 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.8677271604537964, + "style_target": 1, + "support_probability": 0.6794005632400513, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6217597346441454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11223868280649185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3098839819431305, + 0.9344397783279419, + 0.28534796833992004, + 0.22130087018013, + 0.06547515839338303 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.32800596952438354, + "style_target": 0, + "support_probability": 0.6996617317199707, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3479442524817977, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14666154980659485, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23604248464107513, + 0.31052306294441223, + 0.9528970122337341, + 0.1585894078016281, + 0.14305277168750763, + 0.07828308641910553, + 0.17514321208000183, + 0.0916738212108612, + 0.10687624663114548 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.593726396560669, + "style_target": 1, + "support_probability": 0.3514401614665985, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07637966234993167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10731430351734161, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3041534125804901, + 0.3835510015487671, + 0.9706705808639526, + 0.17644670605659485, + 0.1597346067428589, + 0.24129202961921692, + 0.151549831032753, + 0.0959242656826973, + 0.13315075635910034 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.1492508053779602, + "style_target": 0, + "support_probability": 0.43777191638946533, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7726344091109753, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.053323082625865936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39171120524406433, + 0.5738347768783569, + 0.15365104377269745, + 0.9990125894546509, + 0.07565528899431229, + 0.10889670997858047, + 0.0737965777516365, + 0.07549428939819336, + 0.08915568143129349, + 0.03523391857743263, + 0.08176726847887039 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 1, + "style_probability": 0.717463493347168, + "style_target": 1, + "support_probability": 0.8117495775222778, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6787374252128451, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06213542819023132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4065667390823364, + 0.5193224549293518, + 0.12728267908096313, + 0.9934912323951721, + 0.12786804139614105, + 0.0576755553483963, + 0.10032694786787033, + 0.08251500129699707, + 0.12010153383016586, + 0.11741362512111664, + 0.051011014729738235 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 0, + "style_probability": 0.5983120799064636, + "style_target": 0, + "support_probability": 0.7823488712310791, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.494574655947071, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5262385606765747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22007405757904053, + 0.8224533200263977, + 0.21379049122333527, + 0.2955394685268402, + 0.1775542050600052 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.43111640214920044, + "style_target": 1, + "support_probability": 0.832889974117279, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7767216617046966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14807099103927612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2850327789783478, + 0.9347124695777893, + 0.15776973962783813, + 0.2045908123254776, + 0.059023190289735794 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.6470345854759216, + "style_target": 0, + "support_probability": 0.9429460167884827, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.1426036016506367, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.28726932406425476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15950055420398712, + 0.20329998433589935, + 0.15950055420398712, + 0.5903922319412231, + 0.06251994520425797, + 0.09052466601133347 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 0, + "style_probability": 0.3639729619026184, + "style_target": 1, + "support_probability": 0.2107970118522644, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10742522820523116, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3347141742706299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15026924014091492, + 0.23516415059566498, + 0.15026924014091492, + 0.6640105843544006, + 0.07610851526260376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 1, + "style_probability": 0.24421381950378418, + "style_target": 0, + "support_probability": 0.14592736959457397, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5338474467857576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07095257937908173, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32435157895088196, + 0.21290412545204163, + 0.8440003991127014, + 0.2664816081523895, + 0.06261272728443146, + 0.1246505081653595, + 0.07259717583656311, + 0.0932641252875328, + 0.11021872609853745 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 0, + "style_probability": 0.3125673234462738, + "style_target": 1, + "support_probability": 0.854375422000885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47146965016716064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.039557795971632004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44562116265296936, + 0.21943113207817078, + 0.9353111982345581, + 0.34829631447792053, + 0.08817994594573975, + 0.13567404448986053, + 0.11823920160531998, + 0.08701388537883759, + 0.12530255317687988 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 1, + "style_probability": 0.1937926560640335, + "style_target": 0, + "support_probability": 0.9349239468574524, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3162368587973804, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11493344604969025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2482384890317917, + 0.3301007151603699, + 0.2772887051105499, + 0.8692528605461121, + 0.0767412856221199, + 0.04591361805796623, + 0.04459449648857117, + 0.0977059155702591, + 0.07777049392461777, + 0.10142837464809418, + 0.08021140098571777 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 0, + "style_probability": 0.48460501432418823, + "style_target": 1, + "support_probability": 0.49209731817245483, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4122510116332735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12385453283786774, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28403139114379883, + 0.3358287215232849, + 0.28961649537086487, + 0.8485327363014221, + 0.0572395920753479, + 0.02895519696176052, + 0.034238550812006, + 0.03860756754875183, + 0.08224441856145859, + 0.07956936955451965, + 0.05362960323691368 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 0, + "style_probability": 0.803247332572937, + "style_target": 0, + "support_probability": 0.39821454882621765, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5644880996167683, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24846264719963074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3187810480594635, + 0.9628064632415771, + 0.39700591564178467, + 0.30051088333129883, + 0.04002854600548744 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 0, + "style_probability": 0.49874287843704224, + "style_target": 1, + "support_probability": 0.5042183995246887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43251568594614714, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23620489239692688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3418700098991394, + 0.9910021424293518, + 0.46572214365005493, + 0.39186856150627136, + 0.05003887414932251 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 1, + "style_probability": 0.4269169569015503, + "style_target": 0, + "support_probability": 0.7193016409873962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1587685370286261, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41998907923698425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3853009343147278, + 0.6146051287651062, + 0.21027028560638428, + 0.6212238669395447, + 0.04926436394453049, + 0.06480548530817032 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.47060972452163696, + "style_target": 1, + "support_probability": 0.22423578798770905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03262848568609974, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7796705961227417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34271275997161865, + 0.5438243746757507, + 0.22974377870559692, + 0.5736169219017029, + 0.07730559259653091, + 0.08874654769897461 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.5213615894317627, + "style_target": 0, + "support_probability": 0.022203786298632622, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.570044668909934, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09074240922927856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4186:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4487903416156769, + 0.1481076031923294, + 0.9863600134849548, + 0.21001413464546204, + 0.10864847898483276, + 0.08871456980705261, + 0.06628096848726273, + 0.10632234066724777, + 0.10587405413389206 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.48188191652297974, + "style_target": 1, + "support_probability": 0.6750237345695496, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8552135845640834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07802525162696838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4186:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4830504059791565, + 0.13351033627986908, + 0.9937533736228943, + 0.21137596666812897, + 0.0638502761721611, + 0.043029673397541046, + 0.11246143281459808, + 0.055397793650627136, + 0.0650641992688179 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.8833269476890564, + "style_target": 0, + "support_probability": 0.7932866215705872, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6684672862708828, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15486368536949158, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4053482711315155, + 0.08040054142475128, + 0.7747182250022888, + 0.9020722508430481, + 0.07943209260702133, + 0.10803728550672531, + 0.16272111237049103, + 0.09925118088722229, + 0.08736947178840637, + 0.05787646397948265, + 0.09223844856023788 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 0, + "style_probability": 0.379015177488327, + "style_target": 1, + "support_probability": 0.8498207926750183, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2902418775722106, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17518454790115356, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4463171362876892, + 0.08029551804065704, + 0.8450393080711365, + 0.950901985168457, + 0.10482417047023773, + 0.12296696752309799, + 0.3350285589694977, + 0.09860233962535858, + 0.0542694516479969, + 0.13286226987838745 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 1, + "style_probability": 0.17723135650157928, + "style_target": 0, + "support_probability": 0.8296146392822266, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6464139337519265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0759354829788208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8637731671333313, + 0.09086814522743225, + 0.15176211297512054, + 0.34228822588920593, + 0.043070364743471146 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 1, + "style_probability": 0.8366588950157166, + "style_target": 1, + "support_probability": 0.7631693482398987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3868497677459203, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12403014302253723, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7506812810897827, + 0.12262122333049774, + 0.2205648273229599, + 0.4760327935218811, + 0.16209430992603302 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 0, + "style_probability": 0.536682665348053, + "style_target": 0, + "support_probability": 0.6353734731674194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11181772485853791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7198404669761658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3290010094642639, + 0.1319950670003891, + 0.794934868812561, + 0.24310867488384247, + 0.08844748139381409, + 0.059010833501815796 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.8598031997680664, + "style_target": 1, + "support_probability": 0.09700887650251389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06779378790473922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7498108744621277, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3825289011001587, + 0.12814806401729584, + 0.748661994934082, + 0.266695499420166, + 0.18386249244213104 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.4830235540866852, + "style_target": 0, + "support_probability": 0.10306499898433685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0845960019031058, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8006778359413147, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20375430583953857, + 0.2012380063533783, + 0.412736177444458, + 0.3417036235332489, + 0.06250941753387451, + 0.032170314341783524, + 0.22497695684432983, + 0.052739933133125305, + 0.03125065937638283 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.759548008441925, + "style_target": 1, + "support_probability": 0.07897292077541351, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008714169511748722, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8121526837348938, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16559794545173645, + 0.18561676144599915, + 0.41931581497192383, + 0.29882076382637024, + 0.09377562254667282, + 0.16806113719940186, + 0.1308390498161316, + 0.0773455798625946, + 0.1242661327123642 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.15168172121047974, + "style_target": 0, + "support_probability": 0.10286680608987808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1768492964731905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3414728343486786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4096696078777313, + 0.7872719764709473, + 0.2994573712348938, + 0.16916121542453766, + 0.14452888071537018, + 0.11311347782611847, + 0.07647962123155594, + 0.06711430102586746, + 0.08927222341299057, + 0.15026088058948517, + 0.08478278666734695 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 1, + "style_probability": 0.606755256652832, + "style_target": 1, + "support_probability": 0.2449975460767746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17426320559244696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3338746130466461, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4135970175266266, + 0.708990752696991, + 0.25785163044929504, + 0.20662829279899597, + 0.04424440860748291, + 0.10270953178405762, + 0.10994885861873627, + 0.057367365807294846, + 0.06789067387580872, + 0.13715139031410217 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 1, + "style_probability": 0.443343847990036, + "style_target": 0, + "support_probability": 0.2556074261665344, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5948331442780288, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2505349814891815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13898906111717224, + 0.2057657390832901, + 0.4141315817832947, + 0.8792677521705627, + 0.13710691034793854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.23386096954345703, + "style_target": 1, + "support_probability": 0.8256258368492126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8445128065330691, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10343588143587112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11262522637844086, + 0.22993549704551697, + 0.3889884948730469, + 0.9129437208175659, + 0.1471487581729889 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.8843123316764832, + "style_target": 0, + "support_probability": 0.9144675135612488, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008017217600860072, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7905387878417969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20484012365341187, + 0.6759916543960571, + 0.4999563694000244, + 0.3952484726905823, + 0.05870029702782631, + 0.06958794593811035 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 0, + "style_probability": 0.3109779357910156, + "style_target": 1, + "support_probability": 0.006962763145565987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000919168485786853, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8149941563606262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16891908645629883, + 0.6913619041442871, + 0.5464712381362915, + 0.4032423198223114, + 0.13896630704402924, + 0.18754561245441437 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 1, + "style_probability": 0.11093836277723312, + "style_target": 0, + "support_probability": 0.007395267020910978, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.31051397224140115, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2410048544406891, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38402557373046875, + 0.3083561956882477, + 0.41659605503082275, + 0.3083561956882477, + 0.0840185359120369, + 0.08485022932291031, + 0.06489889323711395, + 0.0731138065457344, + 0.04900859296321869 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.3933698534965515, + "style_target": 1, + "support_probability": 0.7291924357414246, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3275648971144398, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32429298758506775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3895346224308014, + 0.33010417222976685, + 0.37743720412254333, + 0.33010417222976685, + 0.10407928377389908, + 0.0380571149289608, + 0.10375989228487015, + 0.07982940971851349, + 0.08749012649059296 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.511409342288971, + "style_target": 0, + "support_probability": 0.7831736207008362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10346444152849388, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12913890182971954, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3120601177215576, + 0.39639320969581604, + 0.3120601177215576, + 0.9537602066993713, + 0.11959188431501389, + 0.12781329452991486, + 0.16835607588291168, + 0.06086611747741699, + 0.054851680994033813, + 0.05876567214727402, + 0.08876747637987137 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.21844305098056793, + "style_target": 1, + "support_probability": 0.13509783148765564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26215010387977944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16289012134075165, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30785563588142395, + 0.3870134651660919, + 0.30785563588142395, + 0.9610410332679749, + 0.04052342474460602, + 0.07766711711883545, + 0.070037342607975, + 0.05452462285757065, + 0.04914896935224533, + 0.11835344135761261, + 0.057968661189079285 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.7673237919807434, + "style_target": 0, + "support_probability": 0.12748926877975464, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.792349527403412, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08617483079433441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3238552510738373, + 0.3396557867527008, + 0.13557438552379608, + 0.9892661571502686, + 0.05806606262922287 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.7497416734695435, + "style_target": 1, + "support_probability": 0.8121090531349182, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3725622095554774, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08328898996114731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38142159581184387, + 0.39114561676979065, + 0.17830035090446472, + 0.9943708777427673, + 0.1305176019668579 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.33660319447517395, + "style_target": 0, + "support_probability": 0.7992688417434692, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7826765263722829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1854138970375061, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8868688941001892, + 0.2756894826889038, + 0.37453335523605347, + 0.5292324423789978, + 0.11839061975479126, + 0.05578593164682388 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.8898987174034119, + "style_target": 1, + "support_probability": 0.8615627288818359, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6045597908440397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2087142914533615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8889020085334778, + 0.36075150966644287, + 0.3214022219181061, + 0.4312359094619751, + 0.09821869432926178, + 0.15281227231025696 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.47474557161331177, + "style_target": 0, + "support_probability": 0.8299630880355835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17307040788839828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9109910726547241, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2565934360027313, + 0.267101913690567, + 0.8767143487930298, + 0.24670429527759552, + 0.06024649366736412, + 0.06559627503156662, + 0.06930884718894958, + 0.18524740636348724, + 0.09132219851016998 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.8547427654266357, + "style_target": 1, + "support_probability": 0.062496282160282135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13133651277403968, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8524281978607178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22678342461585999, + 0.24885328114032745, + 0.8630954623222351, + 0.2629595994949341, + 0.11922627687454224, + 0.20289985835552216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.36186063289642334, + "style_target": 0, + "support_probability": 0.07300646603107452, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6442865047876956, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.016552409157156944, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.294903963804245, + 0.5852651000022888, + 0.3791995346546173, + 0.9865332841873169, + 0.03719676285982132, + 0.027155067771673203, + 0.03869292512536049, + 0.038180988281965256, + 0.05071006715297699, + 0.08888626843690872, + 0.07919891923666 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 0, + "style_probability": 0.4330786466598511, + "style_target": 1, + "support_probability": 0.9004151821136475, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21420712852399362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03260276839137077, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2986633777618408, + 0.5752041935920715, + 0.39608535170555115, + 0.9813756942749023, + 0.11022233217954636, + 0.0823117345571518, + 0.21963492035865784, + 0.0905999094247818, + 0.06253933161497116, + 0.07107178121805191 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 1, + "style_probability": 0.21999819576740265, + "style_target": 0, + "support_probability": 0.8373650908470154, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6773913925861089, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31865909695625305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7963311672210693, + 0.7551615238189697, + 0.37161874771118164, + 0.7963311672210693, + 0.14866653084754944 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 0, + "style_probability": 0.4431161880493164, + "style_target": 1, + "support_probability": 0.9424787163734436, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41566078783340216, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1368982344865799, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9029437899589539, + 0.9327517747879028, + 0.43489208817481995, + 0.9029437899589539, + 0.11536546796560287 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 1, + "style_probability": 0.1843498796224594, + "style_target": 0, + "support_probability": 0.9581683874130249, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.003671810347370307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9892796874046326, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25244253873825073, + 0.4634270966053009, + 0.3536406457424164, + 0.24136723577976227, + 0.05700574070215225, + 0.14772310853004456 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 0, + "style_probability": 0.4748789668083191, + "style_target": 1, + "support_probability": 0.006245599593967199, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017884588960071515, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9862270951271057, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2504027485847473, + 0.4748115539550781, + 0.38575035333633423, + 0.24246889352798462, + 0.16486622393131256, + 0.16877110302448273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 0, + "style_probability": 0.8554310202598572, + "style_target": 0, + "support_probability": 0.01866157166659832, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3472354057003388, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22466586530208588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31815123558044434, + 0.5370181202888489, + 0.3529764413833618, + 0.31815123558044434, + 0.11166829615831375, + 0.10987266153097153, + 0.12318339943885803, + 0.16593703627586365, + 0.06494385004043579 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 0, + "style_probability": 0.4236423075199127, + "style_target": 1, + "support_probability": 0.6377608180046082, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19764739702552916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25840121507644653, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31902793049812317, + 0.5353423953056335, + 0.3516559600830078, + 0.31902793049812317, + 0.07382135093212128, + 0.08733544498682022 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 1, + "style_probability": 0.38807013630867004, + "style_target": 0, + "support_probability": 0.5413022637367249, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008089723387616104, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9032849669456482, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22907616198062897, + 0.5205104351043701, + 0.23385432362556458, + 0.20506533980369568, + 0.06437352299690247, + 0.1576543152332306, + 0.09276757389307022, + 0.08274354040622711, + 0.07687856256961823, + 0.10463493317365646 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 0, + "style_probability": 0.4889512360095978, + "style_target": 1, + "support_probability": 0.010860131122171879, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010016909201113327, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8971700072288513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28006836771965027, + 0.5705615282058716, + 0.2689995765686035, + 0.20877674221992493, + 0.10888846218585968, + 0.13867826759815216, + 0.1033543050289154, + 0.11364489793777466, + 0.10656202584505081, + 0.19649337232112885 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 0, + "style_probability": 0.5334830284118652, + "style_target": 0, + "support_probability": 0.024238232523202896, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4519948533739598, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2866209149360657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6005679965019226, + 0.22158648073673248, + 0.1453271061182022, + 0.9772105813026428, + 0.10666946321725845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 1, + "style_probability": 0.7818028926849365, + "style_target": 1, + "support_probability": 0.37669458985328674, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32643269997673613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27749577164649963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5837576389312744, + 0.20402003824710846, + 0.1532112956047058, + 0.9711534380912781, + 0.09454327821731567, + 0.0408751480281353, + 0.06699463725090027, + 0.06528660655021667, + 0.10226090252399445, + 0.0755467340350151, + 0.11492256075143814 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 0, + "style_probability": 0.5576977133750916, + "style_target": 0, + "support_probability": 0.3685967028141022, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7745941414086096, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1547699123620987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21085847914218903, + 0.9320864081382751, + 0.3436007499694824, + 0.29302892088890076, + 0.22166956961154938, + 0.0696067214012146 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.8253275752067566, + "style_target": 1, + "support_probability": 0.8694647550582886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09776360532192091, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15687108039855957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2234097272157669, + 0.9753360748291016, + 0.46623513102531433, + 0.4560149908065796, + 0.1462462693452835, + 0.2125944048166275 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.14301538467407227, + "style_target": 0, + "support_probability": 0.8390194773674011, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6591528375224838, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26065316796302795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7518706917762756, + 0.8959900140762329, + 0.10915499180555344, + 0.7518706917762756, + 0.09825622290372849, + 0.22687964141368866, + 0.12926171720027924, + 0.18162643909454346, + 0.09679017961025238 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 1, + "style_probability": 0.7870975732803345, + "style_target": 1, + "support_probability": 0.5661640167236328, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5107210729796581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3199407458305359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6787229776382446, + 0.8607664704322815, + 0.12781742215156555, + 0.6787229776382446, + 0.13318444788455963, + 0.10417202115058899, + 0.0417926199734211, + 0.06332816183567047, + 0.08065208047628403 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 0, + "style_probability": 0.632456362247467, + "style_target": 0, + "support_probability": 0.49468424916267395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01321341224200467, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.752856433391571, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5578464865684509, + 0.49159958958625793, + 0.5578464865684509, + 0.7857062220573425, + 0.09300073236227036, + 0.057577867060899734, + 0.17077995836734772, + 0.12612928450107574, + 0.14721961319446564, + 0.13763770461082458 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.5605180859565735, + "style_target": 1, + "support_probability": 0.02191675454378128, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006865057970361638, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8267613053321838, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5095306038856506, + 0.4530661106109619, + 0.5095306038856506, + 0.7414309978485107, + 0.1987835317850113, + 0.09787432849407196, + 0.05269657075405121, + 0.1579495370388031, + 0.055707018822431564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.3320147395133972, + "style_target": 0, + "support_probability": 0.008144896477460861, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05412795292140059, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.47425198554992676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21596994996070862, + 0.43377068638801575, + 0.33407145738601685, + 0.5454496145248413, + 0.11876974254846573 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.8726771473884583, + "style_target": 1, + "support_probability": 0.09228643774986267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0023871864782089875, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5732994675636292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3209209144115448, + 0.4733709692955017, + 0.3798145651817322, + 0.5398851633071899, + 0.08872972428798676 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.14239466190338135, + "style_target": 0, + "support_probability": 0.10260460525751114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13479366429773498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6958209276199341, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7934046387672424, + 0.20988382399082184, + 0.3542340397834778, + 0.4006456434726715, + 0.12888959050178528, + 0.11731927841901779 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 1, + "style_probability": 0.8147206902503967, + "style_target": 1, + "support_probability": 0.14293013513088226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.159436559865025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5492854118347168, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7459940910339355, + 0.23478734493255615, + 0.32188183069229126, + 0.4209091365337372, + 0.18749161064624786, + 0.1645018458366394 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 0, + "style_probability": 0.5983589887619019, + "style_target": 0, + "support_probability": 0.22700729966163635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024170266598844175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49493932723999023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19659805297851562, + 0.8758896589279175, + 0.28031474351882935, + 0.2800286114215851, + 0.06154559180140495, + 0.08281486481428146, + 0.0950925201177597, + 0.04271259531378746, + 0.058746740221977234 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.6406167149543762, + "style_target": 1, + "support_probability": 0.02123488113284111, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008680586844742905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7056092023849487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2120126336812973, + 0.8292499780654907, + 0.2614036202430725, + 0.25478237867355347, + 0.06512945145368576, + 0.07375141233205795 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.3118985593318939, + "style_target": 0, + "support_probability": 0.00751618854701519, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12462099039594711, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7626004219055176, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16032536327838898, + 0.3005256652832031, + 0.9014819860458374, + 0.7001925706863403, + 0.09481719136238098, + 0.23100656270980835, + 0.12311308085918427, + 0.13849206268787384, + 0.09895467013120651, + 0.10145510733127594 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.7218371629714966, + "style_target": 1, + "support_probability": 0.06035351753234863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007765347325285562, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7357138991355896, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2558429539203644, + 0.40473535656929016, + 0.9571917057037354, + 0.7955273389816284, + 0.13263367116451263, + 0.08185423165559769, + 0.26062890887260437, + 0.20487262308597565, + 0.13556230068206787, + 0.13307684659957886 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.039628565311431885, + "style_target": 0, + "support_probability": 0.10364476591348648, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1672200158702993, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23714473843574524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2624216675758362, + 0.2534201443195343, + 0.2761571407318115, + 0.8037992119789124, + 0.13298586010932922 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.5319063067436218, + "style_target": 1, + "support_probability": 0.12727761268615723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029917809091254934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.22760790586471558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21396702527999878, + 0.35374388098716736, + 0.27830785512924194, + 0.8891440033912659, + 0.1539032757282257 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.24030625820159912, + "style_target": 0, + "support_probability": 0.1798284500837326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5291859504400065, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24796944856643677, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8329215049743652, + 0.7012199759483337, + 0.24855273962020874, + 0.562245786190033, + 0.06741254031658173, + 0.05004289373755455 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 0, + "style_probability": 0.3908037841320038, + "style_target": 1, + "support_probability": 0.8737658858299255, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7509551038202973, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1422412097454071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8360174894332886, + 0.7639764547348022, + 0.32447895407676697, + 0.511594831943512, + 0.0975479707121849, + 0.053701434284448624 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 0, + "style_probability": 0.882358968257904, + "style_target": 0, + "support_probability": 0.8870153427124023, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6011110568185366, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2492460161447525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6396544575691223, + 0.3900243639945984, + 0.7103360295295715, + 0.5956197381019592, + 0.06436779350042343, + 0.12323002517223358, + 0.06150094047188759, + 0.2810130715370178, + 0.06847496330738068 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 1, + "style_probability": 0.5497480630874634, + "style_target": 1, + "support_probability": 0.8455162644386292, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5263592806000463, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24099472165107727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6270262598991394, + 0.3709525167942047, + 0.6845769286155701, + 0.6154383420944214, + 0.0739845260977745, + 0.06577003747224808 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 1, + "style_probability": 0.40822288393974304, + "style_target": 0, + "support_probability": 0.7856301069259644, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4902721500940605, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1658509522676468, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5888046622276306, + 0.8877691626548767, + 0.29433736205101013, + 0.37921789288520813, + 0.05857118219137192, + 0.1215701550245285, + 0.03925951570272446, + 0.09740060567855835, + 0.07814770191907883, + 0.1752597838640213, + 0.0860770121216774 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 0, + "style_probability": 0.38826021552085876, + "style_target": 1, + "support_probability": 0.6888546943664551, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12443736229070589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18211358785629272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6367078423500061, + 0.9569963812828064, + 0.4480639696121216, + 0.4175494611263275, + 0.11030508577823639, + 0.10574080795049667, + 0.12773321568965912, + 0.1421160250902176, + 0.08613535761833191, + 0.14265470206737518 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 1, + "style_probability": 0.10991962999105453, + "style_target": 0, + "support_probability": 0.782659649848938, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23412918137096486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22034861147403717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3611566424369812, + 0.24762079119682312, + 0.7651541233062744, + 0.12990601360797882, + 0.1036207303404808 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 0, + "style_probability": 0.4049714505672455, + "style_target": 1, + "support_probability": 0.40096384286880493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3851937546365387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21806268393993378, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41095873713493347, + 0.280429869890213, + 0.8367975950241089, + 0.13971862196922302, + 0.06178108602762222 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 0, + "style_probability": 0.7636551260948181, + "style_target": 0, + "support_probability": 0.3514264225959778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7757355538606701, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0969877764582634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9767817854881287, + 0.4429442882537842, + 0.38329017162323, + 0.34896522760391235, + 0.0874110534787178, + 0.11605504155158997 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.5452107191085815, + "style_target": 1, + "support_probability": 0.793537437915802, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6792039521302486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13246771693229675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9794838428497314, + 0.43155983090400696, + 0.3649178743362427, + 0.29960334300994873, + 0.11806628853082657 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.4680335223674774, + "style_target": 0, + "support_probability": 0.6654974222183228, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4063730435587251, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11017943173646927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8983431458473206, + 0.4818669557571411, + 0.3069435954093933, + 0.22784332931041718, + 0.08066800981760025, + 0.056238725781440735, + 0.05338868498802185, + 0.10226792842149734, + 0.14320045709609985 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 0, + "style_probability": 0.2646270990371704, + "style_target": 1, + "support_probability": 0.5859816670417786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1536302745737584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10730535537004471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9755668640136719, + 0.5049903988838196, + 0.4643212556838989, + 0.2450241595506668, + 0.15071547031402588, + 0.1742066591978073, + 0.11564060300588608, + 0.0707302987575531, + 0.07876890897750854 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 1, + "style_probability": 0.09825563430786133, + "style_target": 0, + "support_probability": 0.7454666495323181, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012219300026717343, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6535602807998657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2903277277946472, + 0.37315574288368225, + 0.40191495418548584, + 0.2404009997844696, + 0.042204178869724274, + 0.09843757748603821, + 0.07540106773376465, + 0.08236411958932877, + 0.06744415313005447, + 0.1985260397195816, + 0.11244218796491623 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.25471073389053345, + "style_target": 1, + "support_probability": 0.07536425441503525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026935758260857798, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7561644911766052, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31419992446899414, + 0.3531630337238312, + 0.3773921728134155, + 0.28211069107055664, + 0.1259964257478714, + 0.05993294343352318, + 0.07807549089193344, + 0.10059445351362228, + 0.05223014950752258, + 0.10882983356714249 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.825606107711792, + "style_target": 0, + "support_probability": 0.029391590505838394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03759402862677727, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5025684237480164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38746464252471924, + 0.19129717350006104, + 0.2526289224624634, + 0.32335346937179565, + 0.07518993318080902 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 0, + "style_probability": 0.47074535489082336, + "style_target": 1, + "support_probability": 0.12704548239707947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028588441032807846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7007355690002441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3986641466617584, + 0.1893121600151062, + 0.21302546560764313, + 0.3574022650718689, + 0.09198125451803207, + 0.18245166540145874, + 0.17264162003993988, + 0.15025277435779572, + 0.0871051698923111, + 0.07310865074396133 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 1, + "style_probability": 0.4690278470516205, + "style_target": 0, + "support_probability": 0.051978860050439835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5530027605385344, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33507031202316284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2106589674949646, + 0.28889739513397217, + 0.2106589674949646, + 0.928320050239563, + 0.06773649156093597, + 0.09607136249542236 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 0, + "style_probability": 0.4116651713848114, + "style_target": 1, + "support_probability": 0.4683423936367035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14490174833985023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.38901251554489136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2791704833507538, + 0.2988872528076172, + 0.2791704833507538, + 0.9654742479324341, + 0.09008820354938507, + 0.11538537591695786 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 1, + "style_probability": 0.21653074026107788, + "style_target": 0, + "support_probability": 0.3240083158016205, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.046927616151079175, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5543290972709656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30616745352745056, + 0.7070469260215759, + 0.3072247803211212, + 0.7792573571205139, + 0.08093974739313126, + 0.0528445765376091, + 0.16609323024749756, + 0.04861309379339218, + 0.10469280183315277 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 1, + "style_probability": 0.555037796497345, + "style_target": 1, + "support_probability": 0.021830623969435692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0373867322985633, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49817442893981934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3205396234989166, + 0.8182405829429626, + 0.3052446246147156, + 0.8399515748023987, + 0.0767010897397995, + 0.051344044506549835, + 0.04642237722873688, + 0.08654718846082687, + 0.044775791466236115 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 0, + "style_probability": 0.6648990511894226, + "style_target": 0, + "support_probability": 0.017948735505342484, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022231405149210768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7994967103004456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6853930354118347, + 0.2705409526824951, + 0.19049958884716034, + 0.26402464509010315, + 0.15441294014453888, + 0.13908690214157104, + 0.13707593083381653, + 0.07507046312093735, + 0.14093400537967682, + 0.08968773484230042, + 0.10300920158624649 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.5375083684921265, + "style_target": 1, + "support_probability": 0.013833796605467796, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04927723178836601, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8479767441749573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7349962592124939, + 0.2544969618320465, + 0.20528092980384827, + 0.24018821120262146, + 0.05660666152834892, + 0.11823413521051407, + 0.08559510111808777, + 0.26196447014808655, + 0.11287748068571091 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.44935962557792664, + "style_target": 0, + "support_probability": 0.02291937917470932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00919906611396123, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9807510375976562, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20070138573646545, + 0.5712717175483704, + 0.24582868814468384, + 0.41667187213897705, + 0.12408196926116943 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.5257181525230408, + "style_target": 1, + "support_probability": 0.01515914872288704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00597137760876558, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.977863609790802, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21110409498214722, + 0.49309226870536804, + 0.26758530735969543, + 0.38247963786125183, + 0.07061686366796494 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.28563162684440613, + "style_target": 0, + "support_probability": 0.02337670885026455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21140469085185068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.053357407450675964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14612305164337158, + 0.3778398931026459, + 0.39924898743629456, + 0.9024989604949951, + 0.14805294573307037, + 0.06019552797079086 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 0, + "style_probability": 0.35948646068573, + "style_target": 1, + "support_probability": 0.41447052359580994, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12779934585393565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.043515078723430634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1353536993265152, + 0.35983115434646606, + 0.4020092487335205, + 0.8795448541641235, + 0.27877458930015564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 1, + "style_probability": 0.11975306272506714, + "style_target": 0, + "support_probability": 0.42089033126831055, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21314054822538267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.849419355392456, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48445120453834534, + 0.26564860343933105, + 0.23106995224952698, + 0.943570613861084, + 0.12223345041275024, + 0.07066558301448822, + 0.04967617988586426, + 0.06794048100709915, + 0.05536222457885742 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.7329656481742859, + "style_target": 1, + "support_probability": 0.1447714865207672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020030084007052907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8621198534965515, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.633988618850708, + 0.37992656230926514, + 0.31704795360565186, + 0.9732636213302612, + 0.09373238682746887, + 0.16991330683231354, + 0.14993150532245636, + 0.1242869570851326, + 0.18678084015846252 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.10334086418151855, + "style_target": 0, + "support_probability": 0.11967601627111435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7801749218078309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17663805186748505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.891360342502594, + 0.43048524856567383, + 0.15971553325653076, + 0.15382331609725952, + 0.0435376800596714, + 0.059053611010313034, + 0.1280265748500824, + 0.0593218170106411, + 0.040981587022542953, + 0.06105245277285576, + 0.04948347061872482 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 1, + "style_probability": 0.8296538591384888, + "style_target": 1, + "support_probability": 0.5803149938583374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.597481538256208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13471916317939758, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8646610975265503, + 0.3761028051376343, + 0.14494965970516205, + 0.19578804075717926, + 0.07769869267940521, + 0.10988382995128632, + 0.11039531975984573, + 0.05177515372633934, + 0.12882642447948456, + 0.10061471909284592 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 0, + "style_probability": 0.5626364350318909, + "style_target": 0, + "support_probability": 0.5151543617248535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20965538976220327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6204767823219299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2065262645483017, + 0.8170217275619507, + 0.35720986127853394, + 0.3150838017463684, + 0.33666083216667175 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.8380128741264343, + "style_target": 1, + "support_probability": 0.047195155173540115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07090226204382749, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5477959513664246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19687408208847046, + 0.7869819402694702, + 0.3575401306152344, + 0.3040238320827484, + 0.10783211886882782, + 0.13049224019050598, + 0.25810134410858154, + 0.26136502623558044, + 0.3434079885482788, + 0.19242648780345917 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.3012717068195343, + "style_target": 0, + "support_probability": 0.03878307715058327, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7753036576950112, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.40979573130607605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3401811718940735, + 0.2163064181804657, + 0.9004303216934204, + 0.2800535559654236, + 0.049348291009664536, + 0.06938393414020538 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.7327628135681152, + "style_target": 1, + "support_probability": 0.7983709573745728, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18487500524057462, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4879009425640106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32087650895118713, + 0.21553179621696472, + 0.9333716630935669, + 0.2452770322561264, + 0.056307919323444366, + 0.10416875034570694 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.16852764785289764, + "style_target": 0, + "support_probability": 0.6009929180145264, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9399806003968447, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11271964758634567, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:4459:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5017882585525513, + 0.48330309987068176, + 0.3256358206272125, + 0.9927268028259277, + 0.03564973920583725, + 0.05946134775876999, + 0.0528825968503952, + 0.06063077226281166, + 0.0695280134677887 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 1, + "style_probability": 0.9186422228813171, + "style_target": 1, + "support_probability": 0.9133824706077576, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8704936563323153, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08274438232183456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4459:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48429346084594727, + 0.4444306492805481, + 0.29488298296928406, + 0.9878535270690918, + 0.11048488318920135, + 0.11072774231433868, + 0.06573165208101273, + 0.0681878924369812, + 0.1873694658279419 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 0, + "style_probability": 0.6122061610221863, + "style_target": 0, + "support_probability": 0.9316326975822449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2052757738988116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8369951248168945, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8433223366737366, + 0.3203178346157074, + 0.3078920543193817, + 0.25128862261772156, + 0.08208925276994705, + 0.04769264906644821, + 0.06048909202218056, + 0.08533335477113724, + 0.0899861603975296, + 0.11423005908727646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 1, + "style_probability": 0.8942624926567078, + "style_target": 1, + "support_probability": 0.06621638685464859, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08147928605726008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8765324354171753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8179367780685425, + 0.3005627691745758, + 0.2987487018108368, + 0.24127691984176636, + 0.09898259490728378, + 0.06950409710407257, + 0.11179880797863007, + 0.08244144171476364, + 0.05461468547582626 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 0, + "style_probability": 0.6015889048576355, + "style_target": 0, + "support_probability": 0.03579357638955116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2159402873578458, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5147132873535156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5379971861839294, + 0.6168584227561951, + 0.4408606290817261, + 0.153621643781662, + 0.10595542192459106 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 1, + "style_probability": 0.5615514516830444, + "style_target": 1, + "support_probability": 0.42591312527656555, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10069079128531172, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7736418843269348, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5707898139953613, + 0.6425275802612305, + 0.4601859450340271, + 0.14037731289863586, + 0.053293626755476, + 0.0680113211274147, + 0.027938978746533394, + 0.05678097531199455, + 0.10739564895629883, + 0.07038954645395279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 1, + "style_probability": 0.45450082421302795, + "style_target": 0, + "support_probability": 0.17181584239006042, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21599544594553569, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4335922300815582, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5815914869308472, + 0.23806524276733398, + 0.512460470199585, + 0.23806524276733398, + 0.05830400064587593, + 0.06637252867221832 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 0, + "style_probability": 0.48823490738868713, + "style_target": 1, + "support_probability": 0.28148356080055237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.051395137492039535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45686569809913635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6897754669189453, + 0.2752041518688202, + 0.5818390250205994, + 0.2752041518688202, + 0.03982008621096611, + 0.08387915790081024 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 1, + "style_probability": 0.2079145908355713, + "style_target": 0, + "support_probability": 0.25325244665145874, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.08033698851869091, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1459914743900299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21341170370578766, + 0.1689632534980774, + 0.9100505113601685, + 0.21341170370578766, + 0.07328779995441437, + 0.06725740432739258, + 0.06477600336074829, + 0.07280099391937256, + 0.08088869601488113 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 0, + "style_probability": 0.3231225311756134, + "style_target": 1, + "support_probability": 0.16976730525493622, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3240164017972202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11440117657184601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2575976848602295, + 0.18421809375286102, + 0.9374910593032837, + 0.2575976848602295, + 0.04389097914099693, + 0.06040806323289871, + 0.04121668264269829, + 0.07291999459266663, + 0.10858850926160812 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 0, + "style_probability": 0.7438532710075378, + "style_target": 0, + "support_probability": 0.23253342509269714, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.36531122858255094, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2039054036140442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1436685472726822, + 0.9313603639602661, + 0.38282468914985657, + 0.2656688392162323, + 0.09498054534196854, + 0.062163107097148895, + 0.08769668638706207, + 0.1470353603363037, + 0.07828875631093979, + 0.1493610143661499 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 0, + "style_probability": 0.405205100774765, + "style_target": 1, + "support_probability": 0.6458849906921387, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41784556928180416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19060972332954407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16536347568035126, + 0.9524853229522705, + 0.42411595582962036, + 0.26604852080345154, + 0.1258741319179535, + 0.11664138734340668, + 0.14102482795715332, + 0.21495318412780762, + 0.21143484115600586 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 1, + "style_probability": 0.31660592555999756, + "style_target": 0, + "support_probability": 0.7155777215957642, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9058887740018307, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.035782087594270706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9683824181556702, + 0.4287056624889374, + 0.5987696051597595, + 0.1486603319644928, + 0.20171324908733368 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 1, + "style_probability": 0.7399545311927795, + "style_target": 1, + "support_probability": 0.9510745406150818, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8468810897730056, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0583086758852005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9455199837684631, + 0.407135546207428, + 0.5501867532730103, + 0.15240122377872467, + 0.10655412077903748 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 0, + "style_probability": 0.5522278547286987, + "style_target": 0, + "support_probability": 0.9376192092895508, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.846917187272946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03105357103049755, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22769539058208466, + 0.9843674302101135, + 0.40029484033584595, + 0.4022810757160187, + 0.07940814644098282, + 0.13478197157382965 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 1, + "style_probability": 0.8823842406272888, + "style_target": 1, + "support_probability": 0.6900616884231567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.452781949777279, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08560691773891449, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24481359124183655, + 0.9635258316993713, + 0.3807264268398285, + 0.3509069085121155, + 0.14038367569446564 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 0, + "style_probability": 0.69810551404953, + "style_target": 0, + "support_probability": 0.4153083860874176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5943933675732599, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3192598521709442, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38091689348220825, + 0.2813062071800232, + 0.9682642817497253, + 0.41391587257385254, + 0.06131809577345848, + 0.03672760725021362, + 0.0648387148976326, + 0.0822010338306427, + 0.23839174211025238 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.7841403484344482, + "style_target": 1, + "support_probability": 0.5653864145278931, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13535567683033234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3417949080467224, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38567817211151123, + 0.3140154778957367, + 0.9780384302139282, + 0.48474863171577454, + 0.06818073242902756, + 0.09095507115125656, + 0.0976596474647522, + 0.06302555650472641, + 0.10945168882608414 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.29682204127311707, + "style_target": 0, + "support_probability": 0.6225243806838989, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6907336804120989, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10764054954051971, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9517669081687927, + 0.645934522151947, + 0.5719960331916809, + 0.665099561214447, + 0.03083011880517006, + 0.07064750790596008, + 0.06915977597236633, + 0.08527890592813492, + 0.12671677768230438, + 0.0513601191341877, + 0.12099669873714447 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 1, + "style_probability": 0.5172227621078491, + "style_target": 1, + "support_probability": 0.8037874698638916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6798307986003973, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08889692276716232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9052258729934692, + 0.6397343873977661, + 0.5898230671882629, + 0.6090033054351807, + 0.1528000682592392, + 0.12911130487918854, + 0.1962677240371704, + 0.08061616122722626, + 0.09595342725515366, + 0.12037546932697296 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 1, + "style_probability": 0.48987889289855957, + "style_target": 0, + "support_probability": 0.7783790826797485, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25798310185972734, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.629548966884613, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4564764201641083, + 0.5588493943214417, + 0.7656197547912598, + 0.2805350124835968, + 0.10979907214641571 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 1, + "style_probability": 0.8151379227638245, + "style_target": 1, + "support_probability": 0.2127881646156311, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12060645461169259, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.89988774061203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4254341721534729, + 0.5212422609329224, + 0.7341362833976746, + 0.307410329580307, + 0.05520837754011154, + 0.05943315103650093, + 0.07759483903646469, + 0.10325811803340912, + 0.24243836104869843, + 0.08583375066518784, + 0.05834430456161499 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 0, + "style_probability": 0.5655132532119751, + "style_target": 0, + "support_probability": 0.07731406390666962, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.593957128502683, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14300772547721863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5503093600273132, + 0.40851205587387085, + 0.7432435750961304, + 0.46913763880729675, + 0.09309699386358261, + 0.055222269147634506 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.6045302152633667, + "style_target": 1, + "support_probability": 0.775427520275116, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08436651724617351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2750742733478546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5589442849159241, + 0.42381322383880615, + 0.8387018442153931, + 0.6368722915649414, + 0.08147116005420685, + 0.07098910957574844 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.07140729576349258, + "style_target": 0, + "support_probability": 0.7479054927825928, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1168943863751581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3689765930175781, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32345449924468994, + 0.32345449924468994, + 0.5022491216659546, + 0.8193677067756653, + 0.1069851964712143, + 0.07233089208602905, + 0.14897386729717255, + 0.1457751989364624, + 0.10067762434482574 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.7129197120666504, + "style_target": 1, + "support_probability": 0.11601753532886505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.072892520447817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39914387464523315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33354824781417847, + 0.33354824781417847, + 0.47136592864990234, + 0.7794544696807861, + 0.08301068097352982, + 0.09214720129966736, + 0.0640435665845871, + 0.12699690461158752, + 0.21974223852157593 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.3782286047935486, + "style_target": 0, + "support_probability": 0.14014972746372223, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013874582777735431, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9511155486106873, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27235063910484314, + 0.5250250101089478, + 0.3813815712928772, + 0.6086065173149109, + 0.12891481816768646, + 0.07585903257131577, + 0.09736118465662003, + 0.09374156594276428, + 0.0938999131321907, + 0.1739409863948822, + 0.14550909399986267 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.6073437333106995, + "style_target": 1, + "support_probability": 0.01544386986643076, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010281560647686844, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9622552990913391, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2843761742115021, + 0.49151766300201416, + 0.38880428671836853, + 0.6044930219650269, + 0.09873557835817337, + 0.08723026514053345, + 0.06980673968791962, + 0.07798245549201965, + 0.11476918309926987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.2840857207775116, + "style_target": 0, + "support_probability": 0.008143486455082893, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7048574313141808, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1591542363166809, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16891787946224213, + 0.5236691832542419, + 0.16891787946224213, + 0.9835684895515442, + 0.09858223050832748 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 1, + "style_probability": 0.6354581713676453, + "style_target": 1, + "support_probability": 0.7427802681922913, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6736664566868987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15765826404094696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14882609248161316, + 0.5849767923355103, + 0.14882609248161316, + 0.9909200668334961, + 0.126719668507576, + 0.07765379548072815, + 0.11262743175029755, + 0.08959325402975082, + 0.11172474920749664, + 0.08075808733701706, + 0.13992004096508026 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 0, + "style_probability": 0.5028197765350342, + "style_target": 0, + "support_probability": 0.7447102665901184, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7305547426718123, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1144273653626442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29117846488952637, + 0.5180931687355042, + 0.3036375343799591, + 0.9011881351470947, + 0.052850399166345596, + 0.06310230493545532 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 0, + "style_probability": 0.45796164870262146, + "style_target": 1, + "support_probability": 0.958257257938385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2842823819358262, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05195024982094765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.405009388923645, + 0.5110942125320435, + 0.3206199109554291, + 0.960118293762207, + 0.11025233566761017, + 0.20651288330554962 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 1, + "style_probability": 0.23132584989070892, + "style_target": 0, + "support_probability": 0.9410635828971863, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02349334970365614, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8994319438934326, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30110234022140503, + 0.16662907600402832, + 0.49042224884033203, + 0.6183617115020752, + 0.1403512805700302, + 0.06807157397270203, + 0.10329852253198624, + 0.14792032539844513, + 0.08694159984588623 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.3685607314109802, + "style_target": 1, + "support_probability": 0.0422481968998909, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08240226215340657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8199207782745361, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.405598908662796, + 0.132547989487648, + 0.6147980690002441, + 0.7467532157897949, + 0.04312116652727127, + 0.10956113785505295, + 0.09989612549543381, + 0.3086282014846802, + 0.06530117243528366 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.6385018825531006, + "style_target": 0, + "support_probability": 0.10542013496160507, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1997199304228502, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14462342858314514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33707544207572937, + 0.6806986927986145, + 0.39668768644332886, + 0.5351333022117615, + 0.040922779589891434, + 0.056432560086250305, + 0.058087967336177826, + 0.03911073878407478, + 0.10512509942054749, + 0.08338171243667603, + 0.04771161079406738 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 1, + "style_probability": 0.5066070556640625, + "style_target": 1, + "support_probability": 0.4011298418045044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15527651954324256, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1535007655620575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3522811233997345, + 0.6885326504707336, + 0.4187260568141937, + 0.5288225412368774, + 0.07234326004981995, + 0.17211870849132538, + 0.1353348344564438, + 0.08454586565494537, + 0.1325371414422989 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 1, + "style_probability": 0.47298678755760193, + "style_target": 0, + "support_probability": 0.2838318645954132, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.11973630076620674, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6302067637443542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8284064531326294, + 0.4750497043132782, + 0.15593838691711426, + 0.46552109718322754, + 0.04830319061875343 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 0, + "style_probability": 0.3461034595966339, + "style_target": 1, + "support_probability": 0.18897533416748047, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08914281795259028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5492039918899536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9256212115287781, + 0.5990198254585266, + 0.20795927941799164, + 0.6713061332702637, + 0.12421728670597076 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 1, + "style_probability": 0.3282061219215393, + "style_target": 0, + "support_probability": 0.21112526953220367, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6729599870790174, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17195981740951538, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.49219653010368347, + 0.4685824513435364, + 0.6885519623756409, + 0.9213083982467651, + 0.11581866443157196, + 0.08469726890325546 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 1, + "style_probability": 0.5887490510940552, + "style_target": 1, + "support_probability": 0.7355947494506836, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7588446305932476, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1552651822566986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5007395148277283, + 0.45785585045814514, + 0.7118899822235107, + 0.9626407027244568, + 0.14169646799564362, + 0.04654815420508385 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 0, + "style_probability": 0.7568479776382446, + "style_target": 0, + "support_probability": 0.7357279658317566, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.34750946512336056, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14809562265872955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3620220422744751, + 0.2884366810321808, + 0.7945799827575684, + 0.23363107442855835, + 0.1529628187417984, + 0.09785491973161697, + 0.08451215922832489, + 0.10620110481977463, + 0.13187731802463531 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 0, + "style_probability": 0.4973531663417816, + "style_target": 1, + "support_probability": 0.38089823722839355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3596356402276331, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15061789751052856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3178277015686035, + 0.30453652143478394, + 0.8040876984596252, + 0.2286556363105774, + 0.08526180684566498, + 0.06651493906974792 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 1, + "style_probability": 0.374605655670166, + "style_target": 0, + "support_probability": 0.41430938243865967, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13756570601818127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9343336224555969, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2973841726779938, + 0.23940709233283997, + 0.7535988092422485, + 0.2363925725221634, + 0.06776942312717438, + 0.055546849966049194, + 0.1598082333803177, + 0.05519270896911621, + 0.07126009464263916, + 0.06604873389005661 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.6505200266838074, + "style_target": 1, + "support_probability": 0.10171108692884445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02612947046465373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9166147708892822, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3795691430568695, + 0.21726849675178528, + 0.8396360278129578, + 0.31243762373924255, + 0.12129264324903488, + 0.06977913528680801, + 0.4288243055343628, + 0.14022141695022583, + 0.0779791995882988, + 0.17203013598918915 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.1866164207458496, + "style_target": 0, + "support_probability": 0.19906063377857208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11172132032997933, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8010998964309692, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3957670032978058, + 0.4279060959815979, + 0.9308021068572998, + 0.3818559944629669, + 0.13494156301021576 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.3901624381542206, + "style_target": 1, + "support_probability": 0.07758096605539322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08750735084430618, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8888135552406311, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.434956431388855, + 0.40586820244789124, + 0.9364322423934937, + 0.38546571135520935, + 0.07487842440605164 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.7481552362442017, + "style_target": 0, + "support_probability": 0.038118358701467514, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6110861565834144, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0223513413220644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4546646475791931, + 0.1897161453962326, + 0.20435066521167755, + 0.9465217590332031, + 0.07688333839178085, + 0.09422841668128967 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 0, + "style_probability": 0.22821593284606934, + "style_target": 1, + "support_probability": 0.9619156718254089, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21875462578562258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03977467492222786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5726552605628967, + 0.2965697944164276, + 0.3173609972000122, + 0.9834589958190918, + 0.0677567645907402, + 0.1485217809677124 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 1, + "style_probability": 0.09967705607414246, + "style_target": 0, + "support_probability": 0.9796140789985657, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6743379256280659, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10237308591604233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30123981833457947, + 0.2047674059867859, + 0.12518785893917084, + 0.9614068269729614, + 0.0743933916091919, + 0.1608094424009323, + 0.0777522623538971, + 0.08395564556121826, + 0.2338407337665558 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 0, + "style_probability": 0.3561350703239441, + "style_target": 1, + "support_probability": 0.7337459325790405, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.660098087467337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1022217720746994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31348010897636414, + 0.18206100165843964, + 0.14890161156654358, + 0.949149489402771, + 0.08314335346221924, + 0.06476148962974548, + 0.0867692083120346, + 0.05144594982266426, + 0.1237896978855133 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 1, + "style_probability": 0.38768303394317627, + "style_target": 0, + "support_probability": 0.687329888343811, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06820678106295006, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3818820118904114, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8026979565620422, + 0.08746469020843506, + 0.42504554986953735, + 0.2393050342798233, + 0.06570379436016083, + 0.06633052229881287, + 0.1058989018201828, + 0.0987153872847557, + 0.07505475729703903, + 0.070024274289608, + 0.1695820540189743 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.3360022306442261, + "style_target": 1, + "support_probability": 0.06691408902406693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.186534688877912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3118869364261627, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8059173226356506, + 0.08305817097425461, + 0.39065274596214294, + 0.27572599053382874, + 0.06729550659656525, + 0.0582035668194294, + 0.0734211653470993, + 0.07298529148101807, + 0.03907053917646408, + 0.10329675674438477, + 0.049526192247867584 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.7885814905166626, + "style_target": 0, + "support_probability": 0.0955607146024704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.037667548703966075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6649759411811829, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6015764474868774, + 0.1453673541545868, + 0.32532694935798645, + 0.26652321219444275, + 0.07564949989318848 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 0, + "style_probability": 0.4513123631477356, + "style_target": 1, + "support_probability": 0.12567748129367828, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005023850993589009, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7390614151954651, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.659243643283844, + 0.17713843286037445, + 0.33455702662467957, + 0.3125878870487213, + 0.09239897131919861 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 1, + "style_probability": 0.06992629915475845, + "style_target": 0, + "support_probability": 0.08882023394107819, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3850680925430547, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22120001912117004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.14483904838562012, + 0.6660060286521912, + 0.5350102782249451, + 0.3751203715801239, + 0.08407532423734665, + 0.042505644261837006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.7807276844978333, + "style_target": 1, + "support_probability": 0.42411959171295166, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06266821637037054, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42335644364356995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13822263479232788, + 0.5651339888572693, + 0.4886167645454407, + 0.34771808981895447, + 0.08566715568304062, + 0.11774399876594543 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.4000401198863983, + "style_target": 0, + "support_probability": 0.10352765023708344, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6639256391588887, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16620904207229614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24486389756202698, + 0.8652748465538025, + 0.2197873294353485, + 0.26705941557884216, + 0.02259495109319687, + 0.06522544473409653, + 0.03907608240842819, + 0.13586677610874176, + 0.0433264784514904 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 1, + "style_probability": 0.8596574664115906, + "style_target": 1, + "support_probability": 0.6985070109367371, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4265657923388194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23922425508499146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23620177805423737, + 0.8471349477767944, + 0.2184518575668335, + 0.2814714014530182, + 0.13125671446323395, + 0.09827008098363876 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 0, + "style_probability": 0.5754691362380981, + "style_target": 0, + "support_probability": 0.6057688593864441, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06863410971646348, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8157381415367126, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15960583090782166, + 0.14094069600105286, + 0.32110896706581116, + 0.2969878911972046, + 0.11652645468711853, + 0.19706304371356964, + 0.032729681581258774, + 0.05098418891429901, + 0.07235671579837799, + 0.0945138931274414, + 0.13585254549980164 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.7925031185150146, + "style_target": 1, + "support_probability": 0.08360900729894638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016175643081979818, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9108644723892212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2166953980922699, + 0.20882254838943481, + 0.3476414084434509, + 0.3756498396396637, + 0.1924675703048706, + 0.09155362099409103, + 0.25630825757980347, + 0.15900228917598724, + 0.17269158363342285, + 0.09564197808504105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.1288624107837677, + "style_target": 0, + "support_probability": 0.025711664929986, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5220883845011508, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37042102217674255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1964578926563263, + 0.8346142172813416, + 0.145379900932312, + 0.28021863102912903, + 0.0470815971493721 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.5343910455703735, + "style_target": 1, + "support_probability": 0.658898651599884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06743027247760364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5258120894432068, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2584795653820038, + 0.8961748480796814, + 0.17586681246757507, + 0.25157222151756287, + 0.15975934267044067 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.12609507143497467, + "style_target": 0, + "support_probability": 0.6330397129058838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0684948174729138, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9235243201255798, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2653084695339203, + 0.1798672378063202, + 0.530857264995575, + 0.4788561761379242, + 0.1290491819381714, + 0.043949346989393234 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 1, + "style_probability": 0.6540311574935913, + "style_target": 1, + "support_probability": 0.03345349803566933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05903704068984388, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.916968047618866, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24486206471920013, + 0.19959865510463715, + 0.542654275894165, + 0.5894731283187866, + 0.07849124819040298, + 0.12312643229961395 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 0, + "style_probability": 0.855605959892273, + "style_target": 0, + "support_probability": 0.04086262732744217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17165708014578068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29059186577796936, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44149690866470337, + 0.34393417835235596, + 0.9343650341033936, + 0.16094759106636047, + 0.1191728338599205, + 0.0775698572397232, + 0.09663382172584534, + 0.13971595466136932, + 0.06132575497031212 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 1, + "style_probability": 0.5780616998672485, + "style_target": 1, + "support_probability": 0.1499321162700653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18527956582953095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2339227944612503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4484676122665405, + 0.3057401180267334, + 0.9494467377662659, + 0.15929275751113892, + 0.1491377055644989, + 0.32500553131103516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 0, + "style_probability": 0.5023661255836487, + "style_target": 0, + "support_probability": 0.1345386654138565, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7087443072088927, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14977402985095978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2920346260070801, + 0.2560718059539795, + 0.07986564934253693, + 0.9400348663330078, + 0.06063162907958031, + 0.055454280227422714, + 0.081940196454525, + 0.08305685967206955, + 0.13810770213603973, + 0.037144944071769714 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.6268265843391418, + "style_target": 1, + "support_probability": 0.687946617603302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5784887917229737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1180398166179657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3240486979484558, + 0.2711124122142792, + 0.08564089983701706, + 0.9878353476524353, + 0.15149858593940735, + 0.10304044932126999, + 0.08480743318796158, + 0.09910169243812561, + 0.05697210878133774, + 0.19347573816776276 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.3092819154262543, + "style_target": 0, + "support_probability": 0.8532209396362305, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.845836490077732, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05188468471169472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3978836238384247, + 0.34758633375167847, + 0.9153980612754822, + 0.16070999205112457, + 0.20058247447013855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.8778663277626038, + "style_target": 1, + "support_probability": 0.9231668710708618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.673878558880682, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05837465077638626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38186970353126526, + 0.32121118903160095, + 0.8493151664733887, + 0.16054187715053558, + 0.0471540242433548, + 0.05102410539984703, + 0.09559804201126099, + 0.19456274807453156, + 0.12748804688453674, + 0.05890775844454765, + 0.3746943771839142 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.3549148440361023, + "style_target": 0, + "support_probability": 0.8805806636810303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02395579599904831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6604472398757935, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7156736254692078, + 0.31794843077659607, + 0.31465983390808105, + 0.3403360843658447, + 0.12197793275117874, + 0.12295465916395187 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 0, + "style_probability": 0.44673654437065125, + "style_target": 1, + "support_probability": 0.042892660945653915, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005104027072263737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8004722595214844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7545820474624634, + 0.3264242708683014, + 0.31208479404449463, + 0.29146549105644226, + 0.08886362612247467, + 0.14601348340511322 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 1, + "style_probability": 0.08410398662090302, + "style_target": 0, + "support_probability": 0.04218361899256706, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3607513085189345, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1758679896593094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2928996980190277, + 0.26376062631607056, + 0.3731536865234375, + 0.9631940126419067, + 0.04166245833039284, + 0.025763317942619324, + 0.039885248988866806, + 0.0524384006857872, + 0.16099432110786438 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 1, + "style_probability": 0.7082059383392334, + "style_target": 1, + "support_probability": 0.19603946805000305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36229247842880025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21377861499786377, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2917326092720032, + 0.30342283844947815, + 0.3548084497451782, + 0.9291133880615234, + 0.05744843930006027, + 0.05049387365579605, + 0.06087843328714371, + 0.03235737234354019, + 0.08937270939350128 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 0, + "style_probability": 0.6458081007003784, + "style_target": 0, + "support_probability": 0.21681758761405945, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7810714675804633, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07877038419246674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3158714473247528, + 0.9487293362617493, + 0.13981236517429352, + 0.1381164938211441, + 0.08507958799600601, + 0.06331157684326172, + 0.03981095179915428, + 0.06501670181751251, + 0.060980357229709625, + 0.0856342762708664, + 0.1304289996623993 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.8179865479469299, + "style_target": 1, + "support_probability": 0.8323970437049866, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42728862806034407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14395257830619812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3011517822742462, + 0.9251559972763062, + 0.16876694560050964, + 0.14676150679588318, + 0.10811653733253479, + 0.085808664560318, + 0.07519695907831192, + 0.12126746773719788, + 0.14245256781578064 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.341034859418869, + "style_target": 0, + "support_probability": 0.612154483795166, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.580243190342143, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14619280397891998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7281284332275391, + 0.29431015253067017, + 0.26919424533843994, + 0.29431015253067017, + 0.10728215426206589 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 0, + "style_probability": 0.43651026487350464, + "style_target": 1, + "support_probability": 0.9531212449073792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4864721317932527, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1593555063009262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.736111044883728, + 0.3062698245048523, + 0.2739427387714386, + 0.3062698245048523, + 0.1088005006313324, + 0.13756924867630005, + 0.09683191031217575, + 0.055449992418289185, + 0.07704459130764008, + 0.08655433356761932, + 0.12482064217329025 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 1, + "style_probability": 0.30496746301651, + "style_target": 0, + "support_probability": 0.8775359392166138, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21616310799992114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16385479271411896, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3311990797519684, + 0.2908181846141815, + 0.18361732363700867, + 0.8940414190292358, + 0.1877371221780777, + 0.14368437230587006 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 0, + "style_probability": 0.3535245656967163, + "style_target": 1, + "support_probability": 0.4616287648677826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021502828756351525, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.32017871737480164, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41274574398994446, + 0.36203616857528687, + 0.2948802709579468, + 0.9629390835762024, + 0.14421194791793823, + 0.17208896577358246 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 1, + "style_probability": 0.09138813614845276, + "style_target": 0, + "support_probability": 0.2807992994785309, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2542919346094201, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24086086452007294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.757996678352356, + 0.12477682530879974, + 0.225753054022789, + 0.5709555149078369, + 0.028237352147698402, + 0.049637001007795334, + 0.09953133016824722, + 0.13027866184711456, + 0.1496899127960205 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 0, + "style_probability": 0.45092639327049255, + "style_target": 1, + "support_probability": 0.4722726047039032, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6916319854851354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11579340696334839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8612722158432007, + 0.11668717116117477, + 0.27313509583473206, + 0.5065664052963257, + 0.05631376430392265, + 0.07399924099445343, + 0.08780314028263092, + 0.18949779868125916, + 0.08845093846321106 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 0, + "style_probability": 0.8576827049255371, + "style_target": 0, + "support_probability": 0.6825346946716309, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40705931779402343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18221266567707062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5017040967941284, + 0.22382637858390808, + 0.9411540031433105, + 0.2493065446615219, + 0.19523262977600098, + 0.04942091554403305, + 0.07694261521100998, + 0.10564449429512024, + 0.11089207231998444, + 0.26936981081962585, + 0.15257395803928375 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.648674488067627, + "style_target": 1, + "support_probability": 0.28552356362342834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48662494611734625, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14132444560527802, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5150260329246521, + 0.22917136549949646, + 0.9584560990333557, + 0.2515026330947876, + 0.151299849152565, + 0.06677345186471939, + 0.13537153601646423, + 0.1470232456922531, + 0.1613568365573883 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.4885336756706238, + "style_target": 0, + "support_probability": 0.3402283787727356, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9102466722874961, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.048765990883111954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4723:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1132526844739914, + 0.21869659423828125, + 0.29835090041160583, + 0.9918099045753479, + 0.12243065983057022 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 1, + "style_probability": 0.7095550894737244, + "style_target": 1, + "support_probability": 0.92423015832901, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8012584515791142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08258320391178131, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1391836702823639, + 0.21721741557121277, + 0.2509630620479584, + 0.9740917086601257, + 0.08423903584480286 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 0, + "style_probability": 0.6278631091117859, + "style_target": 0, + "support_probability": 0.8091827034950256, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.53922530216069, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47379088401794434, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9337345957756042, + 0.24403025209903717, + 0.23565703630447388, + 0.7587152719497681, + 0.11819026619195938, + 0.21347229182720184 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 0, + "style_probability": 0.4926433563232422, + "style_target": 1, + "support_probability": 0.672279417514801, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33176853465593514, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5158682465553284, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9137098789215088, + 0.23925970494747162, + 0.2516019344329834, + 0.7087151408195496, + 0.10421118885278702 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 1, + "style_probability": 0.19770970940589905, + "style_target": 0, + "support_probability": 0.590878427028656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026817081937745346, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9759372472763062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27478402853012085, + 0.3259100019931793, + 0.23071134090423584, + 0.503010630607605, + 0.08241065591573715, + 0.025033095851540565, + 0.13403819501399994, + 0.13952070474624634, + 0.29516473412513733 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.8925102949142456, + "style_target": 1, + "support_probability": 0.01036069355905056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003384277003457615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9833927750587463, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2648935616016388, + 0.3476366698741913, + 0.35614463686943054, + 0.6667709946632385, + 0.0789790078997612, + 0.23930372297763824, + 0.19735923409461975, + 0.0977671667933464, + 0.12011854350566864 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.3921225070953369, + "style_target": 0, + "support_probability": 0.008557314984500408, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6429332977113569, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1767369508743286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9169604182243347, + 0.2644432783126831, + 0.2176484912633896, + 0.27637848258018494, + 0.07145500928163528, + 0.03247959166765213, + 0.04629762843251228, + 0.06664573401212692, + 0.11990400403738022, + 0.1923433393239975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.7584108710289001, + "style_target": 1, + "support_probability": 0.5408738851547241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38338684721426225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16824042797088623, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9205410480499268, + 0.26078885793685913, + 0.17209690809249878, + 0.34068241715431213, + 0.08949568122625351, + 0.09933771193027496, + 0.06798228621482849, + 0.09462370723485947, + 0.25708577036857605, + 0.11456924676895142, + 0.13536231219768524 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.4753558337688446, + "style_target": 0, + "support_probability": 0.44063976407051086, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4010490919193241, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6106243133544922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28244778513908386, + 0.8499159216880798, + 0.455705851316452, + 0.28244778513908386, + 0.1620912104845047 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.592354416847229, + "style_target": 1, + "support_probability": 0.4081742465496063, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17744716766306673, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7298955917358398, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3163491487503052, + 0.7993675470352173, + 0.44097524881362915, + 0.3163491487503052, + 0.1154014840722084, + 0.0669039711356163, + 0.08274973183870316, + 0.12639713287353516, + 0.08414484560489655, + 0.06538153439760208, + 0.062278863042593 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.2685256600379944, + "style_target": 0, + "support_probability": 0.28795474767684937, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7449100434273396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16057023406028748, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24140873551368713, + 0.2723556458950043, + 0.984641969203949, + 0.437101274728775, + 0.08275879919528961, + 0.07360072433948517 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.7685982584953308, + "style_target": 1, + "support_probability": 0.6837853789329529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.263977295954752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15401875972747803, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3370439410209656, + 0.3355802297592163, + 0.9889440536499023, + 0.4845004677772522, + 0.057401981204748154, + 0.10819944739341736 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.27786967158317566, + "style_target": 0, + "support_probability": 0.6913361549377441, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18270226953462768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5170401930809021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20046065747737885, + 0.9341257214546204, + 0.10587434470653534, + 0.3340083062648773, + 0.0651724562048912, + 0.04932104051113129, + 0.08583298325538635, + 0.06344596296548843, + 0.10811062902212143 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 1, + "style_probability": 0.8316394090652466, + "style_target": 1, + "support_probability": 0.07742678374052048, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15992894939013666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40684816241264343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2643445134162903, + 0.9254066944122314, + 0.1454077959060669, + 0.3068578541278839, + 0.05883996933698654, + 0.11592300981283188, + 0.07331372797489166, + 0.08777353912591934, + 0.102432020008564 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 1, + "style_probability": 0.3528333902359009, + "style_target": 0, + "support_probability": 0.18603917956352234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48175309398461685, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6394083499908447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17565278708934784, + 0.17783018946647644, + 0.6565901041030884, + 0.6347323656082153, + 0.0644570142030716, + 0.04439501464366913, + 0.042532969266176224, + 0.03831120952963829, + 0.04674611613154411, + 0.07430469244718552, + 0.03882249817252159 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.8682245016098022, + "style_target": 1, + "support_probability": 0.7584644556045532, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1532597679577481, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6959075331687927, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17074060440063477, + 0.16302426159381866, + 0.589519739151001, + 0.6214310526847839, + 0.08757944405078888, + 0.06212755665183067, + 0.08295290917158127, + 0.028844231739640236, + 0.1544531285762787 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.30337920784950256, + "style_target": 0, + "support_probability": 0.5427085161209106, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7009784977630316, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14218108355998993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8141961693763733, + 0.5829154849052429, + 0.16632236540317535, + 0.48283854126930237, + 0.01816212758421898 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.8025991916656494, + "style_target": 1, + "support_probability": 0.9007521271705627, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04837339602277741, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24094057083129883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9060303568840027, + 0.6249093413352966, + 0.23903147876262665, + 0.5106892585754395, + 0.06774014979600906 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.0669080913066864, + "style_target": 0, + "support_probability": 0.7875047922134399, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3288082675055062, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.369384229183197, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7394998073577881, + 0.9822568297386169, + 0.3993513584136963, + 0.25920385122299194, + 0.0832941085100174, + 0.04760193079710007 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 1, + "style_probability": 0.6647471785545349, + "style_target": 1, + "support_probability": 0.47500547766685486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3263834754048825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27854740619659424, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6690572500228882, + 0.9611831903457642, + 0.4062187075614929, + 0.3120979070663452, + 0.07041426002979279, + 0.06616800278425217 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 0, + "style_probability": 0.5820233821868896, + "style_target": 0, + "support_probability": 0.4838181734085083, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9159871631479822, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09818200021982193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4790:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2106049656867981, + 0.5467630624771118, + 0.9992052912712097, + 0.40405815839767456, + 0.04381831735372543, + 0.03172847256064415, + 0.05704787001013756, + 0.06118231266736984, + 0.053186625242233276 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 1, + "style_probability": 0.7870866060256958, + "style_target": 1, + "support_probability": 0.9393268823623657, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8016825932585486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.124646857380867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4790:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22253650426864624, + 0.5238140821456909, + 0.9975493550300598, + 0.39715033769607544, + 0.06264892965555191, + 0.2346852421760559 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 0, + "style_probability": 0.5533438920974731, + "style_target": 0, + "support_probability": 0.8986849188804626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16680691915450793, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9425126910209656, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4409446120262146, + 0.7334645986557007, + 0.37484613060951233, + 0.4409446120262146, + 0.06867749989032745, + 0.09560814499855042, + 0.044916197657585144, + 0.05048086866736412, + 0.047914229333400726, + 0.0803435817360878, + 0.06413281708955765 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.7689386010169983, + "style_target": 1, + "support_probability": 0.07810215651988983, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02097285766241925, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8867437243461609, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.505881667137146, + 0.8137282133102417, + 0.3668898046016693, + 0.505881667137146, + 0.11871552467346191, + 0.07060801982879639, + 0.4518432021141052, + 0.12516342103481293, + 0.09446797519922256, + 0.13425377011299133 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.11645728349685669, + "style_target": 0, + "support_probability": 0.20047858357429504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01582487537427875, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9291677474975586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4563896954059601, + 0.7403759956359863, + 0.3085052967071533, + 0.3948802351951599, + 0.07346289604902267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 0, + "style_probability": 0.3322288990020752, + "style_target": 1, + "support_probability": 0.008631152100861073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03490821784960768, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.880225658416748, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43765267729759216, + 0.71384197473526, + 0.27903586626052856, + 0.32347819209098816, + 0.10971219837665558 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 0, + "style_probability": 0.5432419180870056, + "style_target": 0, + "support_probability": 0.021572094410657883, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.13654687193990037, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6078348159790039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4114368259906769, + 0.18182946741580963, + 0.19641432166099548, + 0.18182946741580963, + 0.08620025217533112, + 0.148031085729599 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 1, + "style_probability": 0.5358431339263916, + "style_target": 1, + "support_probability": 0.17972438037395477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20972624653565178, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6608812212944031, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4245457649230957, + 0.18932302296161652, + 0.18556120991706848, + 0.18932302296161652, + 0.07926097512245178, + 0.07075156271457672 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 0, + "style_probability": 0.872810423374176, + "style_target": 0, + "support_probability": 0.1523478925228119, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7853396879431559, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0432749018073082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2531794309616089, + 0.12824641168117523, + 0.9946579337120056, + 0.2531794309616089, + 0.04971801117062569, + 0.1121547594666481, + 0.055586572736501694, + 0.07241736352443695, + 0.10530619323253632 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 1, + "style_probability": 0.5309662222862244, + "style_target": 1, + "support_probability": 0.871246337890625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46504065842587844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044124919921159744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28797563910484314, + 0.13878224790096283, + 0.998769223690033, + 0.28797563910484314, + 0.07620161026716232, + 0.16109599173069, + 0.0956125557422638, + 0.11183346062898636, + 0.12774278223514557 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 1, + "style_probability": 0.3125400245189667, + "style_target": 0, + "support_probability": 0.8670704364776611, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08266769525090294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6719430088996887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31945985555648804, + 0.8473582863807678, + 0.4869973361492157, + 0.5888152122497559, + 0.13442014157772064, + 0.04872974753379822, + 0.05112690478563309, + 0.1381578892469406, + 0.06805192679166794, + 0.055034056305885315, + 0.08339636027812958 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.48380032181739807, + "style_target": 1, + "support_probability": 0.09741898626089096, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3057457852738136, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41954997181892395, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3778669834136963, + 0.8757703304290771, + 0.48374876379966736, + 0.6015660762786865, + 0.043756913393735886, + 0.05458875745534897, + 0.19846665859222412, + 0.04155188053846359, + 0.0644441694021225, + 0.0594877265393734, + 0.07636076956987381 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.675480306148529, + "style_target": 0, + "support_probability": 0.3988620340824127, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7200924727082639, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13463889062404633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10416062921285629, + 0.21547462046146393, + 0.4451292157173157, + 0.7826182246208191, + 0.04974251613020897 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.8639422655105591, + "style_target": 1, + "support_probability": 0.927323579788208, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5918591801124986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19145914912223816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.07678021490573883, + 0.16411179304122925, + 0.37427815794944763, + 0.7575876116752625, + 0.10154575854539871, + 0.06916099786758423, + 0.11741627007722855, + 0.0879056453704834, + 0.10712782293558121, + 0.053179848939180374, + 0.0634700283408165 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.4260008633136749, + "style_target": 0, + "support_probability": 0.8674103021621704, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7028940846271295, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17204201221466064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27563202381134033, + 0.2943154573440552, + 0.935208797454834, + 0.43647316098213196, + 0.0645434632897377, + 0.09505073726177216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.8027627468109131, + "style_target": 1, + "support_probability": 0.7097882032394409, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21436289888254875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24287952482700348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3207387924194336, + 0.3932649791240692, + 0.9584448337554932, + 0.5492753982543945, + 0.0415610633790493, + 0.06362473964691162 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.33125898241996765, + "style_target": 0, + "support_probability": 0.6441484093666077, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07605531666416819, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5036526322364807, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2253064513206482, + 0.3489120304584503, + 0.2535971999168396, + 0.2953433394432068, + 0.0890713706612587, + 0.1695219874382019, + 0.07036992162466049, + 0.10049113631248474, + 0.10888183116912842 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 1, + "style_probability": 0.6704511046409607, + "style_target": 1, + "support_probability": 0.11790453642606735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03407713811973389, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45434972643852234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3007561266422272, + 0.3763040006160736, + 0.24821072816848755, + 0.2824418842792511, + 0.1189422532916069, + 0.06060304865241051, + 0.050870008766651154, + 0.03651143237948418, + 0.060034237802028656 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 1, + "style_probability": 0.4751974940299988, + "style_target": 0, + "support_probability": 0.07115881145000458, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6392927054388706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11638425290584564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37966734170913696, + 0.30878934264183044, + 0.9767295122146606, + 0.294283002614975, + 0.09481323510408401, + 0.09028289467096329, + 0.06488872319459915, + 0.14586244523525238, + 0.11591854691505432, + 0.3907593786716461, + 0.10283031314611435 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 1, + "style_probability": 0.7735278010368347, + "style_target": 1, + "support_probability": 0.5876553654670715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44019648420204405, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19803695380687714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4174266755580902, + 0.28823578357696533, + 0.9612048864364624, + 0.27611616253852844, + 0.07716286182403564, + 0.09193667769432068, + 0.504555881023407, + 0.17200715839862823, + 0.20610903203487396 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 1, + "style_probability": 0.49675875902175903, + "style_target": 0, + "support_probability": 0.47534897923469543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6104460018221317, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4973720908164978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22392408549785614, + 0.2334488183259964, + 0.9468404650688171, + 0.17495256662368774, + 0.08233007788658142 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.7452513575553894, + "style_target": 1, + "support_probability": 0.357501357793808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18130913413815847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.551367998123169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29404935240745544, + 0.1868886947631836, + 0.9776960015296936, + 0.17042586207389832, + 0.14692440629005432 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.23657113313674927, + "style_target": 0, + "support_probability": 0.4473249912261963, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8919131658057802, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03963969275355339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4897:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18723823130130768, + 0.9753983020782471, + 0.13208797574043274, + 0.2830795645713806, + 0.08648122847080231, + 0.18939614295959473 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 1, + "style_probability": 0.874572217464447, + "style_target": 1, + "support_probability": 0.8615503907203674, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7487151292407465, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03837825730443001, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4897:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.185983344912529, + 0.9570457935333252, + 0.1269237995147705, + 0.270895779132843, + 0.17946727573871613, + 0.1377861052751541 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 0, + "style_probability": 0.5684487819671631, + "style_target": 0, + "support_probability": 0.8365796208381653, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10843977768528822, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7730877995491028, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5886588096618652, + 0.773523211479187, + 0.2587054669857025, + 0.5886588096618652, + 0.08514958620071411, + 0.24981485307216644, + 0.0716915875673294, + 0.03318503126502037, + 0.16988669335842133 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.7603558301925659, + "style_target": 1, + "support_probability": 0.057792939245700836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0888877620284582, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7108895182609558, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5682088732719421, + 0.7815575003623962, + 0.2715940773487091, + 0.5682088732719421, + 0.1263045221567154, + 0.086484894156456 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.3878856897354126, + "style_target": 0, + "support_probability": 0.092776820063591, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5566551928305997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24010279774665833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15689903497695923, + 0.4541482627391815, + 0.5311266779899597, + 0.900995135307312, + 0.07974104583263397, + 0.08185133337974548, + 0.06535162776708603, + 0.09625107795000076, + 0.08047046512365341, + 0.20681129395961761 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.7304196357727051, + "style_target": 1, + "support_probability": 0.493423730134964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10887055724269647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2074461728334427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.14983291923999786, + 0.4450892508029938, + 0.6201996207237244, + 0.9136472344398499, + 0.10023694485425949, + 0.09493628144264221, + 0.14181844890117645, + 0.07768738269805908, + 0.10271146148443222, + 0.16320005059242249 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.15971268713474274, + "style_target": 0, + "support_probability": 0.6434708833694458, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005376750962287704, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9721531867980957, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20979727804660797, + 0.2039204090833664, + 0.6609163284301758, + 0.19131426513195038, + 0.10949897766113281 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 0, + "style_probability": 0.2884189188480377, + "style_target": 1, + "support_probability": 0.0052567762322723866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005106114023632553, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9582809209823608, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21721534430980682, + 0.2135503888130188, + 0.6695995330810547, + 0.2529279887676239, + 0.14143253862857819 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 1, + "style_probability": 0.44530919194221497, + "style_target": 0, + "support_probability": 0.008676469326019287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09481316571025404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.41921719908714294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39934056997299194, + 0.38335129618644714, + 0.4423641860485077, + 0.7715675830841064, + 0.06759066134691238, + 0.11969482898712158 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.40508055686950684, + "style_target": 1, + "support_probability": 0.13731764256954193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28111889557376246, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.34650570154190063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46302473545074463, + 0.3977227807044983, + 0.4213659167289734, + 0.8233544230461121, + 0.1309259533882141, + 0.08936230093240738 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.8096892237663269, + "style_target": 0, + "support_probability": 0.25798970460891724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1325753575497336, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7662592530250549, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7499956488609314, + 0.35455426573753357, + 0.5562939047813416, + 0.4827757477760315, + 0.11903851479291916, + 0.14235787093639374, + 0.12810656428337097, + 0.11639255285263062, + 0.16236567497253418 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 0, + "style_probability": 0.3842133581638336, + "style_target": 1, + "support_probability": 0.17697684466838837, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.056440014229914445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7733408212661743, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7807795405387878, + 0.3807368874549866, + 0.6068241000175476, + 0.5948798060417175, + 0.20116804540157318, + 0.1401004046201706, + 0.19888411462306976, + 0.1871037632226944, + 0.16689835488796234 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 1, + "style_probability": 0.2679744064807892, + "style_target": 0, + "support_probability": 0.2525925636291504, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.19721481264330865, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4458604156970978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7671639323234558, + 0.34765905141830444, + 0.3214026689529419, + 0.34765905141830444, + 0.17111973464488983, + 0.0699259489774704, + 0.06757482141256332, + 0.09680599719285965, + 0.07958327233791351, + 0.08504307270050049 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 0, + "style_probability": 0.40194329619407654, + "style_target": 1, + "support_probability": 0.12448609620332718, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.385938087290175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23020294308662415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.771953284740448, + 0.38444116711616516, + 0.35005322098731995, + 0.38444116711616516, + 0.18731361627578735, + 0.08483120054006577, + 0.09674764424562454, + 0.17565296590328217, + 0.07183592766523361, + 0.08990614861249924 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 0, + "style_probability": 0.5058135986328125, + "style_target": 0, + "support_probability": 0.37665408849716187, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5788757242043848, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5347110629081726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3044043481349945, + 0.5143280029296875, + 0.7139225006103516, + 0.3044043481349945, + 0.05693556368350983 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.8834075331687927, + "style_target": 1, + "support_probability": 0.6106868982315063, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12964412190746763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6303110718727112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3031008541584015, + 0.48371195793151855, + 0.6910796165466309, + 0.3031008541584015, + 0.050816431641578674, + 0.033812373876571655, + 0.05022424831986427, + 0.38877323269844055, + 0.03670372813940048, + 0.07606810331344604, + 0.13669797778129578 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.3694933354854584, + "style_target": 0, + "support_probability": 0.25485101342201233, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.10300429530857472, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7190098166465759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7917596697807312, + 0.23897673189640045, + 0.17160136997699738, + 0.18280360102653503, + 0.18618306517601013, + 0.045646220445632935 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.8709156513214111, + "style_target": 1, + "support_probability": 0.04233502969145775, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04119708013240918, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27583324909210205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8725897073745728, + 0.234563946723938, + 0.1456495076417923, + 0.18788205087184906, + 0.09642283618450165, + 0.14468523859977722 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.13721473515033722, + "style_target": 0, + "support_probability": 0.3337666392326355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7057003832699963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17817434668540955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16916173696517944, + 0.2218206226825714, + 0.8742382526397705, + 0.16916173696517944, + 0.040270060300827026, + 0.0556386262178421, + 0.040311701595783234, + 0.10591016709804535, + 0.05622178688645363 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 1, + "style_probability": 0.9274636507034302, + "style_target": 1, + "support_probability": 0.5454310774803162, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4672865393451643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16602282226085663, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15491627156734467, + 0.24998658895492554, + 0.8092681169509888, + 0.15491627156734467, + 0.0375470370054245, + 0.04379455745220184, + 0.09243639558553696, + 0.09528427571058273, + 0.2853352129459381 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 0, + "style_probability": 0.6623119711875916, + "style_target": 0, + "support_probability": 0.5288271903991699, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0068451461531128166, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8121941089630127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.46152812242507935, + 0.23623988032341003, + 0.5621752142906189, + 0.5813196897506714, + 0.08785413950681686, + 0.07401525974273682, + 0.04622567445039749, + 0.08638820797204971, + 0.04994305968284607, + 0.07030130922794342, + 0.09724878519773483 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 0, + "style_probability": 0.49399808049201965, + "style_target": 1, + "support_probability": 0.012476309202611446, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009677507494122476, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8681445121765137, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42874592542648315, + 0.2510426640510559, + 0.5569391250610352, + 0.5907443761825562, + 0.1449098140001297, + 0.09624956548213959, + 0.14587362110614777, + 0.13337036967277527, + 0.07235407829284668 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 1, + "style_probability": 0.3079906702041626, + "style_target": 0, + "support_probability": 0.008504625409841537, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3646234505036887, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6457484364509583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7763464450836182, + 0.2507900893688202, + 0.34869736433029175, + 0.4022362530231476, + 0.033797141164541245 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.7806079387664795, + "style_target": 1, + "support_probability": 0.18220095336437225, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012102718275198713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7650551795959473, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8783951997756958, + 0.43230870366096497, + 0.4727126955986023, + 0.47042524814605713, + 0.09985397011041641 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.07365196198225021, + "style_target": 0, + "support_probability": 0.15912139415740967, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8588949411265624, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024290287867188454, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4941:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.443056583404541, + 0.9801488518714905, + 0.41264820098876953, + 0.443056583404541, + 0.07371026277542114, + 0.07022284716367722 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 1, + "style_probability": 0.6461662650108337, + "style_target": 1, + "support_probability": 0.8800159096717834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8204204468716708, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03148283064365387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4941:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.361898273229599, + 0.9509583711624146, + 0.3898654282093048, + 0.361898273229599, + 0.05438898876309395, + 0.24667076766490936 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 1, + "style_probability": 0.44589120149612427, + "style_target": 0, + "support_probability": 0.8784961700439453, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2664127194869439, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6096018552780151, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4748022258281708, + 0.4748022258281708, + 0.3577558100223541, + 0.2180241197347641, + 0.05825790390372276, + 0.052186496555805206, + 0.06628945469856262, + 0.07770849764347076, + 0.04377954825758934 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.8016142249107361, + "style_target": 1, + "support_probability": 0.33252638578414917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04534185199491035, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8317343592643738, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46044015884399414, + 0.46044015884399414, + 0.3581579625606537, + 0.2435241937637329, + 0.07942323386669159, + 0.05016146972775459 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.35462772846221924, + "style_target": 0, + "support_probability": 0.11082998663187027, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9130786040452771, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03644376993179321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4953:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.34478461742401123, + 0.9752927422523499, + 0.3629859983921051, + 0.41951707005500793, + 0.039272990077733994, + 0.06702771782875061, + 0.08427443355321884, + 0.05232848972082138, + 0.06075483560562134, + 0.03852717950940132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.7684415578842163, + "style_target": 1, + "support_probability": 0.9336709976196289, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.568110069583704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04539095610380173, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4953:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.374863862991333, + 0.9822617769241333, + 0.28866058588027954, + 0.3869556188583374, + 0.0889495238661766, + 0.14715132117271423, + 0.11985871940851212, + 0.1758815050125122, + 0.048370636999607086, + 0.08295770734548569 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.224067822098732, + "style_target": 0, + "support_probability": 0.9186974167823792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1250840460185128, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6426254510879517, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7477409839630127, + 0.43983736634254456, + 0.2205134779214859, + 0.24571333825588226, + 0.1131131649017334 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 1, + "style_probability": 0.6235672235488892, + "style_target": 1, + "support_probability": 0.12728168070316315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13847734315732343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6807803511619568, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7098355889320374, + 0.44816839694976807, + 0.2063048630952835, + 0.29526782035827637, + 0.13473568856716156 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 0, + "style_probability": 0.7817888259887695, + "style_target": 0, + "support_probability": 0.141757994890213, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.34008616770649597, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6570537686347961, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6923443078994751, + 0.20357055962085724, + 0.27195659279823303, + 0.40411096811294556, + 0.0444273017346859, + 0.0794597640633583 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.46261000633239746, + "style_target": 1, + "support_probability": 0.3836492598056793, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5501849433017235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5806152820587158, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7206711769104004, + 0.17290152609348297, + 0.2725778818130493, + 0.43899765610694885, + 0.06274250149726868, + 0.056684158742427826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.8924708962440491, + "style_target": 0, + "support_probability": 0.38123154640197754, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06490116000476699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8981528282165527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6862433552742004, + 0.19901587069034576, + 0.40619343519210815, + 0.1966886669397354, + 0.09097126871347427, + 0.07880916446447372, + 0.07807131856679916, + 0.10161268711090088, + 0.1349426507949829 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 0, + "style_probability": 0.48321953415870667, + "style_target": 1, + "support_probability": 0.030674761161208153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012846144056609892, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7399061322212219, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8459924459457397, + 0.32264459133148193, + 0.41528797149658203, + 0.2256377637386322, + 0.06036780774593353, + 0.07910904288291931, + 0.09310713410377502, + 0.06208353489637375, + 0.07734117656946182 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 1, + "style_probability": 0.2208339273929596, + "style_target": 0, + "support_probability": 0.08059340715408325, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6229684479776766, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06547034531831741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9005405306816101, + 0.2006513774394989, + 0.13542303442955017, + 0.28900882601737976, + 0.04087655246257782, + 0.10228231549263, + 0.15409187972545624, + 0.12129199504852295, + 0.1026025265455246, + 0.07203507423400879, + 0.19363485276699066 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.3866683542728424, + "style_target": 1, + "support_probability": 0.6874505281448364, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6212227668789791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0603426955640316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8661032915115356, + 0.20014716684818268, + 0.13260948657989502, + 0.2601732611656189, + 0.07192467898130417, + 0.048000652343034744, + 0.10395011305809021, + 0.09591037780046463, + 0.042114052921533585, + 0.1380281150341034, + 0.20751118659973145 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.5948926210403442, + "style_target": 0, + "support_probability": 0.5784744024276733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6469839687764671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5392778515815735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.859727144241333, + 0.25220245122909546, + 0.14200730621814728, + 0.12737353146076202, + 0.16456381976604462 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 1, + "style_probability": 0.9117944240570068, + "style_target": 1, + "support_probability": 0.43401703238487244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.400760081585198, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5623921751976013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8548486232757568, + 0.23877421021461487, + 0.1565600484609604, + 0.12252264469861984, + 0.0802040547132492, + 0.07539442181587219, + 0.08418396860361099, + 0.071390300989151, + 0.16806401312351227, + 0.22167034447193146, + 0.15154637396335602 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 0, + "style_probability": 0.588392972946167, + "style_target": 0, + "support_probability": 0.3359113931655884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28415444222165204, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1838434785604477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5072447657585144, + 0.09275569021701813, + 0.30219516158103943, + 0.7682682275772095, + 0.1949775069952011, + 0.0693453997373581 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.7023708820343018, + "style_target": 1, + "support_probability": 0.22043874859809875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02684318704175335, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3106194734573364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6258895397186279, + 0.12927202880382538, + 0.48502111434936523, + 0.7954986691474915, + 0.08406537771224976, + 0.10089697688817978 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.10735714435577393, + "style_target": 0, + "support_probability": 0.20984159409999847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22409340732408942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13891158998012543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4224041700363159, + 0.9537147879600525, + 0.15675422549247742, + 0.35010001063346863, + 0.1749112606048584, + 0.07655075192451477, + 0.16017182171344757, + 0.07448723912239075, + 0.15543419122695923 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 1, + "style_probability": 0.622050940990448, + "style_target": 1, + "support_probability": 0.19328877329826355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24586514837224818, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22548875212669373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37984099984169006, + 0.9374034404754639, + 0.168148010969162, + 0.29587841033935547, + 0.2912900447845459, + 0.12166091799736023, + 0.26573315262794495, + 0.4260742664337158, + 0.19740784168243408 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 0, + "style_probability": 0.5015286803245544, + "style_target": 0, + "support_probability": 0.1772034466266632, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7994657177303336, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11636948585510254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41135960817337036, + 0.23226241767406464, + 0.6051222085952759, + 0.8985759615898132, + 0.07374079525470734, + 0.027791853994131088, + 0.06406461447477341, + 0.053695742040872574, + 0.1004946231842041, + 0.061706624925136566, + 0.06643941253423691 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 1, + "style_probability": 0.7678991556167603, + "style_target": 1, + "support_probability": 0.7522308826446533, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5788738976383279, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47056305408477783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41063371300697327, + 0.24023903906345367, + 0.5936709642410278, + 0.8435166478157043, + 0.07270175963640213, + 0.06831507384777069, + 0.11165185272693634, + 0.11081438511610031, + 0.20041345059871674 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 1, + "style_probability": 0.4492897391319275, + "style_target": 0, + "support_probability": 0.47172456979751587, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5143967344523249, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7564709186553955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18244628608226776, + 0.9915591478347778, + 0.30150800943374634, + 0.2299228459596634, + 0.1934029459953308 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.7523518204689026, + "style_target": 1, + "support_probability": 0.2879459261894226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15342831949720517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.89055997133255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25100234150886536, + 0.9969434142112732, + 0.31438809633255005, + 0.3326735198497772, + 0.09391863644123077 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.2386530041694641, + "style_target": 0, + "support_probability": 0.24299968779087067, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19167870747594962, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5643643736839294, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18644076585769653, + 0.16981352865695953, + 0.12328530102968216, + 0.9180663228034973, + 0.10397885739803314, + 0.2542811632156372 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 1, + "style_probability": 0.5873321294784546, + "style_target": 1, + "support_probability": 0.08436767011880875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15900925324813894, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5981219410896301, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21520693600177765, + 0.19339889287948608, + 0.18297244608402252, + 0.865472137928009, + 0.12850119173526764, + 0.044836219400167465 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 0, + "style_probability": 0.5107488036155701, + "style_target": 0, + "support_probability": 0.08525266498327255, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4935541906437191, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30129528045654297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.793265163898468, + 0.1455019861459732, + 0.11483084410429001, + 0.1455019861459732, + 0.08876390755176544, + 0.08238982409238815, + 0.05823004990816116, + 0.060738980770111084, + 0.30625438690185547 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.7323789000511169, + "style_target": 1, + "support_probability": 0.3975506126880646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21963114527110741, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42033815383911133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.730942964553833, + 0.14932918548583984, + 0.12615229189395905, + 0.14932918548583984, + 0.15054944157600403, + 0.10400092601776123 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.3862420916557312, + "style_target": 0, + "support_probability": 0.23563902080059052, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18734144165587158, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.699989378452301, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11588316410779953, + 0.31490588188171387, + 0.6832992434501648, + 0.09581306576728821, + 0.08319486677646637, + 0.054922427982091904, + 0.06971088796854019, + 0.07521364092826843, + 0.05707654356956482, + 0.06439419090747833, + 0.10460467636585236 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.7501545548439026, + "style_target": 1, + "support_probability": 0.1294281780719757, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0050592321852995946, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8108612895011902, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19114375114440918, + 0.3626343309879303, + 0.6102679967880249, + 0.11882740259170532, + 0.10541925579309464, + 0.10388831049203873, + 0.0700572207570076, + 0.14281944930553436, + 0.12109175324440002, + 0.14834675192832947 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.09877172112464905, + "style_target": 0, + "support_probability": 0.09749316424131393, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8108350799407383, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06916212290525436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9880037307739258, + 0.6642667055130005, + 0.526953399181366, + 0.2453015297651291, + 0.11698067933320999 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.5212410688400269, + "style_target": 1, + "support_probability": 0.9272305965423584, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5440287784267918, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03298448771238327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.999544084072113, + 0.7027677297592163, + 0.6651890277862549, + 0.38176220655441284, + 0.09473452717065811 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.1412617564201355, + "style_target": 0, + "support_probability": 0.9791291952133179, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4119083651030854, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1659976989030838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28755301237106323, + 0.47926127910614014, + 0.8082118034362793, + 0.18640609085559845, + 0.07372233271598816, + 0.07544941455125809 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 1, + "style_probability": 0.6597643494606018, + "style_target": 1, + "support_probability": 0.48194509744644165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6285984906131041, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09197735041379929, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3317444920539856, + 0.5209510326385498, + 0.8487540483474731, + 0.23683162033557892, + 0.06365082412958145, + 0.0900307297706604 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 0, + "style_probability": 0.8677362203598022, + "style_target": 0, + "support_probability": 0.5582613945007324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15559880912103843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1114053726196289, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2883886694908142, + 0.2027968019247055, + 0.49360930919647217, + 0.8490007519721985, + 0.14929567277431488, + 0.083692766726017, + 0.06892172992229462, + 0.06744687259197235, + 0.0974293053150177 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 1, + "style_probability": 0.5858226418495178, + "style_target": 1, + "support_probability": 0.11723373085260391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18936731865383583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14011284708976746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2994481325149536, + 0.21807676553726196, + 0.4739573001861572, + 0.877778172492981, + 0.09898532181978226, + 0.12318189442157745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 0, + "style_probability": 0.5695642828941345, + "style_target": 0, + "support_probability": 0.08909524977207184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053413197627667865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7865927815437317, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2544151246547699, + 0.7071519494056702, + 0.5016259551048279, + 0.2898780107498169, + 0.06502755731344223, + 0.146733820438385, + 0.12583677470684052, + 0.1708471179008484, + 0.12004674226045609, + 0.09253998845815659, + 0.17569364607334137 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 1, + "style_probability": 0.5571670532226562, + "style_target": 1, + "support_probability": 0.05499596893787384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017766221501053777, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7001842856407166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17962610721588135, + 0.7403483986854553, + 0.6683148741722107, + 0.23501498997211456, + 0.104425810277462, + 0.1784861981868744, + 0.12361186742782593, + 0.0915159285068512, + 0.13367491960525513, + 0.12154440581798553 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 1, + "style_probability": 0.3315902650356293, + "style_target": 0, + "support_probability": 0.12332053482532501, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0183814057172939, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8518267273902893, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2368629425764084, + 0.3504013419151306, + 0.26274749636650085, + 0.36548665165901184, + 0.2380346655845642 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.8183594346046448, + "style_target": 1, + "support_probability": 0.029854649677872658, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0064126918809576, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.894635796546936, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.2644459307193756, + 0.290331095457077, + 0.2219620943069458, + 0.2104998528957367, + 0.15074676275253296, + 0.1149120107293129, + 0.14371977746486664, + 0.11003891378641129, + 0.0634346678853035, + 0.14480829238891602, + 0.21767212450504303, + 0.04945119470357895 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.2989254593849182, + "style_target": 0, + "support_probability": 0.025630222633481026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23804796996706898, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3549364507198334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3516308665275574, + 0.4393720328807831, + 0.25432464480400085, + 0.8788623809814453, + 0.19413171708583832, + 0.1233372837305069 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.7003857493400574, + "style_target": 1, + "support_probability": 0.10764843225479126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03613554008383576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33839988708496094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3263367712497711, + 0.45939648151397705, + 0.31238222122192383, + 0.9611073136329651, + 0.1673281043767929, + 0.10776297748088837 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.09093498438596725, + "style_target": 0, + "support_probability": 0.1634310632944107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8484072199748827, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09111899882555008, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2547520399093628, + 0.2547520399093628, + 0.9897645711898804, + 0.24595147371292114, + 0.08944197744131088, + 0.09891648590564728, + 0.09007124602794647, + 0.0964585691690445, + 0.22618234157562256 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 1, + "style_probability": 0.7894942760467529, + "style_target": 1, + "support_probability": 0.7730556726455688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7723924476699483, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07642222195863724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2747074365615845, + 0.2747074365615845, + 0.9809479713439941, + 0.2964160442352295, + 0.2123069316148758, + 0.0528278723359108, + 0.06539219617843628, + 0.2394879013299942, + 0.10832527279853821 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 1, + "style_probability": 0.474499374628067, + "style_target": 0, + "support_probability": 0.7693180441856384, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8765304158452848, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14310802519321442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2486017346382141, + 0.26866310834884644, + 0.314858078956604, + 0.9422945976257324, + 0.05688567832112312, + 0.09061586856842041, + 0.06992822885513306, + 0.08380238711833954, + 0.04710475355386734, + 0.0461246483027935, + 0.1435377150774002 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 1, + "style_probability": 0.9003664255142212, + "style_target": 1, + "support_probability": 0.8281155824661255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8157318156073152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14220955967903137, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30374395847320557, + 0.2853766977787018, + 0.2997054159641266, + 0.9295731782913208, + 0.25235557556152344, + 0.1077624186873436, + 0.245153546333313, + 0.08615128695964813, + 0.21501325070858002 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 0, + "style_probability": 0.6004813313484192, + "style_target": 0, + "support_probability": 0.8797508478164673, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08146156611506797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5012852549552917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1950806826353073, + 0.3701220154762268, + 0.6853783130645752, + 0.07361984252929688, + 0.046381644904613495 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.6005858182907104, + "style_target": 1, + "support_probability": 0.07664897292852402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01963241177734491, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.502035915851593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22378365695476532, + 0.4237488806247711, + 0.758587658405304, + 0.10064158588647842, + 0.14841775596141815 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.17752791941165924, + "style_target": 0, + "support_probability": 0.10318353772163391, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7398140528632923, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10455033928155899, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5100560188293457, + 0.9916609525680542, + 0.13373565673828125, + 0.33285146951675415, + 0.046801064163446426, + 0.03251541033387184 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 1, + "style_probability": 0.8284632563591003, + "style_target": 1, + "support_probability": 0.8106802701950073, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6288683820019709, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11532720923423767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5106040239334106, + 0.9808087348937988, + 0.16073723137378693, + 0.3378067910671234, + 0.043141890317201614, + 0.10833138972520828 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 1, + "style_probability": 0.48879945278167725, + "style_target": 0, + "support_probability": 0.8597635626792908, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6613496198760948, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14329418540000916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6874642968177795, + 0.17346028983592987, + 0.8626490235328674, + 0.6874642968177795, + 0.03440903499722481, + 0.06144930049777031, + 0.05165889486670494, + 0.06617574393749237, + 0.031285952776670456 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 1, + "style_probability": 0.8421149253845215, + "style_target": 1, + "support_probability": 0.4152924120426178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2244019086822959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3932429254055023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.67563796043396, + 0.17305943369865417, + 0.8011821508407593, + 0.67563796043396, + 0.18131017684936523, + 0.13256442546844482 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 0, + "style_probability": 0.5372085571289062, + "style_target": 0, + "support_probability": 0.13781698048114777, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7586764239657393, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06578422337770462, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18135473132133484, + 0.6266135573387146, + 0.5267104506492615, + 0.9174959659576416, + 0.10136182606220245, + 0.04894417151808739, + 0.0381118543446064, + 0.06128104403614998, + 0.09000184386968613, + 0.05780567601323128, + 0.025836708024144173 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.766717255115509, + "style_target": 1, + "support_probability": 0.8592326641082764, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13426288240597248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12874698638916016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2762354016304016, + 0.6375077366828918, + 0.6793230175971985, + 0.9406333565711975, + 0.09239920228719711, + 0.09512656182050705, + 0.19490092992782593, + 0.09702100604772568, + 0.10615229606628418, + 0.10787216573953629 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.16749265789985657, + "style_target": 0, + "support_probability": 0.7647682428359985, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0065456616440852725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9600017666816711, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21683724224567413, + 0.3007999062538147, + 0.24245913326740265, + 0.16040989756584167, + 0.03715180978178978 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 1, + "style_probability": 0.5883195996284485, + "style_target": 1, + "support_probability": 0.016650449484586716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004971541095674734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.968593180179596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20168650150299072, + 0.3295873701572418, + 0.2023926079273224, + 0.21670334041118622, + 0.04796932637691498 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 1, + "style_probability": 0.2833545207977295, + "style_target": 0, + "support_probability": 0.014594320207834244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15849209286310822, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5072396397590637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9528393745422363, + 0.23588499426841736, + 0.5179932117462158, + 0.3148048520088196, + 0.12691271305084229, + 0.061309248208999634 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 0, + "style_probability": 0.46348562836647034, + "style_target": 1, + "support_probability": 0.21134087443351746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07964718619632076, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49235275387763977, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9328147172927856, + 0.2542647123336792, + 0.4879925847053528, + 0.31638941168785095, + 0.1062631756067276 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 1, + "style_probability": 0.22828134894371033, + "style_target": 0, + "support_probability": 0.1398313343524933, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8993834545489118, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07831665128469467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5153:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4750320017337799, + 0.9661253094673157, + 0.29671984910964966, + 0.4750320017337799, + 0.03958510607481003, + 0.12397149950265884, + 0.05805739015340805, + 0.08646216988563538, + 0.05548197403550148 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.8179628252983093, + "style_target": 1, + "support_probability": 0.9046369791030884, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.518010237790635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07078031450510025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5153:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5082775354385376, + 0.9822852611541748, + 0.3429430425167084, + 0.5082775354385376, + 0.08177211880683899, + 0.13239972293376923, + 0.13614700734615326, + 0.056981973350048065, + 0.11693237721920013 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.29104146361351013, + "style_target": 0, + "support_probability": 0.906514048576355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.52163605661513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.236653670668602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.269297331571579, + 0.5150004029273987, + 0.35095682740211487, + 0.6241534948348999, + 0.04225623235106468, + 0.07184533774852753, + 0.02949228137731552, + 0.09524954110383987, + 0.033551331609487534, + 0.08383672684431076, + 0.029453439638018608 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 1, + "style_probability": 0.8372611999511719, + "style_target": 1, + "support_probability": 0.6715841889381409, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3018989146100053, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38408058881759644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2819562256336212, + 0.4841998815536499, + 0.31747695803642273, + 0.5976985096931458, + 0.07505780458450317, + 0.04823064059019089, + 0.049183622002601624, + 0.11664258688688278, + 0.09190212935209274, + 0.1868772953748703, + 0.06500133872032166 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 1, + "style_probability": 0.4587109088897705, + "style_target": 0, + "support_probability": 0.5738238096237183, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7906037523701679, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14276881515979767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18918944895267487, + 0.20079033076763153, + 0.8755763173103333, + 0.28339627385139465, + 0.030666077509522438 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.8129516243934631, + "style_target": 1, + "support_probability": 0.9041562676429749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4128308017353513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.426938533782959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1912040412425995, + 0.2170882523059845, + 0.8506956696510315, + 0.2738984525203705, + 0.08523325622081757, + 0.0815248116850853, + 0.07392631471157074, + 0.09557310491800308, + 0.061840929090976715, + 0.11912999302148819, + 0.09390395134687424 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.27845850586891174, + "style_target": 0, + "support_probability": 0.5842370986938477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005968406979301946, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9606135487556458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29963621497154236, + 0.41040438413619995, + 0.4482562243938446, + 0.44183897972106934, + 0.07529090344905853, + 0.05772324651479721 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.5972040891647339, + "style_target": 1, + "support_probability": 0.003258806187659502, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001444086205640527, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9370335340499878, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32970958948135376, + 0.4355068802833557, + 0.4673323333263397, + 0.45194363594055176, + 0.09986749291419983, + 0.08312633633613586 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.1666799783706665, + "style_target": 0, + "support_probability": 0.003149277064949274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0796845426337104, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9050378799438477, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8083903193473816, + 0.5158600211143494, + 0.10172237455844879, + 0.44778141379356384, + 0.15739230811595917, + 0.10441568493843079, + 0.07631479203701019, + 0.03860186040401459, + 0.04443598911166191 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.8423526883125305, + "style_target": 1, + "support_probability": 0.04258332401514053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02528276947008569, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8912725448608398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8360413908958435, + 0.5833263397216797, + 0.11161120235919952, + 0.4064326286315918, + 0.11772637814283371, + 0.1154584139585495, + 0.10169927775859833, + 0.08251529186964035, + 0.09071381390094757 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.33651190996170044, + "style_target": 0, + "support_probability": 0.049327973276376724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10922455526778241, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7692360877990723, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6738150119781494, + 0.5690467953681946, + 0.37752267718315125, + 0.27388548851013184, + 0.06153509393334389, + 0.11604127287864685, + 0.19345885515213013, + 0.12026508152484894, + 0.07950210571289062, + 0.04482119530439377, + 0.03469303250312805 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.7096940279006958, + "style_target": 1, + "support_probability": 0.14955466985702515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032278035180463016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9003248810768127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6850876808166504, + 0.5431222915649414, + 0.3914662301540375, + 0.26913389563560486, + 0.22293488681316376, + 0.05315098911523819, + 0.05253300443291664, + 0.06370839476585388, + 0.12075311690568924 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.30121368169784546, + "style_target": 0, + "support_probability": 0.051701147109270096, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015625499916621776, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8636540770530701, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28029224276542664, + 0.15305843949317932, + 0.3040646016597748, + 0.45863768458366394, + 0.1932637244462967 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 0, + "style_probability": 0.46718084812164307, + "style_target": 1, + "support_probability": 0.05309876799583435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029913367460148433, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9138006567955017, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28246232867240906, + 0.15060916543006897, + 0.3253188133239746, + 0.43890124559402466, + 0.0926516130566597, + 0.07538104057312012, + 0.06720412522554398, + 0.07555920630693436, + 0.04522816091775894, + 0.2517146170139313, + 0.11602983623743057 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 1, + "style_probability": 0.29072484374046326, + "style_target": 0, + "support_probability": 0.06628324091434479, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.44379725790773994, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4548396170139313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8174667954444885, + 0.21638460457324982, + 0.22892208397388458, + 0.26782891154289246, + 0.2522728443145752, + 0.19497910141944885 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.5514996647834778, + "style_target": 1, + "support_probability": 0.5845077633857727, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1766832366568963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3674868643283844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9528928995132446, + 0.18131037056446075, + 0.3059341609477997, + 0.32576170563697815, + 0.10180871188640594, + 0.10862207412719727 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.2741411030292511, + "style_target": 0, + "support_probability": 0.6144978404045105, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2866814649277119, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27715086936950684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32725414633750916, + 0.3729371428489685, + 0.9470828175544739, + 0.41581711173057556, + 0.12470458447933197, + 0.08382559567689896, + 0.13331234455108643, + 0.16483689844608307, + 0.08167079836130142 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 0, + "style_probability": 0.43569326400756836, + "style_target": 1, + "support_probability": 0.2898523807525635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24748210070119186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3328571617603302, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31579840183258057, + 0.35642778873443604, + 0.9829923510551453, + 0.33577701449394226, + 0.056491315364837646, + 0.11772317439317703, + 0.14296042919158936, + 0.06210751086473465, + 0.15257883071899414 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 1, + "style_probability": 0.33364808559417725, + "style_target": 0, + "support_probability": 0.3571021556854248, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3336266863600014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4126778244972229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.909471333026886, + 0.27969348430633545, + 0.10385025292634964, + 0.3570041060447693, + 0.04802078753709793, + 0.041895750910043716, + 0.0567980594933033, + 0.11368772387504578, + 0.043457865715026855, + 0.12462250143289566, + 0.12159579992294312 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.6542993187904358, + "style_target": 1, + "support_probability": 0.18365053832530975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2163701858111045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42631492018699646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9263302683830261, + 0.22709988057613373, + 0.09402067959308624, + 0.33475223183631897, + 0.10682521015405655, + 0.12193701416254044, + 0.04690345749258995, + 0.2047157883644104, + 0.0694442018866539 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.47753745317459106, + "style_target": 0, + "support_probability": 0.17163288593292236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5888122418959518, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23113228380680084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38918569684028625, + 0.6541340947151184, + 0.4475412666797638, + 0.38918569684028625, + 0.07107573002576828 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 1, + "style_probability": 0.8718581795692444, + "style_target": 1, + "support_probability": 0.7066523432731628, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4390238812316432, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19654837250709534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38144683837890625, + 0.6314145922660828, + 0.3779672086238861, + 0.38144683837890625, + 0.2447931468486786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 1, + "style_probability": 0.49494096636772156, + "style_target": 0, + "support_probability": 0.7606058716773987, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8540194083734232, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02419520542025566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9618542790412903, + 0.3566000163555145, + 0.48776161670684814, + 0.8956159949302673, + 0.050098154693841934, + 0.0815909206867218 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.6150804162025452, + "style_target": 1, + "support_probability": 0.935905933380127, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7896746579547527, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03769899532198906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9339391589164734, + 0.3476048409938812, + 0.5046820044517517, + 0.8621401190757751, + 0.1748063713312149 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.40393155813217163, + "style_target": 0, + "support_probability": 0.917911946773529, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06946955416828815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9427419900894165, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3728078305721283, + 0.866341233253479, + 0.22392646968364716, + 0.34372976422309875, + 0.07652597874403, + 0.1295192390680313, + 0.17651492357254028, + 0.11094257980585098, + 0.16427187621593475 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.692665159702301, + "style_target": 1, + "support_probability": 0.03191196173429489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010697437642545488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9405620694160461, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5181037783622742, + 0.9141422510147095, + 0.3228926658630371, + 0.4268077611923218, + 0.13650810718536377, + 0.2831818163394928, + 0.19865059852600098, + 0.14103379845619202, + 0.1799120157957077 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.12165290117263794, + "style_target": 0, + "support_probability": 0.036986734718084335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6746332661337635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46590331196784973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33181267976760864, + 0.3579716086387634, + 0.936653733253479, + 0.4666089415550232, + 0.09053883701562881, + 0.05037390813231468, + 0.09095309674739838, + 0.15168176591396332, + 0.10972374677658081, + 0.18386682868003845 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 1, + "style_probability": 0.8162746429443359, + "style_target": 1, + "support_probability": 0.6423720121383667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4464214552936332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5083341598510742, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31078675389289856, + 0.40970471501350403, + 0.9320529699325562, + 0.43701186776161194, + 0.0972813218832016, + 0.08547250181436539, + 0.12098383903503418, + 0.11829912662506104, + 0.05119464173913002, + 0.3057423233985901 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 0, + "style_probability": 0.5628018379211426, + "style_target": 0, + "support_probability": 0.6698542237281799, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.785541904065525, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05024868994951248, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.896356999874115, + 0.31853482127189636, + 0.40266457200050354, + 0.31853482127189636, + 0.03830629214644432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.7477081418037415, + "style_target": 1, + "support_probability": 0.9427984952926636, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6671539723558438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0763026773929596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8547981977462769, + 0.33858728408813477, + 0.4143128991127014, + 0.33858728408813477, + 0.08414427191019058, + 0.06712297350168228, + 0.1437055468559265, + 0.04458209499716759, + 0.14436273276805878, + 0.1080881729722023 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.41317737102508545, + "style_target": 0, + "support_probability": 0.9367128014564514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41335795020488686, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.25944116711616516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19788265228271484, + 0.18593773245811462, + 0.2799418270587921, + 0.7668949365615845, + 0.07660561054944992, + 0.05856509879231453 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.8957782983779907, + "style_target": 1, + "support_probability": 0.19055283069610596, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04679230128141122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15538835525512695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20756939053535461, + 0.17101259529590607, + 0.2665882408618927, + 0.8528841137886047, + 0.172027125954628, + 0.12422015517950058 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.2435292899608612, + "style_target": 0, + "support_probability": 0.29307135939598083, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8469667474287728, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14486727118492126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16319888830184937, + 0.16057270765304565, + 0.11714095622301102, + 0.9995550513267517, + 0.04889049753546715, + 0.16278919577598572, + 0.06576693058013916, + 0.09735769033432007, + 0.07758264243602753 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 1, + "style_probability": 0.6166031360626221, + "style_target": 1, + "support_probability": 0.8214027285575867, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.72594080178158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12246385961771011, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20130214095115662, + 0.21726740896701813, + 0.12927286326885223, + 0.9937805533409119, + 0.12941424548625946, + 0.0627637654542923, + 0.13829223811626434, + 0.0832168459892273, + 0.19096753001213074 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 1, + "style_probability": 0.49601784348487854, + "style_target": 0, + "support_probability": 0.7114282250404358, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.691274363475921, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40104183554649353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3774068355560303, + 0.8763126730918884, + 0.19601832330226898, + 0.3774068355560303, + 0.059738919138908386, + 0.04661891981959343, + 0.0457611121237278, + 0.11305591464042664, + 0.059068355709314346, + 0.08576642721891403, + 0.13893766701221466 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.8503003120422363, + "style_target": 1, + "support_probability": 0.6714991927146912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2790096361517844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3608821928501129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37161433696746826, + 0.8680931329727173, + 0.19809453189373016, + 0.37161433696746826, + 0.10259062051773071, + 0.06483117491006851, + 0.0595877431333065, + 0.07042448222637177, + 0.07563717663288116 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.2600536346435547, + "style_target": 0, + "support_probability": 0.5434131026268005, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14872361366367, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7500050663948059, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21172600984573364, + 0.5061960816383362, + 0.7944926023483276, + 0.2548103630542755, + 0.09702376276254654 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.5703262090682983, + "style_target": 1, + "support_probability": 0.1504824012517929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03189428423858622, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9509688019752502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19997164607048035, + 0.526154637336731, + 0.7345274090766907, + 0.253312349319458, + 0.1301557421684265, + 0.07925920188426971, + 0.09921824932098389, + 0.0935312882065773, + 0.10110348463058472, + 0.07217452675104141, + 0.07035259157419205 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.4639723002910614, + "style_target": 0, + "support_probability": 0.01981312409043312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37666186511169264, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30340972542762756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8213688731193542, + 0.10291900485754013, + 0.21891817450523376, + 0.10291900485754013, + 0.10605796426534653, + 0.06263744831085205 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.69529789686203, + "style_target": 1, + "support_probability": 0.32010629773139954, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03755728726737347, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5355053544044495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8432803750038147, + 0.18154087662696838, + 0.2664395272731781, + 0.18154087662696838, + 0.10085946321487427, + 0.07350528985261917 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.30022966861724854, + "style_target": 0, + "support_probability": 0.21289587020874023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24283955941261745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40480366349220276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2981418967247009, + 0.34630075097084045, + 0.7519758343696594, + 0.2981418967247009, + 0.1404362916946411, + 0.11984344571828842, + 0.07814615219831467, + 0.05953941494226456, + 0.0996507778763771 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 1, + "style_probability": 0.621345043182373, + "style_target": 1, + "support_probability": 0.1725742220878601, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44790318560583087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3143302798271179, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3516318202018738, + 0.35342782735824585, + 0.7997865676879883, + 0.3516318202018738, + 0.058728113770484924, + 0.12385925650596619, + 0.051077455282211304, + 0.08884080499410629, + 0.06829498708248138 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 0, + "style_probability": 0.8189607858657837, + "style_target": 0, + "support_probability": 0.23063762485980988, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10563055925446664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3621829152107239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2645982801914215, + 0.7226380109786987, + 0.2997710704803467, + 0.3873157203197479, + 0.0689445436000824, + 0.1626828908920288, + 0.22953951358795166, + 0.1160036250948906, + 0.08046369254589081, + 0.10091768205165863 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 1, + "style_probability": 0.5954701900482178, + "style_target": 1, + "support_probability": 0.08131564408540726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09019578182051075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5297418832778931, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2563687860965729, + 0.7292919754981995, + 0.2860356867313385, + 0.3479091227054596, + 0.16628117859363556, + 0.08484520763158798, + 0.20721115171909332, + 0.14187204837799072, + 0.09177359193563461 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 1, + "style_probability": 0.38900208473205566, + "style_target": 0, + "support_probability": 0.04508901759982109, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4926115083241339, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.35975366830825806, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3056548535823822, + 0.49482885003089905, + 0.887407660484314, + 0.16717734932899475, + 0.07529633492231369 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 1, + "style_probability": 0.5566001534461975, + "style_target": 1, + "support_probability": 0.5557688474655151, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4730894942067735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3115265369415283, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3721928596496582, + 0.5235673785209656, + 0.8667905926704407, + 0.17180092632770538, + 0.08311618119478226 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 0, + "style_probability": 0.6340106129646301, + "style_target": 0, + "support_probability": 0.5991328358650208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20194743038514354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26110950112342834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9104680418968201, + 0.3906644582748413, + 0.3626144528388977, + 0.5265989303588867, + 0.22325372695922852, + 0.11885776370763779 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.40228763222694397, + "style_target": 1, + "support_probability": 0.1937519609928131, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5634501344687, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10163262486457825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9396822452545166, + 0.43694034218788147, + 0.39659059047698975, + 0.5517443418502808, + 0.3546295762062073, + 0.14169679582118988 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.7155522704124451, + "style_target": 0, + "support_probability": 0.5464334487915039, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11661339915737051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.752460777759552, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8488861322402954, + 0.13640476763248444, + 0.1455167531967163, + 0.19975645840168, + 0.13203640282154083, + 0.10228502750396729, + 0.07894966751337051, + 0.1011250913143158, + 0.11832728236913681 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 0, + "style_probability": 0.3820424973964691, + "style_target": 1, + "support_probability": 0.10601208359003067, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.043919910063260896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6524210572242737, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9205241799354553, + 0.1389078050851822, + 0.17183120548725128, + 0.2850379943847656, + 0.09132692217826843, + 0.25969213247299194, + 0.08373462408781052, + 0.04424562677741051, + 0.10800734162330627 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 1, + "style_probability": 0.19612926244735718, + "style_target": 0, + "support_probability": 0.25096628069877625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13570978054885904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5154211521148682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23102249205112457, + 0.188118115067482, + 0.8905795812606812, + 0.19518627226352692, + 0.06250306963920593, + 0.05330095440149307, + 0.07398063689470291, + 0.07852528244256973, + 0.07542924582958221, + 0.10352807492017746, + 0.13207022845745087 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.332535058259964, + "style_target": 1, + "support_probability": 0.23115180432796478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24003455263952272, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.392167329788208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24584032595157623, + 0.18446093797683716, + 0.8810343146324158, + 0.20742367208003998, + 0.07715310156345367, + 0.050591059029102325, + 0.034019552171230316, + 0.038642071187496185, + 0.04305088520050049, + 0.0917254313826561 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.6082723736763, + "style_target": 0, + "support_probability": 0.31028708815574646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35425412544543594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21408481895923615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.934165894985199, + 0.30776575207710266, + 0.11195840686559677, + 0.47624412178993225, + 0.27555957436561584 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 1, + "style_probability": 0.5325496196746826, + "style_target": 1, + "support_probability": 0.3077857196331024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2048081589808355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3786095380783081, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9492588639259338, + 0.25304272770881653, + 0.10632982850074768, + 0.42540988326072693, + 0.041338130831718445 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 0, + "style_probability": 0.6477429270744324, + "style_target": 0, + "support_probability": 0.08849348872900009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10754537301479353, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7920641303062439, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4331366717815399, + 0.595323383808136, + 0.2546270489692688, + 0.18906843662261963, + 0.05254942178726196, + 0.14100126922130585 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 1, + "style_probability": 0.5669457912445068, + "style_target": 1, + "support_probability": 0.1699085235595703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07851172041747603, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8824748396873474, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41233599185943604, + 0.6027304530143738, + 0.23050595819950104, + 0.17328974604606628, + 0.15472280979156494 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 1, + "style_probability": 0.3583628535270691, + "style_target": 0, + "support_probability": 0.15115314722061157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6051864340479375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10855551064014435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11718842387199402, + 0.2983405292034149, + 0.41577327251434326, + 0.8838676810264587, + 0.17007076740264893, + 0.06233269348740578, + 0.3451438546180725, + 0.4106072783470154, + 0.1619124412536621 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.6345590353012085, + "style_target": 1, + "support_probability": 0.6615941524505615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30324696767934256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.067232146859169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14091141521930695, + 0.33848837018013, + 0.49899178743362427, + 0.9655435681343079, + 0.1053217351436615, + 0.1638820320367813, + 0.08082864433526993, + 0.12082375586032867, + 0.11002231389284134 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.23835726082324982, + "style_target": 0, + "support_probability": 0.8881940245628357, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5406827152647854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35484614968299866, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9542074799537659, + 0.26300516724586487, + 0.232778862118721, + 0.10575766861438751, + 0.055836062878370285, + 0.0523117259144783, + 0.1311526596546173, + 0.07295030355453491, + 0.07039830833673477, + 0.04979120194911957, + 0.07126541435718536 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 1, + "style_probability": 0.5298200249671936, + "style_target": 1, + "support_probability": 0.4217759668827057, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7989489730089966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45003533363342285, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9895008206367493, + 0.22572606801986694, + 0.16912628710269928, + 0.08883774280548096, + 0.09775888174772263, + 0.12016012519598007, + 0.11483828723430634, + 0.0642964094877243, + 0.155477374792099, + 0.07674812525510788, + 0.06661149859428406 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 0, + "style_probability": 0.7276957035064697, + "style_target": 0, + "support_probability": 0.559977114200592, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5854829286951632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24170634150505066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30006352066993713, + 0.30006352066993713, + 0.7114824056625366, + 0.4324526786804199, + 0.05821317061781883 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 1, + "style_probability": 0.6906150579452515, + "style_target": 1, + "support_probability": 0.7580918669700623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46739487615619063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49908989667892456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31013745069503784, + 0.31013745069503784, + 0.7288599610328674, + 0.41580885648727417, + 0.08046397566795349, + 0.060902923345565796, + 0.09859343618154526, + 0.06127266213297844, + 0.06681785732507706, + 0.04025936499238014, + 0.09722210466861725 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 0, + "style_probability": 0.5521260499954224, + "style_target": 0, + "support_probability": 0.5713087916374207, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015711624713035274, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7141279578208923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3958475589752197, + 0.3668597936630249, + 0.3184131681919098, + 0.5462300777435303, + 0.15690045058727264, + 0.10302284359931946 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 1, + "style_probability": 0.5053475499153137, + "style_target": 1, + "support_probability": 0.017988167703151703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002450612434259328, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7260494828224182, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4894997477531433, + 0.4584227502346039, + 0.3755512833595276, + 0.5721948146820068, + 0.1537812054157257, + 0.151553213596344 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 1, + "style_probability": 0.18746013939380646, + "style_target": 0, + "support_probability": 0.0184149369597435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03867827471044891, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8946095705032349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3445422649383545, + 0.3226521611213684, + 0.3445422649383545, + 0.6724585294723511, + 0.04514966905117035, + 0.030586494132876396, + 0.08000914752483368, + 0.20551280677318573, + 0.09462599456310272 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 1, + "style_probability": 0.512991189956665, + "style_target": 1, + "support_probability": 0.04881446436047554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07414448480860973, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8354994058609009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3068431317806244, + 0.30750441551208496, + 0.3068431317806244, + 0.7556169033050537, + 0.07333806902170181, + 0.19100721180438995, + 0.04909435659646988, + 0.07208015024662018, + 0.104289710521698 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 0, + "style_probability": 0.7164018154144287, + "style_target": 0, + "support_probability": 0.04566614329814911, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13128636218860715, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5498526692390442, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7164114117622375, + 0.47683730721473694, + 0.42504528164863586, + 0.47683730721473694, + 0.07073797285556793, + 0.08213041722774506, + 0.05972811207175255, + 0.05084524676203728, + 0.06598781794309616, + 0.12924489378929138, + 0.09911321848630905 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 1, + "style_probability": 0.5253646969795227, + "style_target": 1, + "support_probability": 0.1305771768093109, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17058427797197073, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6476220488548279, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7246484756469727, + 0.4068775773048401, + 0.4044269621372223, + 0.4068775773048401, + 0.08203112334012985, + 0.04678715392947197, + 0.03199700638651848, + 0.07279635965824127, + 0.1330224722623825 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 0, + "style_probability": 0.5399580001831055, + "style_target": 0, + "support_probability": 0.09329695254564285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13381128896221056, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5829392671585083, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2093191295862198, + 0.5776536464691162, + 0.8435236215591431, + 0.3896723687648773, + 0.13035264611244202 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 0, + "style_probability": 0.4679236114025116, + "style_target": 1, + "support_probability": 0.31520625948905945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023749316401790187, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7003072500228882, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2834145128726959, + 0.4991512894630432, + 0.9116740226745605, + 0.42635202407836914, + 0.13645851612091064 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 1, + "style_probability": 0.19227156043052673, + "style_target": 0, + "support_probability": 0.3521943688392639, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8736460943656303, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.029072823002934456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5887241959571838, + 0.30359014868736267, + 0.35614803433418274, + 0.9463229775428772, + 0.21981987357139587, + 0.05245426669716835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 1, + "style_probability": 0.8243813514709473, + "style_target": 1, + "support_probability": 0.9579477906227112, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8721407605980582, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04627241939306259, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6481632590293884, + 0.29533177614212036, + 0.3806127607822418, + 0.9588773250579834, + 0.1967283934354782, + 0.11674372106790543 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 0, + "style_probability": 0.8792821764945984, + "style_target": 0, + "support_probability": 0.9259542226791382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08987528593212431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20263762772083282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.355894535779953, + 0.183524027466774, + 0.451753705739975, + 0.9064695835113525, + 0.04609724506735802, + 0.10175040364265442, + 0.16105860471725464, + 0.19252285361289978, + 0.11171308904886246 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 0, + "style_probability": 0.4518169164657593, + "style_target": 1, + "support_probability": 0.14227871596813202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10611335622942383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2344595491886139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3493849039077759, + 0.1705460250377655, + 0.4519960880279541, + 0.9130949378013611, + 0.04660085588693619, + 0.07119645178318024 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 1, + "style_probability": 0.36147236824035645, + "style_target": 0, + "support_probability": 0.13057881593704224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047569543740816164, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7808251976966858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24192996323108673, + 0.6454125046730042, + 0.6839398741722107, + 0.32912540435791016, + 0.0745772123336792, + 0.0666913241147995, + 0.06504161655902863, + 0.11091317981481552, + 0.5541607737541199, + 0.24618910253047943, + 0.07457847893238068 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 0, + "style_probability": 0.42050614953041077, + "style_target": 1, + "support_probability": 0.07569640874862671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006042555589193283, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7390336394309998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31344032287597656, + 0.8076594471931458, + 0.7127335071563721, + 0.39677760004997253, + 0.12960603833198547, + 0.09991679340600967, + 0.16663770377635956, + 0.09185963869094849, + 0.1601625233888626, + 0.09695807099342346 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 1, + "style_probability": 0.05123412236571312, + "style_target": 0, + "support_probability": 0.1366865187883377, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6075930490357304, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23528313636779785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6478701233863831, + 0.18562792241573334, + 0.49263012409210205, + 0.8273183703422546, + 0.08484844863414764 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.5779309868812561, + "style_target": 1, + "support_probability": 0.6397984027862549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4085917430742146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2609303891658783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5532486438751221, + 0.1726461499929428, + 0.4057251513004303, + 0.7826973795890808, + 0.18803592026233673, + 0.09443709999322891, + 0.058806296437978745, + 0.09073349833488464, + 0.04163379594683647, + 0.13765935599803925, + 0.07218164950609207 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.21557685732841492, + "style_target": 0, + "support_probability": 0.5950778722763062, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7439942539832074, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0645984634757042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9337821006774902, + 0.23609748482704163, + 0.6937828063964844, + 0.23609748482704163, + 0.02983548492193222, + 0.11061595380306244 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.7782341241836548, + "style_target": 1, + "support_probability": 0.8248434066772461, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07895460952118111, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09998218715190887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.91973876953125, + 0.3068324029445648, + 0.6813809871673584, + 0.3068324029445648, + 0.10593511909246445, + 0.1374145895242691 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.14300617575645447, + "style_target": 0, + "support_probability": 0.731476366519928, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9325469612432336, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025265779346227646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5393:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5969992876052856, + 0.9815691709518433, + 0.4059767425060272, + 0.31471845507621765, + 0.03655849024653435, + 0.037276457995176315, + 0.13746662437915802, + 0.0412277951836586, + 0.28989171981811523 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.8310913443565369, + "style_target": 1, + "support_probability": 0.9648649096488953, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5947280915068006, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1349279284477234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44352787733078003, + 0.9499286413192749, + 0.3853326141834259, + 0.3044775724411011, + 0.028522560372948647, + 0.0448332205414772, + 0.04312081262469292, + 0.036142002791166306, + 0.026488665491342545 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.35222581028938293, + "style_target": 0, + "support_probability": 0.8984848856925964, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.809109152636132, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11370856314897537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3421846032142639, + 0.9349991083145142, + 0.08931276947259903, + 0.3421846032142639, + 0.054010726511478424, + 0.04321892186999321, + 0.041787195950746536, + 0.07467040419578552, + 0.1053776741027832, + 0.033483635634183884, + 0.07517225295305252 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 1, + "style_probability": 0.6968188285827637, + "style_target": 1, + "support_probability": 0.8114596009254456, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3906371568176689, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2822114825248718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35013625025749207, + 0.8838643431663513, + 0.08460880815982819, + 0.35013625025749207, + 0.06658581644296646, + 0.08006513863801956, + 0.06335420161485672, + 0.03930697962641716, + 0.04039541631937027 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 1, + "style_probability": 0.3019951581954956, + "style_target": 0, + "support_probability": 0.5301482081413269, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8880745793471662, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04458274692296982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5064377784729004, + 0.9871088266372681, + 0.24557806551456451, + 0.5064377784729004, + 0.08549550175666809 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.6869550943374634, + "style_target": 1, + "support_probability": 0.9302442669868469, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3776822677721602, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06345617026090622, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6694812178611755, + 0.9937533736228943, + 0.320123553276062, + 0.6694812178611755, + 0.1249479427933693 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.1836947500705719, + "style_target": 0, + "support_probability": 0.9072878956794739, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8438946263042376, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2303045094013214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9914286732673645, + 0.45645272731781006, + 0.5300509333610535, + 0.13829372823238373, + 0.07830667495727539, + 0.08788610994815826 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 1, + "style_probability": 0.8663330674171448, + "style_target": 1, + "support_probability": 0.5610422492027283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6744462465148525, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17470239102840424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9873360991477966, + 0.42751461267471313, + 0.45911693572998047, + 0.1615828573703766, + 0.08298555016517639, + 0.13663090765476227 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 0, + "style_probability": 0.5493384003639221, + "style_target": 0, + "support_probability": 0.7108035683631897, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7567783852583645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2806755602359772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17414861917495728, + 0.17414861917495728, + 0.9064701199531555, + 0.21518085896968842, + 0.10631696879863739, + 0.0725426971912384, + 0.037485796958208084, + 0.11621101200580597, + 0.07302822172641754 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.6853075623512268, + "style_target": 1, + "support_probability": 0.8528353571891785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5347070341201001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.374395489692688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16703282296657562, + 0.16703282296657562, + 0.8804288506507874, + 0.2379208654165268, + 0.07145767658948898, + 0.0902891606092453 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.4651777446269989, + "style_target": 0, + "support_probability": 0.7059701085090637, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.38545813155093367, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6016026735305786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3090572655200958, + 0.5216503143310547, + 0.3352079391479492, + 0.7732780575752258, + 0.11742059141397476, + 0.04526171460747719, + 0.05803583934903145, + 0.07482346892356873, + 0.026391539722681046, + 0.084632508456707, + 0.05875519663095474 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.8246240615844727, + "style_target": 1, + "support_probability": 0.1787211298942566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03949496313397095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43949776887893677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31321877241134644, + 0.5419306755065918, + 0.27459320425987244, + 0.8253552913665771, + 0.1308930516242981, + 0.07928696274757385, + 0.1423235833644867, + 0.055395763367414474, + 0.0825706273317337, + 0.11324050277471542 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.12887117266654968, + "style_target": 0, + "support_probability": 0.3344590663909912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03407959086711247, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.740546703338623, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23861496150493622, + 0.3683793544769287, + 0.19570483267307281, + 0.5306402444839478, + 0.22426564991474152 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 1, + "style_probability": 0.5347574949264526, + "style_target": 1, + "support_probability": 0.07838653773069382, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014323762642089476, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6844054460525513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2149357944726944, + 0.40389880537986755, + 0.2486840933561325, + 0.6573636531829834, + 0.1660539209842682 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 1, + "style_probability": 0.1286470890045166, + "style_target": 0, + "support_probability": 0.31419238448143005, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.34694912995553034, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2832883298397064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26725196838378906, + 0.779839277267456, + 0.23575663566589355, + 0.26573145389556885, + 0.1502726972103119, + 0.1618049144744873 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 1, + "style_probability": 0.581824004650116, + "style_target": 1, + "support_probability": 0.3359847664833069, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6683066515920022, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12965130805969238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.373820424079895, + 0.7949475049972534, + 0.26036903262138367, + 0.2729540169239044, + 0.04303008317947388, + 0.08195122331380844 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 0, + "style_probability": 0.8919156789779663, + "style_target": 0, + "support_probability": 0.6525697708129883, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.12399341159499384, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8357272148132324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18106842041015625, + 0.5521659851074219, + 0.3399253487586975, + 0.34371089935302734, + 0.0579000860452652, + 0.11290962249040604, + 0.12848533689975739, + 0.07988587766885757, + 0.0844821110367775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 0, + "style_probability": 0.3883376717567444, + "style_target": 1, + "support_probability": 0.2705379128456116, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32451955511432184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4578849971294403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18008294701576233, + 0.5966556668281555, + 0.3070741593837738, + 0.295505553483963, + 0.0679205134510994, + 0.04843723401427269 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 1, + "style_probability": 0.33217519521713257, + "style_target": 0, + "support_probability": 0.6698378324508667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014795414025483157, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8129960894584656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38905826210975647, + 0.8233092427253723, + 0.482726514339447, + 0.20749640464782715, + 0.1415601223707199, + 0.05741870403289795, + 0.04735024645924568, + 0.04034631699323654, + 0.17151962220668793, + 0.09546074271202087, + 0.062071025371551514 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.5302691459655762, + "style_target": 1, + "support_probability": 0.008248778060078621, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030893870375132228, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3182986080646515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.45172417163848877, + 0.8902566432952881, + 0.5509716868400574, + 0.1601707935333252, + 0.10689668357372284, + 0.05168420076370239, + 0.11373833566904068, + 0.12729112803936005, + 0.05830749496817589, + 0.08865034580230713 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.14180698990821838, + "style_target": 0, + "support_probability": 0.15729592740535736, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7617538526732102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28275763988494873, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9820663928985596, + 0.5278623700141907, + 0.3480030596256256, + 0.2623865008354187, + 0.06627344340085983 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.8635063171386719, + "style_target": 1, + "support_probability": 0.6123597621917725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39717467873471324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3207530975341797, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9740696549415588, + 0.5282464027404785, + 0.3354285657405853, + 0.27421149611473083, + 0.0870252326130867, + 0.06534907221794128, + 0.10344833135604858, + 0.0821969285607338, + 0.06657657027244568, + 0.12799622118473053, + 0.060193371027708054 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.4310034513473511, + "style_target": 0, + "support_probability": 0.5370677709579468, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7696642187761873, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07814770191907883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5613614916801453, + 0.6587039232254028, + 0.8801103234291077, + 0.9044660925865173, + 0.023806555196642876, + 0.0681162178516388 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.7448246479034424, + "style_target": 1, + "support_probability": 0.8883129954338074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3374003470394275, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09697867184877396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5388057231903076, + 0.7357968091964722, + 0.8745168447494507, + 0.9322905540466309, + 0.04213397949934006, + 0.10405956953763962 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.20696239173412323, + "style_target": 0, + "support_probability": 0.8723456859588623, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5209500882187199, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30322399735450745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4232545793056488, + 0.6686107516288757, + 0.8548315167427063, + 0.4232545793056488, + 0.07073372602462769, + 0.08566848188638687, + 0.06387458741664886, + 0.07288286834955215, + 0.09665750712156296 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 1, + "style_probability": 0.7778311371803284, + "style_target": 1, + "support_probability": 0.41193047165870667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2783385748573059, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4119507670402527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.504082202911377, + 0.6138818264007568, + 0.7998681664466858, + 0.504082202911377, + 0.27411025762557983, + 0.14937299489974976, + 0.0948818027973175, + 0.06630539894104004, + 0.2149922102689743 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 0, + "style_probability": 0.5541585087776184, + "style_target": 0, + "support_probability": 0.23564818501472473, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7953983769661676, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13861364126205444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47905611991882324, + 0.388392835855484, + 0.33881983160972595, + 0.9715092182159424, + 0.03899035230278969, + 0.057427000254392624, + 0.11204657703638077, + 0.050628360360860825, + 0.04533557966351509, + 0.03099958226084709, + 0.06442862004041672 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.7202195525169373, + "style_target": 1, + "support_probability": 0.8767878413200378, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5822551549644239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23833239078521729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43066340684890747, + 0.402965247631073, + 0.366272509098053, + 0.9413895010948181, + 0.07728230208158493, + 0.08805318176746368, + 0.2494613230228424, + 0.055628541857004166, + 0.21205471456050873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.4163600504398346, + "style_target": 0, + "support_probability": 0.7548359632492065, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7682188904025737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16221106052398682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9605913758277893, + 0.1697828322649002, + 0.2920979857444763, + 0.5773191452026367, + 0.0714959055185318 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.8761947751045227, + "style_target": 1, + "support_probability": 0.6386477947235107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23849826968723514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18712785840034485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.973977267742157, + 0.19581562280654907, + 0.3252740502357483, + 0.6557460427284241, + 0.1077805683016777 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.3652670979499817, + "style_target": 0, + "support_probability": 0.6216065287590027, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017140296184977814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6194134950637817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6338880062103271, + 0.3582760691642761, + 0.3927058279514313, + 0.2077578902244568, + 0.0781954899430275, + 0.07834000885486603 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 1, + "style_probability": 0.6497066617012024, + "style_target": 1, + "support_probability": 0.014760277234017849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029151589520088506, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5035715699195862, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6500454545021057, + 0.3437217175960541, + 0.3810933530330658, + 0.2138783484697342, + 0.1809445470571518, + 0.17427222430706024 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 1, + "style_probability": 0.4416672885417938, + "style_target": 0, + "support_probability": 0.030414044857025146, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6583841750010676, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15545496344566345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4123535752296448, + 0.27745145559310913, + 0.5195671319961548, + 0.9180566668510437, + 0.07812708616256714, + 0.09821020066738129, + 0.08906089514493942, + 0.06581773608922958, + 0.06407716870307922 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 0, + "style_probability": 0.4282858073711395, + "style_target": 1, + "support_probability": 0.9487059712409973, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6996627940763891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20648564398288727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42418453097343445, + 0.2925281822681427, + 0.5926706194877625, + 0.9045711755752563, + 0.07239963114261627, + 0.03988025337457657 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 1, + "style_probability": 0.35977163910865784, + "style_target": 0, + "support_probability": 0.9480076432228088, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5554653612579414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.624180793762207, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1903396099805832, + 0.4452127516269684, + 0.713031530380249, + 0.32645660638809204, + 0.08827750384807587, + 0.05830136314034462, + 0.09633558988571167, + 0.06954316049814224, + 0.14058130979537964, + 0.08109467476606369, + 0.12416255474090576 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.8868702054023743, + "style_target": 1, + "support_probability": 0.5319244861602783, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09427482422765793, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5506193041801453, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.228108748793602, + 0.3780045211315155, + 0.9047731757164001, + 0.47269967198371887, + 0.06224582716822624, + 0.07037366926670074, + 0.07605135440826416, + 0.07670319825410843, + 0.04684971645474434, + 0.05978192389011383 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.15029124915599823, + "style_target": 0, + "support_probability": 0.7049165964126587, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4456748012764417, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2914699912071228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12325773388147354, + 0.30819636583328247, + 0.6628835201263428, + 0.9128445982933044, + 0.11431915313005447 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.40219882130622864, + "style_target": 1, + "support_probability": 0.6074075102806091, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5276142505286564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2479832023382187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10625042766332626, + 0.30548831820487976, + 0.6400545835494995, + 0.8968396782875061, + 0.14521518349647522 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.547659158706665, + "style_target": 0, + "support_probability": 0.6460347175598145, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4212152312619527, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3266008794307709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5553:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2303352802991867, + 0.9768641591072083, + 0.17310497164726257, + 0.2994520664215088, + 0.13199438154697418, + 0.07727811485528946 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 0, + "style_probability": 0.4584462344646454, + "style_target": 1, + "support_probability": 0.5830713510513306, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7040953441582193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3388794958591461, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5553:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22936469316482544, + 0.9821871519088745, + 0.17834560573101044, + 0.2952509820461273, + 0.07552698999643326, + 0.08373652398586273 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 0, + "style_probability": 0.8722653388977051, + "style_target": 0, + "support_probability": 0.6259214282035828, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07084637598209738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31236979365348816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7548555731773376, + 0.4431098401546478, + 0.1912236511707306, + 0.4431098401546478, + 0.2026745229959488, + 0.10456379503011703, + 0.10012052953243256, + 0.1951015293598175, + 0.12318143248558044 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 0, + "style_probability": 0.2631834149360657, + "style_target": 1, + "support_probability": 0.11433377861976624, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023206578257544144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.343580037355423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7437434792518616, + 0.42456620931625366, + 0.22965295612812042, + 0.42456620931625366, + 0.06333012878894806, + 0.22765782475471497, + 0.16011196374893188, + 0.21009968221187592, + 0.17340603470802307 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 1, + "style_probability": 0.09763046354055405, + "style_target": 0, + "support_probability": 0.2685607075691223, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3313694269718309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35066238045692444, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36879289150238037, + 0.594562828540802, + 0.8938422799110413, + 0.36879289150238037, + 0.03607293218374252, + 0.06880039721727371, + 0.3053356111049652, + 0.05561056360602379, + 0.05491773784160614, + 0.0838010311126709, + 0.0570276603102684 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 1, + "style_probability": 0.5341483950614929, + "style_target": 1, + "support_probability": 0.33552128076553345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3930761479137016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3865708112716675, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3462691307067871, + 0.6024128198623657, + 0.8812628388404846, + 0.3462691307067871, + 0.08781478554010391, + 0.0883815810084343, + 0.08693692833185196, + 0.0894072875380516, + 0.10407032072544098, + 0.08375551551580429, + 0.05809721723198891 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 0, + "style_probability": 0.6476346850395203, + "style_target": 0, + "support_probability": 0.3415975868701935, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5950766449110994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14553728699684143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8158175349235535, + 0.3072543144226074, + 0.4651799201965332, + 0.30491873621940613, + 0.09270113706588745 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.875624418258667, + "style_target": 1, + "support_probability": 0.395139217376709, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11942505656596229, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4861108064651489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7932851910591125, + 0.284058541059494, + 0.4243580400943756, + 0.26540055871009827, + 0.07654339075088501, + 0.1013403907418251, + 0.09471973031759262, + 0.13275820016860962, + 0.12028653919696808, + 0.07269230484962463 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.4113030433654785, + "style_target": 0, + "support_probability": 0.08576422929763794, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5047345518632005, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26655903458595276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43440088629722595, + 0.9682776927947998, + 0.36858564615249634, + 0.19070227444171906, + 0.0918584018945694, + 0.1020437628030777 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6627568006515503, + "style_target": 1, + "support_probability": 0.5602710247039795, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06221410010993975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36098572611808777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45520317554473877, + 0.9776706695556641, + 0.4427553713321686, + 0.20952466130256653, + 0.13257604837417603, + 0.11887126415967941 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.08292300999164581, + "style_target": 0, + "support_probability": 0.48251819610595703, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7071937466264634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.294206440448761, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.647121012210846, + 0.8797986507415771, + 0.23276762664318085, + 0.44438794255256653, + 0.14383143186569214, + 0.0367702916264534, + 0.158356711268425, + 0.05133244767785072, + 0.03489827737212181 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 1, + "style_probability": 0.8536063432693481, + "style_target": 1, + "support_probability": 0.6604453325271606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26115093169510534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42897334694862366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5510135889053345, + 0.8165453672409058, + 0.28408968448638916, + 0.34732240438461304, + 0.03702671825885773, + 0.08689912408590317, + 0.03623554855585098, + 0.034816108644008636, + 0.06776751577854156 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 0, + "style_probability": 0.5526264905929565, + "style_target": 0, + "support_probability": 0.3979475498199463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07374836672535036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9127889275550842, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4099171757698059, + 0.2254122793674469, + 0.7201399207115173, + 0.23967094719409943, + 0.0862492248415947, + 0.16547630727291107, + 0.20117183029651642, + 0.08224877715110779, + 0.16750630736351013, + 0.14189761877059937, + 0.08406071364879608 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.7950413823127747, + "style_target": 1, + "support_probability": 0.0764983668923378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03198684304836223, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.947027862071991, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42835912108421326, + 0.2127884179353714, + 0.6626728177070618, + 0.24730712175369263, + 0.0747424066066742, + 0.043134696781635284, + 0.06202658265829086, + 0.0809485912322998, + 0.09606786072254181 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.4272386133670807, + "style_target": 0, + "support_probability": 0.043721433728933334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011465436096954666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7218531370162964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39931005239486694, + 0.8546810150146484, + 0.13340109586715698, + 0.3400280177593231, + 0.11150195449590683 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 0, + "style_probability": 0.3421190679073334, + "style_target": 1, + "support_probability": 0.04027780145406723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002825191559396151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7459497451782227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4372040033340454, + 0.8871379494667053, + 0.11271411180496216, + 0.347711443901062, + 0.09107570350170135 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 1, + "style_probability": 0.09224040061235428, + "style_target": 0, + "support_probability": 0.04432098567485809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003395002640040515, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9161160588264465, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3753178119659424, + 0.16470982134342194, + 0.12105992436408997, + 0.35431042313575745, + 0.07118339836597443, + 0.17046548426151276 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 1, + "style_probability": 0.7466686367988586, + "style_target": 1, + "support_probability": 0.002592209493741393, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011716294049438645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9639378190040588, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4050440490245819, + 0.16602322459220886, + 0.1246713399887085, + 0.36730921268463135, + 0.0665208101272583, + 0.046284470707178116 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 1, + "style_probability": 0.4701300263404846, + "style_target": 0, + "support_probability": 0.0010553613537922502, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3690924551839784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10952901095151901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17024223506450653, + 0.2659934461116791, + 0.8039222359657288, + 0.2756207585334778, + 0.049392081797122955, + 0.05104055628180504, + 0.04902845248579979, + 0.09743163734674454, + 0.04713060334324837 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.7656251788139343, + "style_target": 1, + "support_probability": 0.4717647135257721, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2940993660221045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07017436623573303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2080918401479721, + 0.2804100215435028, + 0.7797282338142395, + 0.2937084138393402, + 0.06345489621162415, + 0.064520463347435 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.39148852229118347, + "style_target": 0, + "support_probability": 0.5828122496604919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17043189532281602, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2811325788497925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2535325586795807, + 0.48720160126686096, + 0.9173071384429932, + 0.14229263365268707, + 0.11853081732988358, + 0.08835849910974503, + 0.07154805958271027, + 0.11396975815296173, + 0.06980554759502411, + 0.06759662181138992, + 0.12741759419441223 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.5198695063591003, + "style_target": 1, + "support_probability": 0.21246960759162903, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0177954433126585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4554309844970703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25357088446617126, + 0.4989505410194397, + 0.9177206158638, + 0.19126760959625244, + 0.18992166221141815, + 0.08044802397489548, + 0.18524396419525146, + 0.07825033366680145, + 0.17503710091114044, + 0.10038840770721436 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.05502067133784294, + "style_target": 0, + "support_probability": 0.1823517084121704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5071616643985518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3459796905517578, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9454841613769531, + 0.4240798354148865, + 0.329261839389801, + 0.42351460456848145, + 0.08473993092775345 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 0, + "style_probability": 0.34687408804893494, + "style_target": 1, + "support_probability": 0.7891983389854431, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6153364729883926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21874292194843292, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9513788819313049, + 0.3899464011192322, + 0.3102845251560211, + 0.36361896991729736, + 0.08219676464796066 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 1, + "style_probability": 0.3377928137779236, + "style_target": 0, + "support_probability": 0.8720141649246216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09406667233905752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6045533418655396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7146667838096619, + 0.2911164462566376, + 0.33174967765808105, + 0.3045746386051178, + 0.07466529309749603, + 0.16130110621452332 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.34416458010673523, + "style_target": 1, + "support_probability": 0.07223498076200485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35537310947100387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3617232143878937, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7586594820022583, + 0.34932994842529297, + 0.2560085952281952, + 0.29817721247673035, + 0.051101379096508026, + 0.062102802097797394 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.8623714447021484, + "style_target": 0, + "support_probability": 0.1994909942150116, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4360051525946869, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4691956341266632, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6963257193565369, + 0.5316484570503235, + 0.2536204159259796, + 0.09558827430009842, + 0.09489372372627258, + 0.10016985237598419, + 0.1639324426651001, + 0.030917586758732796, + 0.18139810860157013 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 0, + "style_probability": 0.42369669675827026, + "style_target": 1, + "support_probability": 0.5807814002037048, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11300402847917825, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1783384531736374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8552828431129456, + 0.7410876750946045, + 0.2885097563266754, + 0.2119704782962799, + 0.10966998338699341, + 0.2061837613582611, + 0.07529933750629425, + 0.10951191931962967, + 0.144175186753273 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 1, + "style_probability": 0.10504229366779327, + "style_target": 0, + "support_probability": 0.874410092830658, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.19471645911903307, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7318227887153625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32570743560791016, + 0.778439998626709, + 0.19528886675834656, + 0.08558844029903412, + 0.12801013886928558, + 0.10775933414697647, + 0.09090372920036316, + 0.07689210027456284, + 0.04104607552289963, + 0.06312243640422821 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 0, + "style_probability": 0.2998177707195282, + "style_target": 1, + "support_probability": 0.3555201590061188, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32079494672700437, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7246548533439636, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31697961688041687, + 0.7188451290130615, + 0.18964490294456482, + 0.0728798657655716, + 0.09167271107435226, + 0.14125370979309082, + 0.08471941202878952, + 0.039661917835474014, + 0.11399494111537933, + 0.07632837444543839, + 0.1179811879992485 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 1, + "style_probability": 0.4379061758518219, + "style_target": 0, + "support_probability": 0.47113534808158875, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.612535292536486, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14013078808784485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.058602456003427505, + 0.31612488627433777, + 0.39340633153915405, + 0.8368108868598938, + 0.06964956223964691 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.8091002106666565, + "style_target": 1, + "support_probability": 0.7197278738021851, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5087967283964128, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14105506241321564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06297380477190018, + 0.284640908241272, + 0.37642329931259155, + 0.8399788737297058, + 0.09924020618200302, + 0.2668137848377228, + 0.10612455010414124, + 0.18284744024276733, + 0.06958863139152527, + 0.08680747449398041, + 0.22133997082710266 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.3021204173564911, + "style_target": 0, + "support_probability": 0.7667079567909241, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6406562385863672, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12936292588710785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5053952932357788, + 0.9745434522628784, + 0.5638532042503357, + 0.46230900287628174, + 0.08687205612659454, + 0.04354359954595566 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.5985367894172668, + "style_target": 1, + "support_probability": 0.7872906923294067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3134011539933861, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16466529667377472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.522135317325592, + 0.9843064546585083, + 0.5244333148002625, + 0.5190465450286865, + 0.0590168833732605, + 0.1051335334777832 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.32790958881378174, + "style_target": 0, + "support_probability": 0.8036079406738281, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05562420796848043, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9060665369033813, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46270865201950073, + 0.2063569724559784, + 0.31049519777297974, + 0.2063569724559784, + 0.11067341268062592, + 0.08934565633535385, + 0.15909193456172943, + 0.04081735014915466, + 0.1397055983543396 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 1, + "style_probability": 0.7662847638130188, + "style_target": 1, + "support_probability": 0.03272770345211029, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05369710006196086, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9001460075378418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5221900939941406, + 0.1739191859960556, + 0.29224005341529846, + 0.1739191859960556, + 0.12883895635604858, + 0.07596936821937561, + 0.06260169297456741, + 0.05671951174736023, + 0.213369682431221 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 0, + "style_probability": 0.5369535088539124, + "style_target": 0, + "support_probability": 0.0512675940990448, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.22871784933791497, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7404510378837585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3674708604812622, + 0.8964428901672363, + 0.5432400107383728, + 0.40703117847442627, + 0.05114736780524254, + 0.08317500352859497, + 0.06510976701974869, + 0.10619381815195084, + 0.08554338663816452, + 0.047477733343839645 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.73166424036026, + "style_target": 1, + "support_probability": 0.18332812190055847, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07829140719545959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.783044695854187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3058825433254242, + 0.8631711602210999, + 0.5393630862236023, + 0.3605003356933594, + 0.06109140068292618, + 0.06099603325128555, + 0.1532718986272812, + 0.4271750748157501, + 0.4106985926628113 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.2977360785007477, + "style_target": 0, + "support_probability": 0.10762569308280945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.82768387820917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11343591660261154, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9956415891647339, + 0.2087402641773224, + 0.5025618076324463, + 0.4110402762889862, + 0.1650688797235489 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 1, + "style_probability": 0.8054731488227844, + "style_target": 1, + "support_probability": 0.44358858466148376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4435778270280829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1573694348335266, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9984403252601624, + 0.2462073713541031, + 0.6403186321258545, + 0.4305073022842407, + 0.11898521333932877 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 1, + "style_probability": 0.41439008712768555, + "style_target": 0, + "support_probability": 0.4187345504760742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06815515368926911, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6646440625190735, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2248014211654663, + 0.8052417039871216, + 0.2035137265920639, + 0.33585283160209656, + 0.18956522643566132, + 0.19087280333042145 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 1, + "style_probability": 0.6898459196090698, + "style_target": 1, + "support_probability": 0.08312492072582245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053969193574342905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.773419201374054, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23794035613536835, + 0.744910478591919, + 0.1517142653465271, + 0.28227993845939636, + 0.053165581077337265, + 0.07615111023187637 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 0, + "style_probability": 0.6265361905097961, + "style_target": 0, + "support_probability": 0.052374016493558884, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8542388224977436, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05034603178501129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2645830512046814, + 0.9482817053794861, + 0.21250569820404053, + 0.18016135692596436, + 0.036766670644283295, + 0.04245002195239067, + 0.06476162374019623, + 0.06340397894382477, + 0.23290996253490448 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.8194400668144226, + "style_target": 1, + "support_probability": 0.870370626449585, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6457137000753554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06756426393985748, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2383677363395691, + 0.9330008625984192, + 0.2065601646900177, + 0.15127868950366974, + 0.044940706342458725, + 0.08219531178474426 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.403748095035553, + "style_target": 0, + "support_probability": 0.806939423084259, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4765538005520123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25553011894226074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6863126754760742, + 0.4577859938144684, + 0.13687865436077118, + 0.614675760269165, + 0.036243218928575516, + 0.06739393621683121, + 0.02059137634932995, + 0.02848607674241066, + 0.1428133249282837, + 0.0765116736292839, + 0.05721201002597809 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.7654301524162292, + "style_target": 1, + "support_probability": 0.40543800592422485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12158034059730373, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18368367850780487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8291541337966919, + 0.6275784969329834, + 0.2460557073354721, + 0.7213289141654968, + 0.14637382328510284, + 0.0653546154499054, + 0.30368682742118835, + 0.1401452124118805, + 0.05462804436683655, + 0.1107538565993309 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.18479172885417938, + "style_target": 0, + "support_probability": 0.5458353161811829, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7279896438183089, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13130758702754974, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4837604761123657, + 0.7615060806274414, + 0.9078981876373291, + 0.6676197648048401, + 0.09326142072677612 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 1, + "style_probability": 0.7584627866744995, + "style_target": 1, + "support_probability": 0.7954068779945374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.557437652832193, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14361827075481415, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.527249276638031, + 0.6860898733139038, + 0.8539767265319824, + 0.5915820002555847, + 0.1180119439959526 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 0, + "style_probability": 0.57929527759552, + "style_target": 0, + "support_probability": 0.6801434755325317, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7774002024151142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04852622374892235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4068458080291748, + 0.9766969680786133, + 0.3021446168422699, + 0.3474941551685333, + 0.12589722871780396, + 0.13604088127613068 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.73497474193573, + "style_target": 1, + "support_probability": 0.6690031290054321, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5975180727509983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04102666303515434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3860671818256378, + 0.9729732871055603, + 0.2810656428337097, + 0.30087342858314514, + 0.10909099876880646 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.32798153162002563, + "style_target": 0, + "support_probability": 0.6533645391464233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007083492563430478, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9875401258468628, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45391592383384705, + 0.2324593961238861, + 0.2757834792137146, + 0.45391592383384705, + 0.04727582633495331, + 0.11159723252058029, + 0.14446502923965454, + 0.11108686029911041, + 0.056876420974731445 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.6985582113265991, + "style_target": 1, + "support_probability": 0.01162152923643589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0023566734184880854, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9889007806777954, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4914247691631317, + 0.1633128523826599, + 0.24279123544692993, + 0.4914247691631317, + 0.06882132589817047, + 0.3367902934551239, + 0.21410509943962097, + 0.12127436697483063, + 0.13728196918964386 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.26376399397850037, + "style_target": 0, + "support_probability": 0.00997880194336176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012560960377086783, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8887292742729187, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5304808020591736, + 0.27927666902542114, + 0.1980600655078888, + 0.16188383102416992, + 0.07088357955217361, + 0.06261971592903137, + 0.061983149498701096, + 0.07641740888357162, + 0.08035611361265182, + 0.042657576501369476 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 1, + "style_probability": 0.7529802918434143, + "style_target": 1, + "support_probability": 0.007897601462900639, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014554031824128826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7384129762649536, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5387128591537476, + 0.23747767508029938, + 0.17194025218486786, + 0.15786249935626984, + 0.07997238636016846, + 0.08945026993751526, + 0.13176080584526062, + 0.13125599920749664, + 0.13128364086151123, + 0.21303054690361023, + 0.22009408473968506 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 0, + "style_probability": 0.5086634755134583, + "style_target": 0, + "support_probability": 0.01646069996058941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009263745024693182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.915979266166687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7967255115509033, + 0.24844811856746674, + 0.3819965720176697, + 0.3204348087310791, + 0.08274441212415695 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.22679805755615234, + "style_target": 1, + "support_probability": 0.00860887486487627, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05984296914703797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8919766545295715, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.852756917476654, + 0.2383890450000763, + 0.36924952268600464, + 0.35057303309440613, + 0.10231799632310867 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.8100850582122803, + "style_target": 0, + "support_probability": 0.015974637120962143, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.06202753721503734, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.803963840007782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45626217126846313, + 0.8826354742050171, + 0.24026831984519958, + 0.4149778485298157, + 0.15998706221580505, + 0.08354056626558304 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 0, + "style_probability": 0.28603652119636536, + "style_target": 1, + "support_probability": 0.060103725641965866, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.026330842354575383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6730735301971436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5463286638259888, + 0.9373378157615662, + 0.24527479708194733, + 0.4857412874698639, + 0.08511599153280258, + 0.1408652812242508 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 1, + "style_probability": 0.17642025649547577, + "style_target": 0, + "support_probability": 0.22251743078231812, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.19226668000938982, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6439814567565918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9335597157478333, + 0.16803434491157532, + 0.3777252435684204, + 0.26849132776260376, + 0.048329513520002365, + 0.03622770681977272, + 0.13633601367473602, + 0.11469077318906784, + 0.07534675300121307 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 0, + "style_probability": 0.3300604522228241, + "style_target": 1, + "support_probability": 0.2842436730861664, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25099181184623254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6259855031967163, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.918129026889801, + 0.20265862345695496, + 0.3951534330844879, + 0.24052180349826813, + 0.03842780739068985, + 0.11061487346887589, + 0.14523151516914368, + 0.08110997080802917, + 0.04426735267043114 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 1, + "style_probability": 0.31353557109832764, + "style_target": 0, + "support_probability": 0.4237249791622162, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37989381715416815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.63691645860672, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18852101266384125, + 0.36557984352111816, + 0.22529052197933197, + 0.867765486240387, + 0.05612568557262421, + 0.0757438987493515, + 0.05806894227862358, + 0.05736943334341049, + 0.12141965329647064, + 0.1109221950173378, + 0.0758453905582428 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.48253175616264343, + "style_target": 1, + "support_probability": 0.32272693514823914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5155547639855125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5405846834182739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20500710606575012, + 0.38372284173965454, + 0.20797455310821533, + 0.8839032649993896, + 0.07145798206329346, + 0.04256170988082886, + 0.0979509949684143, + 0.041334815323352814, + 0.05632079020142555, + 0.06529323011636734, + 0.042760785669088364 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.7217171788215637, + "style_target": 0, + "support_probability": 0.43064001202583313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031500530350011113, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.828248918056488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2045716792345047, + 0.23263521492481232, + 0.5341712236404419, + 0.4076566994190216, + 0.13946805894374847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 0, + "style_probability": 0.35201627016067505, + "style_target": 1, + "support_probability": 0.0033846418373286724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005560826492499625, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8304510116577148, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24044133722782135, + 0.21818920969963074, + 0.5042825937271118, + 0.38125625252723694, + 0.1936914026737213, + 0.24631065130233765, + 0.12494233250617981, + 0.21147091686725616, + 0.15766167640686035, + 0.14578142762184143 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 1, + "style_probability": 0.4039348065853119, + "style_target": 0, + "support_probability": 0.0032731988467276096, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5798649803285768, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17163214087486267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8554814457893372, + 0.5190432071685791, + 0.13534078001976013, + 0.29136496782302856, + 0.12704519927501678, + 0.05364253371953964 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.5444583296775818, + "style_target": 1, + "support_probability": 0.7123985290527344, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40447755456742934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15621620416641235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9075100421905518, + 0.5594862103462219, + 0.13345743715763092, + 0.3235999345779419, + 0.0557280033826828, + 0.10558825731277466 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.4040686786174774, + "style_target": 0, + "support_probability": 0.7139791250228882, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.025855283838558504, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8614876866340637, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5243844985961914, + 0.4139865040779114, + 0.35860949754714966, + 0.5571066737174988, + 0.10363032668828964, + 0.11031325161457062, + 0.08931390941143036, + 0.11768698692321777, + 0.1421886682510376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 1, + "style_probability": 0.5242376923561096, + "style_target": 1, + "support_probability": 0.017557378858327866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14402740840993095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7281561493873596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42151767015457153, + 0.46079686284065247, + 0.4287659227848053, + 0.5447254776954651, + 0.0888347402215004, + 0.09032697975635529, + 0.08504673093557358, + 0.1573430299758911, + 0.10752739012241364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 0, + "style_probability": 0.8681091070175171, + "style_target": 0, + "support_probability": 0.039056435227394104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02117065393226372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.923779308795929, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32031387090682983, + 0.5328011512756348, + 0.366615355014801, + 0.6942035555839539, + 0.18468239903450012, + 0.07707007974386215, + 0.11667931824922562, + 0.1063477098941803, + 0.30977559089660645, + 0.07704450935125351, + 0.10480725020170212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 0, + "style_probability": 0.4570585787296295, + "style_target": 1, + "support_probability": 0.025058770552277565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020904816386120597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9282358884811401, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3698543906211853, + 0.5487741827964783, + 0.3441486656665802, + 0.7295283079147339, + 0.18162836134433746, + 0.09371987730264664, + 0.1021750420331955, + 0.07237950712442398, + 0.15020620822906494 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 1, + "style_probability": 0.22804954648017883, + "style_target": 0, + "support_probability": 0.02793850377202034, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7639645585284548, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041513558477163315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9237312078475952, + 0.3849114775657654, + 0.3908267617225647, + 0.20027142763137817, + 0.05579497292637825 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 0, + "style_probability": 0.3707475960254669, + "style_target": 1, + "support_probability": 0.9678174257278442, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09758912279537825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14144077897071838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.932685911655426, + 0.518013060092926, + 0.5593244433403015, + 0.17871583998203278, + 0.17127498984336853 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 1, + "style_probability": 0.039929479360580444, + "style_target": 0, + "support_probability": 0.930656909942627, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3995123674062314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5590851306915283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43940469622612, + 0.8242846727371216, + 0.3842995762825012, + 0.19360554218292236, + 0.1281903237104416, + 0.10898511856794357 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 0, + "style_probability": 0.4987991154193878, + "style_target": 1, + "support_probability": 0.35248810052871704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5546733923065813, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5720728039741516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4754219949245453, + 0.8915954828262329, + 0.35775238275527954, + 0.1742008775472641, + 0.08464264869689941, + 0.077830471098423 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 0, + "style_probability": 0.7809069156646729, + "style_target": 0, + "support_probability": 0.3064408302307129, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05306398708460379, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8929372429847717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25984710454940796, + 0.737217903137207, + 0.3570678234100342, + 0.4053734540939331, + 0.06952525675296783, + 0.055231012403964996, + 0.08845676481723785, + 0.10429859906435013, + 0.04340727999806404 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 0, + "style_probability": 0.3554675579071045, + "style_target": 1, + "support_probability": 0.1390584260225296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04053288470702292, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8614280819892883, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2332136183977127, + 0.7495341897010803, + 0.30030348896980286, + 0.3621335029602051, + 0.046229928731918335, + 0.3133751153945923 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 1, + "style_probability": 0.26225894689559937, + "style_target": 0, + "support_probability": 0.08030256628990173, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4010483494787885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33284398913383484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5626773238182068, + 0.28118234872817993, + 0.47735804319381714, + 0.900033175945282, + 0.10612311959266663, + 0.06263333559036255, + 0.08200105279684067, + 0.0841076597571373, + 0.22081536054611206, + 0.13690367341041565, + 0.0775512307882309 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.7267745733261108, + "style_target": 1, + "support_probability": 0.2510538697242737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07075481350427879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37314125895500183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6102942228317261, + 0.35054558515548706, + 0.42400023341178894, + 0.9255944490432739, + 0.136357381939888, + 0.09682437777519226, + 0.1631481945514679, + 0.1167774498462677, + 0.0820889100432396, + 0.11513924598693848 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.35248497128486633, + "style_target": 0, + "support_probability": 0.1707468032836914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7253333965823687, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.051215630024671555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18739072978496552, + 0.858323872089386, + 0.39900410175323486, + 0.23138071596622467, + 0.07559498399496078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.902680516242981, + "style_target": 1, + "support_probability": 0.6697424054145813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41800754747432145, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1049448624253273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17593340575695038, + 0.8179543614387512, + 0.3681589365005493, + 0.2251800000667572, + 0.04901973530650139, + 0.08261136710643768, + 0.03553004935383797, + 0.0498347282409668, + 0.05846506729722023, + 0.0493202731013298, + 0.16520008444786072 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.4620228111743927, + "style_target": 0, + "support_probability": 0.45839378237724304, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18432929130679193, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6134653687477112, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5491046905517578, + 0.22185346484184265, + 0.5322475433349609, + 0.3240368068218231, + 0.030411528423428535, + 0.05014505982398987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.7062171697616577, + "style_target": 1, + "support_probability": 0.13330808281898499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029817569943581732, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6141385436058044, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5940815806388855, + 0.2558921277523041, + 0.598030149936676, + 0.3274422883987427, + 0.053569767624139786, + 0.11887821555137634 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.25633904337882996, + "style_target": 0, + "support_probability": 0.14231763780117035, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7246079739384005, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12309200316667557, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2950707674026489, + 0.9490718841552734, + 0.21162481606006622, + 0.5276014804840088, + 0.08135754615068436, + 0.07136479765176773, + 0.03312237933278084, + 0.043409496545791626, + 0.046539753675460815 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 1, + "style_probability": 0.8048622608184814, + "style_target": 1, + "support_probability": 0.6647362112998962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7655873861798482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06327582895755768, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3290410041809082, + 0.9457604885101318, + 0.20871245861053467, + 0.5441477298736572, + 0.05858943983912468, + 0.042158614844083786, + 0.04465000703930855, + 0.13888025283813477, + 0.07433897256851196 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 0, + "style_probability": 0.6597164273262024, + "style_target": 0, + "support_probability": 0.7903041839599609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30182669309186616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6935012936592102, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6431217193603516, + 0.21892811357975006, + 0.11022496968507767, + 0.8666555881500244, + 0.03747350722551346, + 0.031785883009433746, + 0.05475126579403877, + 0.0939820185303688, + 0.02421649359166622, + 0.03996015712618828, + 0.07631786912679672 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.8474403619766235, + "style_target": 1, + "support_probability": 0.12239647656679153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11005779926987572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6960049867630005, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.632335364818573, + 0.2099914252758026, + 0.11500144004821777, + 0.8338893055915833, + 0.05688806250691414, + 0.0580136701464653, + 0.09233999252319336, + 0.037349727004766464, + 0.05193212255835533 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.36916032433509827, + "style_target": 0, + "support_probability": 0.09310953319072723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7192105523370538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34297245740890503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9058586359024048, + 0.2967684268951416, + 0.47220560908317566, + 0.5079604387283325, + 0.0599123015999794 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.6974614858627319, + "style_target": 1, + "support_probability": 0.8462267518043518, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038984416820296325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6354686617851257, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8839327096939087, + 0.3360499441623688, + 0.470633864402771, + 0.5278560519218445, + 0.10883020609617233 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.08998347073793411, + "style_target": 0, + "support_probability": 0.5279079079627991, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4528255304484709, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17291325330734253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4506934881210327, + 0.6527217030525208, + 0.31678307056427, + 0.4506934881210327, + 0.04429592564702034, + 0.0371379517018795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 1, + "style_probability": 0.6958045363426208, + "style_target": 1, + "support_probability": 0.6965446472167969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08607232969052885, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4748932719230652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39471182227134705, + 0.5409010052680969, + 0.33845192193984985, + 0.39471182227134705, + 0.06821742653846741, + 0.11076251417398453 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 1, + "style_probability": 0.45428624749183655, + "style_target": 0, + "support_probability": 0.15953081846237183, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6515395951995209, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3060690760612488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27265697717666626, + 0.22992470860481262, + 0.9713957905769348, + 0.7695051431655884, + 0.060518328100442886, + 0.04981553554534912, + 0.06420537084341049, + 0.01925530470907688, + 0.06259704381227493 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.6561862230300903, + "style_target": 1, + "support_probability": 0.4556373357772827, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44744940525636767, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24420662224292755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21965175867080688, + 0.2299836277961731, + 0.957169234752655, + 0.7126687169075012, + 0.05313717573881149, + 0.12549659609794617 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.26126742362976074, + "style_target": 0, + "support_probability": 0.40686264634132385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6119937000898267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14530320465564728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6002209782600403, + 0.16923117637634277, + 0.20315754413604736, + 0.6605364680290222, + 0.12517569959163666, + 0.09107886254787445, + 0.03269937261939049, + 0.1283109188079834, + 0.035665541887283325, + 0.04888587445020676 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.844502329826355, + "style_target": 1, + "support_probability": 0.8074454665184021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10795418329050034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20218728482723236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6175814270973206, + 0.22322551906108856, + 0.2633651793003082, + 0.7848315834999084, + 0.15627893805503845, + 0.1088656410574913, + 0.11361480504274368, + 0.06609974056482315, + 0.11975275725126266, + 0.1586499959230423 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.09150455892086029, + "style_target": 0, + "support_probability": 0.8366899490356445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0266498711147421, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9710926413536072, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6133961081504822, + 0.3551607131958008, + 0.21827977895736694, + 0.5831143260002136, + 0.1450570821762085 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.8224341869354248, + "style_target": 1, + "support_probability": 0.025310363620519638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03055369928369286, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9529637098312378, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5893743634223938, + 0.3588370084762573, + 0.26173868775367737, + 0.5390800833702087, + 0.22645622491836548 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.48365673422813416, + "style_target": 0, + "support_probability": 0.03167949616909027, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5949123919099678, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.42241403460502625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37287765741348267, + 0.36544039845466614, + 0.2636391818523407, + 0.8300765752792358, + 0.03503919392824173, + 0.06380927562713623 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.6691808104515076, + "style_target": 1, + "support_probability": 0.46286189556121826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.303718995335932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5543920397758484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34686699509620667, + 0.38177070021629333, + 0.24717041850090027, + 0.7961780428886414, + 0.09979948401451111 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.28405794501304626, + "style_target": 0, + "support_probability": 0.29306963086128235, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8022251932120413, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15887054800987244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13702821731567383, + 0.16754718124866486, + 0.16372796893119812, + 0.9252346158027649, + 0.05066971108317375, + 0.09897457808256149, + 0.03667877987027168, + 0.028978630900382996, + 0.09378749132156372 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.8070696592330933, + "style_target": 1, + "support_probability": 0.8406501412391663, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2057423573707358, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2173788696527481, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21060052514076233, + 0.20185577869415283, + 0.17924819886684418, + 0.950722336769104, + 0.14082442224025726, + 0.11917929351329803, + 0.08467911928892136, + 0.06921767443418503, + 0.1145562008023262 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.10812316089868546, + "style_target": 0, + "support_probability": 0.8587625622749329, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4419166791170601, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5948039293289185, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5781627893447876, + 0.5221551060676575, + 0.5781627893447876, + 0.46677273511886597, + 0.04316478595137596, + 0.04033563658595085, + 0.08217541873455048, + 0.089166060090065, + 0.01551138050854206, + 0.025507967919111252, + 0.050410956144332886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 1, + "style_probability": 0.8623626828193665, + "style_target": 1, + "support_probability": 0.6113670468330383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11024974517729365, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8320342898368835, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5197833180427551, + 0.4933483302593231, + 0.5197833180427551, + 0.4025615453720093, + 0.06931886076927185, + 0.031258825212717056, + 0.041120294481515884, + 0.05262968689203262, + 0.09226235002279282, + 0.06183822825551033, + 0.10134442150592804 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 1, + "style_probability": 0.42864692211151123, + "style_target": 0, + "support_probability": 0.3979700207710266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16164254385544669, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3365877568721771, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3265858590602875, + 0.47645002603530884, + 0.34388723969459534, + 0.7948306798934937, + 0.110859215259552 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.3309091329574585, + "style_target": 1, + "support_probability": 0.29773297905921936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3270482297325782, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.33807870745658875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3521655797958374, + 0.5026242136955261, + 0.3563123643398285, + 0.7967174649238586, + 0.0521409697830677 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.7255393862724304, + "style_target": 0, + "support_probability": 0.3773023188114166, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05579545897903656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7916484475135803, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3507625460624695, + 0.2955400049686432, + 0.4767855107784271, + 0.29808247089385986, + 0.07018954306840897, + 0.09807584434747696 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 1, + "style_probability": 0.5072246193885803, + "style_target": 1, + "support_probability": 0.07328467816114426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006845701305641802, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5891072750091553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3678598999977112, + 0.25985875725746155, + 0.5517653822898865, + 0.30459949374198914, + 0.10076186060905457, + 0.10638764500617981 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 1, + "style_probability": 0.17780843377113342, + "style_target": 0, + "support_probability": 0.0936223492026329, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09686784961667726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4117162227630615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3533878028392792, + 0.21323062479496002, + 0.13450568914413452, + 0.8279447555541992, + 0.10820923745632172, + 0.07167510688304901, + 0.08708029985427856, + 0.05739225447177887, + 0.06839911639690399 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.42341023683547974, + "style_target": 1, + "support_probability": 0.09003621339797974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15908946557015594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3343780040740967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3467467427253723, + 0.21331557631492615, + 0.12679989635944366, + 0.8431058526039124, + 0.06315938383340836, + 0.0880681723356247, + 0.06363056600093842, + 0.1254596710205078, + 0.1030917838215828 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.6410284638404846, + "style_target": 0, + "support_probability": 0.12726274132728577, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7338015797819452, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05115530267357826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3819452226161957, + 0.3963538408279419, + 0.9214434027671814, + 0.3819452226161957, + 0.08463282138109207, + 0.07155430316925049, + 0.1086953803896904, + 0.05741112679243088, + 0.06886770576238632, + 0.0790560320019722 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.3381137549877167, + "style_target": 1, + "support_probability": 0.9313973784446716, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.890539130948832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.017085326835513115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3744989037513733, + 0.39524197578430176, + 0.950671374797821, + 0.3744989037513733, + 0.07212172448635101, + 0.10939205437898636, + 0.019870834425091743, + 0.03233833983540535, + 0.17552001774311066, + 0.06923329830169678, + 0.029668545350432396 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.7836532592773438, + "style_target": 0, + "support_probability": 0.9803215861320496, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25446746487739985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.372262567281723, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.379177063703537, + 0.9230507612228394, + 0.31143495440483093, + 0.41181236505508423, + 0.10818701982498169 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 1, + "style_probability": 0.7294864058494568, + "style_target": 1, + "support_probability": 0.15389524400234222, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1528621242269356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5108564496040344, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36227306723594666, + 0.9275005459785461, + 0.30080047249794006, + 0.46824556589126587, + 0.0961303859949112, + 0.10167937725782394, + 0.14160536229610443, + 0.046908244490623474, + 0.10227315872907639, + 0.06987825036048889, + 0.1709146648645401 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 0, + "style_probability": 0.5879961848258972, + "style_target": 0, + "support_probability": 0.1224447712302208, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7776412235709447, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06322542577981949, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3261471092700958, + 0.8677747249603271, + 0.30441009998321533, + 0.3077946901321411, + 0.08336227387189865, + 0.12554863095283508 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 1, + "style_probability": 0.6175686717033386, + "style_target": 1, + "support_probability": 0.9702099561691284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2067580996417821, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08246944099664688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3811551034450531, + 0.9304669499397278, + 0.35072097182273865, + 0.2961709797382355, + 0.07261547446250916, + 0.0726144015789032 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 1, + "style_probability": 0.11859505623579025, + "style_target": 0, + "support_probability": 0.9484727382659912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.613437107213308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21600747108459473, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33488380908966064, + 0.33008041977882385, + 0.13416387140750885, + 0.8345417380332947, + 0.07566991448402405, + 0.08849851042032242, + 0.05375909432768822, + 0.09226666390895844, + 0.15327170491218567 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 1, + "style_probability": 0.609576940536499, + "style_target": 1, + "support_probability": 0.8235689401626587, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6926354471876541, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15820956230163574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3031943440437317, + 0.3721906244754791, + 0.13536491990089417, + 0.8673539757728577, + 0.18962246179580688, + 0.04426005855202675, + 0.05806438997387886, + 0.06380587071180344, + 0.12545451521873474 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 0, + "style_probability": 0.8265780806541443, + "style_target": 0, + "support_probability": 0.8300434350967407, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3063301206805698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40852412581443787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3488121032714844, + 0.28550538420677185, + 0.8654540181159973, + 0.3339208662509918, + 0.07459335774183273, + 0.08128464221954346, + 0.1083458736538887, + 0.11909135431051254, + 0.07340925931930542, + 0.10786037147045135, + 0.1668914258480072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.6600214838981628, + "style_target": 1, + "support_probability": 0.1196245327591896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27832388498125205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4496200978755951, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3310859799385071, + 0.26824355125427246, + 0.888758659362793, + 0.29996877908706665, + 0.06862195581197739, + 0.12236928194761276, + 0.06254640966653824, + 0.11442852765321732, + 0.09555543959140778 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.4765828251838684, + "style_target": 0, + "support_probability": 0.16372965276241302, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9058827651829366, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19279363751411438, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9609946608543396, + 0.32208728790283203, + 0.34262049198150635, + 0.26529592275619507, + 0.12973444163799286 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 1, + "style_probability": 0.8290827870368958, + "style_target": 1, + "support_probability": 0.9440735578536987, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7306095613002795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27964159846305847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9232637286186218, + 0.35867807269096375, + 0.3648419976234436, + 0.256356805562973, + 0.13604189455509186 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 0, + "style_probability": 0.5231631398200989, + "style_target": 0, + "support_probability": 0.8648949265480042, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5500426392122029, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2934296131134033, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3904910385608673, + 0.3974691927433014, + 0.58204585313797, + 0.3904910385608673, + 0.03246229141950607, + 0.0818464607000351 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.8976761102676392, + "style_target": 1, + "support_probability": 0.8731308579444885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4269924881781926, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.32242628931999207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3763964772224426, + 0.3724417984485626, + 0.5326995253562927, + 0.3763964772224426, + 0.0933440774679184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.44104546308517456, + "style_target": 0, + "support_probability": 0.8567339181900024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17385778204633523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15353965759277344, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1137961894273758, + 0.6695460677146912, + 0.9542275667190552, + 0.18527346849441528, + 0.04734455421566963, + 0.07481290400028229, + 0.07659487426280975, + 0.06784141808748245, + 0.16725514829158783 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 1, + "style_probability": 0.6889032125473022, + "style_target": 1, + "support_probability": 0.19270648062229156, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08854439945474718, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17415966093540192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11394339799880981, + 0.7103502750396729, + 0.9631722569465637, + 0.20877142250537872, + 0.04215161129832268, + 0.09877029806375504, + 0.047834593802690506, + 0.07628609240055084, + 0.06875883042812347 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 1, + "style_probability": 0.3822903633117676, + "style_target": 0, + "support_probability": 0.31193700432777405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12622997349774964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23961694538593292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23580250144004822, + 0.4722304940223694, + 0.4286314845085144, + 0.704795241355896, + 0.06253822892904282, + 0.05333671718835831, + 0.07382441312074661, + 0.04822747781872749, + 0.04294715076684952, + 0.03419920429587364, + 0.11403125524520874 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8163982033729553, + "style_target": 1, + "support_probability": 0.08262886106967926, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03885943549353799, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39770203828811646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2547542452812195, + 0.49481579661369324, + 0.4108246862888336, + 0.6591478586196899, + 0.19117368757724762, + 0.08926728367805481, + 0.19090695679187775, + 0.18519148230552673, + 0.0595659576356411, + 0.06429428607225418, + 0.09037267416715622 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 0, + "style_probability": 0.5713383555412292, + "style_target": 0, + "support_probability": 0.03805810958147049, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04136731704521442, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9017768502235413, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27416178584098816, + 0.314409077167511, + 0.3827909529209137, + 0.34474217891693115, + 0.31074419617652893 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.8038578629493713, + "style_target": 1, + "support_probability": 0.038037728518247604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032569810716791985, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8984367847442627, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2888096570968628, + 0.29474392533302307, + 0.3870488703250885, + 0.317813515663147, + 0.12641847133636475, + 0.13166305422782898, + 0.08625388890504837, + 0.16316859424114227, + 0.10865917056798935, + 0.08261102437973022, + 0.21680617332458496 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.4360627830028534, + "style_target": 0, + "support_probability": 0.03652290999889374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4566023701468615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4840463697910309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6011795401573181, + 0.4879242181777954, + 0.4727991223335266, + 0.19131839275360107, + 0.05527426302433014, + 0.15297965705394745 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.8981566429138184, + "style_target": 1, + "support_probability": 0.5237239003181458, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018805853809112212, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48281329870224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.768559455871582, + 0.5470015406608582, + 0.49747177958488464, + 0.16288000345230103, + 0.10915350168943405, + 0.2536085247993469 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.06319306790828705, + "style_target": 0, + "support_probability": 0.5279250144958496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17152647369596954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37660709023475647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30725082755088806, + 0.21310575306415558, + 0.30725082755088806, + 0.7760170102119446, + 0.04924680292606354, + 0.04305306449532509, + 0.04216596484184265, + 0.06621221452951431, + 0.1668403297662735 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 1, + "style_probability": 0.7121184468269348, + "style_target": 1, + "support_probability": 0.24963533878326416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0983227382646467, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4263213872909546, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26443949341773987, + 0.2285604178905487, + 0.26443949341773987, + 0.7102675437927246, + 0.1808536946773529, + 0.06726919114589691, + 0.09222257882356644, + 0.1900557279586792, + 0.05571134760975838 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 1, + "style_probability": 0.46574077010154724, + "style_target": 0, + "support_probability": 0.22919723391532898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00459126247412541, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7536473870277405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.49839726090431213, + 0.31173065304756165, + 0.45532533526420593, + 0.619371771812439, + 0.08234332501888275, + 0.10492251813411713, + 0.0656791552901268, + 0.04420154169201851, + 0.02975820004940033, + 0.10210464149713516, + 0.1262974888086319 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.563844621181488, + "style_target": 1, + "support_probability": 0.007008519489318132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007373681165362861, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7796923518180847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4890744984149933, + 0.2949741780757904, + 0.453336238861084, + 0.5578316450119019, + 0.12362737208604813, + 0.07163269817829132, + 0.1242123544216156, + 0.15597067773342133, + 0.0795367956161499 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.31931033730506897, + "style_target": 0, + "support_probability": 0.00864903349429369, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37356167074072544, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2791093587875366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7209662795066833, + 0.5832912921905518, + 0.8608575463294983, + 0.24286790192127228, + 0.07740885019302368 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 1, + "style_probability": 0.5879076719284058, + "style_target": 1, + "support_probability": 0.4860024154186249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.059515750862966024, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3193151354789734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8091531991958618, + 0.6672210097312927, + 0.9283829927444458, + 0.27069082856178284, + 0.13675646483898163 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 1, + "style_probability": 0.18942977488040924, + "style_target": 0, + "support_probability": 0.5728961825370789, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08342467096324879, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8359566926956177, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32029032707214355, + 0.15697123110294342, + 0.18664444983005524, + 0.7037591338157654, + 0.10449081659317017, + 0.08454214036464691 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.35005173087120056, + "style_target": 1, + "support_probability": 0.12408546358346939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.060491852173087064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8226214051246643, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2941787540912628, + 0.16456152498722076, + 0.1881876438856125, + 0.6850656270980835, + 0.14363770186901093, + 0.10883256793022156 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.5589422583580017, + "style_target": 0, + "support_probability": 0.12045303732156754, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.360793213433982, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5157584547996521, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33859679102897644, + 0.36787325143814087, + 0.6979266405105591, + 0.6744938492774963, + 0.02705521695315838, + 0.07355450093746185, + 0.12092949450016022, + 0.10314125567674637, + 0.05818092077970505 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.32961320877075195, + "style_target": 1, + "support_probability": 0.7219807505607605, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6706090721950259, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36673814058303833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36666950583457947, + 0.37860292196273804, + 0.7625914216041565, + 0.7257830500602722, + 0.04710223898291588, + 0.03406049311161041, + 0.025502609089016914, + 0.04536851868033409, + 0.03743072599172592 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.8632162809371948, + "style_target": 0, + "support_probability": 0.8388819694519043, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14269867776073392, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32224494218826294, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5949963331222534, + 0.3734709918498993, + 0.3112177848815918, + 0.834078848361969, + 0.1089433804154396, + 0.04189378023147583, + 0.061859022825956345, + 0.12182652205228806, + 0.20109602808952332, + 0.20684374868869781, + 0.09719124436378479 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 0, + "style_probability": 0.27951353788375854, + "style_target": 1, + "support_probability": 0.1187189370393753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04585328519686025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29781588912010193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6216002106666565, + 0.2954559326171875, + 0.36401936411857605, + 0.8240188956260681, + 0.2280704230070114, + 0.1769641637802124, + 0.2534295618534088, + 0.14314228296279907, + 0.15674041211605072, + 0.20279331505298615 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 1, + "style_probability": 0.20083238184452057, + "style_target": 0, + "support_probability": 0.18046584725379944, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07839134696834943, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7720620036125183, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.314492404460907, + 0.10743038356304169, + 0.5181365609169006, + 0.2803364098072052, + 0.11911207437515259 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 1, + "style_probability": 0.6102402806282043, + "style_target": 1, + "support_probability": 0.12996038794517517, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003883423342250592, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9596380591392517, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3149392902851105, + 0.2464248090982437, + 0.5353230834007263, + 0.36084461212158203, + 0.17310579121112823 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 1, + "style_probability": 0.16143472492694855, + "style_target": 0, + "support_probability": 0.031195534393191338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08031264707103275, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6872237324714661, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6094890236854553, + 0.19110852479934692, + 0.07491222769021988, + 0.19110852479934692, + 0.04801219329237938, + 0.10036607831716537 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.33581802248954773, + "style_target": 1, + "support_probability": 0.3629871904850006, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5172705899460901, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5682670474052429, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6780852675437927, + 0.16967274248600006, + 0.05721047893166542, + 0.16967274248600006, + 0.059155263006687164, + 0.11067695915699005 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.8766696453094482, + "style_target": 0, + "support_probability": 0.4970727562904358, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1977372323397185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28740358352661133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46752628684043884, + 0.22828345000743866, + 0.7880091667175293, + 0.46752628684043884, + 0.03273453563451767, + 0.11728575080633163, + 0.052148230373859406, + 0.0796644538640976, + 0.07564958184957504 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 0, + "style_probability": 0.45835089683532715, + "style_target": 1, + "support_probability": 0.3037269413471222, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2892321139649791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2044377475976944, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5146240592002869, + 0.17276334762573242, + 0.7764202952384949, + 0.5146240592002869, + 0.08106422424316406, + 0.12853099405765533 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 1, + "style_probability": 0.3122866451740265, + "style_target": 0, + "support_probability": 0.39143580198287964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21304104417209285, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6174250841140747, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2861187756061554, + 0.3481341600418091, + 0.6367275714874268, + 0.3513963520526886, + 0.044039513915777206, + 0.04941943287849426, + 0.06270346790552139, + 0.056359026581048965, + 0.08183508366346359, + 0.07356402277946472, + 0.12609201669692993 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 0, + "style_probability": 0.38745805621147156, + "style_target": 1, + "support_probability": 0.314211368560791, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014516738279466044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6002597212791443, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36490869522094727, + 0.4351784586906433, + 0.8264284133911133, + 0.401991605758667, + 0.08293648064136505, + 0.05751115828752518, + 0.14026987552642822, + 0.12649862468242645, + 0.12846554815769196, + 0.15820126235485077 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 1, + "style_probability": 0.07526891678571701, + "style_target": 0, + "support_probability": 0.288343608379364, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.400862942038875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6498113870620728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23213718831539154, + 0.680574357509613, + 0.9305859804153442, + 0.6299008131027222, + 0.07589315623044968 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.7810934782028198, + "style_target": 1, + "support_probability": 0.3250686526298523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2281899911532815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7115580439567566, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26445505023002625, + 0.6486845016479492, + 0.9066528677940369, + 0.5523539185523987, + 0.11720442026853561, + 0.13697020709514618, + 0.07177894562482834, + 0.05472497642040253, + 0.1181105226278305, + 0.10059253871440887, + 0.19574591517448425 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.40113818645477295, + "style_target": 0, + "support_probability": 0.2906862497329712, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025754268719490443, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8142281174659729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5089271068572998, + 0.3516964018344879, + 0.9166982173919678, + 0.3416321277618408, + 0.07325197756290436, + 0.0963652953505516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6992445588111877, + "style_target": 1, + "support_probability": 0.008012622594833374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005209566195239626, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8605596423149109, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5852862000465393, + 0.44105154275894165, + 0.9310097098350525, + 0.320174902677536, + 0.10788409411907196, + 0.11601302772760391 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.23142443597316742, + "style_target": 0, + "support_probability": 0.0070373122580349445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01647038058411047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9599575400352478, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5224746465682983, + 0.2345946729183197, + 0.3264378011226654, + 0.6911837458610535, + 0.07033617794513702, + 0.22064398229122162, + 0.13277676701545715, + 0.09771668165922165, + 0.11671066284179688 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 1, + "style_probability": 0.7564405798912048, + "style_target": 1, + "support_probability": 0.010487191379070282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028405251955430543, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8252576589584351, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49845921993255615, + 0.25824451446533203, + 0.345346063375473, + 0.6584515571594238, + 0.07849953323602676, + 0.04784443974494934, + 0.05574466660618782, + 0.07304926961660385, + 0.09273733198642731 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 1, + "style_probability": 0.39254236221313477, + "style_target": 0, + "support_probability": 0.031598128378391266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0785186257057217, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7893096804618835, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15134423971176147, + 0.8492783308029175, + 0.09854641556739807, + 0.20681947469711304, + 0.21592554450035095, + 0.15420332551002502, + 0.06468411535024643, + 0.11723878979682922, + 0.1317877620458603, + 0.13837338984012604, + 0.07691796123981476 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.6925085186958313, + "style_target": 1, + "support_probability": 0.07621688395738602, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08557833619020791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7565114498138428, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15860731899738312, + 0.8307607769966125, + 0.101335808634758, + 0.21672548353672028, + 0.17200398445129395, + 0.10335938632488251, + 0.05066974461078644, + 0.10273472964763641, + 0.12409137934446335 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.38704657554626465, + "style_target": 0, + "support_probability": 0.09388674795627594, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7090157476316779, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10245974361896515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23080313205718994, + 0.15825025737285614, + 0.33140599727630615, + 0.9960107803344727, + 0.06580659747123718 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.7541981339454651, + "style_target": 1, + "support_probability": 0.7151687145233154, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2881383100736937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12089874595403671, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22535055875778198, + 0.1825142502784729, + 0.35740089416503906, + 0.9964876174926758, + 0.08895190060138702 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.3350352942943573, + "style_target": 0, + "support_probability": 0.7145153880119324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11803855919761179, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47804316878318787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1748153120279312, + 0.8731061220169067, + 0.07800818979740143, + 0.43080174922943115, + 0.12825210392475128, + 0.05188675969839096 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 1, + "style_probability": 0.5160982012748718, + "style_target": 1, + "support_probability": 0.2352301925420761, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11503459162116648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4856221675872803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21926826238632202, + 0.8026060461997986, + 0.10932893306016922, + 0.47884008288383484, + 0.07362080365419388, + 0.15138904750347137 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 1, + "style_probability": 0.43997400999069214, + "style_target": 0, + "support_probability": 0.26860061287879944, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20098078488354099, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2775895595550537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7094691395759583, + 0.39335930347442627, + 0.12634530663490295, + 0.9110453128814697, + 0.024236366152763367, + 0.02812115103006363, + 0.03696899861097336, + 0.19999291002750397, + 0.1237226277589798 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.8504278063774109, + "style_target": 1, + "support_probability": 0.06426911056041718, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13102391638037697, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21456119418144226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7147231101989746, + 0.36549338698387146, + 0.11433333903551102, + 0.9114362597465515, + 0.17818091809749603, + 0.14715895056724548 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.41117292642593384, + "style_target": 0, + "support_probability": 0.07880128175020218, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1750099167295091, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8165721297264099, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14222019910812378, + 0.5365989804267883, + 0.42716655135154724, + 0.2019110918045044, + 0.0642477422952652, + 0.03746035322546959, + 0.056802328675985336, + 0.06209462136030197, + 0.07801191508769989, + 0.21362243592739105, + 0.12445007264614105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.8459616303443909, + "style_target": 1, + "support_probability": 0.11240679770708084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02536130917676633, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6148421168327332, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17255420982837677, + 0.7580468058586121, + 0.3362467885017395, + 0.2694885730743408, + 0.09858943521976471, + 0.07075319439172745, + 0.19689321517944336, + 0.14055901765823364, + 0.11914873868227005, + 0.10527649521827698 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.11311506479978561, + "style_target": 0, + "support_probability": 0.1442214995622635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.520808119299204, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12565526366233826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29732003808021545, + 0.3067902624607086, + 0.22324205935001373, + 0.8852407932281494, + 0.08406322449445724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 0, + "style_probability": 0.42639222741127014, + "style_target": 1, + "support_probability": 0.6665062308311462, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0623701388252763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15538142621517181, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3182624280452728, + 0.4287765324115753, + 0.21313564479351044, + 0.9325870275497437, + 0.16552960872650146 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 1, + "style_probability": 0.12887224555015564, + "style_target": 0, + "support_probability": 0.5033302307128906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5694192070718344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12216965109109879, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8795878291130066, + 0.35033759474754333, + 0.324158638715744, + 0.49769067764282227, + 0.12490660697221756, + 0.11864203959703445 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.42728760838508606, + "style_target": 1, + "support_probability": 0.7850579619407654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.646058079617274, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.118098683655262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8985788226127625, + 0.44670552015304565, + 0.3446368873119354, + 0.5250059962272644, + 0.1133790835738182, + 0.06207061931490898 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.7008075714111328, + "style_target": 0, + "support_probability": 0.6731597781181335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02992638889866764, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8860384225845337, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.607748806476593, + 0.3275030255317688, + 0.6232370734214783, + 0.3275030255317688, + 0.04827796667814255, + 0.11683107912540436, + 0.10697073489427567, + 0.11112985759973526, + 0.08232976496219635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 1, + "style_probability": 0.6033338308334351, + "style_target": 1, + "support_probability": 0.016128845512866974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09923486547469555, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8140074610710144, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.634056806564331, + 0.32050013542175293, + 0.6464317440986633, + 0.32050013542175293, + 0.10991613566875458 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 0, + "style_probability": 0.5350118279457092, + "style_target": 0, + "support_probability": 0.057838600128889084, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6515119406839177, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027234215289354324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2596655786037445, + 0.23899447917938232, + 0.2901514768600464, + 0.9845716953277588, + 0.04533492401242256, + 0.2112320214509964, + 0.11293551325798035, + 0.14702670276165009, + 0.13817563652992249, + 0.17806337773799896 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 1, + "style_probability": 0.5066004395484924, + "style_target": 1, + "support_probability": 0.8024123311042786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2394127654944491, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02851194143295288, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3048074245452881, + 0.27794769406318665, + 0.3334721624851227, + 0.9956308603286743, + 0.1797432154417038, + 0.15656301379203796, + 0.2816666066646576, + 0.2645675837993622, + 0.07510924339294434, + 0.12028372287750244 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 1, + "style_probability": 0.21892118453979492, + "style_target": 0, + "support_probability": 0.7541582584381104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08497015086205452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6531625390052795, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8846127986907959, + 0.27843332290649414, + 0.19629636406898499, + 0.38809987902641296, + 0.1425086259841919 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.29522380232810974, + "style_target": 1, + "support_probability": 0.1207222267985344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11449093887742047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6003004908561707, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9046535491943359, + 0.28301578760147095, + 0.18646208941936493, + 0.4231506586074829, + 0.1027728021144867 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.7341018915176392, + "style_target": 0, + "support_probability": 0.10967448353767395, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6621104451364458, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17548823356628418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9434884786605835, + 0.44013479351997375, + 0.310608446598053, + 0.44013479351997375, + 0.10729283839464188, + 0.10322509706020355 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 1, + "style_probability": 0.5543270707130432, + "style_target": 1, + "support_probability": 0.7892732620239258, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5242736746002871, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08726111799478531, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9831551313400269, + 0.40977558493614197, + 0.290103554725647, + 0.40977558493614197, + 0.11994484066963196, + 0.0945187509059906 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 1, + "style_probability": 0.3984107971191406, + "style_target": 0, + "support_probability": 0.8313564658164978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0060524255888211665, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9942067861557007, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4582962989807129, + 0.2989565432071686, + 0.22883130609989166, + 0.31331512331962585, + 0.13545174896717072, + 0.12082112580537796, + 0.10824679583311081, + 0.11916445940732956, + 0.11177187412977219 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 0, + "style_probability": 0.3487628698348999, + "style_target": 1, + "support_probability": 0.008692472241818905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005055218598923757, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9906119704246521, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4536466896533966, + 0.3765198588371277, + 0.2150815725326538, + 0.34705638885498047, + 0.08145970851182938, + 0.09131715446710587, + 0.09172821044921875, + 0.05871882662177086, + 0.11291956156492233 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 1, + "style_probability": 0.4640130400657654, + "style_target": 0, + "support_probability": 0.006311651784926653, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6599419225973289, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4327259063720703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9717918038368225, + 0.2201043963432312, + 0.32154780626296997, + 0.32922154664993286, + 0.044213809072971344, + 0.1000175029039383, + 0.12975944578647614, + 0.1615823358297348, + 0.09728425741195679, + 0.11374354362487793 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.3668040931224823, + "style_target": 1, + "support_probability": 0.699126660823822, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8554115892602251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39146581292152405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9830152988433838, + 0.1967247724533081, + 0.345498263835907, + 0.3324560225009918, + 0.06733079999685287, + 0.06890993565320969, + 0.09312461316585541, + 0.08586165308952332, + 0.13319875299930573, + 0.11336533725261688, + 0.08513718098402023 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.8414782881736755, + "style_target": 0, + "support_probability": 0.741668701171875, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.638249315392807, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1036190539598465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3617878258228302, + 0.3617878258228302, + 0.8675094246864319, + 0.2551079988479614, + 0.08043769001960754 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 0, + "style_probability": 0.2979011833667755, + "style_target": 1, + "support_probability": 0.9131897687911987, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4881061682544016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05063316226005554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4291982352733612, + 0.4291982352733612, + 0.9584333300590515, + 0.26305437088012695, + 0.20392920076847076 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 1, + "style_probability": 0.19279621541500092, + "style_target": 0, + "support_probability": 0.9355791807174683, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1023488507792807, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7161911725997925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26176607608795166, + 0.24726571142673492, + 0.6289239525794983, + 0.624921441078186, + 0.14222368597984314, + 0.061946142464876175 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 0, + "style_probability": 0.2639019191265106, + "style_target": 1, + "support_probability": 0.19921079277992249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18381906013033425, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5970855951309204, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22598806023597717, + 0.2130793333053589, + 0.6384561657905579, + 0.6095215678215027, + 0.0538954958319664, + 0.23581437766551971 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 1, + "style_probability": 0.35068848729133606, + "style_target": 0, + "support_probability": 0.43628501892089844, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23258770152044667, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6279016733169556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6646118760108948, + 0.7726410627365112, + 0.4059412181377411, + 0.45531460642814636, + 0.03447451815009117, + 0.04751463979482651, + 0.1127251386642456, + 0.07035505771636963, + 0.049102604389190674 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 1, + "style_probability": 0.5504000186920166, + "style_target": 1, + "support_probability": 0.17612232267856598, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4679787189681107, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42585960030555725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6951410174369812, + 0.7701879143714905, + 0.39763256907463074, + 0.46169793605804443, + 0.09283647686243057, + 0.17618681490421295, + 0.10487639158964157, + 0.05042339488863945, + 0.11583920568227768 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 0, + "style_probability": 0.8522223830223083, + "style_target": 0, + "support_probability": 0.23586446046829224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049644649434132083, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.744309663772583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3892768621444702, + 0.48283615708351135, + 0.5978095531463623, + 0.31298714876174927, + 0.10408104211091995, + 0.09031608700752258, + 0.08336631208658218, + 0.14152704179286957, + 0.20748180150985718, + 0.1536286324262619 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 1, + "style_probability": 0.5478273034095764, + "style_target": 1, + "support_probability": 0.035722944885492325, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00753810684314696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5674272775650024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4598574936389923, + 0.6135668754577637, + 0.6935988664627075, + 0.29325810074806213, + 0.0995764285326004, + 0.1026894673705101, + 0.2252739667892456, + 0.15612347424030304, + 0.10899989306926727, + 0.10798260569572449 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 1, + "style_probability": 0.15044505894184113, + "style_target": 0, + "support_probability": 0.1450248807668686, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6541794518194106, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2189311981201172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20535407960414886, + 0.4705061912536621, + 0.9446714520454407, + 0.37634196877479553, + 0.08235111087560654 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.7327569723129272, + "style_target": 1, + "support_probability": 0.7271144986152649, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31827352305316237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.323062539100647, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2223251760005951, + 0.4470260739326477, + 0.9050421118736267, + 0.4079332947731018, + 0.19912764430046082 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.3693092167377472, + "style_target": 0, + "support_probability": 0.5929566621780396, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24536092866541376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4705045223236084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36198341846466064, + 0.30561935901641846, + 0.3143191635608673, + 0.8527383208274841, + 0.12770551443099976, + 0.05855041742324829 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.6768807172775269, + "style_target": 1, + "support_probability": 0.09593880921602249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17385561723648482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4892682433128357, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3190927803516388, + 0.32943472266197205, + 0.32338136434555054, + 0.838085412979126, + 0.11774903535842896, + 0.11893576383590698, + 0.08417269587516785, + 0.06496956944465637, + 0.21129675209522247, + 0.12823304533958435 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.3626645505428314, + "style_target": 0, + "support_probability": 0.1420484185218811, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05610487546602847, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6621921062469482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12350369244813919, + 0.6026782393455505, + 0.11341843754053116, + 0.5370616912841797, + 0.06987661123275757, + 0.06937331706285477, + 0.050973184406757355, + 0.11013074964284897, + 0.05195360630750656 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.6874343156814575, + "style_target": 1, + "support_probability": 0.09452513605356216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011611595621991032, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6817393898963928, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13370546698570251, + 0.6756612658500671, + 0.132507786154747, + 0.6258642077445984, + 0.10140900313854218, + 0.1776968091726303, + 0.07552292197942734, + 0.06483383476734161, + 0.06455367058515549 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.18900330364704132, + "style_target": 0, + "support_probability": 0.13224007189273834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8721630760940369, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20447756350040436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6260:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9792648553848267, + 0.5122328400611877, + 0.09828687459230423, + 0.2472965568304062, + 0.11006911098957062, + 0.034989893436431885, + 0.06530392915010452, + 0.04899551346898079, + 0.0979030653834343, + 0.06957248598337173 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 1, + "style_probability": 0.814677357673645, + "style_target": 1, + "support_probability": 0.7860647439956665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7103248951187731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1497408002614975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6260:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9657485485076904, + 0.481395423412323, + 0.09032279253005981, + 0.2251882255077362, + 0.08748849481344223, + 0.09261111170053482, + 0.07439206540584564, + 0.11470816284418106, + 0.06515554338693619, + 0.0514957569539547, + 0.10201285034418106 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 1, + "style_probability": 0.39861518144607544, + "style_target": 0, + "support_probability": 0.8489333987236023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7562764133138761, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11916139721870422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35788339376449585, + 0.5373671054840088, + 0.9091070294380188, + 0.35788339376449585, + 0.029962193220853806 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.7935285568237305, + "style_target": 1, + "support_probability": 0.7339993119239807, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.504492891874254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09288305044174194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28911516070365906, + 0.5233848094940186, + 0.8904832005500793, + 0.28911516070365906, + 0.025417426601052284, + 0.03178443759679794, + 0.0588640533387661, + 0.07118051499128342, + 0.052012849599123 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.3832378685474396, + "style_target": 0, + "support_probability": 0.5744897723197937, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5447028210931454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11333324760198593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08513259142637253, + 0.30683133006095886, + 0.2972206771373749, + 0.9488210082054138, + 0.03588343411684036, + 0.052770838141441345 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.8292362093925476, + "style_target": 1, + "support_probability": 0.47729387879371643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08079661866742022, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20412318408489227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08923842012882233, + 0.4502737522125244, + 0.47310325503349304, + 0.9843557476997375, + 0.21063369512557983, + 0.11015123873949051 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.20750278234481812, + "style_target": 0, + "support_probability": 0.46533551812171936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17460990936153134, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7933240532875061, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25463196635246277, + 0.17780795693397522, + 0.5257061719894409, + 0.17408102750778198, + 0.03788264840841293, + 0.07270163297653198, + 0.0544833280146122, + 0.0370703786611557, + 0.03835359215736389 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.8567266464233398, + "style_target": 1, + "support_probability": 0.11831820011138916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04087598819858757, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7038902640342712, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24983952939510345, + 0.1973276436328888, + 0.5522320866584778, + 0.1873336285352707, + 0.08804149925708771, + 0.06072993949055672, + 0.044777680188417435, + 0.07727400213479996, + 0.07198672741651535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.38037964701652527, + "style_target": 0, + "support_probability": 0.1313861459493637, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6278312203576752, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0979505404829979, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4353274703025818, + 0.10185020416975021, + 0.5610776543617249, + 0.8569701910018921, + 0.04869138076901436, + 0.05730415880680084, + 0.05505213886499405, + 0.047805845737457275, + 0.06665604561567307, + 0.053754910826683044 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 0, + "style_probability": 0.4959788918495178, + "style_target": 1, + "support_probability": 0.9280242323875427, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49898640879997314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1528891772031784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38905131816864014, + 0.10647621005773544, + 0.5509201884269714, + 0.8070311546325684, + 0.03356727212667465, + 0.05542823299765587 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 1, + "style_probability": 0.3238440454006195, + "style_target": 0, + "support_probability": 0.8538140654563904, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20521662958423784, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7554785013198853, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2509450614452362, + 0.6249911785125732, + 0.35791632533073425, + 0.24794833362102509, + 0.10632140934467316 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.7953638434410095, + "style_target": 1, + "support_probability": 0.1266632229089737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013662783940333469, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8494769930839539, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3109521269798279, + 0.6684531569480896, + 0.44915392994880676, + 0.30503013730049133, + 0.12339919060468674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.2124585658311844, + "style_target": 0, + "support_probability": 0.04896305128931999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4140359542525169, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17674680054187775, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4792863726615906, + 0.37532156705856323, + 0.2560977339744568, + 0.9816368222236633, + 0.22051723301410675, + 0.16118453443050385 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 1, + "style_probability": 0.8083187937736511, + "style_target": 1, + "support_probability": 0.21498964726924896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35808368859991246, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2045823186635971, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4233725070953369, + 0.29454416036605835, + 0.22068877518177032, + 0.9687203764915466, + 0.06017272174358368, + 0.17609640955924988 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 0, + "style_probability": 0.5961616039276123, + "style_target": 0, + "support_probability": 0.3293071389198303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6898549238958821, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17664389312267303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3682580888271332, + 0.11494351178407669, + 0.9843645691871643, + 0.12540504336357117, + 0.052956994622945786, + 0.06245072931051254, + 0.09372716397047043, + 0.2591489851474762, + 0.27717018127441406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.8168109059333801, + "style_target": 1, + "support_probability": 0.5640319585800171, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4846066173709819, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.18887943029403687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32980626821517944, + 0.12579138576984406, + 0.9772167205810547, + 0.1189299076795578, + 0.092721126973629 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.4674800634384155, + "style_target": 0, + "support_probability": 0.5635536313056946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6664050197340032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1555023044347763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42677953839302063, + 0.8714863061904907, + 0.5828280448913574, + 0.3398377299308777, + 0.12053070217370987, + 0.04896427318453789, + 0.03330991417169571, + 0.04538990184664726, + 0.08460500836372375, + 0.09448173642158508, + 0.0725104808807373 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.8488189578056335, + "style_target": 1, + "support_probability": 0.7097768187522888, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1983670652692227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1409873515367508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3773901164531708, + 0.8903980255126953, + 0.6004392504692078, + 0.37289848923683167, + 0.0955084040760994, + 0.06368909776210785, + 0.0924694687128067, + 0.10176748782396317, + 0.09946632385253906, + 0.1066499724984169 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.37480393052101135, + "style_target": 0, + "support_probability": 0.7550153732299805, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07550061847030214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6049429178237915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9511483311653137, + 0.14476820826530457, + 0.16736850142478943, + 0.2628819942474365, + 0.08763556182384491 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 1, + "style_probability": 0.8035055994987488, + "style_target": 1, + "support_probability": 0.07956289499998093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11268081423130359, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4581422805786133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9430131912231445, + 0.18379221856594086, + 0.20320802927017212, + 0.2738531529903412, + 0.15988531708717346 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 0, + "style_probability": 0.559293270111084, + "style_target": 0, + "support_probability": 0.2048417627811432, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8017527507591382, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0837586298584938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9661770462989807, + 0.2787693738937378, + 0.0772259384393692, + 0.1733771711587906, + 0.022286761552095413, + 0.06008528918027878 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 1, + "style_probability": 0.5559923648834229, + "style_target": 1, + "support_probability": 0.8601193428039551, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7985806583844729, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06268855929374695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9399421811103821, + 0.2428503930568695, + 0.08404283225536346, + 0.16181156039237976, + 0.06316767632961273, + 0.05803059786558151, + 0.12900479137897491, + 0.07545103132724762, + 0.0368962362408638, + 0.10927824676036835, + 0.04055766016244888 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 1, + "style_probability": 0.351554274559021, + "style_target": 0, + "support_probability": 0.8916192650794983, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.75644700550695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04044400528073311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10425441712141037, + 0.35765618085861206, + 0.24128974974155426, + 0.9628409147262573, + 0.1575993001461029, + 0.12931016087532043, + 0.07708722352981567, + 0.09798163175582886, + 0.07602612674236298 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.8020363450050354, + "style_target": 1, + "support_probability": 0.5890659093856812, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1341035820522742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0749095007777214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12059001624584198, + 0.3934182822704315, + 0.3349851369857788, + 0.9663854241371155, + 0.18279123306274414, + 0.20306333899497986, + 0.18967309594154358, + 0.12258895486593246, + 0.15897448360919952 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.19654245674610138, + "style_target": 0, + "support_probability": 0.5453287363052368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00414859745486226, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.873539924621582, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3725672662258148, + 0.4733754098415375, + 0.5139358043670654, + 0.2872620224952698, + 0.09219103306531906, + 0.0813632681965828, + 0.0933549553155899, + 0.04833746328949928, + 0.06208423897624016, + 0.05599607899785042, + 0.04559486359357834 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 0, + "style_probability": 0.48752361536026, + "style_target": 1, + "support_probability": 0.006215993780642748, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033489195022319185, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8275099396705627, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40579432249069214, + 0.4616400897502899, + 0.5204259753227234, + 0.28318750858306885, + 0.0670875534415245, + 0.09543028473854065, + 0.19123893976211548, + 0.08328177034854889, + 0.1389136165380478, + 0.24897964298725128 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 1, + "style_probability": 0.20401932299137115, + "style_target": 0, + "support_probability": 0.010060438886284828, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7589508146072926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31317731738090515, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37043920159339905, + 0.9473963379859924, + 0.543904721736908, + 0.34799864888191223, + 0.0647975280880928 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.7715067863464355, + "style_target": 1, + "support_probability": 0.826292097568512, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5072018243515469, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4702303111553192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3481113910675049, + 0.9286887049674988, + 0.5367966294288635, + 0.3640025556087494, + 0.05884414166212082, + 0.10523675382137299, + 0.1063474640250206, + 0.030733978375792503, + 0.06924259662628174 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.4551984965801239, + "style_target": 0, + "support_probability": 0.6442993879318237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1248133934059136, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5918049812316895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46371254324913025, + 0.28779783844947815, + 0.16458021104335785, + 0.41321322321891785, + 0.2055019736289978, + 0.10843649506568909 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.7024913430213928, + "style_target": 1, + "support_probability": 0.33335211873054504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011186049529693437, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7712384462356567, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4352818429470062, + 0.2707493305206299, + 0.1997111439704895, + 0.46851253509521484, + 0.14498376846313477, + 0.136361762881279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.15938714146614075, + "style_target": 0, + "support_probability": 0.16877149045467377, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5367812216214976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21473830938339233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43261998891830444, + 0.9438180923461914, + 0.15816542506217957, + 0.36364540457725525, + 0.057626452296972275, + 0.1774134486913681, + 0.05111493170261383, + 0.1378452330827713, + 0.07409372925758362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 1, + "style_probability": 0.8347436189651489, + "style_target": 1, + "support_probability": 0.3687763810157776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.272520802285886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2546272575855255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4104640483856201, + 0.9261741042137146, + 0.17274677753448486, + 0.37950411438941956, + 0.07845630496740341, + 0.09170138835906982, + 0.14557409286499023, + 0.08515933156013489, + 0.08201777935028076 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 0, + "style_probability": 0.5507486462593079, + "style_target": 0, + "support_probability": 0.2906491756439209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049541584190852445, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9112706184387207, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2664068341255188, + 0.2702646553516388, + 0.23175592720508575, + 0.5054132342338562, + 0.06555358320474625, + 0.11495953053236008, + 0.10012290626764297, + 0.09018021821975708, + 0.26679107546806335, + 0.14490096271038055 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.8022182583808899, + "style_target": 1, + "support_probability": 0.13156403601169586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006863762324825616, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9593263864517212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23252736032009125, + 0.2888037860393524, + 0.22639481723308563, + 0.39502787590026855, + 0.15804380178451538, + 0.20163670182228088 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.24367672204971313, + "style_target": 0, + "support_probability": 0.02362365461885929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8140524785039776, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15604421496391296, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9239299893379211, + 0.6232202053070068, + 0.29431432485580444, + 0.29543179273605347, + 0.09305544942617416 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.8672798275947571, + "style_target": 1, + "support_probability": 0.8457180261611938, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2820583533377386, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18125538527965546, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9719699621200562, + 0.7543829083442688, + 0.36929985880851746, + 0.3044823110103607, + 0.1369209885597229 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.2408139854669571, + "style_target": 0, + "support_probability": 0.8276026844978333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11511965127296088, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5943719744682312, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12342173606157303, + 0.2948024868965149, + 0.639122486114502, + 0.3499760329723358, + 0.04060328006744385, + 0.049192193895578384 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 1, + "style_probability": 0.7356346845626831, + "style_target": 1, + "support_probability": 0.04863165318965912, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08781589918598454, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.550143301486969, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13652493059635162, + 0.2570175230503082, + 0.6408121585845947, + 0.3627816438674927, + 0.039766255766153336, + 0.0684947818517685 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 0, + "style_probability": 0.5896039009094238, + "style_target": 0, + "support_probability": 0.06464909017086029, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06150850073499206, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.562683641910553, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5614217519760132, + 0.7958767414093018, + 0.38549694418907166, + 0.2006995975971222, + 0.1327650249004364, + 0.24380424618721008, + 0.21779568493366241, + 0.07038123905658722, + 0.08847565948963165 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.7566026449203491, + "style_target": 1, + "support_probability": 0.04731352999806404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05923585188197162, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4698548913002014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5645185708999634, + 0.7921044826507568, + 0.3812420070171356, + 0.20567338168621063, + 0.11112835258245468 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.4160950779914856, + "style_target": 0, + "support_probability": 0.07238180935382843, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8868768633243046, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04790358245372772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6419:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26155227422714233, + 0.4412749111652374, + 0.10831282287836075, + 0.9500476121902466, + 0.07549704611301422, + 0.03785514831542969, + 0.05733967944979668, + 0.06597200781106949, + 0.1465667188167572, + 0.04376816004514694 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.9025313258171082, + "style_target": 1, + "support_probability": 0.9474592208862305, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1419839674874874, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08508983254432678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3280726373195648, + 0.6038128733634949, + 0.12537643313407898, + 0.9813081622123718, + 0.14936155080795288, + 0.07015438377857208, + 0.15955308079719543, + 0.067170150578022, + 0.15314863622188568, + 0.16031718254089355 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.08624960482120514, + "style_target": 0, + "support_probability": 0.9133575558662415, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07629835281109083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17919103801250458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28480973839759827, + 0.8292931914329529, + 0.26611730456352234, + 0.3187452256679535, + 0.11995180696249008 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 0, + "style_probability": 0.2766072154045105, + "style_target": 1, + "support_probability": 0.12100286781787872, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020212804093654505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22632721066474915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2388463020324707, + 0.9085291028022766, + 0.20610259473323822, + 0.3603954613208771, + 0.2699888348579407 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 1, + "style_probability": 0.17131668329238892, + "style_target": 0, + "support_probability": 0.14791527390480042, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4857872340695805, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07173553854227066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3272443115711212, + 0.20187798142433167, + 0.9738132953643799, + 0.23849175870418549, + 0.03539580851793289, + 0.08663435280323029 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 1, + "style_probability": 0.6339267492294312, + "style_target": 1, + "support_probability": 0.6511803269386292, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8789896844097516, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05745112523436546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41596928238868713, + 0.1522890031337738, + 0.9896721839904785, + 0.2657036781311035, + 0.1103251576423645, + 0.16112324595451355 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 0, + "style_probability": 0.8977053761482239, + "style_target": 0, + "support_probability": 0.8343862295150757, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.038519489767059774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9465979337692261, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5046678185462952, + 0.38578465580940247, + 0.43908342719078064, + 0.2620338499546051, + 0.08186633139848709, + 0.106717050075531, + 0.08626963198184967, + 0.08600210398435593, + 0.11187753826379776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.6595303416252136, + "style_target": 1, + "support_probability": 0.0330754853785038, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03414612291915176, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9583752751350403, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4950445890426636, + 0.45738428831100464, + 0.3911040723323822, + 0.3208378255367279, + 0.07295481860637665 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.4404728412628174, + "style_target": 0, + "support_probability": 0.043685633689165115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19209816414447878, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46053898334503174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5313554406166077, + 0.8888378143310547, + 0.4515775144100189, + 0.3284006714820862, + 0.14172959327697754, + 0.3141894042491913, + 0.16393114626407623, + 0.11677916347980499, + 0.14690130949020386, + 0.08945413678884506, + 0.1580628603696823 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 0, + "style_probability": 0.48989537358283997, + "style_target": 1, + "support_probability": 0.09222198277711868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07009313248622415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27712664008140564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6322928667068481, + 0.9474259614944458, + 0.51521897315979, + 0.37082287669181824, + 0.09454777091741562, + 0.10453743487596512, + 0.21334385871887207, + 0.15345241129398346, + 0.2004631608724594, + 0.10656130313873291 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 1, + "style_probability": 0.125203475356102, + "style_target": 0, + "support_probability": 0.2392643243074417, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7778404254697329, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08538342267274857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.973063051700592, + 0.22239889204502106, + 0.5037782192230225, + 0.33152472972869873, + 0.29789644479751587 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.4854656755924225, + "style_target": 1, + "support_probability": 0.9286393523216248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8658639320477104, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07858850061893463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9845389723777771, + 0.21230949461460114, + 0.49348509311676025, + 0.3176516592502594, + 0.06749160587787628 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.7702630162239075, + "style_target": 0, + "support_probability": 0.9270970821380615, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8423645679192574, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0315515510737896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15720516443252563, + 0.17753858864307404, + 0.9940130114555359, + 0.23501242697238922, + 0.08983229845762253, + 0.11350200325250626 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 1, + "style_probability": 0.5353975892066956, + "style_target": 1, + "support_probability": 0.9522275924682617, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6533979735773942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04273238778114319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1766771823167801, + 0.23940446972846985, + 0.998930037021637, + 0.24956080317497253, + 0.07422388345003128, + 0.12502458691596985 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 1, + "style_probability": 0.43167948722839355, + "style_target": 0, + "support_probability": 0.9602420330047607, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013929904890323497, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8112799525260925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17874199151992798, + 0.19536033272743225, + 0.12991508841514587, + 0.3554650843143463, + 0.1175934374332428, + 0.12889008224010468, + 0.07794300466775894, + 0.20130905508995056, + 0.19787058234214783 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.40391576290130615, + "style_target": 1, + "support_probability": 0.016060102730989456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0036463227464102, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9135096073150635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2177758663892746, + 0.178376242518425, + 0.1313927173614502, + 0.33684420585632324, + 0.09913984686136246, + 0.060135237872600555, + 0.06812147051095963, + 0.08182168751955032, + 0.04033949226140976 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.5206370949745178, + "style_target": 0, + "support_probability": 0.007320196833461523, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.33841144755669406, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5951635837554932, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23215806484222412, + 0.24260060489177704, + 0.8075109720230103, + 0.3087916374206543, + 0.06584609299898148, + 0.11160482466220856, + 0.09679923951625824, + 0.060864850878715515, + 0.10361126065254211, + 0.10428965091705322, + 0.3068771958351135 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 0, + "style_probability": 0.43751007318496704, + "style_target": 1, + "support_probability": 0.14351238310337067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27434836022050557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7125730514526367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21887873113155365, + 0.2723080813884735, + 0.8199467062950134, + 0.297554075717926, + 0.06966501474380493, + 0.050158437341451645, + 0.11505983769893646, + 0.076837919652462, + 0.09576210379600525, + 0.07161422818899155 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 0, + "style_probability": 0.8476241230964661, + "style_target": 0, + "support_probability": 0.04537948593497276, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23306501344325872, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44732925295829773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.541811466217041, + 0.08500730246305466, + 0.25895583629608154, + 0.3006288707256317, + 0.052762363106012344 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 1, + "style_probability": 0.6360883116722107, + "style_target": 1, + "support_probability": 0.24657894670963287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07264179325675539, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6882871389389038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5347141623497009, + 0.08377908170223236, + 0.28743553161621094, + 0.32134944200515747, + 0.05224734544754028, + 0.11927469074726105, + 0.069086454808712, + 0.09831298142671585, + 0.14117605984210968 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 1, + "style_probability": 0.44019678235054016, + "style_target": 0, + "support_probability": 0.0746772363781929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06959264583735188, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9257374405860901, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1750105917453766, + 0.3175169825553894, + 0.1750105917453766, + 0.4987412393093109, + 0.05242278799414635, + 0.07019186019897461 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 0, + "style_probability": 0.3534073233604431, + "style_target": 1, + "support_probability": 0.13393715023994446, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003983602126516787, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9518477916717529, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20792563259601593, + 0.34785184264183044, + 0.20792563259601593, + 0.5475015640258789, + 0.09834713488817215, + 0.21197780966758728 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 1, + "style_probability": 0.12091441452503204, + "style_target": 0, + "support_probability": 0.031854335218667984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09907746797903191, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7273865938186646, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6537031531333923, + 0.3024311661720276, + 0.19526344537734985, + 0.21881762146949768, + 0.07859303802251816, + 0.052708279341459274, + 0.057911571115255356, + 0.10855347663164139, + 0.1432240754365921 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 1, + "style_probability": 0.5236678719520569, + "style_target": 1, + "support_probability": 0.08728783577680588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09728509273428596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8150466680526733, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7618148326873779, + 0.2649928629398346, + 0.21094971895217896, + 0.2513897120952606, + 0.1672009974718094, + 0.11583729833364487, + 0.07936576753854752, + 0.07477100193500519, + 0.09382400661706924 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 0, + "style_probability": 0.6050578355789185, + "style_target": 0, + "support_probability": 0.0774836465716362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01915914238236343, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.694960355758667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14397713541984558, + 0.33664512634277344, + 0.41455990076065063, + 0.5686875581741333, + 0.12098748236894608, + 0.10663898289203644, + 0.11607059836387634, + 0.09928635507822037, + 0.10294722765684128, + 0.05413791164755821, + 0.05805746465921402 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 0, + "style_probability": 0.43390148878097534, + "style_target": 1, + "support_probability": 0.03891201689839363, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012593211715752194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7539507746696472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1663999706506729, + 0.33645087480545044, + 0.40825212001800537, + 0.5423200130462646, + 0.12394412606954575, + 0.11757156252861023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 1, + "style_probability": 0.33775678277015686, + "style_target": 0, + "support_probability": 0.016969051212072372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019746615468766597, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9437811374664307, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37947317957878113, + 0.366885244846344, + 0.10797187685966492, + 0.366885244846344, + 0.18041935563087463 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 1, + "style_probability": 0.8060321807861328, + "style_target": 1, + "support_probability": 0.04737454280257225, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02847369869686995, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9105390906333923, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.441082239151001, + 0.3279745876789093, + 0.10135103017091751, + 0.3279745876789093, + 0.1393212229013443 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 0, + "style_probability": 0.5738773941993713, + "style_target": 0, + "support_probability": 0.04012244567275047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5495337859119455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07591487467288971, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42540305852890015, + 0.42540305852890015, + 0.1558961421251297, + 0.9535667896270752, + 0.07804582267999649, + 0.21083492040634155 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.7345571517944336, + "style_target": 1, + "support_probability": 0.7712832689285278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.192015194304151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22830982506275177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.439841091632843, + 0.439841091632843, + 0.18117818236351013, + 0.9169646501541138, + 0.09083119034767151, + 0.1404738873243332, + 0.046221230179071426, + 0.09561045467853546, + 0.12973347306251526, + 0.12180990725755692, + 0.06194330379366875 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.3865851163864136, + "style_target": 0, + "support_probability": 0.3843739628791809, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7246382977471547, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05861927196383476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11327706277370453, + 0.9760127663612366, + 0.30036264657974243, + 0.20635592937469482, + 0.05781475454568863, + 0.06171659007668495, + 0.08621815592050552, + 0.0742918998003006, + 0.22082224488258362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.6376327276229858, + "style_target": 1, + "support_probability": 0.8745028972625732, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11910965241395098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09208399057388306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15797658264636993, + 0.9771121144294739, + 0.36693859100341797, + 0.3210402727127075, + 0.07872193306684494, + 0.13294579088687897, + 0.15177549421787262, + 0.15067923069000244, + 0.22281593084335327 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.10080886632204056, + "style_target": 0, + "support_probability": 0.8586603999137878, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2998302532035258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14401498436927795, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.42799362540245056, + 0.8428577780723572, + 0.6047273278236389, + 0.2441447675228119, + 0.13533052802085876, + 0.12602587044239044, + 0.13491769134998322, + 0.07591571658849716, + 0.0916159451007843, + 0.2017570585012436 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 1, + "style_probability": 0.6248171925544739, + "style_target": 1, + "support_probability": 0.29734206199645996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25084570899062725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09082553535699844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43470507860183716, + 0.840756356716156, + 0.6075666546821594, + 0.24019671976566315, + 0.10915318131446838, + 0.12544777989387512, + 0.12932802736759186, + 0.08655445277690887, + 0.11859814077615738, + 0.13597965240478516, + 0.13693274557590485 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 1, + "style_probability": 0.47230610251426697, + "style_target": 0, + "support_probability": 0.39101752638816833, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.30153484253380874, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.021978480741381645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32540208101272583, + 0.3445742726325989, + 0.8696141839027405, + 0.2975965142250061, + 0.1458827406167984 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.3871749937534332, + "style_target": 1, + "support_probability": 0.510295033454895, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6336875566401041, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.01599641889333725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33365869522094727, + 0.35685595870018005, + 0.9195706844329834, + 0.35058528184890747, + 0.17347870767116547 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.6959235072135925, + "style_target": 0, + "support_probability": 0.7379409670829773, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6159332345178514, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29520905017852783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9542280435562134, + 0.3251449465751648, + 0.23174281418323517, + 0.22513657808303833, + 0.13651731610298157, + 0.1080305427312851 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.623684823513031, + "style_target": 1, + "support_probability": 0.5940190553665161, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6998043210472998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19743746519088745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9752834439277649, + 0.35189902782440186, + 0.26087474822998047, + 0.21363860368728638, + 0.07392817735671997, + 0.052792105823755264, + 0.044737424701452255, + 0.0734419971704483, + 0.06287406384944916, + 0.12103317677974701, + 0.06461256742477417 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.4977734684944153, + "style_target": 0, + "support_probability": 0.7727254629135132, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013107643456578506, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8146228194236755, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22193188965320587, + 0.2827669084072113, + 0.2699730396270752, + 0.1963164508342743, + 0.048909302800893784, + 0.17632733285427094, + 0.12686021625995636, + 0.09795772284269333, + 0.11307844519615173 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 0, + "style_probability": 0.47595369815826416, + "style_target": 1, + "support_probability": 0.02763684280216694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007798493785057089, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8048161268234253, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35399946570396423, + 0.29077351093292236, + 0.23231492936611176, + 0.21314376592636108, + 0.09643008559942245, + 0.07323357462882996, + 0.08683996647596359, + 0.04339846223592758, + 0.0778617188334465 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 1, + "style_probability": 0.1778416931629181, + "style_target": 0, + "support_probability": 0.09951485693454742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5907410316014555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19899094104766846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.846953272819519, + 0.5791960954666138, + 0.2562040090560913, + 0.292210191488266, + 0.11755844950675964, + 0.07771104574203491, + 0.16410352289676666, + 0.10609845072031021, + 0.12099496275186539, + 0.1680498719215393 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 0, + "style_probability": 0.4447142779827118, + "style_target": 1, + "support_probability": 0.6991882920265198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7566091276250712, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11559679359197617, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9036590456962585, + 0.6346123218536377, + 0.24769042432308197, + 0.3313346207141876, + 0.07140601426362991, + 0.07843472063541412, + 0.10257513076066971, + 0.04079451411962509, + 0.04311414435505867, + 0.19939489662647247, + 0.18154305219650269 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 0, + "style_probability": 0.5647692680358887, + "style_target": 0, + "support_probability": 0.85547935962677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07853449278163524, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4777785837650299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17292194068431854, + 0.5407349467277527, + 0.31428420543670654, + 0.17292194068431854, + 0.06089925393462181 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 0, + "style_probability": 0.3091367781162262, + "style_target": 1, + "support_probability": 0.27149736881256104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02502647130975468, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6232044100761414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1688499003648758, + 0.5338357090950012, + 0.25097545981407166, + 0.1688499003648758, + 0.08608569204807281, + 0.03963301330804825, + 0.069732166826725, + 0.06388240307569504, + 0.044530726969242096 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 1, + "style_probability": 0.21021458506584167, + "style_target": 0, + "support_probability": 0.0794292539358139, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4330848076091322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1177278533577919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2547362148761749, + 0.9765753149986267, + 0.19852453470230103, + 0.33336585760116577, + 0.04244677349925041, + 0.059434924274683 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.5197502374649048, + "style_target": 1, + "support_probability": 0.5421288013458252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23063052121938385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19635321199893951, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33693891763687134, + 0.9957054257392883, + 0.1968720704317093, + 0.39024850726127625, + 0.059809133410453796, + 0.0824570283293724 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.2571757733821869, + "style_target": 0, + "support_probability": 0.6410672068595886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011286446893596647, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.889249861240387, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3736552894115448, + 0.2301657795906067, + 0.34770384430885315, + 0.4987930655479431, + 0.23395651578903198, + 0.16612301766872406, + 0.1282794177532196, + 0.14615611732006073, + 0.10310263186693192 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.3834407329559326, + "style_target": 1, + "support_probability": 0.026624204590916634, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04819961969289596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7866784334182739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41428112983703613, + 0.23345978558063507, + 0.3205934464931488, + 0.48532602190971375, + 0.07427135854959488, + 0.1381957083940506, + 0.07975833117961884, + 0.06797914206981659, + 0.15878860652446747 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.7063536047935486, + "style_target": 0, + "support_probability": 0.05950859561562538, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7443596033478339, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15069535374641418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8456003069877625, + 0.24515242874622345, + 0.23227237164974213, + 0.12070278823375702, + 0.1752297729253769, + 0.09146950393915176, + 0.1202998012304306, + 0.1261414736509323, + 0.14355814456939697, + 0.1581943929195404 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.6309003829956055, + "style_target": 1, + "support_probability": 0.9627768397331238, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7388518626210303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09792199730873108, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.87022864818573, + 0.23537640273571014, + 0.1719495803117752, + 0.13548120856285095, + 0.08920630067586899, + 0.11576111614704132 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.4744475185871124, + "style_target": 0, + "support_probability": 0.9697701930999756, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6664276722967089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1417423039674759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2236747443675995, + 0.28661394119262695, + 0.11378469318151474, + 0.9438095092773438, + 0.08263883739709854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 1, + "style_probability": 0.7776111960411072, + "style_target": 1, + "support_probability": 0.7488095164299011, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.637355454491252, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1338551938533783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24740269780158997, + 0.2586056888103485, + 0.11583086103200912, + 0.9131672382354736, + 0.04388967528939247 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 0, + "style_probability": 0.6825838088989258, + "style_target": 0, + "support_probability": 0.7973198294639587, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36285411091013564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38421860337257385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9655435681343079, + 0.1655842512845993, + 0.19204746186733246, + 0.375072717666626, + 0.11260102689266205, + 0.12740199267864227 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 1, + "style_probability": 0.8254706859588623, + "style_target": 1, + "support_probability": 0.25664567947387695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3090055671318057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3662641644477844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9525362849235535, + 0.20307700335979462, + 0.20420518517494202, + 0.3800338804721832, + 0.06862219423055649, + 0.10447919368743896, + 0.19887173175811768, + 0.05790748447179794, + 0.1623731553554535, + 0.11591803282499313 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 0, + "style_probability": 0.5185378193855286, + "style_target": 0, + "support_probability": 0.3037341833114624, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7260892893999891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12905991077423096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17192278802394867, + 0.9190850853919983, + 0.2647540867328644, + 0.4246472716331482, + 0.08141171187162399, + 0.08727123588323593, + 0.10863804072141647, + 0.07544231414794922, + 0.13091310858726501 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.7687467932701111, + "style_target": 1, + "support_probability": 0.6558597087860107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3234603214556536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12498170137405396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16639329493045807, + 0.9355595707893372, + 0.3300672769546509, + 0.49923834204673767, + 0.090738944709301, + 0.17097102105617523, + 0.10738511383533478, + 0.06164121627807617, + 0.16444134712219238 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.2572038173675537, + "style_target": 0, + "support_probability": 0.6951856017112732, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6254166232695569, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20214806497097015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22769005596637726, + 0.4233226776123047, + 0.22130152583122253, + 0.9122602343559265, + 0.04387633129954338, + 0.09687480330467224, + 0.07833290100097656, + 0.11625675857067108, + 0.1428026556968689, + 0.14685317873954773, + 0.1581171303987503 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 1, + "style_probability": 0.8264555335044861, + "style_target": 1, + "support_probability": 0.5386795401573181, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4051861163017864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23011291027069092, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21755573153495789, + 0.4233628809452057, + 0.23980630934238434, + 0.8859361410140991, + 0.10646459460258484, + 0.07356781512498856, + 0.08430648595094681, + 0.08227454125881195, + 0.16649438440799713, + 0.1870691031217575, + 0.11506009101867676 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 0, + "style_probability": 0.5941287279129028, + "style_target": 0, + "support_probability": 0.5012608766555786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20104140905320272, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5139933824539185, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3438284397125244, + 0.35267311334609985, + 0.7081099152565002, + 0.3438284397125244, + 0.11890528351068497 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.8599344491958618, + "style_target": 1, + "support_probability": 0.06538621336221695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05139012107566072, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6369124054908752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32495319843292236, + 0.3704742193222046, + 0.7480500936508179, + 0.32495319843292236, + 0.10667385160923004, + 0.08725426346063614, + 0.10084296762943268, + 0.05154568329453468, + 0.09646869450807571 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.3918870985507965, + "style_target": 0, + "support_probability": 0.03357752040028572, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18573223704997965, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4378098249435425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25445371866226196, + 0.13070707023143768, + 0.3842034637928009, + 0.5245952606201172, + 0.031888268887996674, + 0.04083221033215523 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.7397758960723877, + "style_target": 1, + "support_probability": 0.5000649094581604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012221536293334227, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6225190758705139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2993137240409851, + 0.13276484608650208, + 0.47763827443122864, + 0.5503751635551453, + 0.04638868570327759, + 0.10848359018564224 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.17505554854869843, + "style_target": 0, + "support_probability": 0.3314649760723114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18803243616797194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9571071863174438, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09642096608877182, + 0.20290690660476685, + 0.6353771090507507, + 0.6159783005714417, + 0.08916065096855164, + 0.1169644221663475, + 0.12620526552200317, + 0.05756578594446182, + 0.10234174132347107 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 1, + "style_probability": 0.9351161122322083, + "style_target": 1, + "support_probability": 0.10418228060007095, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009949666837318398, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9795368909835815, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08817704766988754, + 0.1749781370162964, + 0.5535975098609924, + 0.5063077211380005, + 0.09561096876859665, + 0.08656555414199829, + 0.08414467424154282, + 0.11547234654426575, + 0.12811507284641266 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 1, + "style_probability": 0.404046893119812, + "style_target": 0, + "support_probability": 0.03113209456205368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1475527154908326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8562568426132202, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.373937726020813, + 0.37952306866645813, + 0.3527753949165344, + 0.42796438932418823, + 0.16167664527893066, + 0.07770436257123947, + 0.07188203930854797, + 0.2155887484550476, + 0.04583384469151497, + 0.11596064269542694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.7805371880531311, + "style_target": 1, + "support_probability": 0.16925473511219025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012958991542873445, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9539223313331604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3601789176464081, + 0.42933782935142517, + 0.30175498127937317, + 0.38833314180374146, + 0.08104480057954788, + 0.17052677273750305 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.24821369349956512, + "style_target": 0, + "support_probability": 0.061435047537088394, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8441393743155667, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04534358158707619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9931083917617798, + 0.28265488147735596, + 0.21427977085113525, + 0.1250569373369217, + 0.08230262249708176 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 1, + "style_probability": 0.5108311176300049, + "style_target": 1, + "support_probability": 0.7113264799118042, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5093726710394475, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045942001044750214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9961088299751282, + 0.2698829770088196, + 0.260811448097229, + 0.13504938781261444, + 0.10213616490364075 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 1, + "style_probability": 0.19586190581321716, + "style_target": 0, + "support_probability": 0.6932082176208496, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28484973753006, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6210886240005493, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6736931204795837, + 0.17160998284816742, + 0.31052422523498535, + 0.16142791509628296, + 0.036435604095458984, + 0.08771862089633942 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 1, + "style_probability": 0.7372183799743652, + "style_target": 1, + "support_probability": 0.35296547412872314, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13160221029088368, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5346996784210205, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7266362905502319, + 0.17974504828453064, + 0.3586174547672272, + 0.1701861023902893, + 0.12775671482086182, + 0.13613571226596832 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 1, + "style_probability": 0.3627564609050751, + "style_target": 0, + "support_probability": 0.41978150606155396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3825110350607596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5669142007827759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.529923141002655, + 0.3446052670478821, + 0.219727024435997, + 0.6669590473175049, + 0.15079250931739807, + 0.22053177654743195, + 0.06795571744441986, + 0.14336781203746796, + 0.12030479311943054 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.786554753780365, + "style_target": 1, + "support_probability": 0.3404587507247925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06598524898875312, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5961032509803772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4547104239463806, + 0.3183909058570862, + 0.2489224225282669, + 0.6312491297721863, + 0.30626916885375977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.27740955352783203, + "style_target": 0, + "support_probability": 0.2020268589258194, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7175129180971865, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21899783611297607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21142299473285675, + 0.8201014995574951, + 0.21539758145809174, + 0.3463161885738373, + 0.14579465985298157, + 0.14799864590168, + 0.08283473551273346, + 0.1759403496980667, + 0.05904064327478409, + 0.06741596758365631, + 0.06227964535355568 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.7305411696434021, + "style_target": 1, + "support_probability": 0.7700368762016296, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46817963039096294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16885614395141602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2439504712820053, + 0.8625216484069824, + 0.23165005445480347, + 0.30779165029525757, + 0.13558997213840485, + 0.17994052171707153, + 0.12325814366340637, + 0.11258668452501297, + 0.08033299446105957, + 0.11438135802745819 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.2809298634529114, + "style_target": 0, + "support_probability": 0.8208516240119934, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0777640883534564, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8469641208648682, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24981556832790375, + 0.3119027614593506, + 0.744553804397583, + 0.7974509000778198, + 0.10390789061784744 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 1, + "style_probability": 0.8052754402160645, + "style_target": 1, + "support_probability": 0.10821771621704102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03291701864074973, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8994172215461731, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2586449086666107, + 0.3118720054626465, + 0.7239102125167847, + 0.759433388710022, + 0.16557784378528595 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 1, + "style_probability": 0.4141315817832947, + "style_target": 0, + "support_probability": 0.029026716947555542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08943261053597856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8297008872032166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6667376160621643, + 0.3423800468444824, + 0.7777536511421204, + 0.44392117857933044, + 0.0736953392624855, + 0.22113879024982452 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.7438492774963379, + "style_target": 1, + "support_probability": 0.12581250071525574, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08180455616100839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8311508893966675, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6012349724769592, + 0.3794275224208832, + 0.7662028670310974, + 0.4161737561225891, + 0.08886227011680603, + 0.08748314529657364, + 0.0678342953324318, + 0.14082832634449005, + 0.10493388772010803, + 0.08362157642841339, + 0.07028888911008835 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.44394364953041077, + "style_target": 0, + "support_probability": 0.10482411831617355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1772425896100922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7350450754165649, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7787889838218689, + 0.2878513038158417, + 0.28231701254844666, + 0.49708136916160583, + 0.05417272821068764, + 0.038940928876399994, + 0.12711696326732635, + 0.040320467203855515, + 0.17675752937793732 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.8057843446731567, + "style_target": 1, + "support_probability": 0.07257955521345139, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.034308814751680836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7830517888069153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8396097421646118, + 0.29442304372787476, + 0.24992628395557404, + 0.5489527583122253, + 0.04025566950440407, + 0.23361627757549286, + 0.07552089542150497, + 0.07325734943151474, + 0.10829707235097885 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.3172360360622406, + "style_target": 0, + "support_probability": 0.12119761854410172, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5780428172672004, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05516393110156059, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3666084408760071, + 0.28639987111091614, + 0.6826891303062439, + 0.923586368560791, + 0.06211017444729805, + 0.0913996696472168, + 0.1531403362751007, + 0.042253799736499786, + 0.1918010711669922, + 0.05423231050372124, + 0.14619645476341248 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 0, + "style_probability": 0.4851250648498535, + "style_target": 1, + "support_probability": 0.896088719367981, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45207401252094725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06561872363090515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3823367655277252, + 0.28843605518341064, + 0.6407080888748169, + 0.8742244243621826, + 0.06588609516620636, + 0.06812284886837006, + 0.10611750930547714, + 0.15971846878528595, + 0.13851916790008545, + 0.07329335808753967, + 0.11660891026258469 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 1, + "style_probability": 0.390129417181015, + "style_target": 0, + "support_probability": 0.7925955057144165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5663589451938319, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20286811888217926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42211437225341797, + 0.8531394600868225, + 0.19472171366214752, + 0.5159278512001038, + 0.1327349841594696 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 1, + "style_probability": 0.5123340487480164, + "style_target": 1, + "support_probability": 0.6835602521896362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7472092343173928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12943018972873688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4453463852405548, + 0.8873045444488525, + 0.1827457696199417, + 0.5647866129875183, + 0.0872235968708992 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 0, + "style_probability": 0.837382972240448, + "style_target": 0, + "support_probability": 0.8037204742431641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4551386964707768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07870720326900482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3702169358730316, + 0.7630112767219543, + 0.19512006640434265, + 0.08802289515733719, + 0.04320788010954857, + 0.06805548071861267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 0, + "style_probability": 0.2011667788028717, + "style_target": 1, + "support_probability": 0.7886815071105957, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14278020107929917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16255418956279755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3355688750743866, + 0.8973316550254822, + 0.20702777802944183, + 0.09080187976360321, + 0.06812446564435959, + 0.08223145455121994 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 1, + "style_probability": 0.10365564376115799, + "style_target": 0, + "support_probability": 0.8033135533332825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06378519554146207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.821040153503418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45605406165122986, + 0.4241831302642822, + 0.4080893397331238, + 0.7747820019721985, + 0.04960823804140091, + 0.05961984023451805, + 0.10230495035648346, + 0.1226956844329834, + 0.1138843223452568 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 0, + "style_probability": 0.29414132237434387, + "style_target": 1, + "support_probability": 0.06406108289957047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04063771566647936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.743881106376648, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4705831706523895, + 0.4241012930870056, + 0.3991556465625763, + 0.7734586000442505, + 0.11208494752645493, + 0.04582253471016884, + 0.15616530179977417, + 0.052693311125040054, + 0.15593688189983368 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 1, + "style_probability": 0.48292726278305054, + "style_target": 0, + "support_probability": 0.04593326151371002, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0056797873723510904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7501328587532043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27626609802246094, + 0.7304971218109131, + 0.26900577545166016, + 0.22929126024246216, + 0.0958692654967308, + 0.09260771423578262, + 0.11051244288682938, + 0.15732859075069427, + 0.3048681318759918, + 0.060274433344602585, + 0.09755674749612808 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.3725477159023285, + "style_target": 1, + "support_probability": 0.0035700087901204824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005799263560322745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6849684715270996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2521580159664154, + 0.741350531578064, + 0.23606830835342407, + 0.2538895606994629, + 0.06974738836288452, + 0.09844794869422913, + 0.05006616935133934, + 0.09633182734251022, + 0.22319012880325317, + 0.08776313811540604, + 0.09425725787878036 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.7523156404495239, + "style_target": 0, + "support_probability": 0.0047183288261294365, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4580027546196785, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19751514494419098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6134637594223022, + 0.38142019510269165, + 0.9687113165855408, + 0.2792392075061798, + 0.06736548990011215 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 0, + "style_probability": 0.44046178460121155, + "style_target": 1, + "support_probability": 0.6604194045066833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17495590002268255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20497991144657135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6752763986587524, + 0.3939407169818878, + 0.9732811450958252, + 0.29696300625801086, + 0.11202012002468109 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 1, + "style_probability": 0.18352307379245758, + "style_target": 0, + "support_probability": 0.6644722819328308, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4984809112644491, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7343443036079407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19196534156799316, + 0.686204195022583, + 0.36556127667427063, + 0.49589139223098755, + 0.0761982873082161, + 0.06224709004163742 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 1, + "style_probability": 0.822562038898468, + "style_target": 1, + "support_probability": 0.4105677306652069, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19021121756468268, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7891706824302673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1738220453262329, + 0.5809168815612793, + 0.3359445631504059, + 0.3704279363155365, + 0.13202640414237976, + 0.18108703196048737 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 1, + "style_probability": 0.4629586935043335, + "style_target": 0, + "support_probability": 0.2688682973384857, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028508007678986393, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9367508888244629, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30362066626548767, + 0.816300630569458, + 0.4934835135936737, + 0.39760658144950867, + 0.13977035880088806, + 0.13650394976139069, + 0.10398310422897339, + 0.08887979388237, + 0.13875889778137207 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.6752768754959106, + "style_target": 1, + "support_probability": 0.022299857810139656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026591321635967308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7523917555809021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27041515707969666, + 0.8102115392684937, + 0.4934394061565399, + 0.3997981548309326, + 0.16948743164539337 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.2429295778274536, + "style_target": 0, + "support_probability": 0.03683698549866676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9733826879763859, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0556781031191349, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:6704:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9949264526367188, + 0.47276392579078674, + 0.3036784827709198, + 0.5623056888580322, + 0.1471247375011444, + 0.13455212116241455, + 0.11328106373548508, + 0.13672412931919098, + 0.07572928071022034, + 0.09272690117359161, + 0.05719389766454697 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.8751139640808105, + "style_target": 1, + "support_probability": 0.9737909436225891, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5359835190124537, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11913652718067169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6704:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9990478157997131, + 0.5403919816017151, + 0.2887418866157532, + 0.6012310981750488, + 0.08510463684797287, + 0.08899004012346268, + 0.3712642788887024, + 0.0960468053817749, + 0.09426285326480865, + 0.09942930936813354 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.1382228285074234, + "style_target": 0, + "support_probability": 0.9567357301712036, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5659720194440927, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1693006008863449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6204466223716736, + 0.44666728377342224, + 0.3740810751914978, + 0.3531193733215332, + 0.036478191614151 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 1, + "style_probability": 0.8871756792068481, + "style_target": 1, + "support_probability": 0.8843607306480408, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2966790572562134, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42664194107055664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5143866539001465, + 0.45424413681030273, + 0.29620203375816345, + 0.3167048692703247, + 0.10814864188432693 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 0, + "style_probability": 0.5082757472991943, + "style_target": 0, + "support_probability": 0.7264455556869507, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2846199969873313, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8711569309234619, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2134459763765335, + 0.5377622842788696, + 0.2669714093208313, + 0.5839788317680359, + 0.12398950010538101, + 0.06061077490448952 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.8453037738800049, + "style_target": 1, + "support_probability": 0.3347741365432739, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030909780509644307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9088804125785828, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17691214382648468, + 0.46270185708999634, + 0.20685403048992157, + 0.588523268699646, + 0.22771257162094116, + 0.20119351148605347, + 0.06316986680030823, + 0.1204732283949852, + 0.2195676863193512, + 0.09636007994413376, + 0.07974349707365036 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.2658234238624573, + "style_target": 0, + "support_probability": 0.10958734899759293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27120461198955786, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1739841103553772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5083853602409363, + 0.8860982656478882, + 0.47784581780433655, + 0.34958750009536743, + 0.16793085634708405, + 0.07634955644607544, + 0.08330542594194412, + 0.043013475835323334, + 0.12721110880374908 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 0, + "style_probability": 0.4916157126426697, + "style_target": 1, + "support_probability": 0.4243088960647583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019087427751523337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3143431842327118, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5779264569282532, + 0.898295521736145, + 0.545547366142273, + 0.31649723649024963, + 0.06211291253566742, + 0.2778668999671936, + 0.09420377761125565, + 0.09463855624198914, + 0.23597221076488495 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 1, + "style_probability": 0.062432460486888885, + "style_target": 0, + "support_probability": 0.33170443773269653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05715115153982211, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7957894802093506, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43625664710998535, + 0.29364386200904846, + 0.7389160990715027, + 0.3344787657260895, + 0.06712516397237778, + 0.12117335200309753, + 0.16613633930683136, + 0.07312095165252686, + 0.09904642403125763, + 0.08457803726196289, + 0.0760369598865509 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 1, + "style_probability": 0.7050490975379944, + "style_target": 1, + "support_probability": 0.025581855326890945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3149139674487634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4583638608455658, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3786731958389282, + 0.3262437582015991, + 0.6894663572311401, + 0.42014279961586, + 0.06537984311580658, + 0.06257891654968262, + 0.07787656038999557, + 0.09771311283111572, + 0.12430395931005478, + 0.4931202232837677, + 0.06120392680168152 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 0, + "style_probability": 0.5581778883934021, + "style_target": 0, + "support_probability": 0.2883519232273102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7097063083853357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0686098113656044, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19581110775470734, + 0.9037602543830872, + 0.06289444863796234, + 0.19581110775470734, + 0.08507250249385834 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 1, + "style_probability": 0.9025325775146484, + "style_target": 1, + "support_probability": 0.5986849069595337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.399221860336186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04328630864620209, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17128369212150574, + 0.9052156209945679, + 0.06456879526376724, + 0.17128369212150574, + 0.03864095360040665, + 0.055228039622306824, + 0.050420504063367844, + 0.0902220755815506, + 0.1886131912469864 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 1, + "style_probability": 0.4130047857761383, + "style_target": 0, + "support_probability": 0.5815309882164001, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09324250549202538, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6294044852256775, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.48612961173057556, + 0.15912221372127533, + 0.1707649976015091, + 0.39015814661979675, + 0.0528867244720459, + 0.09314646571874619 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.8403822183609009, + "style_target": 1, + "support_probability": 0.10121482610702515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009757040197911671, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6361463665962219, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5031263828277588, + 0.16988180577754974, + 0.15881657600402832, + 0.42386341094970703, + 0.08959861099720001, + 0.08199905604124069 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.15144270658493042, + "style_target": 0, + "support_probability": 0.09560874104499817, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.22947556108539735, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7798136472702026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5059019923210144, + 0.12149374932050705, + 0.42161673307418823, + 0.3780895173549652, + 0.045319292694330215, + 0.16671235859394073, + 0.08651667833328247, + 0.060707688331604004, + 0.11853417754173279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 1, + "style_probability": 0.8911290168762207, + "style_target": 1, + "support_probability": 0.10561297833919525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08839704171367915, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7700833678245544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5495694279670715, + 0.1553688645362854, + 0.38276389241218567, + 0.25167691707611084, + 0.06767169386148453, + 0.09210249036550522, + 0.0877884030342102, + 0.09511950612068176, + 0.09086581319570541 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 1, + "style_probability": 0.4462146461009979, + "style_target": 0, + "support_probability": 0.17161840200424194, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7218013103452927, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3669154644012451, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33281826972961426, + 0.5925403237342834, + 0.9346035718917847, + 0.5003581643104553, + 0.04693528264760971, + 0.04401002079248428, + 0.050582367926836014, + 0.1011054739356041, + 0.07987788319587708, + 0.048090167343616486, + 0.06588257849216461 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 1, + "style_probability": 0.8534408807754517, + "style_target": 1, + "support_probability": 0.7041848301887512, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.505598590902518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.35400456190109253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3081584870815277, + 0.5453922152519226, + 0.9107065200805664, + 0.4426219165325165, + 0.0694015845656395, + 0.04979432746767998 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 1, + "style_probability": 0.49626246094703674, + "style_target": 0, + "support_probability": 0.6717824935913086, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6093935941760367, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1070239245891571, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8757094144821167, + 0.12436738610267639, + 0.9891166090965271, + 0.8757094144821167, + 0.09632377326488495 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.5613461136817932, + "style_target": 1, + "support_probability": 0.6835194826126099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1566700744200666, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1754196733236313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9199432134628296, + 0.17928147315979004, + 0.9957063794136047, + 0.9199432134628296, + 0.12901709973812103 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.07336430251598358, + "style_target": 0, + "support_probability": 0.7285657525062561, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4266105897996614, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11399424076080322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36624887585639954, + 0.8933616876602173, + 0.6418359875679016, + 0.1106838807463646, + 0.08674333989620209, + 0.07698331028223038 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 1, + "style_probability": 0.7540993690490723, + "style_target": 1, + "support_probability": 0.38496431708335876, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2929824202992704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21014507114887238, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3396170139312744, + 0.8536440134048462, + 0.5752802491188049, + 0.11798200011253357, + 0.12299887835979462, + 0.07951990514993668 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 0, + "style_probability": 0.583041787147522, + "style_target": 0, + "support_probability": 0.23940922319889069, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7654925549973649, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.048604726791381836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.256143182516098, + 0.9947516322135925, + 0.43517056107521057, + 0.27462777495384216, + 0.08351673185825348, + 0.10042683035135269, + 0.20170380175113678, + 0.08077530562877655, + 0.07879643887281418 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.5353918671607971, + "style_target": 1, + "support_probability": 0.6626313328742981, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6256633148870357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06978963315486908, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24699772894382477, + 0.9910691976547241, + 0.4637807309627533, + 0.27469944953918457, + 0.21605227887630463 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.3181973099708557, + "style_target": 0, + "support_probability": 0.6153052449226379, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44414370892937016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3596053719520569, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7932928800582886, + 0.3335894048213959, + 0.48116689920425415, + 0.244415283203125, + 0.16922897100448608, + 0.05781267583370209, + 0.08097655326128006, + 0.10917457938194275, + 0.08738673478364944, + 0.0973690003156662, + 0.06595277786254883 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.7146000266075134, + "style_target": 1, + "support_probability": 0.42524972558021545, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007360848970612749, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5897289514541626, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8052812218666077, + 0.42237189412117004, + 0.4396806061267853, + 0.3039023280143738, + 0.15186196565628052, + 0.13358305394649506, + 0.26589739322662354, + 0.19615447521209717, + 0.24375803768634796, + 0.09885138273239136 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.046744197607040405, + "style_target": 0, + "support_probability": 0.2593285143375397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.424395691524814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15554924309253693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1563027948141098, + 0.9339458346366882, + 0.20097197592258453, + 0.15846893191337585, + 0.059266962110996246 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 0, + "style_probability": 0.37731561064720154, + "style_target": 1, + "support_probability": 0.5133535861968994, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0935457386370184, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1718347817659378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20927517116069794, + 0.9741265773773193, + 0.20731502771377563, + 0.1866670697927475, + 0.07479115575551987 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 1, + "style_probability": 0.1308707892894745, + "style_target": 0, + "support_probability": 0.4398055374622345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1743286768285653, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.787807285785675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21432486176490784, + 0.3884497582912445, + 0.8110630512237549, + 0.2455582618713379, + 0.10097312182188034, + 0.07561025023460388 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 1, + "style_probability": 0.5974734425544739, + "style_target": 1, + "support_probability": 0.25587669014930725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2018164066361734, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8307961225509644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2123774141073227, + 0.42214563488960266, + 0.8255616426467896, + 0.25561994314193726, + 0.04184228554368019, + 0.11477403342723846 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 0, + "style_probability": 0.7859872579574585, + "style_target": 0, + "support_probability": 0.24636846780776978, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1829864028631794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7384193539619446, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23207569122314453, + 0.24308966100215912, + 0.1899021714925766, + 0.7737109065055847, + 0.2757294476032257, + 0.06957843899726868, + 0.07705507427453995, + 0.0897212028503418, + 0.17247821390628815 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 1, + "style_probability": 0.631132960319519, + "style_target": 1, + "support_probability": 0.21168801188468933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20654857256209386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6879733204841614, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25142520666122437, + 0.22433872520923615, + 0.22178317606449127, + 0.7969651222229004, + 0.1260538250207901 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 0, + "style_probability": 0.5144318342208862, + "style_target": 0, + "support_probability": 0.29054945707321167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05370441699951271, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.774080753326416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2741275727748871, + 0.22983263432979584, + 0.3783338963985443, + 0.24163265526294708, + 0.06434579938650131, + 0.13279716670513153, + 0.24465687572956085, + 0.05181008577346802, + 0.062389228492975235, + 0.042231183499097824, + 0.10402396321296692 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 1, + "style_probability": 0.5227643847465515, + "style_target": 1, + "support_probability": 0.09401320666074753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034664719531777187, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8662731647491455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2556852102279663, + 0.2450307458639145, + 0.4195168614387512, + 0.24317097663879395, + 0.062222689390182495, + 0.04413917288184166, + 0.0658395066857338, + 0.10050688683986664, + 0.05058305338025093, + 0.08479614555835724 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 1, + "style_probability": 0.16452264785766602, + "style_target": 0, + "support_probability": 0.07318121194839478, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8484847324772256, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.053029581904411316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6841:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40254446864128113, + 0.9869596362113953, + 0.23434622585773468, + 0.29679203033447266, + 0.05539379268884659 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 1, + "style_probability": 0.6410166621208191, + "style_target": 1, + "support_probability": 0.8824173808097839, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7012212794115783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06064217537641525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42424482107162476, + 0.9773949980735779, + 0.2674131393432617, + 0.28669801354408264, + 0.0320727676153183, + 0.052218444645404816, + 0.04575398191809654, + 0.08133655786514282, + 0.03861803933978081 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 1, + "style_probability": 0.368026465177536, + "style_target": 0, + "support_probability": 0.8118630051612854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0962460232076949, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7747679948806763, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2369031012058258, + 0.2472214698791504, + 0.5261470079421997, + 0.20561127364635468, + 0.07285186648368835, + 0.05854247882962227 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.8171643614768982, + "style_target": 1, + "support_probability": 0.03163610026240349, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007420295541922741, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8232451677322388, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2147728055715561, + 0.18117350339889526, + 0.5822887420654297, + 0.19641530513763428, + 0.06481418013572693, + 0.08273494988679886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.2168658822774887, + "style_target": 0, + "support_probability": 0.05359961837530136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1489879372035335, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7713738679885864, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8717796206474304, + 0.21083223819732666, + 0.16977334022521973, + 0.2958858013153076, + 0.21845456957817078, + 0.06074431538581848, + 0.18880940973758698, + 0.07197313755750656, + 0.06277977675199509 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 1, + "style_probability": 0.6590953469276428, + "style_target": 1, + "support_probability": 0.12853024899959564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26379560057740037, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5438675284385681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8580708503723145, + 0.2055945098400116, + 0.1529482901096344, + 0.3657448887825012, + 0.11232751607894897, + 0.18136343359947205, + 0.08548977971076965, + 0.03559016436338425, + 0.12009831517934799 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 0, + "style_probability": 0.5494613647460938, + "style_target": 0, + "support_probability": 0.24829332530498505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2576026775052284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3141673803329468, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:positive", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.3788072466850281, + 0.9834682941436768, + 0.16926205158233643, + 0.3795766234397888, + 0.04783325642347336, + 0.06758958846330643, + 0.16940410435199738, + 0.08190831542015076, + 0.11679942905902863, + 0.23060379922389984, + 0.10813185572624207, + 0.21968476474285126, + 0.14831580221652985, + 0.0790124163031578 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.6269071102142334, + "style_target": 1, + "support_probability": 0.12555304169654846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17113888878019592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30107277631759644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4055900573730469, + 0.9741271138191223, + 0.17216762900352478, + 0.364069402217865, + 0.08629480004310608, + 0.14231370389461517 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.31811249256134033, + "style_target": 0, + "support_probability": 0.1364266574382782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.375116629629604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2907402813434601, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2175610214471817, + 0.2270599752664566, + 0.27514269948005676, + 0.9558497071266174, + 0.10393313318490982 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.7620329260826111, + "style_target": 1, + "support_probability": 0.44030433893203735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0980605306418883, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.29230833053588867, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22532939910888672, + 0.28526735305786133, + 0.3527428209781647, + 0.9697431325912476, + 0.07566118985414505 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.3453340530395508, + "style_target": 0, + "support_probability": 0.43167218565940857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47591991474485423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21431565284729004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9816680550575256, + 0.15098963677883148, + 0.2352088838815689, + 0.1609712541103363, + 0.30310043692588806, + 0.11945592612028122 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 1, + "style_probability": 0.658446192741394, + "style_target": 1, + "support_probability": 0.3301924467086792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34494363320244315, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30433014035224915, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9503183960914612, + 0.14515729248523712, + 0.2617177963256836, + 0.18531611561775208, + 0.11800887435674667, + 0.09049765765666962 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 0, + "style_probability": 0.5404860377311707, + "style_target": 0, + "support_probability": 0.2704995274543762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1595003134438034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5512987375259399, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30716148018836975, + 0.34012407064437866, + 0.6714687347412109, + 0.7990851402282715, + 0.04334733635187149, + 0.05136778950691223, + 0.08730998635292053, + 0.12552669644355774, + 0.09614064544439316 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.6789757609367371, + "style_target": 1, + "support_probability": 0.06850173324346542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17005623811341336, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5378118753433228, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2785375416278839, + 0.34457913041114807, + 0.6565684080123901, + 0.7607569098472595, + 0.09531208872795105 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.4812777638435364, + "style_target": 0, + "support_probability": 0.0800091102719307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040223308809295005, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32167741656303406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5241554975509644, + 0.48984652757644653, + 0.1860194355249405, + 0.6384403109550476, + 0.14526990056037903, + 0.09020049124956131, + 0.05460482835769653, + 0.08858159184455872, + 0.13224932551383972, + 0.06981252878904343 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.730241596698761, + "style_target": 1, + "support_probability": 0.04013216868042946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011619689586971288, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3746201992034912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5664405226707458, + 0.519412100315094, + 0.2373020350933075, + 0.6867659091949463, + 0.11462179571390152, + 0.08944272249937057, + 0.07852238416671753, + 0.09520905464887619, + 0.052849072962999344, + 0.11754445731639862 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.25561320781707764, + "style_target": 0, + "support_probability": 0.07566889375448227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6191216873847333, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14529986679553986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23185089230537415, + 0.3236493468284607, + 0.9429702758789062, + 0.31627804040908813, + 0.1652151644229889 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.843114972114563, + "style_target": 1, + "support_probability": 0.6678044199943542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06367643514361208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7486522197723389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.254615843296051, + 0.3491511642932892, + 0.8882039189338684, + 0.3498072922229767, + 0.07224688678979874 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.3967209756374359, + "style_target": 0, + "support_probability": 0.07159124314785004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47818530367146594, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45924022793769836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4461393356323242, + 0.1326291710138321, + 0.66161048412323, + 0.4461393356323242, + 0.10166846215724945, + 0.15215948224067688 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.8338375091552734, + "style_target": 1, + "support_probability": 0.30678969621658325, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0948765114424623, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7948352694511414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3767091929912567, + 0.1451985239982605, + 0.6517454385757446, + 0.3767091929912567, + 0.12718428671360016, + 0.12126987427473068, + 0.09803953021764755, + 0.04752305522561073, + 0.23234428465366364, + 0.0878412052989006, + 0.0912109836935997 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.3655288815498352, + "style_target": 0, + "support_probability": 0.05495365709066391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24268555149185955, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46393585205078125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39555054903030396, + 0.1854192018508911, + 0.9242364764213562, + 0.2656037509441376, + 0.05249118059873581, + 0.07860114425420761, + 0.10532182455062866, + 0.28979191184043884, + 0.3812238276004791 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.7737938165664673, + "style_target": 1, + "support_probability": 0.18874460458755493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05917173725991099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6296447515487671, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47648271918296814, + 0.21616928279399872, + 0.9387871623039246, + 0.324535608291626, + 0.08247672766447067, + 0.08683644980192184, + 0.1050354540348053, + 0.1268777847290039, + 0.15644054114818573 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.47846952080726624, + "style_target": 0, + "support_probability": 0.10360173881053925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5528896693812284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.563460111618042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3604414761066437, + 0.5218239426612854, + 0.3604414761066437, + 0.76309734582901, + 0.07309842854738235, + 0.16378767788410187, + 0.07450635731220245, + 0.11373845487833023, + 0.0628790333867073, + 0.2143314927816391 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 1, + "style_probability": 0.9440830945968628, + "style_target": 1, + "support_probability": 0.22735780477523804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28778232621140215, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49465635418891907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38370969891548157, + 0.5398614406585693, + 0.38370969891548157, + 0.7286761403083801, + 0.05275995656847954, + 0.23370619118213654, + 0.09894952178001404, + 0.08821378648281097, + 0.1293257474899292, + 0.052190426737070084, + 0.090553879737854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 1, + "style_probability": 0.356046199798584, + "style_target": 0, + "support_probability": 0.408920556306839, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.611422202337824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.22163529694080353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3490125238895416, + 0.11256745457649231, + 0.3006660044193268, + 0.7620100378990173, + 0.06215648353099823 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 1, + "style_probability": 0.7425488233566284, + "style_target": 1, + "support_probability": 0.7067859768867493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48205852764183726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28086531162261963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35101601481437683, + 0.11482375115156174, + 0.2656349837779999, + 0.7322930097579956, + 0.06929228454828262, + 0.049858707934617996, + 0.17971979081630707, + 0.0769234150648117, + 0.0774637833237648 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 1, + "style_probability": 0.4148132801055908, + "style_target": 0, + "support_probability": 0.6607718467712402, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8006335157928309, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21051515638828278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2410367727279663, + 0.9753732681274414, + 0.33929452300071716, + 0.5802299976348877, + 0.07325103878974915, + 0.050600770860910416 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.6759110689163208, + "style_target": 1, + "support_probability": 0.5831822752952576, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2559911492493926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2547549307346344, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2162935733795166, + 0.9905948638916016, + 0.30931901931762695, + 0.5815567970275879, + 0.052329663187265396, + 0.0531461276113987 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.15385925769805908, + "style_target": 0, + "support_probability": 0.6402378678321838, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44510342111398477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45108768343925476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4318477511405945, + 0.22767314314842224, + 0.934093177318573, + 0.2837333679199219, + 0.09235415607690811, + 0.08902985602617264, + 0.09743450582027435, + 0.07963486015796661, + 0.05696387216448784 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 1, + "style_probability": 0.7834421396255493, + "style_target": 1, + "support_probability": 0.19866830110549927, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3961803300456149, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41847652196884155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.438486784696579, + 0.19899894297122955, + 0.9080531001091003, + 0.25186634063720703, + 0.06711743772029877, + 0.05469436198472977, + 0.06805933266878128, + 0.04658469557762146, + 0.08870890736579895 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 0, + "style_probability": 0.5362393260002136, + "style_target": 0, + "support_probability": 0.2903312146663666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3566026092552871, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20347364246845245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3003552556037903, + 0.3533692955970764, + 0.8198155164718628, + 0.6483795642852783, + 0.10788297653198242, + 0.059049010276794434, + 0.08577495068311691, + 0.07193212956190109, + 0.05618076026439667, + 0.06751590967178345, + 0.03900070860981941 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 1, + "style_probability": 0.7865119576454163, + "style_target": 1, + "support_probability": 0.10964785516262054, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18500720185387748, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15435436367988586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2925433814525604, + 0.35615667700767517, + 0.7989268898963928, + 0.6393302083015442, + 0.09760259091854095, + 0.13259190320968628 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 1, + "style_probability": 0.34916213154792786, + "style_target": 0, + "support_probability": 0.1442682296037674, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8288965852654337, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12262587994337082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6943:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5199893712997437, + 0.1931694746017456, + 0.5199893712997437, + 0.9628967046737671, + 0.16016216576099396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.8998560905456543, + "style_target": 1, + "support_probability": 0.6814637780189514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0864031935579622, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.169359028339386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6943:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6603842973709106, + 0.22541464865207672, + 0.6603842973709106, + 0.9851072430610657, + 0.12133686989545822 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.11877840757369995, + "style_target": 0, + "support_probability": 0.7621589303016663, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1394419343679776, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.78432297706604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7240442037582397, + 0.275455504655838, + 0.7058502435684204, + 0.7240442037582397, + 0.047728586941957474, + 0.08602948486804962 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 0, + "style_probability": 0.3847653567790985, + "style_target": 1, + "support_probability": 0.32554855942726135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17564469310141492, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8115748167037964, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6850410103797913, + 0.3086300492286682, + 0.6718530654907227, + 0.6850410103797913, + 0.09277244657278061, + 0.09679580479860306 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 1, + "style_probability": 0.41614967584609985, + "style_target": 0, + "support_probability": 0.278209388256073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6139308683985858, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3537324368953705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3503904640674591, + 0.5110425353050232, + 0.27769234776496887, + 0.6995778679847717, + 0.05128950625658035, + 0.0736880972981453, + 0.07614602893590927, + 0.05087227001786232, + 0.07932567596435547 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.8508473038673401, + "style_target": 1, + "support_probability": 0.7361725568771362, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4552440964347579, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3379339873790741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36445537209510803, + 0.4692992568016052, + 0.2694125771522522, + 0.6968876719474792, + 0.07031335681676865 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.449203222990036, + "style_target": 0, + "support_probability": 0.787603497505188, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8732158006039532, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05039764940738678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5366359353065491, + 0.18444186449050903, + 0.49337995052337646, + 0.956502377986908, + 0.12398290634155273, + 0.24823272228240967, + 0.14992749691009521, + 0.07834725826978683, + 0.06523952633142471, + 0.05536629259586334 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.6694770455360413, + "style_target": 1, + "support_probability": 0.9276425242424011, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5029832925181943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044351935386657715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6201664209365845, + 0.2571605145931244, + 0.5590866208076477, + 0.9780628085136414, + 0.1533113569021225, + 0.14947788417339325, + 0.2214978188276291, + 0.16268950700759888, + 0.05317094549536705, + 0.16364805400371552 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.15206313133239746, + "style_target": 0, + "support_probability": 0.9188374280929565, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8017041651349044, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1453012079000473, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5080512762069702, + 0.28451433777809143, + 0.1625349223613739, + 0.9355255961418152, + 0.04859045147895813 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 0, + "style_probability": 0.3310895562171936, + "style_target": 1, + "support_probability": 0.9517036080360413, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29235359403651523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1258549690246582, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5493596196174622, + 0.2082834541797638, + 0.15984658896923065, + 0.9925310015678406, + 0.09981655329465866 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 1, + "style_probability": 0.07477957755327225, + "style_target": 0, + "support_probability": 0.911904513835907, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.40442281146061837, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31720560789108276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2788092792034149, + 0.17696000635623932, + 0.9167200326919556, + 0.8499135971069336, + 0.05680706724524498, + 0.06423596292734146 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 0, + "style_probability": 0.43176791071891785, + "style_target": 1, + "support_probability": 0.5952099561691284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45225454426975975, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2979402542114258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23359818756580353, + 0.14107847213745117, + 0.965745210647583, + 0.8772796988487244, + 0.06047741323709488, + 0.06637782603502274 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 0, + "style_probability": 0.6830713748931885, + "style_target": 0, + "support_probability": 0.5842913389205933, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5948761879509669, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.147907093167305, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7579296231269836, + 0.2651505470275879, + 0.19027313590049744, + 0.5169151425361633, + 0.07663138955831528, + 0.054039116948843, + 0.045807309448719025, + 0.05108677223324776, + 0.06318365037441254 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.5154410600662231, + "style_target": 1, + "support_probability": 0.8603419661521912, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5886263979041644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10416600108146667, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7652941942214966, + 0.23642157018184662, + 0.1750490814447403, + 0.49135687947273254, + 0.0533156581223011 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.46998393535614014, + "style_target": 0, + "support_probability": 0.8786638975143433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.042572263928824086, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6829988956451416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2375691831111908, + 0.2135510891675949, + 0.1960570067167282, + 0.2270619124174118, + 0.11818104982376099, + 0.1651870310306549, + 0.11936672031879425, + 0.07214531302452087, + 0.099732905626297, + 0.07010756433010101, + 0.11680013686418533 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 0, + "style_probability": 0.4748903810977936, + "style_target": 1, + "support_probability": 0.08823787420988083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005223580532763886, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6920886039733887, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2258061021566391, + 0.2288171797990799, + 0.28144046664237976, + 0.33166059851646423, + 0.13578800857067108, + 0.18545931577682495, + 0.10819079726934433, + 0.17850345373153687, + 0.11593872308731079, + 0.13978686928749084 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 1, + "style_probability": 0.1572480946779251, + "style_target": 0, + "support_probability": 0.14092983305454254, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15170341969098397, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5890876054763794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24868766963481903, + 0.25856566429138184, + 0.24624744057655334, + 0.6534945964813232, + 0.04067521542310715 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.182859867811203, + "style_target": 1, + "support_probability": 0.22841329872608185, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3857168570741163, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4574986696243286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2581622898578644, + 0.2604737877845764, + 0.2598886787891388, + 0.6347906589508057, + 0.22565487027168274 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.6565905809402466, + "style_target": 0, + "support_probability": 0.3318169414997101, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18474769913312325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18670935928821564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4742312431335449, + 0.9030314087867737, + 0.3090954124927521, + 0.34420984983444214, + 0.028179146349430084, + 0.03917171433568001 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 0, + "style_probability": 0.3039512634277344, + "style_target": 1, + "support_probability": 0.1617174744606018, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08371979136317385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1449502855539322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.471673846244812, + 0.9551212191581726, + 0.40057259798049927, + 0.36400604248046875, + 0.06827136874198914, + 0.12725408375263214 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 1, + "style_probability": 0.13731078803539276, + "style_target": 0, + "support_probability": 0.29158252477645874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1788171898605288, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45101502537727356, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4377521276473999, + 0.3115798532962799, + 0.05081475153565407, + 0.35138630867004395, + 0.057880207896232605, + 0.08305446803569794, + 0.03494679182767868, + 0.05107901617884636, + 0.06640981137752533 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 0, + "style_probability": 0.2624516785144806, + "style_target": 1, + "support_probability": 0.5636776685714722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3060033006039556, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32539141178131104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46254968643188477, + 0.34407827258110046, + 0.08224278688430786, + 0.3582885265350342, + 0.07219161093235016, + 0.05793342739343643, + 0.038143184036016464, + 0.07478190213441849, + 0.07052989304065704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 1, + "style_probability": 0.37588125467300415, + "style_target": 0, + "support_probability": 0.7526753544807434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08558137718445319, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8559210896492004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.384986013174057, + 0.772688090801239, + 0.1830586940050125, + 0.19019556045532227, + 0.1208789125084877, + 0.09073936194181442, + 0.10245786607265472, + 0.08428511768579483, + 0.05884263664484024, + 0.1134323850274086, + 0.07556512206792831 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.3112691044807434, + "style_target": 1, + "support_probability": 0.14381101727485657, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0718626607863098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9374197125434875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3868882358074188, + 0.7426804304122925, + 0.20710489153862, + 0.17367014288902283, + 0.1275084763765335, + 0.12049500644207001, + 0.12973906099796295, + 0.25486868619918823, + 0.10630294680595398, + 0.08265437930822372, + 0.13600526750087738 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.7779048681259155, + "style_target": 0, + "support_probability": 0.05729631707072258, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5477859166285252, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44082769751548767, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6689463257789612, + 0.7253157496452332, + 0.2692253291606903, + 0.4972362220287323, + 0.0723021849989891 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.7705737948417664, + "style_target": 1, + "support_probability": 0.7579730749130249, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05978844283259936, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6751958131790161, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6807798147201538, + 0.8012166619300842, + 0.3108423352241516, + 0.4713101387023926, + 0.15764814615249634 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.25741979479789734, + "style_target": 0, + "support_probability": 0.4059818387031555, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5731443028392953, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2997210621833801, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5737040042877197, + 0.39863890409469604, + 0.9813808798789978, + 0.6039387583732605, + 0.06397224217653275, + 0.0746464803814888 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 1, + "style_probability": 0.7039923667907715, + "style_target": 1, + "support_probability": 0.434218168258667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5160385956433248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3042169213294983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5428881049156189, + 0.39112693071365356, + 0.9446045756340027, + 0.5999531149864197, + 0.053026579320430756, + 0.07146716862916946 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 0, + "style_probability": 0.6567800641059875, + "style_target": 0, + "support_probability": 0.4727327823638916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4529511212106362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5838580131530762, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22364211082458496, + 0.3743690848350525, + 0.22364211082458496, + 0.9202622175216675, + 0.09275893867015839, + 0.10928189009428024, + 0.09185297042131424, + 0.03895692899823189, + 0.09965091198682785 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.8747137188911438, + "style_target": 1, + "support_probability": 0.3177368938922882, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0836716205848181, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6261992454528809, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27413222193717957, + 0.30712586641311646, + 0.27413222193717957, + 0.8818182349205017, + 0.15663038194179535 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.2802608013153076, + "style_target": 0, + "support_probability": 0.1663540154695511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32711855703290676, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7653021216392517, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3602325916290283, + 0.21866628527641296, + 0.4792705476284027, + 0.6217207908630371, + 0.031266093254089355, + 0.029815636575222015, + 0.029503174126148224, + 0.0614994615316391, + 0.049173276871442795, + 0.0579443983733654, + 0.05316226929426193 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.7540450096130371, + "style_target": 1, + "support_probability": 0.14835602045059204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03148132020359661, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6374675631523132, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33917930722236633, + 0.18140611052513123, + 0.41583797335624695, + 0.6940992474555969, + 0.10691214352846146, + 0.09106741845607758, + 0.21365299820899963, + 0.1025685966014862, + 0.039454419165849686, + 0.15338437259197235 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.13313207030296326, + "style_target": 0, + "support_probability": 0.2400398701429367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007765793480225813, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8674584031105042, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6194275617599487, + 0.7587382197380066, + 0.21128949522972107, + 0.6194275617599487, + 0.12308772653341293 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 1, + "style_probability": 0.78068608045578, + "style_target": 1, + "support_probability": 0.0021233076695352793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0112991766754235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.804577112197876, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6545504331588745, + 0.7564818263053894, + 0.21240313351154327, + 0.6545504331588745, + 0.09986475855112076 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 0, + "style_probability": 0.5646833777427673, + "style_target": 0, + "support_probability": 0.005688664969056845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0775175883280517, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7706369757652283, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1144159808754921, + 0.6069051027297974, + 0.28403744101524353, + 0.05906647816300392, + 0.1766793131828308, + 0.12784729897975922 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.7489246129989624, + "style_target": 1, + "support_probability": 0.07874941825866699, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031857779139605163, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8346073627471924, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1529952585697174, + 0.5452497005462646, + 0.2779075801372528, + 0.06451495736837387, + 0.06686565279960632, + 0.08392568677663803, + 0.1971980333328247, + 0.15885642170906067, + 0.09989991784095764, + 0.20085977017879486, + 0.20402026176452637 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.39121368527412415, + "style_target": 0, + "support_probability": 0.06352810561656952, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7711826317882372, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2976014018058777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4308866560459137, + 0.2448689192533493, + 0.9499883651733398, + 0.2924056947231293, + 0.07541423290967941, + 0.09739185869693756, + 0.14654721319675446, + 0.07250608503818512, + 0.11239629238843918 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.8702756762504578, + "style_target": 1, + "support_probability": 0.6257122755050659, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2955528290299982, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24537329375743866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5233849287033081, + 0.16579532623291016, + 0.9943935871124268, + 0.3812812268733978, + 0.08986542373895645, + 0.14170460402965546, + 0.12050700187683105, + 0.1384330689907074, + 0.15348057448863983 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.2277686893939972, + "style_target": 0, + "support_probability": 0.836528480052948, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34684756405529693, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48311978578567505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3386593759059906, + 0.28679659962654114, + 0.3386593759059906, + 0.9104806780815125, + 0.07568401843309402, + 0.10169089585542679, + 0.04525179788470268, + 0.059592753648757935, + 0.12819485366344452, + 0.05111707001924515, + 0.05507015064358711 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 1, + "style_probability": 0.6482036113739014, + "style_target": 1, + "support_probability": 0.32022541761398315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0446115113190639, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8782238364219666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33808961510658264, + 0.31405937671661377, + 0.33808961510658264, + 0.8253307342529297, + 0.059873972088098526, + 0.06168056279420853, + 0.08664090186357498, + 0.11412610113620758, + 0.07715121656656265, + 0.07449478656053543, + 0.08332380652427673 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 1, + "style_probability": 0.4888383448123932, + "style_target": 0, + "support_probability": 0.034821558743715286, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.49893529259374975, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3904024064540863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.606579601764679, + 0.5018486976623535, + 0.4646196961402893, + 0.22876061499118805, + 0.051575422286987305 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 1, + "style_probability": 0.8411563038825989, + "style_target": 1, + "support_probability": 0.533079206943512, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1834799490817435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8551929593086243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5550985336303711, + 0.4651840329170227, + 0.464611291885376, + 0.21315568685531616, + 0.04858596622943878, + 0.055693622678518295, + 0.08483798801898956, + 0.07266600430011749, + 0.20446155965328217 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 1, + "style_probability": 0.4707847237586975, + "style_target": 0, + "support_probability": 0.1562519371509552, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8417584055751606, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13281157612800598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3406851887702942, + 0.7218883037567139, + 0.9761320948600769, + 0.5238603949546814, + 0.05034828931093216, + 0.03170463442802429 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.7305423021316528, + "style_target": 1, + "support_probability": 0.8185791969299316, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3200196527464467, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21197545528411865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3296869397163391, + 0.7216298580169678, + 0.9828270673751831, + 0.48770153522491455, + 0.061806533485651016, + 0.0928945243358612 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.23274822533130646, + "style_target": 0, + "support_probability": 0.7834952473640442, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7911457852705556, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16589504480361938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30424079298973083, + 0.5368605852127075, + 0.8652878403663635, + 0.4276016056537628, + 0.04877960681915283, + 0.0338771753013134, + 0.08082825690507889, + 0.04175128787755966, + 0.0362517386674881 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 1, + "style_probability": 0.7323535084724426, + "style_target": 1, + "support_probability": 0.9173023700714111, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6366701084973936, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3103622496128082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3024764955043793, + 0.5408031940460205, + 0.7622294425964355, + 0.4319106340408325, + 0.07689467817544937, + 0.060617972165346146, + 0.09305563569068909, + 0.0717189610004425, + 0.10444482415914536 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 1, + "style_probability": 0.44252732396125793, + "style_target": 0, + "support_probability": 0.8655384182929993, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3432184775263036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.784801185131073, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5953264236450195, + 0.3810087740421295, + 0.3578106760978699, + 0.6907619833946228, + 0.05195198953151703, + 0.057493120431900024, + 0.043125368654727936, + 0.03612304478883743, + 0.09630502015352249, + 0.039093468338251114, + 0.0372929610311985 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.8482391834259033, + "style_target": 1, + "support_probability": 0.25175461173057556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11196233296985483, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.85902339220047, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5980377197265625, + 0.3546985983848572, + 0.347123920917511, + 0.646871030330658, + 0.09098496288061142, + 0.11026964336633682 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.3645370304584503, + "style_target": 0, + "support_probability": 0.15596021711826324, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5880526870532033, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0962163507938385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9824015498161316, + 0.1857990175485611, + 0.23291310667991638, + 0.2045242190361023, + 0.1095980554819107 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.8367201089859009, + "style_target": 1, + "support_probability": 0.40048500895500183, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13254396343960764, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14842350780963898, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9806317090988159, + 0.2569425106048584, + 0.27643701434135437, + 0.2340807020664215, + 0.13155671954154968 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.4016578495502472, + "style_target": 0, + "support_probability": 0.36761474609375, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5983503962584368, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05940921604633331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8102536797523499, + 0.49329960346221924, + 0.9838867783546448, + 0.8102536797523499, + 0.08785127103328705, + 0.08642586320638657 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 1, + "style_probability": 0.5405730605125427, + "style_target": 1, + "support_probability": 0.7739207148551941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7755923254087325, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03995887562632561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7955242991447449, + 0.46027591824531555, + 0.9664164781570435, + 0.7955242991447449, + 0.12209676206111908, + 0.1344308704137802 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 1, + "style_probability": 0.4974314272403717, + "style_target": 0, + "support_probability": 0.9046919941902161, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017572073226426354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9827395677566528, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20202365517616272, + 0.3991602957248688, + 0.8741807341575623, + 0.3570416569709778, + 0.1497790664434433, + 0.0941302478313446, + 0.1422719657421112, + 0.14457353949546814, + 0.153389573097229 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.6613723635673523, + "style_target": 1, + "support_probability": 0.014570529572665691, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02417605917358333, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9541943669319153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17752204835414886, + 0.3883078694343567, + 0.8660417795181274, + 0.3137133717536926, + 0.16228356957435608 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.30607539415359497, + "style_target": 0, + "support_probability": 0.021156342700123787, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7382970988813198, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12388487905263901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9575725197792053, + 0.3956839442253113, + 0.5486980080604553, + 0.6338997483253479, + 0.09168393164873123, + 0.04831117019057274, + 0.11961577087640762, + 0.15109829604625702, + 0.18341143429279327, + 0.07719745486974716, + 0.09964457154273987 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.7868114709854126, + "style_target": 1, + "support_probability": 0.7291163802146912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09775371958638576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21241912245750427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9786738753318787, + 0.449462890625, + 0.5121558308601379, + 0.6380624175071716, + 0.2011358141899109, + 0.21966609358787537, + 0.20863837003707886, + 0.09852336347103119, + 0.056487441062927246, + 0.1799464374780655 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.15188679099082947, + "style_target": 0, + "support_probability": 0.682183563709259, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6604973086952293, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09682868421077728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5249795317649841, + 0.3333149552345276, + 0.7459818720817566, + 0.34842413663864136, + 0.09421399235725403 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 1, + "style_probability": 0.8302760124206543, + "style_target": 1, + "support_probability": 0.8825130462646484, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4955146709356617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19143489003181458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4220748245716095, + 0.28570637106895447, + 0.7190142273902893, + 0.30549225211143494, + 0.14416466653347015 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 0, + "style_probability": 0.5875646471977234, + "style_target": 0, + "support_probability": 0.7410094738006592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4103595415486095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3704216182231903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4938163161277771, + 0.5051958560943604, + 0.4294634461402893, + 0.1371137499809265, + 0.059147901833057404, + 0.08048470318317413 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.8724978566169739, + "style_target": 1, + "support_probability": 0.29494380950927734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08791735940952336, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.531613826751709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48129886388778687, + 0.5167151093482971, + 0.37068691849708557, + 0.14081308245658875, + 0.05970025435090065, + 0.17396990954875946, + 0.11605033278465271, + 0.055718857795000076, + 0.047994256019592285, + 0.0926290899515152, + 0.088002510368824 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.17918643355369568, + "style_target": 0, + "support_probability": 0.20384305715560913, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7548299613918654, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03180557116866112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9697782397270203, + 0.274885892868042, + 0.5344899892807007, + 0.5922884941101074, + 0.08888974040746689, + 0.10323818773031235, + 0.07334467768669128, + 0.09403962641954422, + 0.05545584112405777 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.6392823457717896, + "style_target": 1, + "support_probability": 0.8146154284477234, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22205341604978457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03821343183517456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9814110398292542, + 0.262297123670578, + 0.5528970956802368, + 0.6408296823501587, + 0.1448623389005661, + 0.08215760439634323, + 0.08725446462631226, + 0.07308448851108551, + 0.13046815991401672 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.1404820829629898, + "style_target": 0, + "support_probability": 0.824922502040863, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7771736083673269, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09931523352861404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34758713841438293, + 0.9769570827484131, + 0.2776341438293457, + 0.38755279779434204, + 0.08985011279582977, + 0.040905922651290894, + 0.036485377699136734, + 0.06361556053161621, + 0.05613001808524132, + 0.051771655678749084, + 0.08113886415958405 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 1, + "style_probability": 0.7522135972976685, + "style_target": 1, + "support_probability": 0.7563967108726501, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6856800309161827, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09683378785848618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3549984097480774, + 0.9582305550575256, + 0.27381405234336853, + 0.3612842261791229, + 0.046414028853178024, + 0.2286701500415802, + 0.17300747334957123, + 0.1283731758594513, + 0.08093380182981491, + 0.14328770339488983 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 0, + "style_probability": 0.6284920573234558, + "style_target": 0, + "support_probability": 0.8122527599334717, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21235463118705009, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35515716671943665, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8035759925842285, + 0.21668054163455963, + 0.11778522282838821, + 0.747617244720459, + 0.07221195846796036 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 1, + "style_probability": 0.5270686745643616, + "style_target": 1, + "support_probability": 0.6135525107383728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10326346491168305, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5911120176315308, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7950975894927979, + 0.19798532128334045, + 0.12206490337848663, + 0.701908528804779, + 0.04673523083329201 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 0, + "style_probability": 0.759086012840271, + "style_target": 0, + "support_probability": 0.2446693480014801, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.361933252743002, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10353810340166092, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26363590359687805, + 0.18682581186294556, + 0.19244472682476044, + 0.927329421043396, + 0.06762801855802536, + 0.09028387069702148 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 0, + "style_probability": 0.28454479575157166, + "style_target": 1, + "support_probability": 0.4891994893550873, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4680437360669587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09520687907934189, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25301241874694824, + 0.1555674970149994, + 0.18288753926753998, + 0.9368067383766174, + 0.05334630236029625, + 0.09406262636184692, + 0.08364458382129669, + 0.0892011895775795, + 0.08583757281303406, + 0.09655078500509262, + 0.16447162628173828 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 1, + "style_probability": 0.23688769340515137, + "style_target": 0, + "support_probability": 0.49593061208724976, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7773125226910165, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10348048061132431, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2502591907978058, + 0.419900506734848, + 0.9698563814163208, + 0.20516687631607056, + 0.05716002732515335, + 0.05147089064121246, + 0.06811199337244034, + 0.16400907933712006, + 0.10923931002616882 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.7698541283607483, + "style_target": 1, + "support_probability": 0.8033719062805176, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8037975288075812, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.019054753705859184, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2621534764766693, + 0.45925405621528625, + 0.9969942569732666, + 0.18801289796829224, + 0.18030668795108795, + 0.1044066995382309, + 0.13052238523960114, + 0.0542166493833065, + 0.1028912365436554 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.41830432415008545, + "style_target": 0, + "support_probability": 0.9756613969802856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09001533539341752, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41361236572265625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.714008629322052, + 0.1658475250005722, + 0.47730979323387146, + 0.7114021182060242, + 0.09439720958471298, + 0.09708355367183685, + 0.06833431869745255, + 0.14888761937618256, + 0.05006001517176628, + 0.12632371485233307, + 0.10004635155200958 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 1, + "style_probability": 0.5474361777305603, + "style_target": 1, + "support_probability": 0.11873561143875122, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17016284708682314, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1978437453508377, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7394180297851562, + 0.13900931179523468, + 0.5660654902458191, + 0.7553409337997437, + 0.046425290405750275, + 0.06438431888818741, + 0.08263636380434036, + 0.05969807133078575, + 0.08346322178840637, + 0.05814436078071594, + 0.10864173620939255 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 1, + "style_probability": 0.49165868759155273, + "style_target": 0, + "support_probability": 0.3405522406101227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2510992190200787, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5707517862319946, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.412216454744339, + 0.26163920760154724, + 0.412216454744339, + 0.698771059513092, + 0.07251166552305222 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 1, + "style_probability": 0.5138173699378967, + "style_target": 1, + "support_probability": 0.25494447350502014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08066740434252105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.876232385635376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38001275062561035, + 0.310057133436203, + 0.38001275062561035, + 0.6865800023078918, + 0.07879045605659485, + 0.07217615842819214, + 0.12316583096981049, + 0.05459868907928467, + 0.05849802866578102 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 1, + "style_probability": 0.4701089859008789, + "style_target": 0, + "support_probability": 0.060276858508586884, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4648183604996594, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3675430715084076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22443994879722595, + 0.15655852854251862, + 0.3801351487636566, + 0.8909482359886169, + 0.055061787366867065, + 0.07683355361223221 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.5438833832740784, + "style_target": 1, + "support_probability": 0.7490624189376831, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09490746235628077, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.39220720529556274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1999327540397644, + 0.16265897452831268, + 0.3350808918476105, + 0.9493643641471863, + 0.07517731934785843, + 0.10519403964281082 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.22467437386512756, + "style_target": 0, + "support_probability": 0.7235402464866638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1084504849816798, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7686002254486084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18802578747272491, + 0.4509370028972626, + 0.256369411945343, + 0.23410625755786896, + 0.10580866038799286, + 0.11181013286113739, + 0.08971989899873734, + 0.06630788743495941, + 0.06199750676751137 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.47700145840644836, + "style_target": 1, + "support_probability": 0.22025837004184723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3052947071843961, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7421644926071167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17363440990447998, + 0.4831978380680084, + 0.27205589413642883, + 0.3015875220298767, + 0.07727508991956711, + 0.04961332306265831, + 0.04314860329031944, + 0.06614889949560165, + 0.03687907010316849 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.8125757575035095, + "style_target": 0, + "support_probability": 0.39403411746025085, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15307076448848325, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5356633067131042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38387224078178406, + 0.4377342462539673, + 0.8628227710723877, + 0.24406440556049347, + 0.041977137327194214, + 0.06439453363418579, + 0.12858833372592926, + 0.09687017649412155, + 0.08649864792823792, + 0.09115735441446304 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 0, + "style_probability": 0.44518738985061646, + "style_target": 1, + "support_probability": 0.2189864218235016, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37068303086548016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21164768934249878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3642643094062805, + 0.4139639139175415, + 0.87689608335495, + 0.26423004269599915, + 0.07952754199504852, + 0.05378340557217598 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 1, + "style_probability": 0.3411700129508972, + "style_target": 0, + "support_probability": 0.550006091594696, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03162751112135087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18498003482818604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38162681460380554, + 0.7641261219978333, + 0.36492499709129333, + 0.14808501303195953, + 0.08745815604925156 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 0, + "style_probability": 0.19877441227436066, + "style_target": 1, + "support_probability": 0.04566948860883713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025592128386625175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22442615032196045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4080861806869507, + 0.7740526795387268, + 0.41694560647010803, + 0.13107381761074066, + 0.09738337993621826 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 1, + "style_probability": 0.19794614613056183, + "style_target": 0, + "support_probability": 0.06809995323419571, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14997498896826336, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5046782493591309, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16765724122524261, + 0.36231517791748047, + 0.7356551885604858, + 0.2727207839488983, + 0.09242723882198334, + 0.0931149497628212 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.39110392332077026, + "style_target": 1, + "support_probability": 0.18101009726524353, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5424296314347999, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4093429744243622, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15090206265449524, + 0.3760297894477844, + 0.7637403011322021, + 0.25876283645629883, + 0.07458388060331345, + 0.05178234353661537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.8941559195518494, + "style_target": 0, + "support_probability": 0.28072038292884827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09833179535527752, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8625115752220154, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20312190055847168, + 0.25562813878059387, + 0.08733050525188446, + 0.6211822628974915, + 0.13187238574028015, + 0.14715249836444855, + 0.11179734021425247, + 0.09702787548303604, + 0.1492249220609665 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.6017295122146606, + "style_target": 1, + "support_probability": 0.049421004951000214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0383180362747735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7712863683700562, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1752471923828125, + 0.328575998544693, + 0.09666373580694199, + 0.7166581153869629, + 0.13141454756259918, + 0.2032269388437271, + 0.11696048080921173, + 0.06815166771411896, + 0.10378177464008331 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.31379175186157227, + "style_target": 0, + "support_probability": 0.13058610260486603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6525238470024702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12612266838550568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21866533160209656, + 0.46495580673217773, + 0.17743434011936188, + 0.9185097217559814, + 0.08199073374271393, + 0.035498786717653275, + 0.11414890736341476, + 0.08022771030664444, + 0.10613725334405899, + 0.051086656749248505, + 0.06814666092395782 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.45805636048316956, + "style_target": 1, + "support_probability": 0.8619522452354431, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7650902505548032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09086314588785172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24357841908931732, + 0.4119653105735779, + 0.1693725734949112, + 0.8993932008743286, + 0.08389859646558762, + 0.24829311668872833, + 0.1442834436893463, + 0.05242208018898964, + 0.030035769566893578, + 0.024189021438360214, + 0.06841690838336945 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.7292323708534241, + "style_target": 0, + "support_probability": 0.9107334017753601, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6279602978777924, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11293448507785797, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4501994252204895, + 0.7119242548942566, + 0.9528465867042542, + 0.2106095552444458, + 0.08799659460783005 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.6270446181297302, + "style_target": 1, + "support_probability": 0.667666494846344, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5907921805753575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08562038838863373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4271991550922394, + 0.6838957667350769, + 0.9411707520484924, + 0.17694908380508423, + 0.06789293885231018, + 0.07891226559877396, + 0.14185233414173126, + 0.07410566508769989, + 0.07329346984624863 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.37900838255882263, + "style_target": 0, + "support_probability": 0.6769675612449646, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8614936857741091, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0335935615003109, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7259:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.728126049041748, + 0.4016369581222534, + 0.14700064063072205, + 0.9832873344421387, + 0.08189503103494644, + 0.10188613831996918 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.6401572823524475, + "style_target": 1, + "support_probability": 0.9306161999702454, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09332016887782102, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11609146744012833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7259:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7333680391311646, + 0.44601693749427795, + 0.18198584020137787, + 0.9897890686988831, + 0.1428527981042862, + 0.1867707520723343 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.042332351207733154, + "style_target": 0, + "support_probability": 0.8518942594528198, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7917990840608518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04703696072101593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4006008505821228, + 0.900718092918396, + 0.43910181522369385, + 0.17527297139167786, + 0.11671231687068939, + 0.24925419688224792, + 0.09917321801185608, + 0.11687054485082626, + 0.08642815798521042 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 1, + "style_probability": 0.8421121835708618, + "style_target": 1, + "support_probability": 0.854530930519104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7335002871436156, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.046580299735069275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3936046361923218, + 0.8699182868003845, + 0.39144256711006165, + 0.17985552549362183, + 0.12769775092601776, + 0.1597568690776825, + 0.08573459833860397, + 0.08588466793298721, + 0.1187279149889946 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 0, + "style_probability": 0.7473047971725464, + "style_target": 0, + "support_probability": 0.7846201062202454, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5610023518196107, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07178641855716705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3782435953617096, + 0.4214540421962738, + 0.587306022644043, + 0.33676356077194214, + 0.0653134286403656, + 0.06919552385807037, + 0.08576139807701111, + 0.1870025098323822, + 0.059724099934101105, + 0.08668560534715652, + 0.06429801881313324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.8103978633880615, + "style_target": 1, + "support_probability": 0.9376254081726074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4410565676869937, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08972151577472687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3516300618648529, + 0.3636319637298584, + 0.5667912364006042, + 0.36094632744789124, + 0.07465458661317825, + 0.1504170447587967 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.30709484219551086, + "style_target": 0, + "support_probability": 0.8881399631500244, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0441487587582431, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.47204160690307617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2962406277656555, + 0.2359497994184494, + 0.3845957815647125, + 0.3565172553062439, + 0.06553077697753906 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 0, + "style_probability": 0.35947513580322266, + "style_target": 1, + "support_probability": 0.11109672486782074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013838224011859338, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8631496429443359, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2779858112335205, + 0.22491011023521423, + 0.31072792410850525, + 0.3385474681854248, + 0.12824347615242004, + 0.1129913255572319, + 0.1393999308347702, + 0.07243922352790833, + 0.2710735499858856 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 1, + "style_probability": 0.18190498650074005, + "style_target": 0, + "support_probability": 0.040281038731336594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03292222705619326, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31372755765914917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3718409240245819, + 0.8850487470626831, + 0.22631743550300598, + 0.3403308093547821, + 0.16335256397724152, + 0.05251682177186012 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 0, + "style_probability": 0.21498742699623108, + "style_target": 1, + "support_probability": 0.10871008038520813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00981546800872775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29404863715171814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4411146342754364, + 0.9586125016212463, + 0.32743388414382935, + 0.3115268349647522, + 0.1388842910528183, + 0.0886542946100235 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 1, + "style_probability": 0.08185640722513199, + "style_target": 0, + "support_probability": 0.17094168066978455, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7667996460549439, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07361272722482681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9794366955757141, + 0.2597299814224243, + 0.38861483335494995, + 0.3401777148246765, + 0.09127435088157654, + 0.09451211243867874, + 0.1326257437467575, + 0.23333832621574402, + 0.0587814524769783 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 1, + "style_probability": 0.6426563858985901, + "style_target": 1, + "support_probability": 0.7716973423957825, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8951843246585062, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05049457401037216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9956381916999817, + 0.24457034468650818, + 0.3909575939178467, + 0.35658955574035645, + 0.0876394733786583, + 0.04412947595119476, + 0.09058641642332077, + 0.07647007703781128, + 0.10860142856836319 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 0, + "style_probability": 0.7969334721565247, + "style_target": 0, + "support_probability": 0.861920952796936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1528301229218827, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19113457202911377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4016876220703125, + 0.9153876900672913, + 0.2782725393772125, + 0.37676337361335754, + 0.11711648851633072, + 0.06282957643270493, + 0.15253400802612305, + 0.12184668332338333, + 0.1330207735300064, + 0.2065030187368393, + 0.06265474855899811 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 0, + "style_probability": 0.4985559582710266, + "style_target": 1, + "support_probability": 0.18509000539779663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0818531296002707, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2071046233177185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45793044567108154, + 0.9311575293540955, + 0.2377757877111435, + 0.37811240553855896, + 0.13331295549869537, + 0.29868459701538086 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 1, + "style_probability": 0.2949405610561371, + "style_target": 0, + "support_probability": 0.15278437733650208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1085883272332208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33253687620162964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3485592305660248, + 0.9509003758430481, + 0.25255653262138367, + 0.3485592305660248, + 0.07793071866035461 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 0, + "style_probability": 0.39067012071609497, + "style_target": 1, + "support_probability": 0.09898213297128677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04284559401486834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3475828170776367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44590091705322266, + 0.9875537157058716, + 0.28544268012046814, + 0.44590091705322266, + 0.17674554884433746 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 1, + "style_probability": 0.19226312637329102, + "style_target": 0, + "support_probability": 0.1256628930568695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3098478638814953, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23596855998039246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.691234827041626, + 0.2730896770954132, + 0.6292362809181213, + 0.691234827041626, + 0.16408804059028625, + 0.10047883540391922 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 1, + "style_probability": 0.706071138381958, + "style_target": 1, + "support_probability": 0.3796405494213104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44459495919349223, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.34266355633735657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6944792866706848, + 0.27392393350601196, + 0.662371814250946, + 0.6944792866706848, + 0.1285979300737381, + 0.22357217967510223 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 0, + "style_probability": 0.9434770345687866, + "style_target": 0, + "support_probability": 0.18345458805561066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6986706759033167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05078394338488579, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43778127431869507, + 0.9632436633110046, + 0.5481859445571899, + 0.45884591341018677, + 0.12757590413093567, + 0.054029833525419235, + 0.1474924087524414, + 0.10006443411111832, + 0.034330956637859344 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 0, + "style_probability": 0.4237890839576721, + "style_target": 1, + "support_probability": 0.7705944776535034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7301970652419751, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05259983241558075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4319859445095062, + 0.9755222797393799, + 0.4906585216522217, + 0.5017369985580444, + 0.034078437834978104 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 1, + "style_probability": 0.33556586503982544, + "style_target": 0, + "support_probability": 0.8437226414680481, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32433259615174515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47004687786102295, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8868116140365601, + 0.5508765578269958, + 0.21814346313476562, + 0.17612279951572418, + 0.05347408354282379, + 0.06080315634608269, + 0.2729921340942383, + 0.07316399365663528, + 0.13959826529026031, + 0.16860821843147278, + 0.04909077286720276 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.6343233585357666, + "style_target": 1, + "support_probability": 0.2583167254924774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11438922816812233, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20657138526439667, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9184182286262512, + 0.6268585324287415, + 0.2023385763168335, + 0.20557639002799988, + 0.07812876999378204, + 0.09796128422021866, + 0.09339220821857452, + 0.07005035877227783, + 0.08076605200767517, + 0.08905858546495438 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.11388896405696869, + "style_target": 0, + "support_probability": 0.7491446137428284, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.452406207304753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3379846513271332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22488388419151306, + 0.30327534675598145, + 0.21719007194042206, + 0.9921308159828186, + 0.165883406996727 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.7549118399620056, + "style_target": 1, + "support_probability": 0.3077563941478729, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2090935510904524, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4538846015930176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21326954662799835, + 0.3138398230075836, + 0.2203478217124939, + 0.9833198189735413, + 0.055955130606889725, + 0.11362218856811523, + 0.1376236379146576, + 0.12160861492156982, + 0.08561962097883224 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.4217582046985626, + "style_target": 0, + "support_probability": 0.18413367867469788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2575463148781393, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2304796427488327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38093188405036926, + 0.15416479110717773, + 0.36632460355758667, + 0.6074809432029724, + 0.025183996185660362, + 0.0899212658405304 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.8435848951339722, + "style_target": 1, + "support_probability": 0.13967612385749817, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027376794658955157, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3112647831439972, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3349307179450989, + 0.22111566364765167, + 0.4427718222141266, + 0.6700515151023865, + 0.08059077709913254, + 0.08861422538757324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.21837599575519562, + "style_target": 0, + "support_probability": 0.3514072597026825, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8018000867228547, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08303982764482498, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19502536952495575, + 0.9260694980621338, + 0.2928810715675354, + 0.23244786262512207, + 0.03702221438288689, + 0.04495873302221298, + 0.0664106234908104, + 0.06122887134552002, + 0.04640161991119385 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 1, + "style_probability": 0.7585407495498657, + "style_target": 1, + "support_probability": 0.9349636435508728, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6824528405366944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11710144579410553, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18710458278656006, + 0.8629753589630127, + 0.3785971701145172, + 0.2632398307323456, + 0.03955236077308655, + 0.06546198576688766, + 0.053916700184345245, + 0.1364801675081253, + 0.10750775039196014 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 0, + "style_probability": 0.5353307127952576, + "style_target": 0, + "support_probability": 0.9369556307792664, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8799820227990205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07629317790269852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43451988697052, + 0.3035337030887604, + 0.4300374388694763, + 0.9843342900276184, + 0.06052006781101227, + 0.15687349438667297, + 0.18846888840198517, + 0.05654999241232872, + 0.0988914743065834, + 0.07086160033941269, + 0.07942981272935867 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.8687623739242554, + "style_target": 1, + "support_probability": 0.8860214352607727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6851319094853032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07110273092985153, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4409002959728241, + 0.27912086248397827, + 0.4067850708961487, + 0.976989209651947, + 0.24841612577438354, + 0.1288580745458603 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.40959712862968445, + "style_target": 0, + "support_probability": 0.8916208744049072, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.944958704698099, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.021854029968380928, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:7316:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9945476055145264, + 0.4504503011703491, + 0.4554840624332428, + 0.39422914385795593, + 0.05613013356924057 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.7481319904327393, + "style_target": 1, + "support_probability": 0.976107120513916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7851816281351028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022346964105963707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7316:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9979082345962524, + 0.45485919713974, + 0.5233643651008606, + 0.3890928328037262, + 0.14651748538017273 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.25765833258628845, + "style_target": 0, + "support_probability": 0.9841015934944153, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5263601140267719, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2697063684463501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4975138008594513, + 0.8625146746635437, + 0.2603658139705658, + 0.4975138008594513, + 0.0685691088438034, + 0.061536554247140884 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 1, + "style_probability": 0.8754560947418213, + "style_target": 1, + "support_probability": 0.4280335009098053, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30073594045654023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2780701220035553, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4931293725967407, + 0.8369174599647522, + 0.27785080671310425, + 0.4931293725967407, + 0.07776375859975815, + 0.12537384033203125 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 1, + "style_probability": 0.454837828874588, + "style_target": 0, + "support_probability": 0.3496280014514923, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2917454813109433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1067093163728714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9646428823471069, + 0.10513236373662949, + 0.39042437076568604, + 0.35997191071510315, + 0.05176956206560135, + 0.039039187133312225, + 0.2450261116027832, + 0.08405444771051407, + 0.0616617314517498 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 1, + "style_probability": 0.7775462865829468, + "style_target": 1, + "support_probability": 0.0963534340262413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1667629646560506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.151376873254776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9501929879188538, + 0.10859129577875137, + 0.381053626537323, + 0.33503249287605286, + 0.18959282338619232 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 1, + "style_probability": 0.3676385283470154, + "style_target": 0, + "support_probability": 0.10294513404369354, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.46884950238003237, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09481405466794968, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36379215121269226, + 0.2454969733953476, + 0.9931464791297913, + 0.36379215121269226, + 0.09793481975793839, + 0.05823851004242897, + 0.08171658962965012, + 0.03350038453936577, + 0.3757413625717163, + 0.21912820637226105, + 0.041696514934301376 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.5841710567474365, + "style_target": 1, + "support_probability": 0.2872926890850067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1654993283390034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11046755313873291, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3742302656173706, + 0.2732989490032196, + 0.9968454241752625, + 0.3742302656173706, + 0.11151058971881866, + 0.05231393873691559, + 0.16077831387519836, + 0.11236478388309479, + 0.12272429466247559, + 0.15376441180706024 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.15214338898658752, + "style_target": 0, + "support_probability": 0.3900313973426819, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8027104628063535, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10596459358930588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9789605140686035, + 0.42611539363861084, + 0.1661945879459381, + 0.3870195150375366, + 0.05850248038768768 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 0, + "style_probability": 0.4467249810695648, + "style_target": 1, + "support_probability": 0.864821195602417, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7966762493212249, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09082505106925964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9705042839050293, + 0.42926493287086487, + 0.20885097980499268, + 0.4360940456390381, + 0.07417278736829758 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 0, + "style_probability": 0.5710251331329346, + "style_target": 0, + "support_probability": 0.7989997267723083, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42024742226641937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41421133279800415, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3305864930152893, + 0.9072358012199402, + 0.5368435382843018, + 0.3385879695415497, + 0.04843641817569733, + 0.08068466186523438 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.3484813868999481, + "style_target": 1, + "support_probability": 0.5383577346801758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5556711059499156, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35798925161361694, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37707051634788513, + 0.9424331784248352, + 0.506675660610199, + 0.31139135360717773, + 0.2873901128768921, + 0.07467127591371536 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.7346738576889038, + "style_target": 0, + "support_probability": 0.5563951730728149, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4956298754091374, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20117372274398804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28957048058509827, + 0.9243798851966858, + 0.4759516716003418, + 0.3067699670791626, + 0.10751047730445862, + 0.18500424921512604, + 0.13575050234794617, + 0.07149641215801239, + 0.1491616666316986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.5681139826774597, + "style_target": 1, + "support_probability": 0.49422794580459595, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48505800831937484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0888410210609436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3815368413925171, + 0.9665220379829407, + 0.577445924282074, + 0.37812602519989014, + 0.22512607276439667, + 0.18498045206069946, + 0.1717054843902588, + 0.1215309426188469, + 0.16877508163452148 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.4363040328025818, + "style_target": 0, + "support_probability": 0.7439267039299011, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7168907520019632, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05026296526193619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1677231788635254, + 0.2745242416858673, + 0.954888641834259, + 0.2890339195728302, + 0.0660349652171135, + 0.07929743826389313, + 0.10243026167154312, + 0.031998246908187866, + 0.09945686161518097, + 0.03147701546549797, + 0.02737327851355076 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 0, + "style_probability": 0.3601072430610657, + "style_target": 1, + "support_probability": 0.8858155608177185, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5936518342467619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05110527575016022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1619863063097, + 0.2376805543899536, + 0.9306373596191406, + 0.23408114910125732, + 0.05444008857011795, + 0.07180051505565643, + 0.05914994329214096, + 0.08465302735567093, + 0.04095048829913139, + 0.09362880885601044, + 0.05136175826191902 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 1, + "style_probability": 0.4224606156349182, + "style_target": 0, + "support_probability": 0.7821854948997498, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6284911141478391, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1784476488828659, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2709062099456787, + 0.16946445405483246, + 0.2726536989212036, + 0.9128111004829407, + 0.05105040967464447 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.7508862018585205, + "style_target": 1, + "support_probability": 0.6383992433547974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48807338172222986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16758808493614197, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26717424392700195, + 0.17491215467453003, + 0.26916736364364624, + 0.9053775072097778, + 0.045979611575603485, + 0.061040084809064865, + 0.03721713274717331, + 0.047752998769283295, + 0.05213172361254692 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.3970145881175995, + "style_target": 0, + "support_probability": 0.6876229643821716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3988987360861959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6266230344772339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44540339708328247, + 0.36728131771087646, + 0.49621719121932983, + 0.8783974051475525, + 0.054652661085128784, + 0.08062797039747238 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.781330943107605, + "style_target": 1, + "support_probability": 0.1549336165189743, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05616988488310841, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7256309986114502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4999498128890991, + 0.439431369304657, + 0.5377551913261414, + 0.921669065952301, + 0.07060977071523666, + 0.07365290820598602 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.27975016832351685, + "style_target": 0, + "support_probability": 0.1523428112268448, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0638949909961119, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8078579306602478, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.789222776889801, + 0.10820795595645905, + 0.30595287680625916, + 0.2526651620864868, + 0.07726580649614334, + 0.07183048874139786, + 0.07521279156208038, + 0.05850677564740181, + 0.06795261800289154 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 1, + "style_probability": 0.6688793897628784, + "style_target": 1, + "support_probability": 0.13210861384868622, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.042476537671194775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8444548845291138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7018063068389893, + 0.1480361819267273, + 0.33921802043914795, + 0.2858870327472687, + 0.13368846476078033, + 0.18004940450191498, + 0.13519704341888428, + 0.20980273187160492, + 0.09194856137037277 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 1, + "style_probability": 0.44768473505973816, + "style_target": 0, + "support_probability": 0.08859893679618835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9114693338749282, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04324008896946907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33657678961753845, + 0.5592432022094727, + 0.98569655418396, + 0.5734083652496338, + 0.07560619711875916, + 0.030039938166737556, + 0.05628681555390358, + 0.0655907467007637, + 0.056848980486392975, + 0.04028351604938507, + 0.03766103461384773 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.8363065719604492, + "style_target": 1, + "support_probability": 0.9021235704421997, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6994789180637159, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06022685393691063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3352506458759308, + 0.5828080177307129, + 0.9780128598213196, + 0.585999608039856, + 0.050330791622400284, + 0.06563063710927963 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.41783958673477173, + "style_target": 0, + "support_probability": 0.8768820762634277, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.546131331793401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5983498096466064, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36799323558807373, + 0.40385523438453674, + 0.23447726666927338, + 0.8601570725440979, + 0.09416840225458145 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.9273016452789307, + "style_target": 1, + "support_probability": 0.4060780704021454, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018195791267363724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8729166984558105, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4840959906578064, + 0.563071608543396, + 0.22753609716892242, + 0.9040377736091614, + 0.07337073981761932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.19121146202087402, + "style_target": 0, + "support_probability": 0.3038855791091919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3353531664688215, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6128813624382019, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24914990365505219, + 0.3552171587944031, + 0.3925226628780365, + 0.24914990365505219, + 0.16598929464817047, + 0.0580759160220623 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 1, + "style_probability": 0.8631934523582458, + "style_target": 1, + "support_probability": 0.39873766899108887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06471289332136826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6305253505706787, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2534335255622864, + 0.39159825444221497, + 0.3636462986469269, + 0.2534335255622864, + 0.09297823905944824, + 0.16209156811237335 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 0, + "style_probability": 0.5418426394462585, + "style_target": 0, + "support_probability": 0.12448092550039291, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05128955453154354, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.818328857421875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36485132575035095, + 0.6663744449615479, + 0.33047163486480713, + 0.46718326210975647, + 0.0912202000617981, + 0.06847398728132248, + 0.10411713272333145, + 0.2623271346092224, + 0.19934584200382233 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.7689007520675659, + "style_target": 1, + "support_probability": 0.02509881742298603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03697197403667917, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7674775719642639, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35832324624061584, + 0.6507152915000916, + 0.3149011433124542, + 0.444364458322525, + 0.3004913330078125 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.3408117890357971, + "style_target": 0, + "support_probability": 0.05800845846533775, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23237061683072507, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6262394785881042, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3445934057235718, + 0.5880918502807617, + 0.7385666966438293, + 0.48416873812675476, + 0.05233430489897728, + 0.05439400672912598, + 0.05655280500650406, + 0.049274884164333344, + 0.03626452013850212, + 0.07117753475904465, + 0.1440775841474533 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 1, + "style_probability": 0.6612305045127869, + "style_target": 1, + "support_probability": 0.30704161524772644, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14535480968430098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6320281624794006, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35208478569984436, + 0.5895465016365051, + 0.7487761378288269, + 0.4749071002006531, + 0.07454309612512589, + 0.04421229660511017, + 0.1568576842546463, + 0.08723209798336029, + 0.0684659481048584, + 0.12257618457078934 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 1, + "style_probability": 0.43197324872016907, + "style_target": 0, + "support_probability": 0.41750088334083557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7612876681224388, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.06520938128232956, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19720835983753204, + 0.4674203395843506, + 0.7203871607780457, + 0.9674745202064514, + 0.15277498960494995 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.6661204695701599, + "style_target": 1, + "support_probability": 0.8084685206413269, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23134039555863087, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07652151584625244, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18649142980575562, + 0.41000381112098694, + 0.76677405834198, + 0.9884883761405945, + 0.1542481780052185 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.22006593644618988, + "style_target": 0, + "support_probability": 0.7308835983276367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012956258313302441, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7949197888374329, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3104206621646881, + 0.5257487893104553, + 0.2521471679210663, + 0.5115637183189392, + 0.06507126241922379, + 0.14519929885864258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 0, + "style_probability": 0.3125205934047699, + "style_target": 1, + "support_probability": 0.03669482469558716, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008672701949111404, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8984371423721313, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31738436222076416, + 0.43384021520614624, + 0.20756027102470398, + 0.49357956647872925, + 0.10044652968645096, + 0.12404379993677139 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 1, + "style_probability": 0.4706769585609436, + "style_target": 0, + "support_probability": 0.020415982231497765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6137127881651984, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.072937972843647, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2284117043018341, + 0.953719437122345, + 0.04124682769179344, + 0.5367794632911682, + 0.10450659692287445, + 0.1351177841424942, + 0.09804690629243851, + 0.07923354208469391, + 0.10830533504486084 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 0, + "style_probability": 0.4260508120059967, + "style_target": 1, + "support_probability": 0.6407649517059326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41118248024196075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1117314025759697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22955283522605896, + 0.9604213237762451, + 0.04371359199285507, + 0.46958351135253906, + 0.1599334180355072 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 1, + "style_probability": 0.27926257252693176, + "style_target": 0, + "support_probability": 0.5716753005981445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4041680340570111, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23010069131851196, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9818888306617737, + 0.44404855370521545, + 0.4096263647079468, + 0.2036593109369278, + 0.10166852921247482, + 0.13006363809108734, + 0.12730564177036285, + 0.08054240047931671, + 0.06773035228252411, + 0.16683395206928253 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 0, + "style_probability": 0.3411842882633209, + "style_target": 1, + "support_probability": 0.4131101071834564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28233011770889843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1972479224205017, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9984661340713501, + 0.46209728717803955, + 0.45224374532699585, + 0.25843092799186707, + 0.07069633156061172, + 0.07187609374523163, + 0.0702710971236229, + 0.07030089944601059, + 0.15911047160625458, + 0.16874685883522034 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 1, + "style_probability": 0.1344098448753357, + "style_target": 0, + "support_probability": 0.6285569667816162, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.034593044248267424, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5690543055534363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2554573118686676, + 0.5685951113700867, + 0.24668554961681366, + 0.35485732555389404, + 0.06701827794313431 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.758236825466156, + "style_target": 1, + "support_probability": 0.0715658962726593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0068844165645753335, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8800204396247864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2500287592411041, + 0.5341245532035828, + 0.2274188995361328, + 0.3414234220981598, + 0.06815998256206512, + 0.15679901838302612, + 0.16120435297489166, + 0.061925262212753296, + 0.0994761660695076 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.2616368532180786, + "style_target": 0, + "support_probability": 0.012806586921215057, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4489952938178268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1912354826927185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2588975429534912, + 0.9718196392059326, + 0.14776286482810974, + 0.2006789892911911, + 0.07827325910329819, + 0.09584450721740723 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.5350563526153564, + "style_target": 1, + "support_probability": 0.750056266784668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23822545798136438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22618748247623444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31015726923942566, + 0.9729542136192322, + 0.16964873671531677, + 0.2462882697582245, + 0.04183386638760567, + 0.09177158772945404 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.3594105541706085, + "style_target": 0, + "support_probability": 0.760881781578064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12736263362260075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5559094548225403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09214582294225693, + 0.4122063219547272, + 0.6806923151016235, + 0.13982470333576202, + 0.049885064363479614, + 0.07889275252819061, + 0.07314574718475342, + 0.0824371874332428, + 0.05178378149867058 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 1, + "style_probability": 0.7611212730407715, + "style_target": 1, + "support_probability": 0.11279202252626419, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06579033340816304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6122242212295532, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09255066514015198, + 0.3852021396160126, + 0.665029764175415, + 0.15304246544837952, + 0.034221913665533066, + 0.2034655660390854, + 0.12416327744722366, + 0.07055476307868958, + 0.07685965299606323 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 1, + "style_probability": 0.4350382685661316, + "style_target": 0, + "support_probability": 0.07199226319789886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2854170356212381, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6889948844909668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15429742634296417, + 0.16721397638320923, + 0.7939556241035461, + 0.7592675685882568, + 0.08560174703598022, + 0.048279762268066406, + 0.03072691522538662, + 0.06742190569639206, + 0.03299477696418762, + 0.052315499633550644, + 0.06255118548870087 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 1, + "style_probability": 0.859829843044281, + "style_target": 1, + "support_probability": 0.15253835916519165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26403377000112793, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5874154567718506, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1753493696451187, + 0.19599413871765137, + 0.7926533818244934, + 0.7362544536590576, + 0.09957632422447205, + 0.06353024393320084 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 0, + "style_probability": 0.5174860954284668, + "style_target": 0, + "support_probability": 0.33568787574768066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6875604523237825, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1873871386051178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3150199055671692, + 0.12000177800655365, + 0.8934906125068665, + 0.46850666403770447, + 0.14805501699447632 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.6541340947151184, + "style_target": 1, + "support_probability": 0.683192253112793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2902897664610036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21321351826190948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3538157045841217, + 0.16045433282852173, + 0.9122039079666138, + 0.4453134536743164, + 0.12166088074445724 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.2637595236301422, + "style_target": 0, + "support_probability": 0.7151700854301453, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2616698184723849, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8756303787231445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7226037979125977, + 0.42357495427131653, + 0.28994062542915344, + 0.39876681566238403, + 0.13700737059116364, + 0.1191248893737793 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 1, + "style_probability": 0.8123593926429749, + "style_target": 1, + "support_probability": 0.22623302042484283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31530806034513353, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8327295780181885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.669888436794281, + 0.34745755791664124, + 0.2817722260951996, + 0.3702811896800995, + 0.08846106380224228, + 0.058427661657333374 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 0, + "style_probability": 0.5492731332778931, + "style_target": 0, + "support_probability": 0.3974197506904602, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12783165695329315, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6052064299583435, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2268858104944229, + 0.4859020411968231, + 0.5005114674568176, + 0.2268858104944229, + 0.061002761125564575, + 0.10280787944793701, + 0.05245907977223396, + 0.38251084089279175, + 0.13119637966156006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.6353226900100708, + "style_target": 1, + "support_probability": 0.2473539412021637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026623230340044844, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.909205973148346, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26566025614738464, + 0.45388656854629517, + 0.49599072337150574, + 0.26566025614738464, + 0.16755564510822296 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.29362159967422485, + "style_target": 0, + "support_probability": 0.0324719212949276, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1267562491455534, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8120753765106201, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34083718061447144, + 0.18869316577911377, + 0.34083718061447144, + 0.6440934538841248, + 0.07406792044639587, + 0.06957953423261642, + 0.06804170459508896, + 0.06349188834428787, + 0.030715137720108032, + 0.12594996392726898, + 0.07423987239599228 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.8114964962005615, + "style_target": 1, + "support_probability": 0.06592679768800735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06853845432286759, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6690095067024231, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.442465215921402, + 0.19131112098693848, + 0.442465215921402, + 0.8148985505104065, + 0.07034637033939362, + 0.06985902041196823, + 0.2133963704109192, + 0.061611734330654144, + 0.05743343010544777, + 0.07190447300672531 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.14551323652267456, + "style_target": 0, + "support_probability": 0.3820260763168335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5021970424581916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.28446218371391296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3305782973766327, + 0.36990591883659363, + 0.3305782973766327, + 0.8449145555496216, + 0.1893223226070404 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 1, + "style_probability": 0.7149068117141724, + "style_target": 1, + "support_probability": 0.5361649990081787, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4003451694502651, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.35230880975723267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3766022026538849, + 0.42032501101493835, + 0.3766022026538849, + 0.8026547431945801, + 0.13215461373329163 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 1, + "style_probability": 0.3829464912414551, + "style_target": 0, + "support_probability": 0.5880138278007507, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4932822147930618, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26648271083831787, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9668946862220764, + 0.2958953082561493, + 0.29058241844177246, + 0.2806928753852844, + 0.06847147643566132, + 0.0670262798666954 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 1, + "style_probability": 0.6915321350097656, + "style_target": 1, + "support_probability": 0.5098364949226379, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3300006583913802, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2715626358985901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9510599970817566, + 0.29889947175979614, + 0.294757217168808, + 0.24336181581020355, + 0.05362536013126373, + 0.06601966172456741, + 0.03690863400697708, + 0.16814705729484558, + 0.08769931644201279, + 0.049810830503702164, + 0.07502733916044235 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 1, + "style_probability": 0.49434390664100647, + "style_target": 0, + "support_probability": 0.4163209795951843, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1420799992433982, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3484302759170532, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7685777544975281, + 0.109754778444767, + 0.18452996015548706, + 0.38694754242897034, + 0.08578131347894669, + 0.05525073781609535, + 0.06051492691040039, + 0.12419982254505157, + 0.20856280624866486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.6347558498382568, + "style_target": 1, + "support_probability": 0.11043110489845276, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013010489555826439, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4373667240142822, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.811503529548645, + 0.12608322501182556, + 0.2157530039548874, + 0.45129528641700745, + 0.11730509996414185, + 0.2017722725868225, + 0.19421319663524628, + 0.16149044036865234, + 0.15762220323085785 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.13324590027332306, + "style_target": 0, + "support_probability": 0.09692348539829254, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8235936058341959, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13425420224666595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20370587706565857, + 0.10578757524490356, + 0.9678382277488708, + 0.23872703313827515, + 0.08608797937631607, + 0.20500344038009644, + 0.04494588449597359, + 0.05939732491970062, + 0.027692023664712906, + 0.06471024453639984, + 0.10247036069631577 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 1, + "style_probability": 0.7364183068275452, + "style_target": 1, + "support_probability": 0.7432061433792114, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5501717310987473, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24462208151817322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1776159405708313, + 0.11788316816091537, + 0.9553238749504089, + 0.2323870211839676, + 0.09751776605844498, + 0.10892530530691147, + 0.09255519509315491, + 0.29270249605178833, + 0.10341594368219376, + 0.12095118314027786, + 0.06425585597753525 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 0, + "style_probability": 0.5177358388900757, + "style_target": 0, + "support_probability": 0.5457647442817688, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14900567791491567, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3511919677257538, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4021223187446594, + 0.31439685821533203, + 0.6980429291725159, + 0.3503641188144684, + 0.05877360329031944 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.5217899084091187, + "style_target": 1, + "support_probability": 0.2829781472682953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0328489879544715, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8371944427490234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3721451163291931, + 0.3219383656978607, + 0.6194069981575012, + 0.34854814410209656, + 0.09347344934940338, + 0.1152070164680481, + 0.058449555188417435, + 0.05667775124311447, + 0.1333262324333191 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.2689547836780548, + "style_target": 0, + "support_probability": 0.033083707094192505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010257908566101138, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.992296040058136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5810899138450623, + 0.2950666546821594, + 0.33489149808883667, + 0.2640220820903778, + 0.08279416710138321, + 0.061908237636089325 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.7915531396865845, + "style_target": 1, + "support_probability": 0.004578916821628809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028509187581370107, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.985499382019043, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6133182048797607, + 0.2910732328891754, + 0.40918129682540894, + 0.31242212653160095, + 0.13756904006004333, + 0.12348132580518723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.15120162069797516, + "style_target": 0, + "support_probability": 0.012003503739833832, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8846002075520595, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07437371462583542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1634833961725235, + 0.9351451396942139, + 0.19693365693092346, + 0.1634833961725235, + 0.06320373713970184, + 0.0686473622918129, + 0.041974738240242004, + 0.06581452488899231, + 0.09013895690441132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 1, + "style_probability": 0.9057380557060242, + "style_target": 1, + "support_probability": 0.9568122029304504, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8386945606721664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04756595939397812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7553:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17022928595542908, + 0.9374963045120239, + 0.16257363557815552, + 0.17022928595542908, + 0.057948313653469086, + 0.07868468016386032, + 0.04318220913410187, + 0.14926370978355408, + 0.10282887518405914 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 0, + "style_probability": 0.5106726884841919, + "style_target": 0, + "support_probability": 0.964504063129425, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5386757059874941, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4982423782348633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25791293382644653, + 0.6772528290748596, + 0.630280613899231, + 0.5716387033462524, + 0.033636048436164856, + 0.02855610102415085, + 0.04988192394375801, + 0.07268687337636948, + 0.030161814764142036, + 0.02706908993422985, + 0.032616402953863144 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 1, + "style_probability": 0.811955988407135, + "style_target": 1, + "support_probability": 0.5108646154403687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.34780421272953727, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6369521021842957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2385033667087555, + 0.5608224272727966, + 0.635012149810791, + 0.5061441659927368, + 0.0638907179236412, + 0.03589481860399246 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 1, + "style_probability": 0.44150713086128235, + "style_target": 0, + "support_probability": 0.4888727068901062, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.30697419137892334, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.21702544391155243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6328140497207642, + 0.4653277099132538, + 0.16583143174648285, + 0.6628366708755493, + 0.20667409896850586 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 0, + "style_probability": 0.4541701376438141, + "style_target": 1, + "support_probability": 0.6008239984512329, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2193471940634572, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30255842208862305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.71004718542099, + 0.401848167181015, + 0.1718696504831314, + 0.6536427140235901, + 0.1028144508600235, + 0.14002442359924316, + 0.08279917389154434, + 0.09061400592327118, + 0.12722879648208618 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 1, + "style_probability": 0.3053836524486542, + "style_target": 0, + "support_probability": 0.5122846961021423, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.279450725798257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09468162804841995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26821255683898926, + 0.12190108001232147, + 0.3776499330997467, + 0.7474470138549805, + 0.08226045966148376, + 0.12574121356010437 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.5363014340400696, + "style_target": 1, + "support_probability": 0.3268852233886719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015625989460128574, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2032398134469986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41782957315444946, + 0.14491212368011475, + 0.4410175383090973, + 0.7925465703010559, + 0.08295568823814392, + 0.13889336585998535 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.12273673713207245, + "style_target": 0, + "support_probability": 0.22158484160900116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1593701876667834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.524877667427063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3968895971775055, + 0.33649781346321106, + 0.7616363167762756, + 0.6031803488731384, + 0.04361455887556076, + 0.1053246334195137, + 0.05549177899956703, + 0.06058364361524582, + 0.14113174378871918 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 1, + "style_probability": 0.5187662243843079, + "style_target": 1, + "support_probability": 0.12674830853939056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3159310752376807, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5795243382453918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34358662366867065, + 0.30418941378593445, + 0.784013569355011, + 0.613811194896698, + 0.03728131949901581, + 0.06301740556955338, + 0.04320960491895676, + 0.07371653616428375, + 0.08418145030736923 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 0, + "style_probability": 0.8631197214126587, + "style_target": 0, + "support_probability": 0.08856120705604553, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.21762086337096848, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3985169529914856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12371328473091125, + 0.31465184688568115, + 0.8848890066146851, + 0.31845277547836304, + 0.07679776102304459, + 0.06690096110105515, + 0.0872269794344902, + 0.043355949223041534, + 0.05886975675821304, + 0.06688424944877625, + 0.10831273347139359 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 0, + "style_probability": 0.48651984333992004, + "style_target": 1, + "support_probability": 0.16113239526748657, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29839314176830606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25866061449050903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10237253457307816, + 0.24119536578655243, + 0.9314649105072021, + 0.23727239668369293, + 0.06567589938640594, + 0.17882764339447021 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 1, + "style_probability": 0.25039011240005493, + "style_target": 0, + "support_probability": 0.42833250761032104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.466140662474459, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2463967204093933, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4032483398914337, + 0.12787239253520966, + 0.13885313272476196, + 0.8658447861671448, + 0.1284911334514618 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 1, + "style_probability": 0.6754510402679443, + "style_target": 1, + "support_probability": 0.42375943064689636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44719767826305096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.18111693859100342, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3505176603794098, + 0.164037823677063, + 0.17166173458099365, + 0.8526512384414673, + 0.045567721128463745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 0, + "style_probability": 0.5291754603385925, + "style_target": 0, + "support_probability": 0.4713995158672333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.152197173504099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5009689927101135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47559893131256104, + 0.8532658815383911, + 0.13968610763549805, + 0.6286439299583435, + 0.06413105130195618, + 0.09888011962175369 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.6052886843681335, + "style_target": 1, + "support_probability": 0.14247381687164307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09927911518851537, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6240795850753784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4779093563556671, + 0.806611955165863, + 0.12891249358654022, + 0.6240558624267578, + 0.10521551966667175, + 0.046083103865385056, + 0.11921567469835281, + 0.07586252689361572, + 0.06997312605381012, + 0.10725323110818863 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.33870381116867065, + "style_target": 0, + "support_probability": 0.1267879456281662, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25338771050267006, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7227895855903625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5610076189041138, + 0.295075386762619, + 0.2993580102920532, + 0.40807852149009705, + 0.040287379175424576, + 0.04229516535997391, + 0.08312134444713593, + 0.09797502309083939, + 0.06929248571395874 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 1, + "style_probability": 0.9079865217208862, + "style_target": 1, + "support_probability": 0.07154541462659836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32449140596848874, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4409637749195099, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5478886961936951, + 0.378658264875412, + 0.38804858922958374, + 0.42606058716773987, + 0.05669183284044266, + 0.1449441909790039, + 0.08775459229946136, + 0.06447027623653412, + 0.09892803430557251 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 0, + "style_probability": 0.742224395275116, + "style_target": 0, + "support_probability": 0.3362184762954712, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8624922595946587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0396450012922287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30321452021598816, + 0.19944989681243896, + 0.30321452021598816, + 0.9154038429260254, + 0.07447248697280884, + 0.058652617037296295, + 0.05278473719954491, + 0.047711580991744995, + 0.1280369758605957, + 0.10487961769104004 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 1, + "style_probability": 0.8242329955101013, + "style_target": 1, + "support_probability": 0.9718995690345764, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8126324285577572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045650385320186615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2696155607700348, + 0.2168883979320526, + 0.2696155607700348, + 0.8755195140838623, + 0.1637105494737625, + 0.10273297876119614, + 0.11345595121383667, + 0.10042809695005417, + 0.07987053692340851, + 0.05875645950436592 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 0, + "style_probability": 0.6186118721961975, + "style_target": 0, + "support_probability": 0.9739491939544678, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46414229102045584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48595479130744934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8193011283874512, + 0.5342864990234375, + 0.19253571331501007, + 0.289750874042511, + 0.17846010625362396 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 0, + "style_probability": 0.4282133877277374, + "style_target": 1, + "support_probability": 0.46975022554397583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6078805760087709, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49428316950798035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8498413562774658, + 0.5174527168273926, + 0.15772047638893127, + 0.24945710599422455, + 0.0817936435341835 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 0, + "style_probability": 0.7122974395751953, + "style_target": 0, + "support_probability": 0.4311002790927887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0419405504466759, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8882602453231812, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.81647127866745, + 0.552544891834259, + 0.3413560688495636, + 0.20750650763511658, + 0.1794053465127945, + 0.16733905673027039 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.5745024085044861, + "style_target": 1, + "support_probability": 0.01991201378405094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004130900635414592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9870753884315491, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7719219923019409, + 0.4774247109889984, + 0.3548150360584259, + 0.17863208055496216, + 0.12442564964294434, + 0.1595870405435562, + 0.1395082026720047, + 0.16981177031993866, + 0.08063657581806183, + 0.155276820063591, + 0.11551295220851898 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.2786934971809387, + "style_target": 0, + "support_probability": 0.0015213100705295801, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6620708912108599, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04147546365857124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09694773703813553, + 0.3671540319919586, + 0.09694773703813553, + 0.9791223406791687, + 0.07950004190206528, + 0.09031444787979126, + 0.15867789089679718, + 0.06562530249357224, + 0.12458562105894089 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.6142085194587708, + "style_target": 1, + "support_probability": 0.6056514382362366, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2286887121580623, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05683412775397301, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.10245341062545776, + 0.4371420741081238, + 0.10245341062545776, + 0.992122232913971, + 0.05809233710169792, + 0.12264587730169296, + 0.08566887676715851, + 0.06582477688789368, + 0.09108725190162659 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.19721660017967224, + "style_target": 0, + "support_probability": 0.660075306892395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26409560382062713, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4004163146018982, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38262221217155457, + 0.2577284872531891, + 0.9549719095230103, + 0.8241574764251709, + 0.08714288473129272, + 0.07076992094516754, + 0.07236764580011368, + 0.08881095051765442, + 0.06335171312093735, + 0.12621313333511353 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 0, + "style_probability": 0.45535701513290405, + "style_target": 1, + "support_probability": 0.4099532663822174, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5465188619218692, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47004997730255127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37352505326271057, + 0.2527611553668976, + 0.9703572988510132, + 0.8727902770042419, + 0.09193480014801025, + 0.046464961022138596, + 0.0626193955540657, + 0.09179654717445374, + 0.0971672460436821, + 0.043479111045598984, + 0.030410833656787872 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 0, + "style_probability": 0.6946881413459778, + "style_target": 0, + "support_probability": 0.4854358434677124, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7754653748189047, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13545580208301544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3091566562652588, + 0.45151349902153015, + 0.6290165185928345, + 0.9960357546806335, + 0.07020410150289536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.714123010635376, + "style_target": 1, + "support_probability": 0.6870384812355042, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.397189136819295, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20098936557769775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3420726954936981, + 0.5407261848449707, + 0.689555823802948, + 0.9987598657608032, + 0.0897003710269928 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.20830588042736053, + "style_target": 0, + "support_probability": 0.7367422580718994, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7766304696386674, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04208926483988762, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3952137529850006, + 0.2697979807853699, + 0.29227352142333984, + 0.9954518675804138, + 0.06365654617547989, + 0.05325666442513466 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 1, + "style_probability": 0.718401312828064, + "style_target": 1, + "support_probability": 0.6459082365036011, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7870574577470163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04829973727464676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3714202046394348, + 0.2692968249320984, + 0.3096544146537781, + 0.9889035820960999, + 0.0844135582447052, + 0.04134771227836609 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 0, + "style_probability": 0.5433145761489868, + "style_target": 0, + "support_probability": 0.7183388471603394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07431948687507689, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7233569025993347, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27262571454048157, + 0.662226140499115, + 0.6021513938903809, + 0.7603881359100342, + 0.07749906182289124, + 0.06438402086496353, + 0.12532708048820496, + 0.06890921294689178, + 0.059320803731679916 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.7451793551445007, + "style_target": 1, + "support_probability": 0.169879749417305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0671829781245101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.686572790145874, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32919853925704956, + 0.6446571946144104, + 0.5646310448646545, + 0.715753972530365, + 0.07490190118551254 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.4384884238243103, + "style_target": 0, + "support_probability": 0.17785042524337769, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8002399290267149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17104066908359528, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9818560481071472, + 0.5600292682647705, + 0.214927539229393, + 0.541719377040863, + 0.04770820960402489, + 0.09365351498126984, + 0.04693320393562317, + 0.11245796084403992, + 0.05767187848687172, + 0.052266933023929596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.8082349896430969, + "style_target": 1, + "support_probability": 0.3967231512069702, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37319397669223986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17952190339565277, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9894305467605591, + 0.5641262531280518, + 0.23652534186840057, + 0.6175698637962341, + 0.09885118156671524, + 0.14168070256710052, + 0.104454904794693, + 0.09648458659648895, + 0.07775532454252243, + 0.13225755095481873 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.26741018891334534, + "style_target": 0, + "support_probability": 0.620582640171051, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04829353863776632, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7004526257514954, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7156481146812439, + 0.3636791706085205, + 0.7348534464836121, + 0.595940351486206, + 0.05723557621240616 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 0, + "style_probability": 0.405087947845459, + "style_target": 1, + "support_probability": 0.13928335905075073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01094751909016712, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6256994009017944, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8211371898651123, + 0.3593122959136963, + 0.8672587275505066, + 0.7438927888870239, + 0.13494661450386047 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 1, + "style_probability": 0.11813054978847504, + "style_target": 0, + "support_probability": 0.22886033356189728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6526039325291078, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1176728680729866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5967532992362976, + 0.8001968860626221, + 0.18068847060203552, + 0.8255152106285095, + 0.04699081555008888, + 0.03816895931959152 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 1, + "style_probability": 0.5147046446800232, + "style_target": 1, + "support_probability": 0.8342099785804749, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7993555142534015, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09840258955955505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7039247155189514, + 0.8346011638641357, + 0.13324187695980072, + 0.8929909467697144, + 0.05610668286681175, + 0.04338561370968819 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 0, + "style_probability": 0.8291130661964417, + "style_target": 0, + "support_probability": 0.8522314429283142, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6052251631379141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2778402864933014, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22705256938934326, + 0.2889745533466339, + 0.8483485579490662, + 0.36980950832366943, + 0.08099415153265, + 0.08754511922597885, + 0.11498446762561798, + 0.10666325688362122, + 0.10235278308391571 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.521348774433136, + "style_target": 1, + "support_probability": 0.761936604976654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4940022909700423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.41136083006858826, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24836860597133636, + 0.25363343954086304, + 0.892684280872345, + 0.35443851351737976, + 0.09979791194200516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.44951558113098145, + "style_target": 0, + "support_probability": 0.7109320163726807, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5523174850149801, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1678360402584076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9115887880325317, + 0.36032330989837646, + 0.25586411356925964, + 0.2555146813392639, + 0.15712805092334747, + 0.07591166347265244, + 0.09285537153482437, + 0.17231450974941254, + 0.1180286556482315, + 0.0803089365363121, + 0.09189333766698837 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 0, + "style_probability": 0.3512616455554962, + "style_target": 1, + "support_probability": 0.4670161306858063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5943889467142924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14900678396224976, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9620384573936462, + 0.42965349555015564, + 0.2811838686466217, + 0.2688797414302826, + 0.09215093404054642, + 0.11909767985343933, + 0.17399293184280396, + 0.12231055647134781, + 0.05999472364783287, + 0.18294966220855713 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 1, + "style_probability": 0.4343765079975128, + "style_target": 0, + "support_probability": 0.5933564901351929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3824110396936946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.45835208892822266, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4041333794593811, + 0.19213145971298218, + 0.869727611541748, + 0.5002902746200562, + 0.17916317284107208 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 1, + "style_probability": 0.6356801390647888, + "style_target": 1, + "support_probability": 0.20292448997497559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21418688449012002, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6176442503929138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36525070667266846, + 0.2091645896434784, + 0.9062382578849792, + 0.5245187282562256, + 0.22782659530639648 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 0, + "style_probability": 0.6119162440299988, + "style_target": 0, + "support_probability": 0.09954190999269485, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6015095221849546, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08917659521102905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3204233944416046, + 0.8866938352584839, + 0.4137750566005707, + 0.22875410318374634, + 0.06641899049282074, + 0.11967037618160248 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.5566611886024475, + "style_target": 1, + "support_probability": 0.7923164963722229, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6968651872149284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09472623467445374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2424168437719345, + 0.9089802503585815, + 0.35055631399154663, + 0.16581077873706818, + 0.06909903138875961, + 0.06305129081010818, + 0.059885140508413315, + 0.1024860218167305, + 0.17064495384693146, + 0.042057786136865616, + 0.05585026368498802 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.37678301334381104, + "style_target": 0, + "support_probability": 0.883799135684967, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01273764781847543, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9497723579406738, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30609598755836487, + 0.3375927209854126, + 0.3073183298110962, + 0.7150557041168213, + 0.19604894518852234, + 0.15410766005516052, + 0.3913082182407379, + 0.10473202913999557, + 0.3046138882637024 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 0, + "style_probability": 0.3559279441833496, + "style_target": 1, + "support_probability": 0.02688576653599739, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031692243260875337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9817016124725342, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2771739065647125, + 0.2622135579586029, + 0.4645993113517761, + 0.7759986519813538, + 0.17992918193340302, + 0.245240718126297, + 0.1763780415058136, + 0.14183254539966583, + 0.22806799411773682 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 1, + "style_probability": 0.17889438569545746, + "style_target": 0, + "support_probability": 0.016594719141721725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004246873371867178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9219415783882141, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23239091038703918, + 0.7444709539413452, + 0.23239091038703918, + 0.19338198006153107, + 0.09428432583808899, + 0.04550168290734291, + 0.07072211802005768, + 0.10953152924776077, + 0.2267911434173584, + 0.09885253012180328, + 0.1620943695306778 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 0, + "style_probability": 0.43488043546676636, + "style_target": 1, + "support_probability": 0.003416074439883232, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007263182178275967, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9088521599769592, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22908145189285278, + 0.7804268002510071, + 0.22908145189285278, + 0.19873835146427155, + 0.053495943546295166, + 0.05375049635767937, + 0.06989038735628128, + 0.053953733295202255, + 0.07762295007705688, + 0.20557856559753418 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 0, + "style_probability": 0.7969930171966553, + "style_target": 0, + "support_probability": 0.002573789795860648, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.45984646338017454, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.283153772354126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28482064604759216, + 0.22083579003810883, + 0.36220353841781616, + 0.8591589331626892, + 0.08903954178094864 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.6213300228118896, + "style_target": 1, + "support_probability": 0.4391416311264038, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11749522055483155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3211049735546112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27478498220443726, + 0.21823832392692566, + 0.3878380060195923, + 0.9164090752601624, + 0.12063607573509216 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.2987017035484314, + "style_target": 0, + "support_probability": 0.34486597776412964, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14138661207485212, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5672381520271301, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8702017068862915, + 0.500415563583374, + 0.10230116546154022, + 0.500415563583374, + 0.06013837829232216, + 0.08418934792280197 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 0, + "style_probability": 0.2298814356327057, + "style_target": 1, + "support_probability": 0.2878986895084381, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1398230310627345, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5872862935066223, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7950268387794495, + 0.44993162155151367, + 0.1314016431570053, + 0.44993162155151367, + 0.10877905040979385, + 0.0568702407181263 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 1, + "style_probability": 0.38849449157714844, + "style_target": 0, + "support_probability": 0.22917775809764862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2539527936804131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4047290086746216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47390738129615784, + 0.2019357979297638, + 0.28037020564079285, + 0.9224392175674438, + 0.11360019445419312, + 0.0386384092271328, + 0.04648429527878761, + 0.07229086011648178, + 0.08569536358118057 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 0, + "style_probability": 0.43511056900024414, + "style_target": 1, + "support_probability": 0.38922470808029175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5247162603606954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4084395468235016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48629966378211975, + 0.2201196402311325, + 0.28957194089889526, + 0.9343187212944031, + 0.02425423637032509, + 0.1365187168121338, + 0.0725177749991417, + 0.03463684394955635, + 0.062417495995759964 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 0, + "style_probability": 0.7793829441070557, + "style_target": 0, + "support_probability": 0.39097288250923157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.042974892763218565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5617172718048096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5481313467025757, + 0.2226274162530899, + 0.8139955401420593, + 0.6194401383399963, + 0.15075469017028809, + 0.13890504837036133, + 0.14545047283172607, + 0.10675321519374847, + 0.07301799207925797, + 0.09340430051088333, + 0.25129759311676025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 0, + "style_probability": 0.3908357620239258, + "style_target": 1, + "support_probability": 0.031269099563360214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04452774457320372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3733811676502228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5750083923339844, + 0.2248365879058838, + 0.9150676131248474, + 0.6416448950767517, + 0.08055335283279419, + 0.12215645611286163, + 0.1390392780303955, + 0.0924823135137558, + 0.1607784628868103, + 0.10801103711128235 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 1, + "style_probability": 0.3610036373138428, + "style_target": 0, + "support_probability": 0.12783031165599823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044079644625732, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4907117784023285, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14519919455051422, + 0.34579694271087646, + 0.22230620682239532, + 0.7151330709457397, + 0.12190554291009903 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 0, + "style_probability": 0.3566433787345886, + "style_target": 1, + "support_probability": 0.18282702565193176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0445481613527345, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5344804525375366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14018820226192474, + 0.3262936770915985, + 0.19290639460086823, + 0.6375635266304016, + 0.08734799176454544 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 0, + "style_probability": 0.5028688907623291, + "style_target": 0, + "support_probability": 0.12195061147212982, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.47378378588730735, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05966656282544136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9576130509376526, + 0.2169661670923233, + 0.3067626655101776, + 0.3838227689266205, + 0.11382979899644852, + 0.07803180813789368 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 0, + "style_probability": 0.3564898371696472, + "style_target": 1, + "support_probability": 0.742954432964325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5898367666717828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07058510184288025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9637749791145325, + 0.2213408201932907, + 0.27516135573387146, + 0.38597428798675537, + 0.05017470568418503, + 0.1342114508152008 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 0, + "style_probability": 0.7159336805343628, + "style_target": 0, + "support_probability": 0.6782851219177246, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28960646622216407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17490632832050323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2500233054161072, + 0.42313745617866516, + 0.13439001142978668, + 0.8791226744651794, + 0.14726369082927704, + 0.11806121468544006, + 0.058003928512334824, + 0.20706434547901154, + 0.11971532553434372 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 0, + "style_probability": 0.32589754462242126, + "style_target": 1, + "support_probability": 0.4717842936515808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06180221812383352, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24451851844787598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27141690254211426, + 0.4955673813819885, + 0.1432758867740631, + 0.9345518946647644, + 0.17999491095542908, + 0.09942302852869034, + 0.10686414688825607, + 0.11542166024446487, + 0.09255170077085495 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 1, + "style_probability": 0.09474185109138489, + "style_target": 0, + "support_probability": 0.5226548910140991, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.622930476524515, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04437769204378128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.997945249080658, + 0.28751304745674133, + 0.5137587189674377, + 0.20835614204406738, + 0.11792522668838501, + 0.15692256391048431, + 0.16896766424179077, + 0.1817411482334137, + 0.06223604455590248, + 0.10219409316778183, + 0.07360812276601791 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.4141084849834442, + "style_target": 1, + "support_probability": 0.8354928493499756, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6416980241260908, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.033369794487953186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9946349859237671, + 0.31630969047546387, + 0.5112016797065735, + 0.2530614137649536, + 0.07711758464574814, + 0.15491369366645813, + 0.1626472771167755, + 0.08617734909057617, + 0.09008490294218063, + 0.05737214908003807 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.5443932414054871, + "style_target": 0, + "support_probability": 0.791658878326416, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007321125032574761, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9807989597320557, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2141178995370865, + 0.37238433957099915, + 0.3274117410182953, + 0.19289229810237885, + 0.07449575513601303 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.7919353246688843, + "style_target": 1, + "support_probability": 0.004830982536077499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004822653726711263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9807538390159607, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23106159269809723, + 0.3545607328414917, + 0.3200451135635376, + 0.2197446972131729, + 0.08203212916851044, + 0.10955645143985748, + 0.09699195623397827, + 0.0831969827413559, + 0.1581670641899109 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.21636448800563812, + "style_target": 0, + "support_probability": 0.006656765006482601, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1754628581341734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8422337174415588, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3196473717689514, + 0.15190723538398743, + 0.08422459661960602, + 0.4743861258029938, + 0.10178644210100174, + 0.058885566890239716 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.8240754008293152, + "style_target": 1, + "support_probability": 0.1278565376996994, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02344069727773237, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3770586848258972, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38583624362945557, + 0.21129652857780457, + 0.1578158736228943, + 0.638013482093811, + 0.2187514454126358, + 0.12307640165090561 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.10785146802663803, + "style_target": 0, + "support_probability": 0.4421214759349823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015100901165802063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5446702241897583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33183491230010986, + 0.18483403325080872, + 0.2061668038368225, + 0.7800682783126831, + 0.19442418217658997, + 0.07459615170955658, + 0.06517902761697769, + 0.10567241162061691, + 0.038597337901592255 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.5945035219192505, + "style_target": 1, + "support_probability": 0.008555728942155838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021861251171931478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3935255706310272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3374939560890198, + 0.1786622703075409, + 0.2304166555404663, + 0.7692660689353943, + 0.2929372191429138, + 0.16265122592449188, + 0.06076284497976303, + 0.134475439786911, + 0.1782965064048767 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.29215189814567566, + "style_target": 0, + "support_probability": 0.02545492723584175, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7640020578806488, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1359303593635559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9357185959815979, + 0.1914612352848053, + 0.150174081325531, + 0.45810243487358093, + 0.10374993085861206, + 0.06649493426084518, + 0.12362180650234222, + 0.03405981883406639, + 0.0740860253572464, + 0.08368963748216629, + 0.03971755877137184 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8489772081375122, + "style_target": 1, + "support_probability": 0.3944419026374817, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43636259726875737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15863089263439178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9050424098968506, + 0.22012929618358612, + 0.15740588307380676, + 0.49108633399009705, + 0.06443305313587189, + 0.14555326104164124 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.4275875687599182, + "style_target": 0, + "support_probability": 0.35434243083000183, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4391639171097688, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14357736706733704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3066568970680237, + 0.8457769155502319, + 0.17036204040050507, + 0.4758237898349762, + 0.0729244127869606 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 0, + "style_probability": 0.3213579058647156, + "style_target": 1, + "support_probability": 0.6779966354370117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37968848107668035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36704346537590027, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3211789131164551, + 0.8575246334075928, + 0.15413911640644073, + 0.5300666093826294, + 0.10165178775787354, + 0.1370503157377243, + 0.033148251473903656, + 0.056757472455501556, + 0.08118470758199692 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 1, + "style_probability": 0.39022547006607056, + "style_target": 0, + "support_probability": 0.49364718794822693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2822272292736077, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5072415471076965, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.683231770992279, + 0.23581302165985107, + 0.1400616317987442, + 0.23581302165985107, + 0.033334679901599884, + 0.05892203003168106 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 0, + "style_probability": 0.38787177205085754, + "style_target": 1, + "support_probability": 0.4804215431213379, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02628772604708922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.531683087348938, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7931219339370728, + 0.37899982929229736, + 0.21288210153579712, + 0.37899982929229736, + 0.06372543424367905, + 0.08006182312965393 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 1, + "style_probability": 0.11295820772647858, + "style_target": 0, + "support_probability": 0.3285641372203827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3658367353089298, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40997233986854553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24385839700698853, + 0.3341231942176819, + 0.6804296970367432, + 0.33351218700408936, + 0.07033145427703857, + 0.05594358593225479, + 0.06126565486192703, + 0.09747501462697983, + 0.052615270018577576 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 1, + "style_probability": 0.561355710029602, + "style_target": 1, + "support_probability": 0.3966413736343384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30695388377513144, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4884154200553894, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27504244446754456, + 0.24609659612178802, + 0.6854899525642395, + 0.33703410625457764, + 0.08457500487565994, + 0.08341247588396072, + 0.052029263228178024, + 0.10810782760381699, + 0.07255232334136963 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 0, + "style_probability": 0.6261841058731079, + "style_target": 0, + "support_probability": 0.38209766149520874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038441815942034836, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7639061808586121, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22018231451511383, + 0.1522611379623413, + 0.23102356493473053, + 0.665061354637146, + 0.05283650755882263, + 0.036364104598760605, + 0.05630580335855484, + 0.04083612188696861, + 0.03449120372533798, + 0.07620947062969208 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.5152700543403625, + "style_target": 1, + "support_probability": 0.05392921715974808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03175618130219149, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8040164709091187, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22013364732265472, + 0.12555474042892456, + 0.2706843316555023, + 0.5854288935661316, + 0.03490724787116051, + 0.046760961413383484 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.34341153502464294, + "style_target": 0, + "support_probability": 0.04685986042022705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31424828139474753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.19663190841674805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4047142565250397, + 0.5221837759017944, + 0.4047142565250397, + 0.9518952965736389, + 0.17462877929210663 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.767148494720459, + "style_target": 1, + "support_probability": 0.35807058215141296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19515764637214084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.13599158823490143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3797913193702698, + 0.5258398652076721, + 0.3797913193702698, + 0.9341107606887817, + 0.0949481874704361 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.4986913502216339, + "style_target": 0, + "support_probability": 0.4029785096645355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05565073766164623, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7129388451576233, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17034322023391724, + 0.3760921061038971, + 0.44213640689849854, + 0.2607172131538391, + 0.12006491422653198, + 0.40272319316864014 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 1, + "style_probability": 0.7909795045852661, + "style_target": 1, + "support_probability": 0.01703333668410778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0077340757621599835, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8987533450126648, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16741982102394104, + 0.37148019671440125, + 0.44765448570251465, + 0.2557953894138336, + 0.22400452196598053, + 0.10743898153305054, + 0.11514412611722946, + 0.06268341094255447, + 0.07105851173400879, + 0.10703796148300171 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 1, + "style_probability": 0.4588903486728668, + "style_target": 0, + "support_probability": 0.0056305937469005585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26729270306035263, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7860414981842041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1375054121017456, + 0.4284664988517761, + 0.5184050798416138, + 0.23616789281368256, + 0.03335227072238922, + 0.04915350675582886, + 0.07717617601156235, + 0.02088937535881996, + 0.05293738469481468 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.8801460862159729, + "style_target": 1, + "support_probability": 0.15224982798099518, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02006494215389249, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6823927760124207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19114075601100922, + 0.418499231338501, + 0.6120487451553345, + 0.27601516246795654, + 0.060421764850616455, + 0.23205989599227905, + 0.06331965327262878, + 0.06729040294885635, + 0.057241830974817276 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.20144279301166534, + "style_target": 0, + "support_probability": 0.14694230258464813, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8633117619412901, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07049927860498428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.167585089802742, + 0.40549853444099426, + 0.4614328444004059, + 0.9689241647720337, + 0.08655109256505966, + 0.06340852379798889, + 0.07249951362609863, + 0.07808221876621246, + 0.13208121061325073, + 0.06035015732049942, + 0.13938307762145996 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 1, + "style_probability": 0.8787527680397034, + "style_target": 1, + "support_probability": 0.7608802318572998, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5259949129234138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12316137552261353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16890954971313477, + 0.3521001935005188, + 0.3595755994319916, + 0.954311192035675, + 0.07601381838321686, + 0.15197771787643433, + 0.17862479388713837, + 0.10572627186775208, + 0.20232553780078888, + 0.033346351236104965, + 0.05262255668640137 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 0, + "style_probability": 0.5271285772323608, + "style_target": 0, + "support_probability": 0.6389154195785522, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09587966595981096, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5745182037353516, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6631375551223755, + 0.30454516410827637, + 0.5943453311920166, + 0.5572406053543091, + 0.12265518307685852 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.7006322145462036, + "style_target": 1, + "support_probability": 0.0767771378159523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05505522282800285, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8217681050300598, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.669681966304779, + 0.2989157438278198, + 0.5654775500297546, + 0.5379320979118347, + 0.13925383985042572, + 0.12335141748189926, + 0.11639171838760376, + 0.07058433443307877, + 0.11436232179403305 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.396548867225647, + "style_target": 0, + "support_probability": 0.062498122453689575, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3055425504416571, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5133823752403259, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9322929382324219, + 0.1265680342912674, + 0.20559604465961456, + 0.5628589987754822, + 0.07939104735851288, + 0.17245744168758392 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.7119548916816711, + "style_target": 1, + "support_probability": 0.17158421874046326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06582976149675387, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5568839311599731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9710952043533325, + 0.1807129830121994, + 0.3056388795375824, + 0.6998982429504395, + 0.13739457726478577, + 0.21136793494224548 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.23297402262687683, + "style_target": 0, + "support_probability": 0.17013618350028992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7224490658023581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12088931351900101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.854891300201416, + 0.146670863032341, + 0.21910594403743744, + 0.2832598388195038, + 0.2980605363845825, + 0.09515995532274246, + 0.09532223641872406, + 0.04446886107325554, + 0.049438536167144775 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 1, + "style_probability": 0.7906924486160278, + "style_target": 1, + "support_probability": 0.7622482180595398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4849233035186386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13062092661857605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8097504377365112, + 0.13673700392246246, + 0.23698030412197113, + 0.3430761992931366, + 0.052613262087106705, + 0.04594559594988823, + 0.034730467945337296, + 0.061617787927389145, + 0.08425099402666092 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 1, + "style_probability": 0.4510045051574707, + "style_target": 0, + "support_probability": 0.6012166142463684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24622030663347877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6460756063461304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19648629426956177, + 0.3451838493347168, + 0.6661579012870789, + 0.9340174794197083, + 0.06593729555606842, + 0.057165518403053284, + 0.2266988903284073, + 0.11470915377140045, + 0.15008515119552612, + 0.07188991457223892, + 0.07359015941619873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 1, + "style_probability": 0.8219872713088989, + "style_target": 1, + "support_probability": 0.197344571352005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1404910965829742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6523900032043457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22687284648418427, + 0.3510897159576416, + 0.6529985666275024, + 0.9052687883377075, + 0.09135816991329193, + 0.11541654914617538 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 1, + "style_probability": 0.45046791434288025, + "style_target": 0, + "support_probability": 0.23392623662948608, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5711171640171374, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07879921048879623, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44360285997390747, + 0.880584180355072, + 0.23432667553424835, + 0.36247768998146057, + 0.2546618580818176 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 0, + "style_probability": 0.34541240334510803, + "style_target": 1, + "support_probability": 0.5959191918373108, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3779715952274959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08863311260938644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4370073080062866, + 0.928812563419342, + 0.30233821272850037, + 0.3894428312778473, + 0.1648447960615158 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 1, + "style_probability": 0.2771981954574585, + "style_target": 0, + "support_probability": 0.639326274394989, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03468613722228305, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9183520674705505, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23327292501926422, + 0.47748371958732605, + 0.38659238815307617, + 0.5408064126968384, + 0.25809794664382935, + 0.17075926065444946 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 0, + "style_probability": 0.28998202085494995, + "style_target": 1, + "support_probability": 0.05114623159170151, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011980196825777271, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8722599744796753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2157878577709198, + 0.40341266989707947, + 0.3554285764694214, + 0.4782768785953522, + 0.1434774398803711, + 0.1641061156988144 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 1, + "style_probability": 0.40098991990089417, + "style_target": 0, + "support_probability": 0.024149972945451736, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12065125015093613, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20664829015731812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5876665115356445, + 0.19929571449756622, + 0.7537202835083008, + 0.735451340675354, + 0.1589667797088623, + 0.19519948959350586, + 0.24021363258361816, + 0.09781508147716522, + 0.23965583741664886 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.4703572988510132, + "style_target": 1, + "support_probability": 0.16603213548660278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11751872321849892, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17147743701934814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6263962388038635, + 0.17594364285469055, + 0.8235865831375122, + 0.7192589044570923, + 0.10061480104923248, + 0.11238120496273041, + 0.12156351655721664, + 0.055033959448337555, + 0.2084904909133911 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.7417454123497009, + "style_target": 0, + "support_probability": 0.1868332177400589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03235117743968437, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6875497102737427, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5329285264015198, + 0.27525314688682556, + 0.33046993613243103, + 0.10767867416143417, + 0.061495210975408554, + 0.052563585340976715, + 0.13590751588344574, + 0.3262682855129242, + 0.19349344074726105, + 0.11915674060583115, + 0.44819942116737366 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 0, + "style_probability": 0.2870287001132965, + "style_target": 1, + "support_probability": 0.03414144366979599, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012677550650445105, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.919267475605011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.455653578042984, + 0.25356993079185486, + 0.33358100056648254, + 0.14219634234905243, + 0.10654548555612564, + 0.1076759621500969, + 0.25778600573539734, + 0.12393251061439514, + 0.13538894057273865, + 0.15101902186870575 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 1, + "style_probability": 0.08087891340255737, + "style_target": 0, + "support_probability": 0.012665660120546818, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2428105430782153, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8539716005325317, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3710949122905731, + 0.48643362522125244, + 0.47696301341056824, + 0.19324195384979248, + 0.06192995607852936 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 1, + "style_probability": 0.8374528884887695, + "style_target": 1, + "support_probability": 0.35654863715171814, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10386436698958165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8995981812477112, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30745723843574524, + 0.49066251516342163, + 0.4992001950740814, + 0.2208438515663147, + 0.06520595401525497 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 0, + "style_probability": 0.5156819820404053, + "style_target": 0, + "support_probability": 0.18707682192325592, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6966439743334621, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.145615816116333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4149705767631531, + 0.3013542890548706, + 0.9336127042770386, + 0.21587350964546204, + 0.036337707191705704, + 0.06426781415939331 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.6709609627723694, + "style_target": 1, + "support_probability": 0.7870079874992371, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41850596269180595, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3277078866958618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4070490300655365, + 0.2927915155887604, + 0.8889362812042236, + 0.23839130997657776, + 0.05130511894822121, + 0.050187740474939346, + 0.25013890862464905, + 0.13799993693828583, + 0.06955942511558533, + 0.0825934037566185, + 0.06535962969064713 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.3908195197582245, + "style_target": 0, + "support_probability": 0.5108476877212524, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8763632797797644, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09695896506309509, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9415802359580994, + 0.05676230788230896, + 0.20564568042755127, + 0.38522401452064514, + 0.06565027683973312, + 0.07851867377758026, + 0.04210951551795006, + 0.026797987520694733, + 0.0288047157227993 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.7959417104721069, + "style_target": 1, + "support_probability": 0.9549970626831055, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6383476138684603, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11661971360445023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.976443886756897, + 0.09976054728031158, + 0.26295778155326843, + 0.4864577651023865, + 0.12521596252918243, + 0.10785716027021408, + 0.18931527435779572, + 0.07339226454496384, + 0.10822861641645432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.27203285694122314, + "style_target": 0, + "support_probability": 0.9502399563789368, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6233426671180315, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.055829472839832306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.41840600967407227, + 0.18434081971645355, + 0.9091403484344482, + 0.781397819519043, + 0.06755635142326355, + 0.06892238557338715, + 0.0548611618578434, + 0.05109681189060211, + 0.04308906942605972, + 0.06752849370241165 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 1, + "style_probability": 0.679695725440979, + "style_target": 1, + "support_probability": 0.5722121000289917, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1653198461472639, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18469423055648804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4292236864566803, + 0.16349078714847565, + 0.818120539188385, + 0.7106248736381531, + 0.0865282490849495, + 0.09458865970373154, + 0.10167059302330017, + 0.19604423642158508, + 0.2605940103530884, + 0.06948153674602509, + 0.08208081126213074 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 1, + "style_probability": 0.42853784561157227, + "style_target": 0, + "support_probability": 0.2262057065963745, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2425348427746383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.595089852809906, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4221670627593994, + 0.3226549029350281, + 0.4221670627593994, + 0.7857192158699036, + 0.19437259435653687 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.877640962600708, + "style_target": 1, + "support_probability": 0.3086813986301422, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030208893032404344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8400238156318665, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3754047453403473, + 0.31441909074783325, + 0.3754047453403473, + 0.736973226070404, + 0.10040119290351868, + 0.13773077726364136, + 0.151832714676857, + 0.05475788190960884, + 0.06475138664245605 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.32674890756607056, + "style_target": 0, + "support_probability": 0.04069482535123825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35331569905512017, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4429084360599518, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35094478726387024, + 0.28954440355300903, + 0.9114324450492859, + 0.47826051712036133, + 0.09241665154695511, + 0.08252924680709839 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.6390838623046875, + "style_target": 1, + "support_probability": 0.5525787472724915, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04103653348299373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.49157825112342834, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3649681806564331, + 0.34234854578971863, + 0.9275583624839783, + 0.4699292778968811, + 0.10175178200006485, + 0.10812577605247498 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.14494699239730835, + "style_target": 0, + "support_probability": 0.5179569125175476, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7614346077133448, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.160835400223732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9569045305252075, + 0.3456927239894867, + 0.32386714220046997, + 0.17064303159713745, + 0.16727814078330994, + 0.13138417899608612, + 0.0509280189871788, + 0.08474283665418625, + 0.09067641943693161 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 1, + "style_probability": 0.8190048336982727, + "style_target": 1, + "support_probability": 0.6703376770019531, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4873145467895962, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15479177236557007, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9490247368812561, + 0.3288348913192749, + 0.32292425632476807, + 0.15283378958702087, + 0.12273130565881729, + 0.20068877935409546, + 0.12191805988550186, + 0.13197818398475647, + 0.08268488198518753 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 0, + "style_probability": 0.5232424736022949, + "style_target": 0, + "support_probability": 0.6077598929405212, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018501181701660242, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9613772630691528, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47281020879745483, + 0.31021106243133545, + 0.5552278757095337, + 0.7508563995361328, + 0.05935077369213104, + 0.15979072451591492, + 0.3573370575904846, + 0.20705829560756683, + 0.08984766155481339, + 0.07172488421201706, + 0.09789978712797165 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.6028117537498474, + "style_target": 1, + "support_probability": 0.02421407774090767, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007343803245294024, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9808945059776306, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4773962199687958, + 0.3040452003479004, + 0.5491712689399719, + 0.6991839408874512, + 0.08363278955221176, + 0.053163718432188034 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.1450638473033905, + "style_target": 0, + "support_probability": 0.011778652667999268, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1929827560003555, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2949885129928589, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2527530789375305, + 0.8572049736976624, + 0.2772887051105499, + 0.323134183883667, + 0.07648703455924988 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.507451593875885, + "style_target": 1, + "support_probability": 0.22566454112529755, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19983520301913593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27906858921051025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24596019089221954, + 0.8723735809326172, + 0.2905690371990204, + 0.31341394782066345, + 0.042765501886606216, + 0.058331213891506195, + 0.10706707090139389, + 0.09984248131513596, + 0.08956152945756912 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.358316034078598, + "style_target": 0, + "support_probability": 0.2422344982624054, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5917250393204215, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06565201282501221, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12956753373146057, + 0.18846723437309265, + 0.10102076828479767, + 0.9044241309165955, + 0.05529474467039108, + 0.06799642741680145 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 0, + "style_probability": 0.37293151021003723, + "style_target": 1, + "support_probability": 0.7724306583404541, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19339999893228554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06943102926015854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15293192863464355, + 0.21009273827075958, + 0.14568105340003967, + 0.9745662212371826, + 0.08044832199811935, + 0.09056450426578522 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 1, + "style_probability": 0.11662167310714722, + "style_target": 0, + "support_probability": 0.8785034418106079, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4029816144144351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3101062774658203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7656283974647522, + 0.15861175954341888, + 0.29913225769996643, + 0.2718774676322937, + 0.11538802832365036, + 0.09150271117687225, + 0.23567833006381989, + 0.13719360530376434, + 0.08353271335363388 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 1, + "style_probability": 0.531580924987793, + "style_target": 1, + "support_probability": 0.4108169674873352, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6678699313663115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22148238122463226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.812881588935852, + 0.18314693868160248, + 0.3136942982673645, + 0.2775617241859436, + 0.07138325273990631, + 0.08093281090259552, + 0.16396082937717438, + 0.13162027299404144, + 0.08955147862434387 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 0, + "style_probability": 0.763102650642395, + "style_target": 0, + "support_probability": 0.6882756352424622, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3850881350154915, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4768535792827606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7488197088241577, + 0.17489071190357208, + 0.5907888412475586, + 0.4172584116458893, + 0.09355837851762772, + 0.08697395026683807, + 0.09815603494644165, + 0.09929686784744263, + 0.0840984582901001, + 0.0884559154510498, + 0.05572185665369034 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 0, + "style_probability": 0.4244859516620636, + "style_target": 1, + "support_probability": 0.6516985893249512, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36380524868707315, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.38658177852630615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7268899083137512, + 0.17845304310321808, + 0.5588924288749695, + 0.3596836030483246, + 0.06619707494974136, + 0.08298838138580322 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 1, + "style_probability": 0.30789196491241455, + "style_target": 0, + "support_probability": 0.6176995635032654, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.495209447748573, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3914351761341095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7269360423088074, + 0.4602956175804138, + 0.27709826827049255, + 0.4602956175804138, + 0.13244065642356873 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 1, + "style_probability": 0.5249189734458923, + "style_target": 1, + "support_probability": 0.7045767307281494, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2354755623755801, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3093012571334839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8163073062896729, + 0.4900408685207367, + 0.3851083219051361, + 0.4900408685207367, + 0.08362095803022385 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 1, + "style_probability": 0.23642945289611816, + "style_target": 0, + "support_probability": 0.7531190514564514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22318625881738505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20315925776958466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5336629748344421, + 0.43177640438079834, + 0.5336629748344421, + 0.7756898403167725, + 0.07030793279409409, + 0.061763063073158264 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.45106223225593567, + "style_target": 1, + "support_probability": 0.3155667185783386, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2676933950958837, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1829746812582016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.48961323499679565, + 0.4159712791442871, + 0.48961323499679565, + 0.7859671115875244, + 0.10680612921714783, + 0.09799758344888687 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.6778574585914612, + "style_target": 0, + "support_probability": 0.27858781814575195, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5270335737616801, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15991121530532837, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43140318989753723, + 0.44633057713508606, + 0.6770495176315308, + 0.6532624363899231, + 0.06284233182668686, + 0.06366059929132462, + 0.03543418273329735, + 0.04592243209481239, + 0.047735873609781265 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 1, + "style_probability": 0.5085112452507019, + "style_target": 1, + "support_probability": 0.923662006855011, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.566192666654239, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08170977234840393, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39952653646469116, + 0.4048640727996826, + 0.7449148297309875, + 0.6362370252609253, + 0.04688180610537529 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 1, + "style_probability": 0.3679461181163788, + "style_target": 0, + "support_probability": 0.9334319233894348, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7002255568983289, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12683013081550598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22490860521793365, + 0.25423625111579895, + 0.9751816391944885, + 0.2569127678871155, + 0.24699799716472626, + 0.06666197627782822, + 0.09927520900964737, + 0.05783858895301819, + 0.051872771233320236, + 0.09079567342996597 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.5600031614303589, + "style_target": 1, + "support_probability": 0.6487204432487488, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4241881583837756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1224001795053482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20623674988746643, + 0.2537989914417267, + 0.9962342381477356, + 0.2912609875202179, + 0.08880064636468887, + 0.06644292920827866, + 0.2246316373348236, + 0.09255968034267426, + 0.10886698961257935, + 0.11513902992010117 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.21594998240470886, + "style_target": 0, + "support_probability": 0.6225134134292603, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08469992193464633, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.852597177028656, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3178558647632599, + 0.4797472059726715, + 0.190957173705101, + 0.2938048243522644, + 0.044318798929452896 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.8724417686462402, + "style_target": 1, + "support_probability": 0.05675527825951576, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027664494592740275, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8985071182250977, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3168239891529083, + 0.39910537004470825, + 0.16439515352249146, + 0.2832208573818207, + 0.058713894337415695, + 0.12933996319770813, + 0.07761728018522263, + 0.11639342457056046, + 0.09093122184276581 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.44101062417030334, + "style_target": 0, + "support_probability": 0.0402425192296505, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.820583037645356, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09117698669433594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2099694311618805, + 0.9061622619628906, + 0.34577977657318115, + 0.2970685362815857, + 0.03427830711007118, + 0.02538902871310711 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.9052749276161194, + "style_target": 1, + "support_probability": 0.9074387550354004, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28609609375336403, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1189703643321991, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.285082072019577, + 0.9678373336791992, + 0.45466887950897217, + 0.3353433310985565, + 0.05863845348358154, + 0.09221330285072327 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.23602283000946045, + "style_target": 0, + "support_probability": 0.8572178483009338, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8484964684506682, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07244042307138443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.942168116569519, + 0.10572123527526855, + 0.2082699090242386, + 0.48859548568725586, + 0.02339732088148594, + 0.04029085487127304, + 0.0475655272603035, + 0.04981661215424538, + 0.03306933864951134 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 1, + "style_probability": 0.8780171275138855, + "style_target": 1, + "support_probability": 0.8701449632644653, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7891642730530619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09183710813522339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9081111550331116, + 0.10888563096523285, + 0.182390034198761, + 0.4341862201690674, + 0.060750339180231094, + 0.03831498697400093, + 0.05089878663420677, + 0.1474711149930954, + 0.05469786375761032 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 0, + "style_probability": 0.7178374528884888, + "style_target": 0, + "support_probability": 0.8740573525428772, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7049201813232444, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12406772375106812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24673444032669067, + 0.39169809222221375, + 0.33756983280181885, + 0.7786404490470886, + 0.05140770971775055, + 0.04623842611908913, + 0.05112960562109947, + 0.08295485377311707, + 0.043153997510671616, + 0.038439683616161346, + 0.03352902829647064 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.871506929397583, + "style_target": 1, + "support_probability": 0.9010605812072754, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5321345237775493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20466600358486176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2255241721868515, + 0.33816075325012207, + 0.3150525987148285, + 0.7572917938232422, + 0.09438256919384003, + 0.14124028384685516 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.43674352765083313, + "style_target": 0, + "support_probability": 0.7685051560401917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23158173935017798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12407136708498001, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33139365911483765, + 0.7514857649803162, + 0.17980997264385223, + 0.28388527035713196, + 0.12210814654827118 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.7106941342353821, + "style_target": 1, + "support_probability": 0.4518641233444214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047954358580100276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17042720317840576, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37461236119270325, + 0.7956176400184631, + 0.2316979169845581, + 0.31509724259376526, + 0.15012295544147491 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.3080393075942993, + "style_target": 0, + "support_probability": 0.4429336190223694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8530341222523629, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07936376333236694, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33629170060157776, + 0.2350434809923172, + 0.9956272840499878, + 0.3200990557670593, + 0.16759394109249115, + 0.07254081964492798 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 1, + "style_probability": 0.8273324966430664, + "style_target": 1, + "support_probability": 0.8300604820251465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7774573611606768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10456984490156174, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32549622654914856, + 0.2728997468948364, + 0.9807158708572388, + 0.3447469472885132, + 0.16944967210292816, + 0.05834450572729111 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 0, + "style_probability": 0.7920145392417908, + "style_target": 0, + "support_probability": 0.706797182559967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3233406995101528, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8332202434539795, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42845624685287476, + 0.3753300607204437, + 0.24771960079669952, + 0.3753300607204437, + 0.0790189877152443, + 0.10096641629934311, + 0.0987439826130867, + 0.20104822516441345, + 0.11472031474113464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.9027687907218933, + "style_target": 1, + "support_probability": 0.23488517105579376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03703013266692184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9088852405548096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4927840828895569, + 0.3096012473106384, + 0.2584713101387024, + 0.3096012473106384, + 0.0871986597776413 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.24222001433372498, + "style_target": 0, + "support_probability": 0.08079439401626587, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7945755177941775, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0549369677901268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.49086064100265503, + 0.7200971841812134, + 0.8538098335266113, + 0.41227585077285767, + 0.12564244866371155, + 0.0683048740029335, + 0.048838190734386444, + 0.06380289047956467, + 0.07830758392810822, + 0.17815884947776794, + 0.06876322627067566 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.8531522154808044, + "style_target": 1, + "support_probability": 0.9663217067718506, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18143537596074594, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10412878543138504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.535054087638855, + 0.7954216599464417, + 0.9138607978820801, + 0.5747014284133911, + 0.13867822289466858, + 0.04853722080588341, + 0.1465061753988266, + 0.110587477684021, + 0.11344600468873978, + 0.19998857378959656 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.055539119988679886, + "style_target": 0, + "support_probability": 0.9396896958351135, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20307178916255772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.872918426990509, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4127967655658722, + 0.9285984039306641, + 0.6833730340003967, + 0.12652356922626495, + 0.05059295892715454 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 0, + "style_probability": 0.4666527211666107, + "style_target": 1, + "support_probability": 0.2735280990600586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2268939213793323, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7457410097122192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5142130255699158, + 0.8695662021636963, + 0.6287608742713928, + 0.15327142179012299, + 0.21003587543964386 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 1, + "style_probability": 0.2907411456108093, + "style_target": 0, + "support_probability": 0.38391196727752686, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9243638193085285, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11427247524261475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5531871318817139, + 0.3599194586277008, + 0.19455847144126892, + 0.9893769025802612, + 0.09024090319871902, + 0.07441222667694092 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 1, + "style_probability": 0.9250239133834839, + "style_target": 1, + "support_probability": 0.850470244884491, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6875837572886022, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23672328889369965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5236514806747437, + 0.3581617474555969, + 0.18316517770290375, + 0.9748285412788391, + 0.09551527351140976, + 0.06234637647867203, + 0.08901908248662949, + 0.08009450882673264, + 0.05950477346777916, + 0.06417421996593475 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 0, + "style_probability": 0.6941925883293152, + "style_target": 0, + "support_probability": 0.6608573794364929, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6431381718456007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19150714576244354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22962713241577148, + 0.15078622102737427, + 0.2293778359889984, + 0.9440234303474426, + 0.05890730768442154, + 0.10485188663005829, + 0.11145912110805511, + 0.09782623499631882, + 0.05093405395746231 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.7500709891319275, + "style_target": 1, + "support_probability": 0.7247055172920227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16171247789700635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20595617592334747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2539556324481964, + 0.19384019076824188, + 0.29977092146873474, + 0.9700888991355896, + 0.19004397094249725, + 0.23837131261825562, + 0.133419468998909, + 0.05036112293601036, + 0.11343245953321457 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.20589134097099304, + "style_target": 0, + "support_probability": 0.7973464131355286, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11548816198273511, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8347119688987732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2124348282814026, + 0.7630913257598877, + 0.17627044022083282, + 0.5695540904998779, + 0.07338101416826248, + 0.05148995667695999, + 0.1260930597782135, + 0.28581884503364563, + 0.081573985517025, + 0.2280728965997696, + 0.032157305628061295 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.8559471368789673, + "style_target": 1, + "support_probability": 0.042068105190992355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023777651879492052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9093636870384216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1877424716949463, + 0.7736678123474121, + 0.19726260006427765, + 0.49951767921447754, + 0.052193302661180496, + 0.04970564320683479, + 0.24999332427978516, + 0.04917031154036522, + 0.08569122850894928, + 0.06931939721107483, + 0.16527192294597626 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.4041920304298401, + "style_target": 0, + "support_probability": 0.02255273424088955, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7708138360166118, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21179336309432983, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.892450213432312, + 0.11987441033124924, + 0.4322100877761841, + 0.6827334761619568, + 0.06759663671255112 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.8350620269775391, + "style_target": 1, + "support_probability": 0.8906817436218262, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6637401793083878, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17585110664367676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8552811741828918, + 0.15772736072540283, + 0.4283809959888458, + 0.7299063801765442, + 0.045883264392614365, + 0.12836813926696777, + 0.19931413233280182, + 0.13329839706420898, + 0.10804864019155502 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.4468801021575928, + "style_target": 0, + "support_probability": 0.9161656498908997, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.775820650902795, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.023019343614578247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4408383071422577, + 0.18771767616271973, + 0.9233245849609375, + 0.24802061915397644, + 0.12627193331718445, + 0.047344885766506195 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.8213422894477844, + "style_target": 1, + "support_probability": 0.7253298759460449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1854437503778995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.021297745406627655, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41833043098449707, + 0.24420702457427979, + 0.9547783136367798, + 0.22446146607398987, + 0.07819739729166031, + 0.11008022725582123 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.22019849717617035, + "style_target": 0, + "support_probability": 0.7842974066734314, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9050818401003432, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08160705864429474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8213:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32393574714660645, + 0.994440495967865, + 0.2541693150997162, + 0.14235074818134308, + 0.08383061736822128, + 0.0824371725320816, + 0.07301240414381027, + 0.19046397507190704, + 0.07782672345638275 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 1, + "style_probability": 0.7347007989883423, + "style_target": 1, + "support_probability": 0.8974354267120361, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8273865221519842, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057959992438554764, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8213:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31340333819389343, + 0.9942373037338257, + 0.25121578574180603, + 0.1758306622505188, + 0.12766332924365997, + 0.0992504358291626, + 0.1422220766544342, + 0.15249483287334442, + 0.09212137758731842 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 1, + "style_probability": 0.38465026021003723, + "style_target": 0, + "support_probability": 0.9114146828651428, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.046845473590663815, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9337498545646667, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36023691296577454, + 0.4291425347328186, + 0.3269292712211609, + 0.450609028339386, + 0.07988534867763519, + 0.12592341005802155, + 0.1981736719608307, + 0.060258638113737106, + 0.0491873100399971, + 0.35429003834724426, + 0.041848041117191315 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.7270337343215942, + "style_target": 1, + "support_probability": 0.11545949429273605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031200251290857163, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.865797758102417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33088135719299316, + 0.4245138168334961, + 0.29761016368865967, + 0.49301373958587646, + 0.12492744624614716, + 0.08072366565465927 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.2993684411048889, + "style_target": 0, + "support_probability": 0.1210903450846672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03480564691880903, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8017892241477966, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28895434737205505, + 0.6854875087738037, + 0.12735514342784882, + 0.2348756194114685, + 0.04172295704483986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.6353992223739624, + "style_target": 1, + "support_probability": 0.11381485313177109, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001667723751630765, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.868453323841095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2899848520755768, + 0.6458116769790649, + 0.15258188545703888, + 0.18755902349948883, + 0.0464712455868721 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.06947233527898788, + "style_target": 0, + "support_probability": 0.08081452548503876, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9529007943801765, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.010550513863563538, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8231:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4705720841884613, + 0.34805941581726074, + 0.38260525465011597, + 0.9978278279304504, + 0.10582256317138672, + 0.08806690573692322 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 1, + "style_probability": 0.8446581363677979, + "style_target": 1, + "support_probability": 0.9127662181854248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8896409437584376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.012230181135237217, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8231:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.48197659850120544, + 0.30245068669319153, + 0.39981895685195923, + 0.9935789108276367, + 0.09304054081439972, + 0.09349139034748077 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 1, + "style_probability": 0.4926738440990448, + "style_target": 0, + "support_probability": 0.9341469407081604, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5035430329392686, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1547631472349167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.975274920463562, + 0.3912004232406616, + 0.2706272602081299, + 0.3190002143383026, + 0.15058548748493195, + 0.08691678941249847, + 0.16813954710960388, + 0.1430196315050125, + 0.1784714162349701 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.6827290058135986, + "style_target": 1, + "support_probability": 0.5131670236587524, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4195316165160941, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1424509435892105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9651961922645569, + 0.3967016935348511, + 0.2697591185569763, + 0.30791690945625305, + 0.09745384007692337 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.36237233877182007, + "style_target": 0, + "support_probability": 0.532168447971344, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8409478988056658, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1410076767206192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2310711145401001, + 0.20057480037212372, + 0.9369550347328186, + 0.14358055591583252, + 0.052996668964624405, + 0.03363659977912903, + 0.05425911024212837, + 0.1285967081785202, + 0.07414700090885162, + 0.03837256133556366 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.8327021598815918, + "style_target": 1, + "support_probability": 0.7097502946853638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3127777047011032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0814923420548439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2764974534511566, + 0.19822920858860016, + 0.9669623374938965, + 0.1878223568201065, + 0.08604544401168823, + 0.06865251809358597, + 0.19443419575691223, + 0.09284918755292892, + 0.043454743921756744, + 0.08054672926664352 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.17922988533973694, + "style_target": 0, + "support_probability": 0.8226253986358643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.024878890532984112, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6764611601829529, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22079169750213623, + 0.22336812317371368, + 0.5646308064460754, + 0.45246008038520813, + 0.10872601717710495 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 1, + "style_probability": 0.7484232783317566, + "style_target": 1, + "support_probability": 0.02589554525911808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012323965450204146, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6748220324516296, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2566412389278412, + 0.21595703065395355, + 0.6001370549201965, + 0.40585857629776, + 0.13113945722579956 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 1, + "style_probability": 0.25611406564712524, + "style_target": 0, + "support_probability": 0.028486652299761772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04259409061337438, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6414980292320251, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4192233383655548, + 0.2357361912727356, + 0.5609250068664551, + 0.3086094260215759, + 0.10881711542606354, + 0.11484468728303909 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.817962646484375, + "style_target": 1, + "support_probability": 0.022507457062602043, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011180140433174083, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6353991031646729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36118876934051514, + 0.23354573547840118, + 0.5216841697692871, + 0.2985207736492157, + 0.10719969868659973, + 0.05897603556513786, + 0.11428723484277725, + 0.13667066395282745, + 0.09135793149471283, + 0.09442959725856781, + 0.058330636471509933 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.312791109085083, + "style_target": 0, + "support_probability": 0.013077029958367348, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019542481002164214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9825180172920227, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2588143050670624, + 0.6954744458198547, + 0.16746369004249573, + 0.3172212541103363, + 0.14893929660320282, + 0.09922801703214645, + 0.0877138152718544, + 0.08839695900678635, + 0.12755626440048218 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.752731442451477, + "style_target": 1, + "support_probability": 0.014599239453673363, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005352635281067446, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9635521769523621, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3228963613510132, + 0.7353410720825195, + 0.17872413992881775, + 0.4206370413303375, + 0.10317802429199219, + 0.2608066499233246, + 0.17415037751197815, + 0.08534489572048187, + 0.16956827044487 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.2114042341709137, + "style_target": 0, + "support_probability": 0.03181138262152672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005339821719777975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.632311999797821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22581033408641815, + 0.4893859028816223, + 0.212490975856781, + 0.6073247790336609, + 0.22991493344306946, + 0.08997029066085815, + 0.08094869554042816, + 0.11452128738164902, + 0.07502678781747818, + 0.09263020753860474, + 0.1033119261264801 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.5478441119194031, + "style_target": 1, + "support_probability": 0.005730860400944948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004044666254702659, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7058600783348083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23693245649337769, + 0.469983845949173, + 0.2311827391386032, + 0.5802657008171082, + 0.16807107627391815, + 0.11009278893470764, + 0.12254469841718674, + 0.15142224729061127, + 0.11159446835517883, + 0.16472157835960388, + 0.16652509570121765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.3579739034175873, + "style_target": 0, + "support_probability": 0.005425973329693079, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3220378212421051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31279513239860535, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8785435557365417, + 0.1822957694530487, + 0.10876384377479553, + 0.13846467435359955, + 0.04057372733950615 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.8003957271575928, + "style_target": 1, + "support_probability": 0.4574141502380371, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12031206781058904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46542778611183167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8552921414375305, + 0.1965750753879547, + 0.1195085346698761, + 0.13730141520500183, + 0.09279365837574005, + 0.2254599779844284, + 0.1314770132303238, + 0.1763206422328949, + 0.09947478026151657 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.4610992968082428, + "style_target": 0, + "support_probability": 0.18801504373550415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02406792017328658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9419105052947998, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25566810369491577, + 0.5525805950164795, + 0.24322736263275146, + 0.25566810369491577, + 0.07391100376844406, + 0.024675162509083748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.7344666719436646, + "style_target": 1, + "support_probability": 0.014479908160865307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028827455087978206, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9439957737922668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3019690215587616, + 0.6252731084823608, + 0.23179300129413605, + 0.3019690215587616, + 0.10302991420030594, + 0.14695730805397034 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.19426336884498596, + "style_target": 0, + "support_probability": 0.02274257503449917, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7687580741977624, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13482531905174255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9542210698127747, + 0.2565462291240692, + 0.35734066367149353, + 0.2565462291240692, + 0.0712013766169548, + 0.037893425673246384, + 0.01783999614417553, + 0.05453028902411461, + 0.0199219211935997 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 1, + "style_probability": 0.6120060682296753, + "style_target": 1, + "support_probability": 0.6710104942321777, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5820585069064664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24655677378177643, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9049267768859863, + 0.3040439188480377, + 0.37896955013275146, + 0.3040439188480377, + 0.1305413842201233, + 0.07191161066293716, + 0.04698937013745308, + 0.060169849544763565, + 0.07874751836061478 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 0, + "style_probability": 0.51161128282547, + "style_target": 0, + "support_probability": 0.5689278841018677, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05134667841636336, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9675745368003845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2204514741897583, + 0.23337647318840027, + 0.4938725531101227, + 0.2204514741897583, + 0.04736074060201645, + 0.049420375376939774, + 0.06592809408903122, + 0.05868130549788475, + 0.04692010581493378, + 0.04524673894047737, + 0.02411302551627159 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.8964062333106995, + "style_target": 1, + "support_probability": 0.030615471303462982, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015448250100588775, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9531379342079163, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22425490617752075, + 0.1860417127609253, + 0.4582531452178955, + 0.22425490617752075, + 0.030557433143258095, + 0.04758117347955704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.2765788435935974, + "style_target": 0, + "support_probability": 0.03862452134490013, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6122420518878025, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11757639050483704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3503471910953522, + 0.3412034809589386, + 0.1899309605360031, + 0.9260038733482361, + 0.08364792168140411 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.5807704925537109, + "style_target": 1, + "support_probability": 0.8080898523330688, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05139978974745896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2372841238975525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3620512783527374, + 0.37370017170906067, + 0.170512855052948, + 0.9211536049842834, + 0.11392813920974731 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.0855182334780693, + "style_target": 0, + "support_probability": 0.5787087678909302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7972972146470738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03156164288520813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19189558923244476, + 0.20402464270591736, + 0.16797101497650146, + 0.932029664516449, + 0.20143091678619385, + 0.17542718350887299 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 1, + "style_probability": 0.8403577208518982, + "style_target": 1, + "support_probability": 0.7534183859825134, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3692452281010462, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09771483391523361, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18166016042232513, + 0.22055299580097198, + 0.2029365748167038, + 0.8547225594520569, + 0.19626112282276154, + 0.08992114663124084 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 0, + "style_probability": 0.6278066635131836, + "style_target": 0, + "support_probability": 0.35624614357948303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20245174317346318, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8917774558067322, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32297953963279724, + 0.11821864545345306, + 0.25225645303726196, + 0.28541457653045654, + 0.08503580838441849, + 0.1096162274479866, + 0.0666651800274849, + 0.30886733531951904, + 0.15414318442344666 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.828894853591919, + "style_target": 1, + "support_probability": 0.2530822157859802, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030524396322895497, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9010584354400635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2911670207977295, + 0.15022261440753937, + 0.23175691068172455, + 0.28131911158561707, + 0.2112877368927002 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.17178118228912354, + "style_target": 0, + "support_probability": 0.2326933890581131, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2985234097877501, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7485268712043762, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1420893669128418, + 0.21379360556602478, + 0.4685373306274414, + 0.7855814099311829, + 0.09380871057510376, + 0.09548283368349075, + 0.22451356053352356, + 0.1225561574101448, + 0.14541225135326385, + 0.26827049255371094, + 0.1914721578359604 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.7964416742324829, + "style_target": 1, + "support_probability": 0.2196408063173294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09719702889502259, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7490213513374329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1692562699317932, + 0.2639906406402588, + 0.6517526507377625, + 0.8318087458610535, + 0.2726503014564514, + 0.15349049866199493, + 0.2604061961174011, + 0.17174625396728516, + 0.08480007201433182, + 0.22899901866912842 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.251467764377594, + "style_target": 0, + "support_probability": 0.5343158841133118, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12089326451802962, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7373825907707214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6588783264160156, + 0.33164194226264954, + 0.21907870471477509, + 0.25944703817367554, + 0.1500619500875473 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 1, + "style_probability": 0.751660168170929, + "style_target": 1, + "support_probability": 0.14042897522449493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11570910171723298, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6110031604766846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6362189650535583, + 0.35558491945266724, + 0.2018633633852005, + 0.24234741926193237, + 0.13571307063102722 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 1, + "style_probability": 0.4552094042301178, + "style_target": 0, + "support_probability": 0.2497873604297638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5394554504553054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4302482306957245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17536702752113342, + 0.2744867503643036, + 0.2365301549434662, + 0.7480064034461975, + 0.06021314486861229, + 0.09853804856538773 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.7859416007995605, + "style_target": 1, + "support_probability": 0.5399640798568726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28225296142176787, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6370292901992798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18683582544326782, + 0.2854505777359009, + 0.2490343302488327, + 0.7055830359458923, + 0.058464694768190384, + 0.09235688298940659, + 0.09766539931297302, + 0.04037697613239288, + 0.03611527755856514, + 0.056569915264844894, + 0.1699831187725067 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.48335519433021545, + "style_target": 0, + "support_probability": 0.28578829765319824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04968343022847519, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8541628122329712, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3106566369533539, + 0.28969815373420715, + 0.3771592974662781, + 0.26872360706329346, + 0.08866044133901596, + 0.053195711225271225, + 0.07010424137115479, + 0.07299607992172241, + 0.07944654673337936 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.714767575263977, + "style_target": 1, + "support_probability": 0.055085934698581696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009558893467718581, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8509989976882935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3139839768409729, + 0.34378278255462646, + 0.43327799439430237, + 0.28410953283309937, + 0.06733346730470657, + 0.34350815415382385, + 0.08661171793937683, + 0.07431995868682861, + 0.09955062717199326 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.2769638001918793, + "style_target": 0, + "support_probability": 0.055864118039608, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6355431625723601, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13494907319545746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5091708302497864, + 0.6819835901260376, + 0.4420122802257538, + 0.3981139063835144, + 0.1418946236371994, + 0.07846780121326447, + 0.05841154232621193, + 0.0635465458035469, + 0.050284020602703094, + 0.06483615934848785, + 0.06505001336336136 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 1, + "style_probability": 0.8322133421897888, + "style_target": 1, + "support_probability": 0.9520773887634277, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4680130794010718, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3879202604293823, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43388739228248596, + 0.5950964689254761, + 0.51194167137146, + 0.44866228103637695, + 0.060771483927965164, + 0.09403729438781738, + 0.07718836516141891, + 0.10094986110925674, + 0.039777062833309174, + 0.053240157663822174, + 0.0915178433060646 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 1, + "style_probability": 0.4454230070114136, + "style_target": 0, + "support_probability": 0.9138625264167786, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2971679681909759, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48623859882354736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8139849901199341, + 0.2569175362586975, + 0.25276097655296326, + 0.2359633892774582, + 0.1528700292110443 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 0, + "style_probability": 0.4639642536640167, + "style_target": 1, + "support_probability": 0.3926500082015991, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5586034288322922, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16285410523414612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8866227269172668, + 0.2557848393917084, + 0.24724586308002472, + 0.22553808987140656, + 0.0747830718755722 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 0, + "style_probability": 0.6916795372962952, + "style_target": 0, + "support_probability": 0.6821459531784058, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03431049506663542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9305098056793213, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5869501233100891, + 0.25783225893974304, + 0.15721583366394043, + 0.48590683937072754, + 0.3040310740470886, + 0.08613559603691101 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 0, + "style_probability": 0.430776983499527, + "style_target": 1, + "support_probability": 0.04706251993775368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018692869956970577, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9776414036750793, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6157476305961609, + 0.23496779799461365, + 0.14228200912475586, + 0.4698230028152466, + 0.08272559940814972, + 0.1460515856742859, + 0.11458607017993927, + 0.13219282031059265, + 0.05665811896324158, + 0.11171243339776993, + 0.09695877879858017 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 1, + "style_probability": 0.2815673053264618, + "style_target": 0, + "support_probability": 0.026278605684638023, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6356841184564072, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08694086223840714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8384:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8951818943023682, + 0.3225470781326294, + 0.19754305481910706, + 0.3516487777233124, + 0.07828666269779205, + 0.10240433365106583, + 0.08354562520980835, + 0.05529769882559776, + 0.06405918300151825 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.5198332667350769, + "style_target": 1, + "support_probability": 0.7866069674491882, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11235082800705308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09342091530561447, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8384:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9757716059684753, + 0.43785998225212097, + 0.2260427474975586, + 0.2983434796333313, + 0.08059331774711609, + 0.130304217338562, + 0.11397094279527664, + 0.13438725471496582, + 0.09742356091737747 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.1379079967737198, + "style_target": 0, + "support_probability": 0.7479478120803833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005438140115907972, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9668892025947571, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42811310291290283, + 0.3832493722438812, + 0.2451879382133484, + 0.516665518283844, + 0.10477388650178909, + 0.0867304652929306, + 0.14647604525089264, + 0.09505512565374374, + 0.13195613026618958, + 0.1432826966047287, + 0.08742315322160721 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 1, + "style_probability": 0.5040132999420166, + "style_target": 1, + "support_probability": 0.008087515830993652, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0070274345476333926, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9521809220314026, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4189651310443878, + 0.31622079014778137, + 0.2421594262123108, + 0.5601881742477417, + 0.11368008702993393, + 0.1511581391096115, + 0.07952570915222168, + 0.11238684505224228, + 0.08867934346199036, + 0.07294783741235733, + 0.1890084445476532 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 0, + "style_probability": 0.6271222233772278, + "style_target": 0, + "support_probability": 0.011717159301042557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022083017564794544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9217410683631897, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36179059743881226, + 0.43208402395248413, + 0.5111001133918762, + 0.5779616832733154, + 0.20243464410305023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.6933889389038086, + "style_target": 1, + "support_probability": 0.034773655235767365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0027842397039670196, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.947810173034668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3641604483127594, + 0.48990243673324585, + 0.6324867010116577, + 0.6660512685775757, + 0.09191536158323288 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.11044943332672119, + "style_target": 0, + "support_probability": 0.03163726255297661, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3278168293933007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11355981230735779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29049989581108093, + 0.456966757774353, + 0.38015684485435486, + 0.8657592535018921, + 0.07925166934728622, + 0.09060677140951157 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 1, + "style_probability": 0.6634437441825867, + "style_target": 1, + "support_probability": 0.44679102301597595, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3119976791846383, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10329605638980865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28530439734458923, + 0.46346884965896606, + 0.36045774817466736, + 0.8479059338569641, + 0.04077053442597389, + 0.08422410488128662 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 0, + "style_probability": 0.5463653206825256, + "style_target": 0, + "support_probability": 0.4206458032131195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.434486815813003, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7349311113357544, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2361539602279663, + 0.6263381242752075, + 0.5357786417007446, + 0.3929685652256012, + 0.12305571883916855, + 0.03439411148428917, + 0.027667254209518433, + 0.03701382502913475, + 0.07892753183841705 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 1, + "style_probability": 0.8573111891746521, + "style_target": 1, + "support_probability": 0.29453685879707336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3159024182474053, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6407167315483093, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26322141289711, + 0.6052823066711426, + 0.45487794280052185, + 0.4220874011516571, + 0.041948579251766205 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 0, + "style_probability": 0.5086808204650879, + "style_target": 0, + "support_probability": 0.38009390234947205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4974504420421068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08012699335813522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24073801934719086, + 0.383935809135437, + 0.48442816734313965, + 0.9442786574363708, + 0.09996148943901062, + 0.09704653918743134, + 0.06285293400287628, + 0.061125461012125015, + 0.07872812449932098, + 0.11262714117765427 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.6572098135948181, + "style_target": 1, + "support_probability": 0.6650698781013489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22256634544072226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08961459994316101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25653427839279175, + 0.44477954506874084, + 0.5293517112731934, + 0.9653122425079346, + 0.11800198256969452, + 0.08339804410934448, + 0.2006252259016037, + 0.1449609398841858, + 0.08499127626419067, + 0.12448569387197495 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.29747214913368225, + "style_target": 0, + "support_probability": 0.7632521986961365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06809517233190578, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5820977687835693, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3899897336959839, + 0.2320707142353058, + 0.37752896547317505, + 0.3875475525856018, + 0.05560177192091942 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 1, + "style_probability": 0.5010667443275452, + "style_target": 1, + "support_probability": 0.14795294404029846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006614600091875795, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7698612213134766, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4908921718597412, + 0.24525325000286102, + 0.46641966700553894, + 0.31029778718948364, + 0.08374088257551193 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 1, + "style_probability": 0.18905076384544373, + "style_target": 0, + "support_probability": 0.08406051248311996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07054180731125173, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4091729521751404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3254576027393341, + 0.7930467128753662, + 0.6914513111114502, + 0.2828282415866852, + 0.07867833226919174, + 0.18037953972816467 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 1, + "style_probability": 0.7051845788955688, + "style_target": 1, + "support_probability": 0.06045762822031975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14011342804753657, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43088778853416443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32467976212501526, + 0.8069689273834229, + 0.6921457648277283, + 0.27543628215789795, + 0.06360913068056107, + 0.06146054342389107 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 0, + "style_probability": 0.8841983079910278, + "style_target": 0, + "support_probability": 0.04811335355043411, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.23611959214296993, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29240715503692627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7040466666221619, + 0.6022951006889343, + 0.26615196466445923, + 0.5155214667320251, + 0.06795941293239594, + 0.024386459961533546, + 0.09702228754758835, + 0.05101342871785164, + 0.036478493362665176 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 0, + "style_probability": 0.33335599303245544, + "style_target": 1, + "support_probability": 0.30649393796920776, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31152980516816736, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18009032309055328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7142587900161743, + 0.5942012071609497, + 0.23165534436702728, + 0.5945844650268555, + 0.06759660691022873 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 1, + "style_probability": 0.19937489926815033, + "style_target": 0, + "support_probability": 0.5280762314796448, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041436969946441624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.743114709854126, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7902919054031372, + 0.354955792427063, + 0.5452895760536194, + 0.25473806262016296, + 0.07006406038999557, + 0.06583651900291443, + 0.08953481912612915, + 0.051050953567028046, + 0.2625773847103119, + 0.13310572504997253, + 0.08990606665611267 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.5139186382293701, + "style_target": 1, + "support_probability": 0.03359663486480713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036223078929234265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45345914363861084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8817197680473328, + 0.3686404228210449, + 0.5715968608856201, + 0.21281524002552032, + 0.1304250955581665, + 0.11804154515266418, + 0.35561543703079224, + 0.11492649465799332, + 0.08304331451654434, + 0.10246867686510086 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.16789820790290833, + "style_target": 0, + "support_probability": 0.20788900554180145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15198735102560687, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8648622035980225, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2827594578266144, + 0.36864784359931946, + 0.18563710153102875, + 0.7875210642814636, + 0.09786766022443771 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.7236611843109131, + "style_target": 1, + "support_probability": 0.09558757394552231, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07012403356079633, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8577163219451904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2746780216693878, + 0.32042258977890015, + 0.15919889509677887, + 0.7763793468475342, + 0.1781916469335556, + 0.12240584939718246, + 0.16270478069782257, + 0.13615496456623077, + 0.35167282819747925 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.21021611988544464, + "style_target": 0, + "support_probability": 0.03967771306633949, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04304868957868479, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8861796259880066, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5769684314727783, + 0.40899658203125, + 0.8303611278533936, + 0.1580355316400528, + 0.3186272382736206, + 0.03861059620976448 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.560789942741394, + "style_target": 1, + "support_probability": 0.06015696004033089, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008887056785693785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9350144863128662, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5414375066757202, + 0.46186894178390503, + 0.8558971881866455, + 0.221034437417984, + 0.12171468138694763, + 0.08218308538198471 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.14068424701690674, + "style_target": 0, + "support_probability": 0.06542299687862396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049637780367878366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45531412959098816, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6895949840545654, + 0.3413473963737488, + 0.26862025260925293, + 0.37875303626060486, + 0.1722012460231781, + 0.1911211907863617, + 0.12299451977014542, + 0.16991926729679108, + 0.21194738149642944 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.789048433303833, + "style_target": 1, + "support_probability": 0.03932952135801315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012395004725690092, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6977097988128662, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5907400846481323, + 0.2993840277194977, + 0.2491026669740677, + 0.3521316945552826, + 0.13568180799484253, + 0.06037456914782524, + 0.05594667047262192, + 0.07181736081838608, + 0.10886459797620773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.2974694073200226, + "style_target": 0, + "support_probability": 0.015136413276195526, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4997272702197246, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06020834296941757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9350429773330688, + 0.12614215910434723, + 0.1663888692855835, + 0.14936548471450806, + 0.032013483345508575, + 0.026980845257639885, + 0.09889335185289383, + 0.12660431861877441, + 0.09358763694763184, + 0.14940008521080017, + 0.1332729309797287 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.620736837387085, + "style_target": 1, + "support_probability": 0.4215148687362671, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2771208231119552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.049556516110897064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9098281264305115, + 0.13108442723751068, + 0.1885918378829956, + 0.17731830477714539, + 0.044749267399311066, + 0.03947705030441284 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.1833869218826294, + "style_target": 0, + "support_probability": 0.4803304970264435, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09754044809886986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7686787843704224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5036196112632751, + 0.5093608498573303, + 0.4199932813644409, + 0.5036196112632751, + 0.05157852917909622 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.6990214586257935, + "style_target": 1, + "support_probability": 0.05149706453084946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014933207291330143, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8056207299232483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7115082740783691, + 0.5489365458488464, + 0.46299365162849426, + 0.7115082740783691, + 0.07853423804044724 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.16161467134952545, + "style_target": 0, + "support_probability": 0.04633890092372894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007981238380864767, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9507272839546204, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27657419443130493, + 0.3042804002761841, + 0.35850977897644043, + 0.34609025716781616, + 0.11525840312242508, + 0.08864010125398636 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 1, + "style_probability": 0.6190003156661987, + "style_target": 1, + "support_probability": 0.020252563059329987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03537212861590756, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.806564211845398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2331712543964386, + 0.3054867684841156, + 0.4058383107185364, + 0.41960421204566956, + 0.21086657047271729, + 0.07663588970899582 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 0, + "style_probability": 0.5123075246810913, + "style_target": 0, + "support_probability": 0.06548814475536346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6809905118817209, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14747975766658783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3429534137248993, + 0.2530471086502075, + 0.3429534137248993, + 0.8147397637367249, + 0.02556137926876545, + 0.08975493162870407, + 0.056423865258693695, + 0.058666713535785675, + 0.04695500433444977 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.7564720511436462, + "style_target": 1, + "support_probability": 0.7701396942138672, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4071658733865481, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.22981369495391846, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34639856219291687, + 0.2504071891307831, + 0.34639856219291687, + 0.7936545610427856, + 0.08460785448551178 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.26794368028640747, + "style_target": 0, + "support_probability": 0.7119472026824951, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6516413728092524, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6679378151893616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15568101406097412, + 0.9099713563919067, + 0.15568101406097412, + 0.30977532267570496, + 0.06462566554546356, + 0.035098444670438766, + 0.058379556983709335, + 0.03120926208794117, + 0.05465785041451454, + 0.07129217684268951, + 0.07922801375389099 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.6139377951622009, + "style_target": 1, + "support_probability": 0.5249464511871338, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29606420199459416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6727240681648254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1716298758983612, + 0.9622430801391602, + 0.1716298758983612, + 0.3489944338798523, + 0.14021775126457214, + 0.06165977194905281, + 0.4018106162548065, + 0.09988583624362946, + 0.03621182590723038, + 0.08086968213319778 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.21875625848770142, + "style_target": 0, + "support_probability": 0.6566591858863831, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12060733047742532, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8569837808609009, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2974327802658081, + 0.2911827862262726, + 0.8097519874572754, + 0.2974327802658081, + 0.09414348006248474 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.7148776650428772, + "style_target": 1, + "support_probability": 0.10385779291391373, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0683793358545413, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8136503100395203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3292762339115143, + 0.3388898968696594, + 0.9183952808380127, + 0.3292762339115143, + 0.10661749541759491 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.42677024006843567, + "style_target": 0, + "support_probability": 0.16813115775585175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3767669790763982, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2869510352611542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33805572986602783, + 0.19324491918087006, + 0.20766127109527588, + 0.9825709462165833, + 0.06918495148420334, + 0.11609318107366562 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 0, + "style_probability": 0.30815577507019043, + "style_target": 1, + "support_probability": 0.655059814453125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31715674091821067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3766030967235565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3475337624549866, + 0.21690718829631805, + 0.2018970251083374, + 0.9927998781204224, + 0.03593500703573227, + 0.09060372412204742 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 1, + "style_probability": 0.4122268259525299, + "style_target": 0, + "support_probability": 0.5683074593544006, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.10012605967031643, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45276564359664917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2444484978914261, + 0.1425696462392807, + 0.3652854561805725, + 0.8868860602378845, + 0.07407160103321075, + 0.10676794499158859, + 0.08416447043418884, + 0.06130695343017578, + 0.17221428453922272 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 0, + "style_probability": 0.30825117230415344, + "style_target": 1, + "support_probability": 0.15385614335536957, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07402989694618699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5567033886909485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24626238644123077, + 0.1607847958803177, + 0.376939982175827, + 0.8915743231773376, + 0.09967358410358429 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 1, + "style_probability": 0.27728697657585144, + "style_target": 0, + "support_probability": 0.13508924841880798, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07565980387009476, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.721116840839386, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5854969024658203, + 0.3032402992248535, + 0.31249693036079407, + 0.3032402992248535, + 0.07386015355587006, + 0.0916205495595932, + 0.08939401060342789, + 0.11811160296201706, + 0.05411917343735695, + 0.15358954668045044 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 1, + "style_probability": 0.521277904510498, + "style_target": 1, + "support_probability": 0.19218511879444122, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03319801765964351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5869215726852417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7818465232849121, + 0.33743834495544434, + 0.3904300332069397, + 0.33743834495544434, + 0.10965458303689957, + 0.1621537059545517, + 0.12600137293338776, + 0.05764217674732208, + 0.08216983079910278, + 0.11079008877277374 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 1, + "style_probability": 0.3238922953605652, + "style_target": 0, + "support_probability": 0.361354798078537, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32000757490076603, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7224524617195129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3720413148403168, + 0.8902963399887085, + 0.2957473397254944, + 0.31786876916885376, + 0.0839599072933197 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.901622474193573, + "style_target": 1, + "support_probability": 0.1480208933353424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05284157645614737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9087748527526855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37033605575561523, + 0.8591713309288025, + 0.30840691924095154, + 0.30642253160476685, + 0.1229851245880127, + 0.08958182483911514, + 0.14126750826835632, + 0.08001435548067093, + 0.14703519642353058 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.4220413267612457, + "style_target": 0, + "support_probability": 0.039477184414863586, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.66466324971291, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3273216784000397, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9705916047096252, + 0.23443962633609772, + 0.5674598217010498, + 0.23443962633609772, + 0.09576887637376785, + 0.09622344374656677 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.7509779930114746, + "style_target": 1, + "support_probability": 0.6037242412567139, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17546730191763943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3780885338783264, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.984430730342865, + 0.24488599598407745, + 0.6482710838317871, + 0.24488599598407745, + 0.049651481211185455, + 0.13059177994728088 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.3265882134437561, + "style_target": 0, + "support_probability": 0.5867114663124084, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8400702892832058, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1643325686454773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9124308228492737, + 0.46893852949142456, + 0.4804278314113617, + 0.39933863282203674, + 0.03257017582654953, + 0.04036220535635948, + 0.09466017037630081, + 0.07803918421268463, + 0.10374905169010162 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 1, + "style_probability": 0.884731650352478, + "style_target": 1, + "support_probability": 0.8664126396179199, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5049356883914671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6487690806388855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8292626738548279, + 0.3583596646785736, + 0.38305768370628357, + 0.33112311363220215, + 0.12303651869297028, + 0.13701407611370087, + 0.06182829663157463, + 0.07485365867614746, + 0.12136467546224594 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 0, + "style_probability": 0.6200574040412903, + "style_target": 0, + "support_probability": 0.45026442408561707, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7275105072169935, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16506101191043854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9020463228225708, + 0.1527596116065979, + 0.1527596116065979, + 0.2016214281320572, + 0.039660971611738205, + 0.04465837776660919, + 0.03520222008228302, + 0.030847884714603424, + 0.09246860444545746, + 0.04553315415978432, + 0.05515221506357193 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 1, + "style_probability": 0.9077996611595154, + "style_target": 1, + "support_probability": 0.45657140016555786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48316253898438594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10685679316520691, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9145937561988831, + 0.16849659383296967, + 0.16849659383296967, + 0.19241410493850708, + 0.09889239072799683, + 0.05781906470656395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 0, + "style_probability": 0.5026492476463318, + "style_target": 0, + "support_probability": 0.4925156235694885, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7689268471074047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08181065320968628, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15948206186294556, + 0.9656955599784851, + 0.2677235007286072, + 0.48696351051330566, + 0.08864761143922806 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.7306032180786133, + "style_target": 1, + "support_probability": 0.6860274076461792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5215827401363127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04925195872783661, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1978226602077484, + 0.9865249395370483, + 0.3200630843639374, + 0.5766008496284485, + 0.18375594913959503 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.31795063614845276, + "style_target": 0, + "support_probability": 0.7542293071746826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008815808043069625, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9901987314224243, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5384836196899414, + 0.2810940742492676, + 0.41311290860176086, + 0.2810940742492676, + 0.13116104900836945, + 0.06787265837192535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 1, + "style_probability": 0.546042799949646, + "style_target": 1, + "support_probability": 0.0016826140927150846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026999943968549402, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9934531450271606, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5082048773765564, + 0.23635101318359375, + 0.4020887017250061, + 0.23635101318359375, + 0.12631985545158386, + 0.12674455344676971 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 1, + "style_probability": 0.39757299423217773, + "style_target": 0, + "support_probability": 0.004413839429616928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5087658272854751, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15665073692798615, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5315827131271362, + 0.6072003841400146, + 0.3908465504646301, + 0.5418075919151306, + 0.05688488855957985, + 0.04032566025853157, + 0.09841109067201614, + 0.13049672544002533, + 0.08644243329763412 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.7702336311340332, + "style_target": 1, + "support_probability": 0.6943235993385315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24566708564181852, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16685397922992706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5197635889053345, + 0.5369405150413513, + 0.38595446944236755, + 0.4913831353187561, + 0.20337283611297607 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.26620328426361084, + "style_target": 0, + "support_probability": 0.5907123684883118, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5038394377970146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06310537457466125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14599114656448364, + 0.9317062497138977, + 0.47547006607055664, + 0.18959122896194458, + 0.03155069053173065, + 0.09182571619749069, + 0.038620200008153915, + 0.0643625557422638, + 0.02892445959150791, + 0.09225402027368546, + 0.03881620988249779 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.7067354321479797, + "style_target": 1, + "support_probability": 0.6377326846122742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12007065441558176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08452558517456055, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17464716732501984, + 0.9542284607887268, + 0.5414862036705017, + 0.17836689949035645, + 0.044933564960956573, + 0.051542967557907104, + 0.14442910254001617, + 0.08302856236696243, + 0.050728414207696915, + 0.07184043526649475 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.19249701499938965, + "style_target": 0, + "support_probability": 0.6534879803657532, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7248735339764205, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14639811217784882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.264617919921875, + 0.3687477111816406, + 0.919417679309845, + 0.3155943751335144, + 0.06615880131721497 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 1, + "style_probability": 0.7405351996421814, + "style_target": 1, + "support_probability": 0.844805896282196, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5200929946964882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2560974359512329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27953702211380005, + 0.3373138904571533, + 0.8783804774284363, + 0.31555864214897156, + 0.11806769669055939 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 0, + "style_probability": 0.5149227380752563, + "style_target": 0, + "support_probability": 0.6842896938323975, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.454531616615627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.581360936164856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8513584136962891, + 0.4649272561073303, + 0.2249850183725357, + 0.3303007483482361, + 0.08372659981250763, + 0.09531491994857788 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.8709529638290405, + "style_target": 1, + "support_probability": 0.20634406805038452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11806178300868225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7389165759086609, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8555616140365601, + 0.4892561137676239, + 0.24337249994277954, + 0.30670052766799927, + 0.06728900969028473, + 0.18336154520511627, + 0.05239500105381012, + 0.0596931055188179, + 0.05546975135803223, + 0.10280730575323105, + 0.14574992656707764 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.474583238363266, + "style_target": 0, + "support_probability": 0.09806443005800247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6751686577330692, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25479230284690857, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9436631798744202, + 0.23699793219566345, + 0.13976074755191803, + 0.4318234622478485, + 0.07776353508234024, + 0.044831179082393646, + 0.06813224405050278, + 0.05839575454592705, + 0.07659056782722473 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.762661337852478, + "style_target": 1, + "support_probability": 0.7319918870925903, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18632557441801115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.210880309343338, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9688898921012878, + 0.2647588849067688, + 0.1746431291103363, + 0.4406822919845581, + 0.086455799639225, + 0.05832235515117645, + 0.09266586601734161, + 0.10284055769443512, + 0.1782180517911911 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.25816425681114197, + "style_target": 0, + "support_probability": 0.7600136399269104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6168171867070195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44123461842536926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2274424284696579, + 0.9134688377380371, + 0.13245877623558044, + 0.24178676307201385, + 0.14527995884418488, + 0.08726174384355545, + 0.07863325625658035, + 0.06929196417331696, + 0.20124472677707672, + 0.11652612686157227 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 1, + "style_probability": 0.8280163407325745, + "style_target": 1, + "support_probability": 0.5080990791320801, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4389827393887052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37995368242263794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21069511771202087, + 0.9078148603439331, + 0.14849205315113068, + 0.23621362447738647, + 0.08015903830528259, + 0.07022599130868912, + 0.12893778085708618, + 0.07074231654405594, + 0.0877181813120842, + 0.3224821090698242 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 1, + "style_probability": 0.49166297912597656, + "style_target": 0, + "support_probability": 0.5846377611160278, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8310445159349644, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2956746816635132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13841120898723602, + 0.18550124764442444, + 0.9257667660713196, + 0.30514711141586304, + 0.04343413561582565 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 1, + "style_probability": 0.8354691863059998, + "style_target": 1, + "support_probability": 0.7671486735343933, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.620573102033351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4284527003765106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15182778239250183, + 0.16646231710910797, + 0.9050372838973999, + 0.2760377526283264, + 0.1107764020562172, + 0.0931490808725357, + 0.14295613765716553, + 0.16832216084003448, + 0.10429354757070541 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 1, + "style_probability": 0.34295567870140076, + "style_target": 0, + "support_probability": 0.6747997999191284, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8468250034224323, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04496549814939499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30493777990341187, + 0.23945273458957672, + 0.998295247554779, + 0.25803592801094055, + 0.04741773381829262, + 0.053196996450424194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.6999678611755371, + "style_target": 1, + "support_probability": 0.8623499274253845, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7286233009178607, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03360661491751671, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3202069401741028, + 0.2100404053926468, + 0.9993649125099182, + 0.2677898705005646, + 0.04413244500756264, + 0.0723685473203659 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.4386464059352875, + "style_target": 0, + "support_probability": 0.9267452359199524, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7615422415394029, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04469648376107216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2518918216228485, + 0.302631676197052, + 0.21962910890579224, + 0.9980021119117737, + 0.06064430996775627, + 0.12021768093109131, + 0.05997108668088913, + 0.04820510372519493, + 0.05519434064626694 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 1, + "style_probability": 0.6813800930976868, + "style_target": 1, + "support_probability": 0.8180754780769348, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6324050557278338, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060650020837783813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27368056774139404, + 0.30964216589927673, + 0.21731987595558167, + 0.9928365349769592, + 0.11167117208242416, + 0.0781276598572731, + 0.1026841551065445, + 0.05041610449552536, + 0.15546247363090515 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 0, + "style_probability": 0.6422524452209473, + "style_target": 0, + "support_probability": 0.737678050994873, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23469057089863885, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8376479744911194, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25308483839035034, + 0.3037881553173065, + 0.4751606583595276, + 0.3539028763771057, + 0.039389848709106445, + 0.10855529457330704, + 0.11410511285066605, + 0.09878577291965485, + 0.06846767663955688, + 0.08286720514297485, + 0.06504420191049576 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.8408654928207397, + "style_target": 1, + "support_probability": 0.18836960196495056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015485898297467288, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9628473520278931, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23721899092197418, + 0.3032699525356293, + 0.48173490166664124, + 0.29511600732803345, + 0.1294727325439453, + 0.09825503826141357 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.4473881721496582, + "style_target": 0, + "support_probability": 0.028807617723941803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3931131976297305, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5893232822418213, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.279278039932251, + 0.3694740831851959, + 0.4740417003631592, + 0.9001424908638, + 0.11910010129213333 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.790562093257904, + "style_target": 1, + "support_probability": 0.23049861192703247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016361460367104876, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7380053997039795, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24886977672576904, + 0.427942156791687, + 0.3932030200958252, + 0.9126244187355042, + 0.16313157975673676 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.15458084642887115, + "style_target": 0, + "support_probability": 0.0930386483669281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08941729506220075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6021131873130798, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41130325198173523, + 0.5144842863082886, + 0.23795682191848755, + 0.5033413171768188, + 0.11080322414636612, + 0.06723470240831375 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.9068995118141174, + "style_target": 1, + "support_probability": 0.10539229959249496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00883760648965616, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6846162676811218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3952883183956146, + 0.46514999866485596, + 0.23226137459278107, + 0.4332064390182495, + 0.17655228078365326, + 0.12646420300006866 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.4579274356365204, + "style_target": 0, + "support_probability": 0.050227534025907516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20961618379414648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5596380829811096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20658628642559052, + 0.8918689489364624, + 0.2401994913816452, + 0.45315393805503845, + 0.14343631267547607, + 0.10676290094852448, + 0.057342927902936935, + 0.0551505908370018, + 0.20692412555217743 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 1, + "style_probability": 0.5630136728286743, + "style_target": 1, + "support_probability": 0.17798395454883575, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16807686590841375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.547569215297699, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20723463594913483, + 0.8711142539978027, + 0.25109240412712097, + 0.39302071928977966, + 0.0959460437297821 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 1, + "style_probability": 0.37843191623687744, + "style_target": 0, + "support_probability": 0.19811183214187622, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5045022389829547, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37313175201416016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.549507200717926, + 0.34870633482933044, + 0.549507200717926, + 0.6452308893203735, + 0.04865275323390961, + 0.05680616945028305, + 0.039209041744470596, + 0.027683725580573082, + 0.028665930032730103, + 0.04186281934380531 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.7154714465141296, + "style_target": 1, + "support_probability": 0.706465482711792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029182866049331548, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5186778903007507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5746802687644958, + 0.3971292972564697, + 0.5746802687644958, + 0.7271303534507751, + 0.08040206134319305, + 0.08955784887075424, + 0.2541150152683258, + 0.10047725588083267, + 0.08178295195102692, + 0.10424564778804779 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.08219430595636368, + "style_target": 0, + "support_probability": 0.5234825611114502, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015546134624124502, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5827465057373047, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8633553981781006, + 0.5314426422119141, + 0.39223161339759827, + 0.24815301597118378, + 0.08572877198457718 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.6414344310760498, + "style_target": 1, + "support_probability": 0.017080601304769516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05134929567723853, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30825337767601013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8589996099472046, + 0.575958788394928, + 0.43571406602859497, + 0.27906328439712524, + 0.0781133845448494 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.49644723534584045, + "style_target": 0, + "support_probability": 0.06717430800199509, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02461673798826794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9586476683616638, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.329122394323349, + 0.4895874559879303, + 0.7272099256515503, + 0.2179444581270218, + 0.19576558470726013, + 0.09919405728578568 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.8664560317993164, + "style_target": 1, + "support_probability": 0.010286194272339344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005068611151762725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9886404275894165, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31463202834129333, + 0.4760136604309082, + 0.7043993473052979, + 0.21263264119625092, + 0.1296323835849762, + 0.12446620315313339, + 0.09970563650131226, + 0.09750199317932129, + 0.1897428035736084, + 0.10636412352323532, + 0.12812592089176178 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.48057815432548523, + "style_target": 0, + "support_probability": 0.0028114174492657185, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5092868254737304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27452805638313293, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4602125883102417, + 0.23203687369823456, + 0.36259883642196655, + 0.6483210325241089, + 0.10626576095819473, + 0.0443556047976017, + 0.039350371807813644, + 0.11224813759326935, + 0.054130975157022476 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.8629248142242432, + "style_target": 1, + "support_probability": 0.3735731840133667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.043325205875225606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29519420862197876, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6065520644187927, + 0.33180853724479675, + 0.5113794207572937, + 0.8055546879768372, + 0.11568477004766464, + 0.16319094598293304, + 0.1854540854692459, + 0.11218626797199249, + 0.09372394531965256 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.08735080063343048, + "style_target": 0, + "support_probability": 0.2634035050868988, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6729981531552595, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06511154770851135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4607205390930176, + 0.9587557911872864, + 0.5002822875976562, + 0.24752457439899445, + 0.12770196795463562, + 0.11586792767047882, + 0.11317086964845657, + 0.029192762449383736, + 0.05762413516640663, + 0.13774323463439941, + 0.07082539796829224 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 1, + "style_probability": 0.7982999682426453, + "style_target": 1, + "support_probability": 0.5403008460998535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4300369510317724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09457103908061981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.446132630109787, + 0.9270426630973816, + 0.4940093159675598, + 0.2312273383140564, + 0.0679798349738121, + 0.09501831978559494, + 0.03891746699810028, + 0.11951692402362823, + 0.16955700516700745, + 0.1966637670993805, + 0.04958666115999222 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 0, + "style_probability": 0.690305769443512, + "style_target": 0, + "support_probability": 0.3170776069164276, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2948806278417203, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5144094824790955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4760099947452545, + 0.5146018862724304, + 0.5600588917732239, + 0.3154442012310028, + 0.17801621556282043 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 1, + "style_probability": 0.5112399458885193, + "style_target": 1, + "support_probability": 0.4833575189113617, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5048481969772525, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37381109595298767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4972822070121765, + 0.5351826548576355, + 0.5655323266983032, + 0.3358265161514282, + 0.08312401175498962 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 0, + "style_probability": 0.9148573875427246, + "style_target": 0, + "support_probability": 0.6764214634895325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.016151811055692574, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9313536286354065, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.311181902885437, + 0.29274222254753113, + 0.2957879602909088, + 0.2539942264556885, + 0.10039469599723816, + 0.16988149285316467 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 0, + "style_probability": 0.4681946635246277, + "style_target": 1, + "support_probability": 0.04424325004220009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008376436342198791, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9843234419822693, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.3409945070743561, + 0.318146288394928, + 0.32477882504463196, + 0.23813775181770325, + 0.11246486008167267, + 0.12826570868492126, + 0.08595919609069824, + 0.08191690593957901, + 0.1082097738981247, + 0.13855215907096863, + 0.09969475120306015, + 0.08176855742931366 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 1, + "style_probability": 0.43647870421409607, + "style_target": 0, + "support_probability": 0.020421164110302925, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6133447653697948, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0693572461605072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2568899393081665, + 0.33145684003829956, + 0.2859978973865509, + 0.9727208018302917, + 0.11459754407405853, + 0.12647263705730438, + 0.08295082300901413, + 0.07123713940382004, + 0.15424925088882446 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 1, + "style_probability": 0.6952755451202393, + "style_target": 1, + "support_probability": 0.603351354598999, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14767176289346182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11911142617464066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2249182015657425, + 0.3284343481063843, + 0.23618917167186737, + 0.9910633563995361, + 0.06643064320087433, + 0.24523630738258362, + 0.09386074542999268, + 0.11463580280542374, + 0.25814518332481384 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 1, + "style_probability": 0.2556132972240448, + "style_target": 0, + "support_probability": 0.6924797892570496, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8131339242118771, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03144385665655136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9769740700721741, + 0.3261134922504425, + 0.3261134922504425, + 0.3833969533443451, + 0.06817808002233505, + 0.08926989883184433, + 0.08734607696533203, + 0.038075998425483704, + 0.04568837210536003, + 0.14815808832645416 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 0, + "style_probability": 0.38721996545791626, + "style_target": 1, + "support_probability": 0.9201112389564514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8427189339279362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0520252026617527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9771348834037781, + 0.338270366191864, + 0.338270366191864, + 0.40613940358161926, + 0.09474382549524307, + 0.05052180960774422, + 0.18077154457569122, + 0.24079889059066772, + 0.07121557742357254, + 0.08269509673118591 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 0, + "style_probability": 0.7313151359558105, + "style_target": 0, + "support_probability": 0.8642210960388184, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06973954974166142, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.571922779083252, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37035828828811646, + 0.5814355611801147, + 0.45927658677101135, + 0.3864896297454834, + 0.32456454634666443 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.6303294897079468, + "style_target": 1, + "support_probability": 0.10185764729976654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004095260789170414, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7780978679656982, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3813352584838867, + 0.5903285145759583, + 0.5164837837219238, + 0.3465772271156311, + 0.11453398317098618 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.16133080422878265, + "style_target": 0, + "support_probability": 0.040398161858320236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2825913720610484, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11126822978258133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9811866283416748, + 0.7064977884292603, + 0.32442137598991394, + 0.5198696851730347, + 0.13702303171157837, + 0.40417250990867615 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 0, + "style_probability": 0.3054575026035309, + "style_target": 1, + "support_probability": 0.5244433283805847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22280815247569366, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10306299477815628, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9675801992416382, + 0.6423883438110352, + 0.4022919535636902, + 0.43346351385116577, + 0.08136788010597229, + 0.12411694973707199 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 1, + "style_probability": 0.3364362418651581, + "style_target": 0, + "support_probability": 0.3820511996746063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.662369665771225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7346451878547668, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20927120745182037, + 0.14913605153560638, + 0.2562043070793152, + 0.9637665748596191, + 0.1048760786652565, + 0.22290100157260895, + 0.2110416740179062, + 0.10941428691148758, + 0.1524817943572998 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 1, + "style_probability": 0.8747730255126953, + "style_target": 1, + "support_probability": 0.3465854823589325, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4030033824895539, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7597623467445374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19609497487545013, + 0.1640671342611313, + 0.24963851273059845, + 0.950147807598114, + 0.10931331664323807 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 0, + "style_probability": 0.6213085651397705, + "style_target": 0, + "support_probability": 0.2784663736820221, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7342314933747399, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0463290810585022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28944966197013855, + 0.9801247119903564, + 0.5808457136154175, + 0.5626553893089294, + 0.15642929077148438, + 0.1255917102098465, + 0.12521523237228394, + 0.1118825227022171, + 0.18619811534881592, + 0.08752890676259995, + 0.12792930006980896 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 1, + "style_probability": 0.850212037563324, + "style_target": 1, + "support_probability": 0.38972073793411255, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.354263909053234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04079296439886093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28872013092041016, + 0.9853472709655762, + 0.5418969988822937, + 0.5309945344924927, + 0.17611722648143768, + 0.08776313811540604, + 0.2372434139251709, + 0.10179302096366882, + 0.08266425132751465, + 0.19022974371910095 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 0, + "style_probability": 0.5434911251068115, + "style_target": 0, + "support_probability": 0.429098904132843, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0626625176232214, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7552480697631836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2070523053407669, + 0.7902815341949463, + 0.19460181891918182, + 0.1528908610343933, + 0.1111079528927803 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 0, + "style_probability": 0.43900588154792786, + "style_target": 1, + "support_probability": 0.056854669004678726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012621676201878923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7948670387268066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1842983514070511, + 0.8766783475875854, + 0.20948614180088043, + 0.16083012521266937, + 0.08352317661046982 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 1, + "style_probability": 0.1262054294347763, + "style_target": 0, + "support_probability": 0.08246047049760818, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4885250718834828, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.40999794006347656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3168542981147766, + 0.2376686930656433, + 0.2578709125518799, + 0.7710346579551697, + 0.05296866223216057, + 0.044161710888147354 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 1, + "style_probability": 0.6123132109642029, + "style_target": 1, + "support_probability": 0.8470287919044495, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7693162510051221, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13042864203453064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3378325402736664, + 0.2789919972419739, + 0.25341829657554626, + 0.871627926826477, + 0.05845961719751358, + 0.04655769094824791 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 0, + "style_probability": 0.8941034078598022, + "style_target": 0, + "support_probability": 0.9046650528907776, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24585604928382018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6637215614318848, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8122272491455078, + 0.38977909088134766, + 0.5730170607566833, + 0.09339559823274612, + 0.08139965683221817, + 0.059319254010915756, + 0.037715379148721695, + 0.03934931755065918, + 0.053088363260030746 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.6136912107467651, + "style_target": 1, + "support_probability": 0.26627475023269653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11854070554753045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9034380912780762, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.762093722820282, + 0.33416372537612915, + 0.5599020719528198, + 0.08790943026542664, + 0.09614868462085724 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.4070538580417633, + "style_target": 0, + "support_probability": 0.09211946278810501, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.059946809419571104, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8170331120491028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16062448918819427, + 0.9172667264938354, + 0.3550702929496765, + 0.18852944672107697, + 0.11465662717819214, + 0.09484704583883286, + 0.11877744644880295, + 0.13069050014019012, + 0.05243348702788353, + 0.06674636900424957, + 0.07547999173402786 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.5195105671882629, + "style_target": 1, + "support_probability": 0.04693925753235817, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04448981765843986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5117480158805847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19053508341312408, + 0.9618625044822693, + 0.33737456798553467, + 0.1858256459236145, + 0.16463322937488556, + 0.06660519540309906, + 0.2765677273273468, + 0.10380064696073532, + 0.08159208297729492, + 0.13342899084091187 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.2667776048183441, + "style_target": 0, + "support_probability": 0.1871744841337204, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7762264657286516, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.106174536049366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4630342423915863, + 0.8377108573913574, + 0.3181610107421875, + 0.5721475481987, + 0.06775543838739395 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.8733109831809998, + "style_target": 1, + "support_probability": 0.9203659296035767, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6365465252605044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1564861685037613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3819268047809601, + 0.8322502374649048, + 0.31599900126457214, + 0.5253692269325256, + 0.07051028311252594, + 0.09489083290100098, + 0.10873865336179733, + 0.07485045492649078, + 0.07779496908187866 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.28029337525367737, + "style_target": 0, + "support_probability": 0.8717349767684937, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8067810618879986, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1300552487373352, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9499401450157166, + 0.09824065864086151, + 0.5556378960609436, + 0.2508032023906708, + 0.14161410927772522, + 0.08316752314567566 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.834877610206604, + "style_target": 1, + "support_probability": 0.7707222700119019, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3087457062382939, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10172642022371292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9907061457633972, + 0.14589424431324005, + 0.7371277213096619, + 0.49145177006721497, + 0.15611113607883453, + 0.15710176527500153 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.1554785668849945, + "style_target": 0, + "support_probability": 0.8298476338386536, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006916412278117509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.982876718044281, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6029202342033386, + 0.2649514675140381, + 0.32164230942726135, + 0.25842636823654175, + 0.07627716660499573, + 0.11408954113721848, + 0.26935356855392456, + 0.05961378663778305, + 0.08229172974824905 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 1, + "style_probability": 0.721076488494873, + "style_target": 1, + "support_probability": 0.007774434983730316, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011486224137896189, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9492130875587463, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.537301778793335, + 0.3019695281982422, + 0.3560425043106079, + 0.2804098427295685, + 0.10971339046955109, + 0.13888435065746307, + 0.1012406200170517, + 0.13465723395347595, + 0.15485942363739014 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 0, + "style_probability": 0.5879306793212891, + "style_target": 0, + "support_probability": 0.010389069095253944, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48112501666991997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2444002330303192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2798793315887451, + 0.7056059837341309, + 0.46103742718696594, + 0.2798793315887451, + 0.1286582052707672, + 0.054089631885290146, + 0.06754100322723389, + 0.213682159781456, + 0.11131179332733154, + 0.06313056498765945 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 1, + "style_probability": 0.7867333889007568, + "style_target": 1, + "support_probability": 0.43429920077323914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43957244205806334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.220646932721138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27328959107398987, + 0.7005414366722107, + 0.42763954401016235, + 0.27328959107398987, + 0.09667014330625534, + 0.05328286066651344 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 1, + "style_probability": 0.49489879608154297, + "style_target": 0, + "support_probability": 0.5648016929626465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4046887596701169, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4257047772407532, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5105538964271545, + 0.2984256148338318, + 0.2423921823501587, + 0.2984256148338318, + 0.07474151253700256 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.7646391987800598, + "style_target": 1, + "support_probability": 0.5423328876495361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04607715079181717, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5369745492935181, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5810137391090393, + 0.47845226526260376, + 0.3020445704460144, + 0.47845226526260376, + 0.10896863788366318 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.17806610465049744, + "style_target": 0, + "support_probability": 0.43060535192489624, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17067977246763397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2122010439634323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3165934383869171, + 0.22555306553840637, + 0.24170394241809845, + 0.6907439231872559, + 0.03655841574072838, + 0.08406733721494675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 1, + "style_probability": 0.8083736300468445, + "style_target": 1, + "support_probability": 0.1514313966035843, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0741101093858588, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23673638701438904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36169305443763733, + 0.25640615820884705, + 0.26300927996635437, + 0.6598840355873108, + 0.03593795374035835, + 0.14079953730106354 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 0, + "style_probability": 0.580999493598938, + "style_target": 0, + "support_probability": 0.09875267744064331, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6596795643663335, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23779508471488953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5860865116119385, + 0.21605922281742096, + 0.9240676760673523, + 0.2315784990787506, + 0.04543453827500343, + 0.041191186755895615, + 0.0690552294254303, + 0.05334150046110153, + 0.10027734935283661 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.7860142588615417, + "style_target": 1, + "support_probability": 0.5573108196258545, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37882736554601265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2783488929271698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5604384541511536, + 0.2062152475118637, + 0.9041308164596558, + 0.2064439356327057, + 0.20134900510311127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.3933642506599426, + "style_target": 0, + "support_probability": 0.4396185576915741, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9531002266159541, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03417937830090523, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8822:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12281286716461182, + 0.2756531536579132, + 0.9866608381271362, + 0.2617029547691345, + 0.07620589435100555, + 0.0581706240773201, + 0.0486726388335228, + 0.06453759223222733, + 0.17819052934646606, + 0.04468664154410362, + 0.0949699729681015 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.8411023616790771, + "style_target": 1, + "support_probability": 0.9456799626350403, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7412538847803217, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060842953622341156, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8822:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21091420948505402, + 0.2802346646785736, + 0.9976784586906433, + 0.25537174940109253, + 0.11356661468744278, + 0.10901614278554916, + 0.13360010087490082, + 0.251628577709198, + 0.09738268703222275, + 0.16154071688652039 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.2991906404495239, + "style_target": 0, + "support_probability": 0.9422558546066284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06891742407884531, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4346970021724701, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3988233506679535, + 0.3152284324169159, + 0.4283096492290497, + 0.19946381449699402, + 0.08868169784545898 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 1, + "style_probability": 0.6573429107666016, + "style_target": 1, + "support_probability": 0.142166405916214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021441388827513293, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5551969408988953, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3866884708404541, + 0.2620032727718353, + 0.3533918559551239, + 0.20895901322364807, + 0.141855850815773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 1, + "style_probability": 0.41204044222831726, + "style_target": 0, + "support_probability": 0.0515589602291584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.295236939433682, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29333600401878357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1358312964439392, + 0.6060415506362915, + 0.15423426032066345, + 0.1358312964439392, + 0.06182682514190674, + 0.06280265003442764 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.7404140830039978, + "style_target": 1, + "support_probability": 0.17231982946395874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07548323609473771, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6199237108230591, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14720575511455536, + 0.5892958641052246, + 0.15265721082687378, + 0.14720575511455536, + 0.07046496868133545, + 0.09972327947616577, + 0.13988198339939117, + 0.07506715506315231, + 0.16342918574810028, + 0.06975413113832474, + 0.1284274011850357 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.326688289642334, + "style_target": 0, + "support_probability": 0.07248832285404205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.735404081873614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15374785661697388, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13638918101787567, + 0.9363034963607788, + 0.1409132182598114, + 0.3777291178703308, + 0.06324772536754608, + 0.06257536262273788, + 0.1430526077747345, + 0.10257403552532196, + 0.14245085418224335 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.8620862364768982, + "style_target": 1, + "support_probability": 0.5753494501113892, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5213783030503026, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06281284242868423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17657846212387085, + 0.9784243106842041, + 0.21435324847698212, + 0.45480412244796753, + 0.19998374581336975, + 0.24066241085529327, + 0.13255341351032257, + 0.10174310952425003, + 0.2278701663017273 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.28162145614624023, + "style_target": 0, + "support_probability": 0.8854920268058777, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9208263244396967, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023476503789424896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8835:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3657160699367523, + 0.1559901088476181, + 0.16320225596427917, + 0.9610612988471985, + 0.11398562788963318, + 0.06727319955825806, + 0.06824437528848648, + 0.03751341253519058, + 0.15586523711681366, + 0.07083616405725479, + 0.0658087357878685 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 1, + "style_probability": 0.8837404847145081, + "style_target": 1, + "support_probability": 0.98288893699646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8497390390356808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.037248797714710236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8835:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3806031048297882, + 0.198684424161911, + 0.19947662949562073, + 0.9352589845657349, + 0.07652296870946884, + 0.0761038213968277, + 0.12518775463104248, + 0.1729101985692978, + 0.12076036632061005, + 0.11243823170661926 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 1, + "style_probability": 0.46012061834335327, + "style_target": 0, + "support_probability": 0.9700671434402466, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39563315259092546, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.616772472858429, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2869800329208374, + 0.4429750442504883, + 0.522393524646759, + 0.31688442826271057, + 0.04889728128910065 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.9263541102409363, + "style_target": 1, + "support_probability": 0.3555610179901123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04768564311629331, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7705639600753784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25384676456451416, + 0.45825961232185364, + 0.4559749662876129, + 0.3241662383079529, + 0.0866813138127327, + 0.07890339195728302, + 0.08043774962425232, + 0.0952228382229805, + 0.12305378168821335 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.3825557827949524, + "style_target": 0, + "support_probability": 0.061970554292201996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11084521064709642, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7483392953872681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15187408030033112, + 0.3719717860221863, + 0.277295857667923, + 0.6428337693214417, + 0.20946530997753143, + 0.052091509103775024 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.6442630887031555, + "style_target": 1, + "support_probability": 0.15817341208457947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015686342762079496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.766982913017273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19551855325698853, + 0.39559897780418396, + 0.3511335849761963, + 0.7462872266769409, + 0.09924853593111038, + 0.14461268484592438 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.18367794156074524, + "style_target": 0, + "support_probability": 0.100856713950634, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.724493606871081, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08520887047052383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36003613471984863, + 0.8551053404808044, + 0.14856405556201935, + 0.21189214289188385, + 0.04448584467172623, + 0.12040689587593079, + 0.04467229172587395, + 0.02216435596346855, + 0.02800576016306877 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.6563624143600464, + "style_target": 1, + "support_probability": 0.8974837064743042, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5196408779287416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1558130532503128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31061217188835144, + 0.7678909301757812, + 0.20096547901630402, + 0.24708794057369232, + 0.09093474596738815, + 0.12782832980155945, + 0.07732236385345459, + 0.04672285541892052, + 0.13866645097732544 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.352248877286911, + "style_target": 0, + "support_probability": 0.8720924258232117, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8372899688749057, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16015906631946564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19403553009033203, + 0.289326548576355, + 0.33694151043891907, + 0.9615373015403748, + 0.07344836741685867, + 0.14489313960075378, + 0.046769678592681885, + 0.09228771179914474, + 0.051122378557920456, + 0.10273253917694092, + 0.06127404421567917 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.8309792280197144, + "style_target": 1, + "support_probability": 0.7749432325363159, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5141034282436601, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2067415565252304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17839065194129944, + 0.2608288824558258, + 0.3457935154438019, + 0.9506497979164124, + 0.05390814319252968, + 0.052763283252716064 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.4139155149459839, + "style_target": 0, + "support_probability": 0.6407103538513184, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29056451981084613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5173999667167664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2498827576637268, + 0.8728227019309998, + 0.21062801778316498, + 0.20348533987998962, + 0.047973401844501495 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.8814155459403992, + "style_target": 1, + "support_probability": 0.15313418209552765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015984650180168373, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6831771731376648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26814138889312744, + 0.9295719861984253, + 0.22689059376716614, + 0.18980509042739868, + 0.12437977641820908 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.20067210495471954, + "style_target": 0, + "support_probability": 0.1442987471818924, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19982863995491976, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5299317240715027, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23976677656173706, + 0.1696796864271164, + 0.21465426683425903, + 0.585410475730896, + 0.05448422208428383, + 0.06894175708293915 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.8367053866386414, + "style_target": 1, + "support_probability": 0.1830320805311203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02146218222502007, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6240756511688232, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20719657838344574, + 0.1668415516614914, + 0.21379928290843964, + 0.5435543060302734, + 0.057996317744255066, + 0.04712431877851486 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.24304340779781342, + "style_target": 0, + "support_probability": 0.09740959107875824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46134039933584303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06380601972341537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2431098371744156, + 0.40922918915748596, + 0.8748276829719543, + 0.5200733542442322, + 0.06149034947156906, + 0.09426373243331909, + 0.04854960739612579, + 0.08483171463012695, + 0.06194394826889038 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.6898519396781921, + "style_target": 1, + "support_probability": 0.46742555499076843, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45681816208889675, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04155515879392624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28456202149391174, + 0.3920915722846985, + 0.8574832677841187, + 0.5156363844871521, + 0.09139174968004227 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.36662834882736206, + "style_target": 0, + "support_probability": 0.5573931932449341, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8450566875224504, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13006862998008728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9602786302566528, + 0.42159250378608704, + 0.29920053482055664, + 0.42159250378608704, + 0.07498686015605927, + 0.03945576399564743, + 0.06625451892614365, + 0.20704850554466248, + 0.0767960399389267, + 0.20929919183254242, + 0.09196837246417999 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.7376254200935364, + "style_target": 1, + "support_probability": 0.9078695774078369, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1941691049105874, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2083619236946106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9804115295410156, + 0.5463162660598755, + 0.3128124177455902, + 0.5463162660598755, + 0.15922844409942627, + 0.19950780272483826, + 0.23317265510559082, + 0.13743892312049866, + 0.10569363832473755, + 0.13686667382717133 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.10266389697790146, + "style_target": 0, + "support_probability": 0.8767054677009583, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5947020864605138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4403374493122101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25784602761268616, + 0.6100842356681824, + 0.9726408123970032, + 0.22479258477687836, + 0.07327083498239517 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 1, + "style_probability": 0.8364202976226807, + "style_target": 1, + "support_probability": 0.47254082560539246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.654930287290566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20138011872768402, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26305365562438965, + 0.5881027579307556, + 0.9331070184707642, + 0.22389213740825653, + 0.10019302368164062 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 0, + "style_probability": 0.6125284433364868, + "style_target": 0, + "support_probability": 0.6505717635154724, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.875277026107786, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.061906300485134125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41064924001693726, + 0.10646504908800125, + 0.9648340940475464, + 0.17765219509601593, + 0.10086575895547867, + 0.11473792046308517 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.8328347206115723, + "style_target": 1, + "support_probability": 0.819976270198822, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5445614939954093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1228320300579071, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35971158742904663, + 0.1382056623697281, + 0.925186038017273, + 0.15817992389202118, + 0.10848704725503922, + 0.07282020896673203, + 0.13443975150585175, + 0.07597586512565613, + 0.04476417228579521, + 0.06246937811374664, + 0.10193119943141937 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.33271071314811707, + "style_target": 0, + "support_probability": 0.6128110885620117, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8700546722945048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1610327661037445, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3590056598186493, + 0.2849452793598175, + 0.9563296437263489, + 0.47238418459892273, + 0.0851258859038353, + 0.10572266578674316, + 0.06727667152881622, + 0.08603791147470474, + 0.10988225042819977 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.894967257976532, + "style_target": 1, + "support_probability": 0.7811315655708313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39556839400747806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11675604432821274, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4350791573524475, + 0.3657083511352539, + 0.9815253019332886, + 0.46646156907081604, + 0.14579257369041443, + 0.16408652067184448, + 0.12999527156352997, + 0.11418680846691132, + 0.11616170406341553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.2892177402973175, + "style_target": 0, + "support_probability": 0.7491202354431152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4702939469402405, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47394347190856934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41422244906425476, + 0.3233514726161957, + 0.3912801444530487, + 0.7153490781784058, + 0.11132500320672989, + 0.0814642384648323, + 0.06207335367798805, + 0.16784903407096863, + 0.053306546062231064, + 0.1050691232085228, + 0.23519131541252136 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.8168647289276123, + "style_target": 1, + "support_probability": 0.46763932704925537, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2827130767925665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3989885747432709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4019392132759094, + 0.34174099564552307, + 0.4240848124027252, + 0.7025395035743713, + 0.0752405896782875, + 0.0722511038184166, + 0.10020839422941208, + 0.07487864792346954, + 0.08237142115831375, + 0.12388461828231812 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.4130043685436249, + "style_target": 0, + "support_probability": 0.5029828548431396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6669248956463923, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36347079277038574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8435889482498169, + 0.7664390802383423, + 0.24305523931980133, + 0.8435889482498169, + 0.13138946890830994 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 1, + "style_probability": 0.8889645338058472, + "style_target": 1, + "support_probability": 0.4100678265094757, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29030439270020614, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.641238808631897, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8125180602073669, + 0.7245184779167175, + 0.2521098256111145, + 0.8125180602073669, + 0.06835559010505676, + 0.2049095183610916, + 0.09250561892986298, + 0.19236566126346588, + 0.10992397367954254 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 0, + "style_probability": 0.5427327156066895, + "style_target": 0, + "support_probability": 0.17554917931556702, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22145037933611295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16867554187774658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7658125162124634, + 0.2340756356716156, + 0.29556161165237427, + 0.2438984513282776, + 0.06840156018733978, + 0.12798768281936646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.5814282298088074, + "style_target": 1, + "support_probability": 0.22457638382911682, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07781683928482123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16101287305355072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8256365656852722, + 0.3095948398113251, + 0.3466152846813202, + 0.2553179860115051, + 0.07168595492839813, + 0.13975168764591217 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.272341251373291, + "style_target": 0, + "support_probability": 0.3260928988456726, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8666194859776551, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026916075497865677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9456496238708496, + 0.616908609867096, + 0.2777734696865082, + 0.5275536179542542, + 0.032114703208208084, + 0.041078969836235046, + 0.10903669148683548, + 0.07519777119159698, + 0.07316643744707108 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.7788537740707397, + "style_target": 1, + "support_probability": 0.9102997779846191, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6418876661249903, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07826388627290726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9070109724998474, + 0.6086103320121765, + 0.26850447058677673, + 0.47341349720954895, + 0.1174486055970192, + 0.06892254948616028, + 0.04763951897621155, + 0.08041520416736603, + 0.09539080411195755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.4688780903816223, + "style_target": 0, + "support_probability": 0.7910285592079163, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38634566657991865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28959399461746216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4798525869846344, + 0.8600248098373413, + 0.4864521324634552, + 0.25339794158935547, + 0.036531172692775726, + 0.023908473551273346, + 0.024529755115509033, + 0.03608999773859978, + 0.10881303250789642, + 0.055938296020030975, + 0.06204662099480629 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.818751871585846, + "style_target": 1, + "support_probability": 0.25120851397514343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16473780083521108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32913005352020264, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4263289272785187, + 0.8250009417533875, + 0.5256404280662537, + 0.24885325133800507, + 0.0954497680068016, + 0.2256547212600708 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.4655005633831024, + "style_target": 0, + "support_probability": 0.2439565807580948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16544242062987388, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5184706449508667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8030831217765808, + 0.45480453968048096, + 0.3906748592853546, + 0.45480453968048096, + 0.15324001014232635 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 1, + "style_probability": 0.8175587058067322, + "style_target": 1, + "support_probability": 0.05001313239336014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12321758147142432, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28796541690826416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8559231162071228, + 0.4398258626461029, + 0.3736388087272644, + 0.4398258626461029, + 0.1077614575624466 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 1, + "style_probability": 0.2740584909915924, + "style_target": 0, + "support_probability": 0.15393854677677155, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8590443216014458, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.100711889564991, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24229702353477478, + 0.979671061038971, + 0.1467999368906021, + 0.18236440420150757, + 0.045211587101221085, + 0.059056006371974945 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 1, + "style_probability": 0.8709947466850281, + "style_target": 1, + "support_probability": 0.8757479190826416, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4759585003519064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17347411811351776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27285677194595337, + 0.9592538475990295, + 0.18348193168640137, + 0.19615328311920166, + 0.06237421929836273, + 0.09952259063720703 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 0, + "style_probability": 0.5176275372505188, + "style_target": 0, + "support_probability": 0.7092000842094421, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6817678095166158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18271401524543762, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8945904970169067, + 0.3056926131248474, + 0.15164455771446228, + 0.3056926131248474, + 0.1420166790485382, + 0.08274348825216293, + 0.05795476958155632, + 0.06295832991600037, + 0.07969619333744049 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.8793658018112183, + "style_target": 1, + "support_probability": 0.5503520965576172, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3505901992826319, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19746100902557373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8593937754631042, + 0.2940203547477722, + 0.15375205874443054, + 0.2940203547477722, + 0.09918976575136185 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.4143485724925995, + "style_target": 0, + "support_probability": 0.5397593379020691, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016541058717339907, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8593123555183411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16189873218536377, + 0.4430112838745117, + 0.44377800822257996, + 0.5229944586753845, + 0.09165132790803909, + 0.19223065674304962, + 0.11333341896533966, + 0.061845697462558746, + 0.12855590879917145, + 0.12352979928255081, + 0.1373499631881714 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 1, + "style_probability": 0.8246140480041504, + "style_target": 1, + "support_probability": 0.003149318741634488, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010561136576116468, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.731221079826355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15169145166873932, + 0.5098358988761902, + 0.5238367319107056, + 0.552937388420105, + 0.1338052749633789, + 0.033470168709754944, + 0.09749175608158112, + 0.13635674118995667, + 0.15689893066883087, + 0.1748531460762024 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 1, + "style_probability": 0.41163235902786255, + "style_target": 0, + "support_probability": 0.010989139787852764, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3109130916849447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4201180636882782, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3553502857685089, + 0.20124751329421997, + 0.7840774655342102, + 0.3553502857685089, + 0.07308058440685272 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 0, + "style_probability": 0.31447315216064453, + "style_target": 1, + "support_probability": 0.4536082446575165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5164457083182628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3629520833492279, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3795498013496399, + 0.21246087551116943, + 0.7827391028404236, + 0.3795498013496399, + 0.06677509844303131 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 0, + "style_probability": 0.5436199307441711, + "style_target": 0, + "support_probability": 0.6010957360267639, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.569067178294863, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09858310222625732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36784279346466064, + 0.41519075632095337, + 0.274043470621109, + 0.7852101922035217, + 0.07938723266124725, + 0.14126305282115936 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.32591331005096436, + "style_target": 1, + "support_probability": 0.845570981502533, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7233702298653707, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08048468083143234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3664954900741577, + 0.3297543525695801, + 0.28367534279823303, + 0.7847277522087097, + 0.07337691634893417, + 0.05227059870958328 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.9069210290908813, + "style_target": 0, + "support_probability": 0.9146210551261902, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.1324677826484912, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7979626655578613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6475896239280701, + 0.26224902272224426, + 0.2393674999475479, + 0.26556867361068726, + 0.19873064756393433, + 0.07806465774774551, + 0.08649412542581558, + 0.1179913580417633, + 0.10853739827871323 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 0, + "style_probability": 0.3052416145801544, + "style_target": 1, + "support_probability": 0.13851651549339294, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06405942483746169, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48328739404678345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7975438833236694, + 0.3224082589149475, + 0.16734746098518372, + 0.25220340490341187, + 0.10973572731018066, + 0.10946134477853775, + 0.07572849839925766, + 0.10696346312761307, + 0.07519150525331497 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 1, + "style_probability": 0.06924346834421158, + "style_target": 0, + "support_probability": 0.5411218404769897, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.3787789171082793, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5267825722694397, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29726988077163696, + 0.19372478127479553, + 0.34209251403808594, + 0.6451950669288635, + 0.12814506888389587, + 0.13609084486961365, + 0.05802265554666519, + 0.05171217396855354, + 0.07547270506620407, + 0.1054777279496193, + 0.061752572655677795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 0, + "style_probability": 0.2727087438106537, + "style_target": 1, + "support_probability": 0.6952584981918335, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5148434198323163, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47342169284820557, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3022046983242035, + 0.1689668446779251, + 0.3097681999206543, + 0.6837366223335266, + 0.1418176293373108, + 0.041167888790369034, + 0.058845870196819305, + 0.19045914709568024, + 0.08371097594499588, + 0.061769891530275345, + 0.07667448371648788 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 1, + "style_probability": 0.4393649697303772, + "style_target": 0, + "support_probability": 0.816611647605896, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.058748626951637226, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8737252354621887, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6816738843917847, + 0.22038054466247559, + 0.11985961347818375, + 0.4870256781578064, + 0.05423568934202194 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 1, + "style_probability": 0.7255115509033203, + "style_target": 1, + "support_probability": 0.022090449929237366, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026053230623653147, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9317839741706848, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6088635921478271, + 0.20811517536640167, + 0.1176513060927391, + 0.46713876724243164, + 0.0730089545249939, + 0.0850377157330513, + 0.09094055742025375, + 0.10334232449531555, + 0.0954604521393776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 0, + "style_probability": 0.5239454507827759, + "style_target": 0, + "support_probability": 0.008043655194342136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0544741896822871, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5376863479614258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44106507301330566, + 0.5191843509674072, + 0.23877307772636414, + 0.897111713886261, + 0.11759581416845322, + 0.1209411770105362 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.6514723300933838, + "style_target": 1, + "support_probability": 0.05629291012883186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028529894816538986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5442278981208801, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5047309398651123, + 0.49062418937683105, + 0.27610012888908386, + 0.9273906946182251, + 0.1368958204984665, + 0.08590693026781082 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.37346065044403076, + "style_target": 0, + "support_probability": 0.0659809485077858, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05380910463293687, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7146100997924805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4939461350440979, + 0.35904499888420105, + 0.37631967663764954, + 0.23722800612449646, + 0.0731111466884613, + 0.05075264349579811, + 0.08123165369033813, + 0.07969530671834946, + 0.07731202244758606 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 1, + "style_probability": 0.8336906433105469, + "style_target": 1, + "support_probability": 0.017330219969153404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05098779859137953, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6629454493522644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5737180709838867, + 0.3554314076900482, + 0.3485703766345978, + 0.34501591324806213, + 0.07807621359825134, + 0.09872201085090637, + 0.05269854888319969, + 0.1263338029384613, + 0.06701026111841202 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 1, + "style_probability": 0.4639975130558014, + "style_target": 0, + "support_probability": 0.053550660610198975, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8329941825410514, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05920318514108658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.935337483882904, + 0.3469463586807251, + 0.473760187625885, + 0.366499125957489, + 0.03730276972055435, + 0.04635823890566826, + 0.04136097431182861, + 0.03890744224190712, + 0.07134000957012177, + 0.05608515813946724 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.7646034955978394, + "style_target": 1, + "support_probability": 0.9250385761260986, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6406454152901196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07145041972398758, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9077355861663818, + 0.31671229004859924, + 0.466484934091568, + 0.3074352443218231, + 0.049717921763658524, + 0.0570596344769001 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.37853893637657166, + "style_target": 0, + "support_probability": 0.9086607694625854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33778168693867955, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14146706461906433, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29273298382759094, + 0.631202220916748, + 0.29273298382759094, + 0.9792059659957886, + 0.18379941582679749 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 0, + "style_probability": 0.4594608247280121, + "style_target": 1, + "support_probability": 0.43262994289398193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3016126465840397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1491674780845642, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21739421784877777, + 0.5935250520706177, + 0.21739421784877777, + 0.9894943833351135, + 0.06829674541950226, + 0.22229528427124023, + 0.06758367270231247, + 0.06861191242933273, + 0.08060368150472641 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 1, + "style_probability": 0.23368458449840546, + "style_target": 0, + "support_probability": 0.5299134850502014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11267692819904757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7874727845191956, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7606592774391174, + 0.19926665723323822, + 0.4470632076263428, + 0.38224175572395325, + 0.14344321191310883, + 0.2593631148338318 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.6330562233924866, + "style_target": 1, + "support_probability": 0.1507045179605484, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013425742163397852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8829236626625061, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8214403390884399, + 0.21321547031402588, + 0.463763564825058, + 0.43179261684417725, + 0.10239370912313461, + 0.09961965680122375 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.2875976860523224, + "style_target": 0, + "support_probability": 0.14039039611816406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004271048398391377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8964998126029968, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19910557568073273, + 0.36376824975013733, + 0.2711813151836395, + 0.3400510549545288, + 0.10889704525470734, + 0.05745387822389603, + 0.08139513432979584, + 0.10443967580795288, + 0.24059073626995087 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.3669079542160034, + "style_target": 1, + "support_probability": 0.005595483351498842, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0039046125690171875, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9369565844535828, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21110394597053528, + 0.33136799931526184, + 0.2710161507129669, + 0.35163041949272156, + 0.0725347250699997, + 0.11487849801778793, + 0.059639401733875275, + 0.09322573989629745, + 0.06754087656736374 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.6680718660354614, + "style_target": 0, + "support_probability": 0.003299231408163905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006981166427575704, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7937750816345215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16394564509391785, + 0.5298229455947876, + 0.6585769653320312, + 0.24522501230239868, + 0.05921519175171852, + 0.08390330523252487, + 0.11888270080089569, + 0.07721510529518127, + 0.07656541466712952, + 0.13082174956798553, + 0.2630035877227783 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 1, + "style_probability": 0.5240399241447449, + "style_target": 1, + "support_probability": 0.0036515635438263416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015451364809050232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7583454847335815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1583024561405182, + 0.5588879585266113, + 0.7076996564865112, + 0.23704314231872559, + 0.08459682762622833, + 0.1764364093542099 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 1, + "style_probability": 0.41915035247802734, + "style_target": 0, + "support_probability": 0.013217235915362835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7537851834734445, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08714893460273743, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3234311640262604, + 0.9469454288482666, + 0.18564748764038086, + 0.34177473187446594, + 0.05754953995347023 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 0, + "style_probability": 0.3978116512298584, + "style_target": 1, + "support_probability": 0.9565334320068359, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49043270286221213, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05802592262625694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.344868928194046, + 0.9870217442512512, + 0.21036383509635925, + 0.4350634515285492, + 0.19814568758010864 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 1, + "style_probability": 0.23831255733966827, + "style_target": 0, + "support_probability": 0.9447218179702759, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8676651940887686, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04980355501174927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8994:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3353085219860077, + 0.29859650135040283, + 0.9594817757606506, + 0.12033657729625702, + 0.048546019941568375, + 0.06571461260318756 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 0, + "style_probability": 0.4191054403781891, + "style_target": 1, + "support_probability": 0.9829189777374268, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9430234536209753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.033188603818416595, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8994:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3306748569011688, + 0.27136537432670593, + 0.9868049621582031, + 0.11761955916881561, + 0.04877559840679169, + 0.05956599861383438 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 0, + "style_probability": 0.8018380403518677, + "style_target": 0, + "support_probability": 0.9664535522460938, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1373333545380024, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7153974771499634, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.305509090423584, + 0.6152827143669128, + 0.27697500586509705, + 0.7033612728118896, + 0.07881392538547516, + 0.12985534965991974, + 0.038020629435777664, + 0.08182388544082642, + 0.07717876881361008 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.5223035216331482, + "style_target": 1, + "support_probability": 0.16997221112251282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44969659184034566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.25568389892578125, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28111812472343445, + 0.5982169508934021, + 0.2935068905353546, + 0.693899929523468, + 0.1723124384880066 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.32034486532211304, + "style_target": 0, + "support_probability": 0.6501907110214233, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6173518411838647, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19415582716464996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24735897779464722, + 0.21379749476909637, + 0.28018155694007874, + 0.8466585874557495, + 0.036826085299253464, + 0.053798940032720566, + 0.10584785044193268, + 0.026804067194461823, + 0.04858909547328949, + 0.05476515367627144 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 0, + "style_probability": 0.43831172585487366, + "style_target": 1, + "support_probability": 0.7097148299217224, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07201098125697829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39430609345436096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28524166345596313, + 0.42694011330604553, + 0.4200403690338135, + 0.9004145860671997, + 0.08336398750543594, + 0.11504961550235748, + 0.14904740452766418, + 0.1467548906803131, + 0.04031506925821304, + 0.1042214035987854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 1, + "style_probability": 0.10790544003248215, + "style_target": 0, + "support_probability": 0.641715943813324, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25594115369953435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5185497999191284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.264240026473999, + 0.36948198080062866, + 0.1465601623058319, + 0.6249691843986511, + 0.054040007293224335 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.9122646450996399, + "style_target": 1, + "support_probability": 0.09722159802913666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015301319441837078, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.56297367811203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20039361715316772, + 0.3791377544403076, + 0.10540909320116043, + 0.48667463660240173, + 0.11214251071214676, + 0.0951877161860466, + 0.0777449831366539, + 0.06357403099536896, + 0.15859416127204895 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.12259353697299957, + "style_target": 0, + "support_probability": 0.033703140914440155, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.19322390317575813, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.149851992726326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.382657915353775, + 0.16806749999523163, + 0.29379716515541077, + 0.9297345280647278, + 0.1240520179271698, + 0.16934901475906372 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.6904705762863159, + "style_target": 1, + "support_probability": 0.0993531346321106, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07533943374888889, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1420072764158249, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4261009693145752, + 0.18435421586036682, + 0.27053672075271606, + 0.9604849815368652, + 0.07000087201595306, + 0.11543441563844681 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.2774752676486969, + "style_target": 0, + "support_probability": 0.12255328893661499, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1646657271961427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7955102920532227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08403446525335312, + 0.2885964512825012, + 0.7523611187934875, + 0.16370268166065216, + 0.0371503010392189, + 0.14045946300029755, + 0.041826412081718445, + 0.0561743825674057, + 0.04883025586605072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 1, + "style_probability": 0.8534965515136719, + "style_target": 1, + "support_probability": 0.07618343830108643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09095249594156127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7607167363166809, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07038723677396774, + 0.24144846200942993, + 0.7703714370727539, + 0.1641073226928711, + 0.09745079278945923, + 0.05209687352180481, + 0.13905742764472961, + 0.08848954737186432, + 0.04593076929450035 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 0, + "style_probability": 0.5006855130195618, + "style_target": 0, + "support_probability": 0.0958249643445015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12830681456608026, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9351458549499512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11136044561862946, + 0.37405097484588623, + 0.5265095233917236, + 0.5436767339706421, + 0.04886305332183838, + 0.040957845747470856, + 0.08094518631696701, + 0.05989353358745575, + 0.08636165410280228, + 0.06393179297447205 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.7201523184776306, + "style_target": 1, + "support_probability": 0.21463879942893982, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04235472613960567, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9778785705566406, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1203307956457138, + 0.3531532883644104, + 0.46051663160324097, + 0.45505836606025696, + 0.12233038991689682, + 0.08327022939920425 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.4034818112850189, + "style_target": 0, + "support_probability": 0.104288250207901, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5538054704594081, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29964539408683777, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9868332743644714, + 0.4766235649585724, + 0.18259508907794952, + 0.09800418466329575, + 0.06697478890419006 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.6864872574806213, + "style_target": 1, + "support_probability": 0.507382333278656, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20461196397755507, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.315218985080719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9960476756095886, + 0.5359469652175903, + 0.18030093610286713, + 0.13726110756397247, + 0.18628902733325958 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.26139312982559204, + "style_target": 0, + "support_probability": 0.5460794568061829, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9118906286820538, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.044159185141325, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1341707706451416, + 0.5729169845581055, + 0.9697120785713196, + 0.25613200664520264, + 0.1971864104270935, + 0.0577244907617569 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 1, + "style_probability": 0.5150455236434937, + "style_target": 1, + "support_probability": 0.9827898740768433, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8783856351170063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05182979628443718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15443329513072968, + 0.5281350612640381, + 0.950801432132721, + 0.2966671288013458, + 0.06390204280614853, + 0.07814362645149231 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 1, + "style_probability": 0.3937082588672638, + "style_target": 0, + "support_probability": 0.9856875538825989, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005307191120900886, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9938994646072388, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4366297721862793, + 0.3780781626701355, + 0.2888906002044678, + 0.3780781626701355, + 0.07429277151823044, + 0.0612100213766098, + 0.08515491336584091, + 0.22508203983306885, + 0.07982221245765686 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.7175993919372559, + "style_target": 1, + "support_probability": 0.003606656799092889, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003445110868104408, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9879702925682068, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44961774349212646, + 0.38850483298301697, + 0.2866668403148651, + 0.38850483298301697, + 0.12946727871894836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.2819364070892334, + "style_target": 0, + "support_probability": 0.007556751370429993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6940285129349175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2510787546634674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2766910195350647, + 0.34201279282569885, + 0.34125107526779175, + 0.8983795046806335, + 0.08771269023418427, + 0.2413773387670517, + 0.05016268417239189, + 0.06409770995378494, + 0.06848311424255371, + 0.22893023490905762, + 0.07432980090379715 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.8543790578842163, + "style_target": 1, + "support_probability": 0.45128682255744934, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06777919910072486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2573426067829132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38926059007644653, + 0.2858620584011078, + 0.3620474636554718, + 0.9347954988479614, + 0.08960722386837006, + 0.09470371901988983, + 0.11895157396793365, + 0.13650985062122345, + 0.09791125357151031, + 0.14074911177158356 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.19908303022384644, + "style_target": 0, + "support_probability": 0.4988146126270294, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6568225703827892, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06949929893016815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9607893824577332, + 0.24691441655158997, + 0.24691441655158997, + 0.20940005779266357, + 0.038180604577064514 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 1, + "style_probability": 0.7313756346702576, + "style_target": 1, + "support_probability": 0.5360325574874878, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5716170225736722, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10663007944822311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9308297634124756, + 0.27630171179771423, + 0.27630171179771423, + 0.24522905051708221, + 0.06783178448677063 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 0, + "style_probability": 0.6449493169784546, + "style_target": 0, + "support_probability": 0.5364764928817749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6280620297678854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09667807072401047, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21712839603424072, + 0.30407193303108215, + 0.9544250965118408, + 0.5670816898345947, + 0.053283657878637314, + 0.034422390162944794 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 1, + "style_probability": 0.693285346031189, + "style_target": 1, + "support_probability": 0.4270711839199066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44206291365442096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09473644942045212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23328427970409393, + 0.30701926350593567, + 0.9394700527191162, + 0.5672227740287781, + 0.04713532328605652, + 0.048299167305231094, + 0.06938029080629349, + 0.07744156569242477, + 0.0321546271443367, + 0.032563671469688416 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 0, + "style_probability": 0.5126533508300781, + "style_target": 0, + "support_probability": 0.3314971625804901, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12977737889391314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9176545739173889, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32786405086517334, + 0.3897700011730194, + 0.3600268065929413, + 0.7048523426055908, + 0.09091683477163315, + 0.08416437357664108, + 0.18053488433361053, + 0.07031749933958054, + 0.06521710008382797 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.9180182218551636, + "style_target": 1, + "support_probability": 0.08185561746358871, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015810756116381698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8751596212387085, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3651551306247711, + 0.46965160965919495, + 0.38606539368629456, + 0.809856116771698, + 0.086903415620327, + 0.33753618597984314, + 0.17732363939285278, + 0.11991836130619049, + 0.2768453061580658 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.3902088403701782, + "style_target": 0, + "support_probability": 0.10720837861299515, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9027303639183754, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07935906201601028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9080:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9767381548881531, + 0.418192058801651, + 0.3354600965976715, + 0.2363283783197403, + 0.09040787816047668, + 0.3000023365020752, + 0.06627020239830017, + 0.041099924594163895, + 0.13776709139347076, + 0.07409750670194626, + 0.04062720388174057 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 1, + "style_probability": 0.8972261548042297, + "style_target": 1, + "support_probability": 0.9091150164604187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7094343330894225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10216212272644043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9080:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9603384137153625, + 0.42642608284950256, + 0.3341630697250366, + 0.2548888325691223, + 0.08834017813205719, + 0.09918885678052902, + 0.0471012182533741, + 0.17785726487636566, + 0.20605014264583588, + 0.09061456471681595, + 0.1366211175918579 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 0, + "style_probability": 0.6467499136924744, + "style_target": 0, + "support_probability": 0.811481237411499, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6404912188637849, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25306573510169983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40932169556617737, + 0.917601466178894, + 0.41409388184547424, + 0.4069352149963379, + 0.11441989988088608 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.31151774525642395, + "style_target": 1, + "support_probability": 0.8116846084594727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8301144089082388, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2677024304866791, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4388744831085205, + 0.937002956867218, + 0.4030928909778595, + 0.424220472574234, + 0.045365672558546066 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.8340391516685486, + "style_target": 0, + "support_probability": 0.753393292427063, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5502755458772947, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4627832770347595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24220366775989532, + 0.4307836592197418, + 0.4497094750404358, + 0.7951055765151978, + 0.1558884233236313, + 0.17151865363121033 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 0, + "style_probability": 0.46468856930732727, + "style_target": 1, + "support_probability": 0.5980250835418701, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16580243265196903, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.49518632888793945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25828132033348083, + 0.5545910596847534, + 0.5562626123428345, + 0.9343555569648743, + 0.07227598875761032, + 0.09880657494068146 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 1, + "style_probability": 0.19613979756832123, + "style_target": 0, + "support_probability": 0.6615731716156006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06326564706782664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3405926525592804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7633636593818665, + 0.42758792638778687, + 0.32205140590667725, + 0.2701442241668701, + 0.06912538409233093, + 0.05287516862154007, + 0.061807550489902496, + 0.13063180446624756, + 0.07244705408811569 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 0, + "style_probability": 0.18413904309272766, + "style_target": 1, + "support_probability": 0.060587696731090546, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.057997815318321384, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3152868151664734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7664042711257935, + 0.4380316734313965, + 0.34237852692604065, + 0.3170470595359802, + 0.06189717352390289, + 0.09431710094213486, + 0.11907728761434555, + 0.09289658814668655, + 0.13844388723373413 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 1, + "style_probability": 0.2593119740486145, + "style_target": 0, + "support_probability": 0.09222234785556793, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.17309153110814682, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.306037962436676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.484702467918396, + 0.3709973096847534, + 0.4318660795688629, + 0.9165342450141907, + 0.16210582852363586, + 0.08612242341041565, + 0.0835084617137909, + 0.11378409713506699, + 0.21657034754753113, + 0.23387432098388672 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 0, + "style_probability": 0.3497827649116516, + "style_target": 1, + "support_probability": 0.16898146271705627, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.338371723288873, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.352550208568573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4998989999294281, + 0.38954541087150574, + 0.4225143790245056, + 0.9324252009391785, + 0.1513117402791977, + 0.055799949914216995, + 0.13328613340854645, + 0.041954636573791504, + 0.0979079082608223, + 0.045562900602817535 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 0, + "style_probability": 0.7271021604537964, + "style_target": 0, + "support_probability": 0.21576720476150513, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07863401542408575, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6620287895202637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35595911741256714, + 0.27346742153167725, + 0.575127124786377, + 0.4055512249469757, + 0.17218802869319916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.9260159134864807, + "style_target": 1, + "support_probability": 0.01900322549045086, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031447275321999764, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.41759127378463745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44674262404441833, + 0.29510822892189026, + 0.6457716822624207, + 0.4427042603492737, + 0.09469100832939148 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.49521738290786743, + "style_target": 0, + "support_probability": 0.06620518863201141, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.34490995222143195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16950838267803192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39008158445358276, + 0.4341719150543213, + 0.39008158445358276, + 0.9973474740982056, + 0.10021068900823593, + 0.1128075048327446 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 0, + "style_probability": 0.44673579931259155, + "style_target": 1, + "support_probability": 0.5260511636734009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32724283047245545, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19351433217525482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42617061734199524, + 0.4222429096698761, + 0.42617061734199524, + 0.9915492534637451, + 0.0649731457233429, + 0.04720722883939743 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 1, + "style_probability": 0.3800238072872162, + "style_target": 0, + "support_probability": 0.5137883424758911, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8334024574211178, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14498937129974365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.154267355799675, + 0.9699634909629822, + 0.2657144069671631, + 0.26986777782440186, + 0.036639224737882614, + 0.11777425557374954, + 0.06148310378193855, + 0.08999615907669067, + 0.0999053418636322 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 1, + "style_probability": 0.676671028137207, + "style_target": 1, + "support_probability": 0.8954535722732544, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5875979745413116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19968749582767487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16537459194660187, + 0.9595049023628235, + 0.2673855125904083, + 0.2714002728462219, + 0.05886102840304375 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 1, + "style_probability": 0.28359222412109375, + "style_target": 0, + "support_probability": 0.8817114233970642, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03903143905221513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9414262771606445, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2179112732410431, + 0.7949478626251221, + 0.5426486134529114, + 0.2456466257572174, + 0.12229221314191818, + 0.11666596680879593, + 0.16566549241542816, + 0.16333435475826263, + 0.10095856338739395, + 0.08977501839399338, + 0.0666937306523323 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.7484976053237915, + "style_target": 1, + "support_probability": 0.02289501205086708, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003129416226493738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9651064872741699, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2692764699459076, + 0.8463353514671326, + 0.5898642539978027, + 0.2694781720638275, + 0.10078462958335876, + 0.13611137866973877, + 0.23623354732990265, + 0.13354754447937012, + 0.15079113841056824, + 0.08513157814741135 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.11658886820077896, + "style_target": 0, + "support_probability": 0.018813371658325195, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3987803308472346, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07996571809053421, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39089053869247437, + 0.8638945817947388, + 0.3109684884548187, + 0.22621427476406097, + 0.2116008996963501 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.625360369682312, + "style_target": 1, + "support_probability": 0.4433579444885254, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09081562388270567, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06267458200454712, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.430618017911911, + 0.9315007925033569, + 0.34309661388397217, + 0.2114550918340683, + 0.16251926124095917 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.1706768274307251, + "style_target": 0, + "support_probability": 0.6635774970054626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5439438128733374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19061870872974396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4996952414512634, + 0.936434268951416, + 0.25327715277671814, + 0.2173406332731247, + 0.07742363214492798, + 0.17395268380641937 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 1, + "style_probability": 0.5358049869537354, + "style_target": 1, + "support_probability": 0.5841397047042847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6960032038069134, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20342402160167694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46071678400039673, + 0.9734396934509277, + 0.24995489418506622, + 0.2057756632566452, + 0.0893147811293602, + 0.38731372356414795 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 0, + "style_probability": 0.7741363644599915, + "style_target": 0, + "support_probability": 0.54547518491745, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.434725197353562, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5928462743759155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7170062065124512, + 0.20647995173931122, + 0.22457121312618256, + 0.461025208234787, + 0.04968733713030815, + 0.15647226572036743, + 0.0637885257601738, + 0.07208606600761414, + 0.19965220987796783 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 1, + "style_probability": 0.5395981073379517, + "style_target": 1, + "support_probability": 0.5132592916488647, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17708769286268597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7425664067268372, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.749534010887146, + 0.24677811563014984, + 0.201859712600708, + 0.49147894978523254, + 0.1549205183982849 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 1, + "style_probability": 0.266591340303421, + "style_target": 0, + "support_probability": 0.2192537486553192, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5624535142919453, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13703668117523193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.47674664855003357, + 0.1618671864271164, + 0.949297308921814, + 0.47674664855003357, + 0.10394672304391861, + 0.21255889534950256, + 0.16495497524738312, + 0.11264863610267639, + 0.12674196064472198, + 0.1678277999162674 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 0, + "style_probability": 0.4144519567489624, + "style_target": 1, + "support_probability": 0.6930140852928162, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2504076742683417, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08006621152162552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6598709225654602, + 0.2418292611837387, + 0.9901524186134338, + 0.6598709225654602, + 0.11176470667123795, + 0.09481015056371689, + 0.09456413984298706, + 0.14865580201148987, + 0.2244919389486313, + 0.1657651662826538 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 1, + "style_probability": 0.14011690020561218, + "style_target": 0, + "support_probability": 0.8664553165435791, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06665237380186759, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2016223967075348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12289557605981827, + 0.7274922132492065, + 0.17332227528095245, + 0.15814265608787537, + 0.1281641721725464 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.45339494943618774, + "style_target": 1, + "support_probability": 0.07249198853969574, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2375630489246862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14716632664203644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12388839572668076, + 0.7483571171760559, + 0.1971924901008606, + 0.1611073762178421, + 0.09923909604549408 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.8393803834915161, + "style_target": 0, + "support_probability": 0.1464800387620926, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5589713160240447, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3688773810863495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1681765466928482, + 0.31174153089523315, + 0.8700512051582336, + 0.4006960690021515, + 0.08966385573148727, + 0.09118745476007462 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 0, + "style_probability": 0.44403165578842163, + "style_target": 1, + "support_probability": 0.6309155821800232, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12396880286745993, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.32878848910331726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20627903938293457, + 0.41690412163734436, + 0.950244665145874, + 0.4464872479438782, + 0.11186982691287994, + 0.10817885398864746 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 1, + "style_probability": 0.11872966587543488, + "style_target": 0, + "support_probability": 0.7081950902938843, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01896017048007987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9426164627075195, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2601172626018524, + 0.2685690224170685, + 0.4276774823665619, + 0.6855866312980652, + 0.11549527198076248, + 0.09693293273448944, + 0.09456824511289597, + 0.11700119823217392, + 0.07977800071239471 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 0, + "style_probability": 0.4608769714832306, + "style_target": 1, + "support_probability": 0.016025250777602196, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11279761243738541, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6998606324195862, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25275030732154846, + 0.3548879027366638, + 0.4523772597312927, + 0.7484976053237915, + 0.10799913108348846, + 0.08690977096557617, + 0.11184559762477875, + 0.13210240006446838, + 0.08444524556398392 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 0, + "style_probability": 0.6373627781867981, + "style_target": 0, + "support_probability": 0.09162595123052597, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10858279165593972, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7905598282814026, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18278466165065765, + 0.5680301785469055, + 0.3971719443798065, + 0.511807918548584, + 0.16470547020435333, + 0.10513299703598022, + 0.16021089255809784, + 0.13597866892814636, + 0.15802349150180817, + 0.15507912635803223, + 0.11768149584531784 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.3983991742134094, + "style_target": 1, + "support_probability": 0.14047327637672424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27179550286162774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6256147623062134, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17298075556755066, + 0.5377329587936401, + 0.38411659002304077, + 0.516258716583252, + 0.032256193459033966, + 0.06794828921556473, + 0.07643716782331467, + 0.10113681852817535, + 0.24365836381912231, + 0.14626114070415497, + 0.13739138841629028 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.8176369667053223, + "style_target": 0, + "support_probability": 0.19785559177398682, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47879783765341455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1893504559993744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.810829222202301, + 0.4080762565135956, + 0.26118823885917664, + 0.19584335386753082, + 0.0528678260743618 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.7221329212188721, + "style_target": 1, + "support_probability": 0.5354185700416565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023351449344807814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4455610513687134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8478262424468994, + 0.4741956889629364, + 0.41989579796791077, + 0.20017611980438232, + 0.13733963668346405 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.06364858895540237, + "style_target": 0, + "support_probability": 0.40960416197776794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03777925587382282, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9541557431221008, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21135523915290833, + 0.6249262094497681, + 0.3905784785747528, + 0.4540204405784607, + 0.16258543729782104, + 0.10412117093801498 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 1, + "style_probability": 0.7861518263816833, + "style_target": 1, + "support_probability": 0.02549547515809536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09057754255865601, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8614221215248108, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25197187066078186, + 0.6103693842887878, + 0.3659578263759613, + 0.42682114243507385, + 0.090135358273983, + 0.07901697605848312 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 0, + "style_probability": 0.6763690710067749, + "style_target": 0, + "support_probability": 0.0614713653922081, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.29523300844368244, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6850177049636841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.227146714925766, + 0.31150636076927185, + 0.8057375550270081, + 0.1948089450597763, + 0.018933171406388283, + 0.04973356053233147, + 0.07017023116350174, + 0.16788558661937714, + 0.04349594935774803 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.8786670565605164, + "style_target": 1, + "support_probability": 0.12459567189216614, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10395268961367359, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7174898982048035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23656366765499115, + 0.3088415265083313, + 0.7476633191108704, + 0.19884490966796875, + 0.05160486698150635 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.49262017011642456, + "style_target": 0, + "support_probability": 0.13550536334514618, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8732528594962062, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03958815336227417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6497237086296082, + 0.4645702838897705, + 0.5556673407554626, + 0.9874899983406067, + 0.0379679799079895, + 0.03402843326330185, + 0.084466353058815, + 0.13149529695510864, + 0.15361686050891876, + 0.09226400405168533 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.7712506651878357, + "style_target": 1, + "support_probability": 0.9325063228607178, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3808678236705368, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03613986074924469, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7003149390220642, + 0.4104757010936737, + 0.5075986385345459, + 0.9906041026115417, + 0.09956065565347672, + 0.10287803411483765, + 0.17675930261611938, + 0.19759880006313324, + 0.14827580749988556, + 0.11709827929735184 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.1599101573228836, + "style_target": 0, + "support_probability": 0.9468276500701904, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.843641957166833, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07151436060667038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13532297313213348, + 0.9591647982597351, + 0.1501966267824173, + 0.3933824300765991, + 0.07787708193063736 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 1, + "style_probability": 0.8440013527870178, + "style_target": 1, + "support_probability": 0.7564201951026917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7158729346980538, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10799752920866013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1420852541923523, + 0.911888062953949, + 0.1765071153640747, + 0.3456241488456726, + 0.09472548961639404 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 0, + "style_probability": 0.6144742369651794, + "style_target": 0, + "support_probability": 0.7759060263633728, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10787692416497728, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8771511912345886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4430009722709656, + 0.6680775284767151, + 0.6630496978759766, + 0.4430009722709656, + 0.0866161435842514, + 0.10741706192493439 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.6430738568305969, + "style_target": 1, + "support_probability": 0.11794453859329224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05284442247752218, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.949069619178772, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3771347403526306, + 0.6131245493888855, + 0.6388790011405945, + 0.3771347403526306, + 0.06343141198158264, + 0.04647849500179291, + 0.09464948624372482, + 0.08160421997308731, + 0.09293195605278015, + 0.12894311547279358, + 0.10474329441785812 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.3016221523284912, + "style_target": 0, + "support_probability": 0.0646456852555275, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8408628135295515, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.032457903027534485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1361149549484253, + 0.30848631262779236, + 0.9714435338973999, + 0.2909001111984253, + 0.09902096539735794, + 0.11963099986314774, + 0.05504622682929039, + 0.0597643218934536, + 0.07268545031547546 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 0, + "style_probability": 0.31367629766464233, + "style_target": 1, + "support_probability": 0.9791989922523499, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5102220810260396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03678014129400253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1786467730998993, + 0.38855618238449097, + 0.9808620810508728, + 0.38415300846099854, + 0.10792724788188934, + 0.08419880270957947, + 0.11266288161277771, + 0.1004452258348465, + 0.1486208438873291 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 1, + "style_probability": 0.07945247739553452, + "style_target": 0, + "support_probability": 0.9786438345909119, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6296926171647325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3782086670398712, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7956787347793579, + 0.2568538784980774, + 0.2619488537311554, + 0.2568538784980774, + 0.09907158464193344, + 0.12504658102989197, + 0.06900378316640854, + 0.11469647288322449, + 0.1809239387512207, + 0.14409326016902924, + 0.08944209665060043 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 1, + "style_probability": 0.8475436568260193, + "style_target": 1, + "support_probability": 0.5382327437400818, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6278470061418678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2645080089569092, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.792235791683197, + 0.31169450283050537, + 0.2516867518424988, + 0.31169450283050537, + 0.07993965595960617, + 0.1488184779882431, + 0.09461406618356705, + 0.08650977164506912, + 0.09645475447177887, + 0.07155748456716537 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 0, + "style_probability": 0.675147294998169, + "style_target": 0, + "support_probability": 0.6816926598548889, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22575801503651372, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6720342040061951, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9749382138252258, + 0.396611750125885, + 0.3139660656452179, + 0.15151828527450562, + 0.08048661798238754 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 0, + "style_probability": 0.4674789011478424, + "style_target": 1, + "support_probability": 0.44700658321380615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3111680596540624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4928864538669586, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9681417942047119, + 0.3397539556026459, + 0.27493777871131897, + 0.15223447978496552, + 0.05473461002111435, + 0.042240191251039505, + 0.046684928238391876, + 0.05031434819102287, + 0.08731015026569366 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 1, + "style_probability": 0.2029915750026703, + "style_target": 0, + "support_probability": 0.6123179793357849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0061992812955319766, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9873900413513184, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35697510838508606, + 0.22231324017047882, + 0.24931780993938446, + 0.23167410492897034, + 0.08603111654520035, + 0.09929925203323364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 1, + "style_probability": 0.786046028137207, + "style_target": 1, + "support_probability": 0.005044695921242237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002202814395978525, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9961764812469482, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3940693438053131, + 0.2777429223060608, + 0.22582823038101196, + 0.29280397295951843, + 0.12793388962745667, + 0.09705214947462082 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 1, + "style_probability": 0.3928285837173462, + "style_target": 0, + "support_probability": 0.004234433639794588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18249509628917845, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6603195071220398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34689608216285706, + 0.6570094227790833, + 0.2846575677394867, + 0.487216591835022, + 0.09294091910123825, + 0.09295740723609924, + 0.15178632736206055, + 0.05446672812104225, + 0.14588265120983124 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 1, + "style_probability": 0.8071287870407104, + "style_target": 1, + "support_probability": 0.10364308208227158, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08552891222828253, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5421306490898132, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3868182599544525, + 0.6192145347595215, + 0.27367153763771057, + 0.4553678631782532, + 0.1817474216222763, + 0.11550392210483551, + 0.15191160142421722, + 0.06995604932308197, + 0.07853557914495468 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 1, + "style_probability": 0.4917796552181244, + "style_target": 0, + "support_probability": 0.10236594080924988, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7517834848998318, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17942564189434052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5040414333343506, + 0.33275842666625977, + 0.8695501685142517, + 0.3671749532222748, + 0.057521164417266846, + 0.057214945554733276, + 0.058943718671798706, + 0.31290680170059204, + 0.08069577068090439, + 0.05132871866226196, + 0.08164936304092407 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.6879904866218567, + "style_target": 1, + "support_probability": 0.8984715938568115, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6202656463467003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.247773215174675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4869452714920044, + 0.33638399839401245, + 0.8401306867599487, + 0.36653801798820496, + 0.21317264437675476, + 0.06603749841451645 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.23294001817703247, + "style_target": 0, + "support_probability": 0.889590859413147, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8495577719854893, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09652870148420334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9485403299331665, + 0.2745368778705597, + 0.12255257368087769, + 0.2027132362127304, + 0.08504187315702438 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.9373167157173157, + "style_target": 1, + "support_probability": 0.7522302269935608, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5184771129663026, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.015308590605854988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9968885779380798, + 0.3323935270309448, + 0.15556670725345612, + 0.12864674627780914, + 0.24620842933654785 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.2108197659254074, + "style_target": 0, + "support_probability": 0.9446784853935242, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5935834557735742, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1588752716779709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6699265837669373, + 0.9398706555366516, + 0.21382692456245422, + 0.38405901193618774, + 0.045828770846128464, + 0.10167641192674637 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 1, + "style_probability": 0.6470418572425842, + "style_target": 1, + "support_probability": 0.752720057964325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7507931220208205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0772692933678627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6004176735877991, + 0.9230424761772156, + 0.2713470160961151, + 0.3411864936351776, + 0.07398971915245056, + 0.0765785500407219 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 0, + "style_probability": 0.5903221964836121, + "style_target": 0, + "support_probability": 0.8479985594749451, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8428544920845015, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17803901433944702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9835808277130127, + 0.421260267496109, + 0.3816087245941162, + 0.6472523212432861, + 0.06995052844285965, + 0.05567139759659767, + 0.030836831778287888, + 0.06090578809380531, + 0.10238301008939743 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.8189014792442322, + "style_target": 1, + "support_probability": 0.7770287394523621, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5915958096745655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1694323867559433, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9796436429023743, + 0.40545955300331116, + 0.3693992495536804, + 0.6148268580436707, + 0.3253096342086792 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.4237247705459595, + "style_target": 0, + "support_probability": 0.7098202705383301, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2918864524742877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43769699335098267, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16339321434497833, + 0.735454797744751, + 0.6584107279777527, + 0.2860088348388672, + 0.07842834293842316, + 0.029929962009191513, + 0.06864672154188156, + 0.04212802276015282, + 0.09365955740213394, + 0.03663942590355873 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7041690945625305, + "style_target": 1, + "support_probability": 0.23210258781909943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03256914929878929, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5423703789710999, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2198161780834198, + 0.7856089472770691, + 0.6882179379463196, + 0.3272792100906372, + 0.09770660847425461, + 0.12410270422697067, + 0.17335397005081177, + 0.13197441399097443, + 0.09356248378753662, + 0.13838240504264832 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.1696273386478424, + "style_target": 0, + "support_probability": 0.18742403388023376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20682488010631786, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.44302764534950256, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14210329949855804, + 0.6161070466041565, + 0.3230682611465454, + 0.645611047744751, + 0.1399872899055481 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 0, + "style_probability": 0.26413869857788086, + "style_target": 1, + "support_probability": 0.38203123211860657, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12766508254188444, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.49387064576148987, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12931948900222778, + 0.5543708205223083, + 0.2746785581111908, + 0.6129261255264282, + 0.07072972506284714 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 1, + "style_probability": 0.3502439260482788, + "style_target": 0, + "support_probability": 0.3147434592247009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008657145744312264, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9257142543792725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5737345218658447, + 0.1593170166015625, + 0.18290352821350098, + 0.2571199834346771, + 0.05900684744119644, + 0.11982113867998123 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.4005115032196045, + "style_target": 1, + "support_probability": 0.00941797997802496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016610526477068777, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9376344084739685, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5789328813552856, + 0.1388828009366989, + 0.1787881851196289, + 0.2725701630115509, + 0.031207207590341568, + 0.1321922093629837 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.8492416739463806, + "style_target": 0, + "support_probability": 0.008607706986367702, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2671285933023483, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29738983511924744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4234254062175751, + 0.6357629299163818, + 0.2870620787143707, + 0.28366780281066895, + 0.1006900742650032, + 0.09311114251613617, + 0.1760827898979187, + 0.030211783945560455, + 0.08432929217815399 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 1, + "style_probability": 0.5320950746536255, + "style_target": 1, + "support_probability": 0.35108160972595215, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17562352242358992, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11796807497739792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4516944885253906, + 0.8039202690124512, + 0.39724233746528625, + 0.20578724145889282, + 0.06578300148248672, + 0.11432985216379166, + 0.06940456479787827, + 0.08278399705886841, + 0.12267430126667023 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 1, + "style_probability": 0.34763985872268677, + "style_target": 0, + "support_probability": 0.6636211276054382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12503164967005298, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7670007944107056, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43398433923721313, + 0.2225196659564972, + 0.14988718926906586, + 0.9443266987800598, + 0.17215558886528015, + 0.39485567808151245, + 0.14471164345741272, + 0.20869556069374084, + 0.20707151293754578, + 0.0785260796546936, + 0.1887074112892151 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 0, + "style_probability": 0.3285294771194458, + "style_target": 1, + "support_probability": 0.18134334683418274, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42350249625564806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5624719858169556, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.42738062143325806, + 0.20408405363559723, + 0.15242105722427368, + 0.9410952925682068, + 0.16031120717525482, + 0.13776515424251556, + 0.0940970927476883, + 0.15693862736225128, + 0.10540259629487991, + 0.12672124803066254 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 0, + "style_probability": 0.537192702293396, + "style_target": 0, + "support_probability": 0.3653251528739929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21793073587048725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4755517840385437, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.0671173632144928, + 0.13342027366161346, + 0.39557427167892456, + 0.32700860500335693, + 0.23964907228946686 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.38505861163139343, + "style_target": 1, + "support_probability": 0.49016502499580383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.328815571751793, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5222733020782471, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.06170468032360077, + 0.13204838335514069, + 0.39951595664024353, + 0.2932395040988922, + 0.057701658457517624 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.8476259112358093, + "style_target": 0, + "support_probability": 0.46993139386177063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07127182282999521, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8177329301834106, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2580815851688385, + 0.22875113785266876, + 0.5310022234916687, + 0.3626794219017029, + 0.06753367185592651, + 0.12351120263338089 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 1, + "style_probability": 0.5186295509338379, + "style_target": 1, + "support_probability": 0.10993978381156921, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06426066542228731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43106067180633545, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3336784839630127, + 0.35070282220840454, + 0.7705776691436768, + 0.627543568611145, + 0.11474647372961044, + 0.13308051228523254 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 1, + "style_probability": 0.36886200308799744, + "style_target": 0, + "support_probability": 0.4196068048477173, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014823088940783276, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9687851667404175, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35874900221824646, + 0.5329184532165527, + 0.46420782804489136, + 0.38732266426086426, + 0.1133236214518547, + 0.11782209575176239, + 0.10456548631191254, + 0.05115222930908203, + 0.1253305822610855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 0, + "style_probability": 0.25356295704841614, + "style_target": 1, + "support_probability": 0.02260787971317768, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02779790682677905, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9356546401977539, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3612103760242462, + 0.6005069613456726, + 0.4922586977481842, + 0.3387121260166168, + 0.1314278393983841, + 0.11375388503074646, + 0.12690384685993195, + 0.08561474829912186, + 0.15585751831531525 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 1, + "style_probability": 0.3967232406139374, + "style_target": 0, + "support_probability": 0.04270815849304199, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4625302899412938, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37530753016471863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16505871713161469, + 0.1810750812292099, + 0.319524347782135, + 0.9253734946250916, + 0.07858555018901825, + 0.07332024723291397, + 0.06905901432037354, + 0.12956322729587555, + 0.07690394669771194, + 0.10349461436271667 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.2903192341327667, + "style_target": 1, + "support_probability": 0.6971977353096008, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7437945729461481, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3166942298412323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1871722787618637, + 0.15970835089683533, + 0.3440554738044739, + 0.9395221471786499, + 0.05636037141084671, + 0.14940840005874634, + 0.06756572425365448, + 0.18636226654052734, + 0.07144030928611755, + 0.07438557595014572, + 0.08926647156476974 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.7929202914237976, + "style_target": 0, + "support_probability": 0.7195414304733276, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.24373372063799112, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3364637494087219, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3030449151992798, + 0.14857758581638336, + 0.35716187953948975, + 0.7773342728614807, + 0.036171965301036835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.806462287902832, + "style_target": 1, + "support_probability": 0.07766471058130264, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0422616096019987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4458693861961365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3123236894607544, + 0.14854727685451508, + 0.3383753001689911, + 0.8163251876831055, + 0.08948556333780289 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.3887474834918976, + "style_target": 0, + "support_probability": 0.06483656167984009, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.051543489849880435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6044468879699707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6534082889556885, + 0.8413305282592773, + 0.18474972248077393, + 0.5510281324386597, + 0.17416352033615112, + 0.09647800028324127 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 0, + "style_probability": 0.44874417781829834, + "style_target": 1, + "support_probability": 0.12191726267337799, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09140507015449209, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3872249126434326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6327952742576599, + 0.8535103797912598, + 0.21487320959568024, + 0.49627619981765747, + 0.038265276700258255, + 0.047833506017923355 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 1, + "style_probability": 0.3031982481479645, + "style_target": 0, + "support_probability": 0.2734520733356476, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2898885311449255, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30703121423721313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2824958562850952, + 0.4979516565799713, + 0.9772818088531494, + 0.7045108675956726, + 0.09057387709617615, + 0.08642099052667618, + 0.07075190544128418, + 0.14996923506259918, + 0.0922951027750969 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.5696055889129639, + "style_target": 1, + "support_probability": 0.33289557695388794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2484587854773323, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22695286571979523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2780921757221222, + 0.47189104557037354, + 0.9677422046661377, + 0.656478762626648, + 0.0823613852262497 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.27001017332077026, + "style_target": 0, + "support_probability": 0.39764168858528137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13363508106565986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8657553195953369, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6322107911109924, + 0.13055159151554108, + 0.28053370118141174, + 0.15286488831043243, + 0.05108566954731941, + 0.12125089019536972, + 0.06460742652416229, + 0.0929933488368988, + 0.15502117574214935, + 0.1560327112674713, + 0.0737900510430336 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.875386655330658, + "style_target": 1, + "support_probability": 0.053993504494428635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017676605964452235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8962023258209229, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7893408536911011, + 0.21933864057064056, + 0.38449567556381226, + 0.176610067486763, + 0.10294269025325775, + 0.07798758149147034, + 0.19630743563175201, + 0.09785294532775879, + 0.098934106528759, + 0.121702179312706 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.24124547839164734, + "style_target": 0, + "support_probability": 0.06920182704925537, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016159895837551508, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6334003210067749, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4987453818321228, + 0.19563984870910645, + 0.4987453818321228, + 0.3652784526348114, + 0.09196645766496658 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 1, + "style_probability": 0.5453671813011169, + "style_target": 1, + "support_probability": 0.0103753712028265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03264790088113709, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4961990416049957, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4890836477279663, + 0.25034570693969727, + 0.4890836477279663, + 0.35042861104011536, + 0.17800089716911316 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 1, + "style_probability": 0.4675173759460449, + "style_target": 0, + "support_probability": 0.028416607528924942, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8075699699891674, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026988551020622253, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7896209955215454, + 0.8933150768280029, + 0.2929624617099762, + 0.19294743239879608, + 0.09307481348514557, + 0.10929200053215027 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 1, + "style_probability": 0.8257238864898682, + "style_target": 1, + "support_probability": 0.9074217677116394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6059898117552309, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0703035220503807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7144099473953247, + 0.8350361585617065, + 0.35021257400512695, + 0.227084681391716, + 0.13407357037067413, + 0.07369554042816162, + 0.13398168981075287, + 0.115680992603302, + 0.1548442244529724, + 0.10346700996160507 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 1, + "style_probability": 0.3498271107673645, + "style_target": 0, + "support_probability": 0.8353369235992432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10470143975238723, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7307108044624329, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1825564056634903, + 0.37784910202026367, + 0.24177956581115723, + 0.47175082564353943, + 0.08872486650943756, + 0.03578998148441315, + 0.043216388672590256, + 0.06303960829973221, + 0.04801483824849129 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.7702053189277649, + "style_target": 1, + "support_probability": 0.09752164781093597, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034437049466615457, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8118278980255127, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24474722146987915, + 0.3802458345890045, + 0.2941163182258606, + 0.5152416229248047, + 0.11444396525621414, + 0.11698409914970398, + 0.09099733829498291, + 0.09754561632871628, + 0.12928861379623413 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.06708841025829315, + "style_target": 0, + "support_probability": 0.08242755383253098, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9006859465418081, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11771611869335175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4510040283203125, + 0.18711026012897491, + 0.21152885258197784, + 0.9790462255477905, + 0.091795913875103, + 0.048653990030288696, + 0.054192546755075455, + 0.04068683832883835, + 0.04716363549232483, + 0.04930947348475456, + 0.062158193439245224 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 1, + "style_probability": 0.9036675691604614, + "style_target": 1, + "support_probability": 0.8734586834907532, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7363984321805679, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.243360698223114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9347:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4262523651123047, + 0.15010792016983032, + 0.17070306837558746, + 0.9508985280990601, + 0.053196828812360764, + 0.07319861650466919, + 0.08734136074781418, + 0.09172245860099792, + 0.048658158630132675, + 0.07572504132986069, + 0.27245089411735535 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 0, + "style_probability": 0.6665228009223938, + "style_target": 0, + "support_probability": 0.7681005597114563, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9301331861366363, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10687633603811264, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9349:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21933400630950928, + 0.9830528497695923, + 0.5252367258071899, + 0.30668699741363525, + 0.19219912588596344 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.7669233679771423, + "style_target": 1, + "support_probability": 0.9101737141609192, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8177305651350342, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1198754534125328, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19167537987232208, + 0.9772558808326721, + 0.5042211413383484, + 0.3408910632133484, + 0.21228577196598053, + 0.15759789943695068, + 0.09401921182870865, + 0.05197993293404579, + 0.13435305655002594 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.21814629435539246, + "style_target": 0, + "support_probability": 0.9018739461898804, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020272588603261334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9371567368507385, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3755360543727875, + 0.7039424777030945, + 0.18489976227283478, + 0.1320021152496338, + 0.12486746162176132, + 0.08191438764333725 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.6105648875236511, + "style_target": 1, + "support_probability": 0.014442985877394676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004096464991567567, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9063153266906738, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4396032691001892, + 0.7775833606719971, + 0.22134754061698914, + 0.1753527969121933, + 0.0897129476070404, + 0.14634336531162262 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.2447967231273651, + "style_target": 0, + "support_probability": 0.017732219770550728, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7617337914775032, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1555810272693634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3406473994255066, + 0.9586718678474426, + 0.11408811807632446, + 0.579988956451416, + 0.0789179876446724, + 0.06868097931146622, + 0.09631091356277466, + 0.060100145637989044, + 0.07752683013677597 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 1, + "style_probability": 0.7363128662109375, + "style_target": 1, + "support_probability": 0.6800042986869812, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5971135036538952, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25955313444137573, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33636829257011414, + 0.9419533610343933, + 0.12243761122226715, + 0.5912852883338928, + 0.07509984076023102, + 0.200630322098732, + 0.07017503678798676, + 0.11683529615402222, + 0.07026686519384384 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 0, + "style_probability": 0.6257544755935669, + "style_target": 0, + "support_probability": 0.5111215114593506, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19724308325357065, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8655248284339905, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11482229083776474, + 0.21944448351860046, + 0.29711613059043884, + 0.11482229083776474, + 0.11542793363332748, + 0.04852229356765747, + 0.029644787311553955, + 0.05052771046757698, + 0.1028747707605362, + 0.09104137122631073, + 0.04612058773636818 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.8390530943870544, + "style_target": 1, + "support_probability": 0.2013060748577118, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11691891525933701, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7716588377952576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13024373352527618, + 0.21271198987960815, + 0.2573338747024536, + 0.13024373352527618, + 0.0321941077709198, + 0.2738402783870697 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.4456920921802521, + "style_target": 0, + "support_probability": 0.17496775090694427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6745207352383605, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1328885406255722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12613636255264282, + 0.31158292293548584, + 0.13599197566509247, + 0.9956901669502258, + 0.2701816260814667 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.7998241186141968, + "style_target": 1, + "support_probability": 0.6265893578529358, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20865071042217115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1337994635105133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.11472023278474808, + 0.2985686659812927, + 0.13535287976264954, + 0.9982855916023254, + 0.10462378710508347 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.25395631790161133, + "style_target": 0, + "support_probability": 0.6601280570030212, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7471336134270148, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2048034965991974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.941131591796875, + 0.2021329253911972, + 0.3234034776687622, + 0.15329761803150177, + 0.2208227664232254, + 0.21643605828285217 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.69913250207901, + "style_target": 1, + "support_probability": 0.7585418820381165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6637895073410478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19945017993450165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9056919813156128, + 0.29089033603668213, + 0.29185354709625244, + 0.16434422135353088, + 0.08383830636739731, + 0.27390938997268677 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.43399304151535034, + "style_target": 0, + "support_probability": 0.8000070452690125, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7578742751853891, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14780093729496002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2772628962993622, + 0.3068983852863312, + 0.9008833765983582, + 0.2597256898880005, + 0.05270536616444588, + 0.08477652072906494, + 0.06762374937534332, + 0.09884043782949448, + 0.1636705845594406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.6625335812568665, + "style_target": 1, + "support_probability": 0.7770717144012451, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6542155717454969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.18307290971279144, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27113714814186096, + 0.3692653477191925, + 0.887055516242981, + 0.25233691930770874, + 0.21677392721176147 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.32598182559013367, + "style_target": 0, + "support_probability": 0.7755828499794006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1508983310009308, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6960567235946655, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3863728642463684, + 0.6076923608779907, + 0.2824896574020386, + 0.30702412128448486, + 0.02693016082048416, + 0.06880653649568558, + 0.07905209064483643, + 0.06457201391458511, + 0.04463672265410423, + 0.1348429024219513, + 0.1607457399368286 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.8804379105567932, + "style_target": 1, + "support_probability": 0.1712213158607483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024749689231743033, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49703672528266907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2893083691596985, + 0.7030126452445984, + 0.2336607575416565, + 0.31776976585388184, + 0.07495430111885071, + 0.05621880665421486, + 0.28701210021972656, + 0.07422355562448502, + 0.06367111951112747, + 0.18297715485095978 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.15075169503688812, + "style_target": 0, + "support_probability": 0.2877172529697418, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.442214451481251, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4550155699253082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4725399315357208, + 0.7962141633033752, + 0.20707282423973083, + 0.1964120864868164, + 0.056278787553310394 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 1, + "style_probability": 0.8281283974647522, + "style_target": 1, + "support_probability": 0.2679842710494995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3455634776270764, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42093849182128906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48188644647598267, + 0.7780261635780334, + 0.18908607959747314, + 0.21226432919502258, + 0.0728476420044899 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 0, + "style_probability": 0.517073929309845, + "style_target": 0, + "support_probability": 0.2873837649822235, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8495939455063792, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.052179619669914246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9413:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06537871062755585, + 0.9867016673088074, + 0.745316207408905, + 0.06537871062755585, + 0.043062444776296616, + 0.042884040623903275 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.7314107418060303, + "style_target": 1, + "support_probability": 0.8942853212356567, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6558865992375693, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.051364023238420486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9413:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06692342460155487, + 0.97700035572052, + 0.7176727056503296, + 0.06692342460155487, + 0.08506409823894501, + 0.24339410662651062, + 0.06370176374912262, + 0.05275410786271095, + 0.08640853315591812, + 0.033703792840242386, + 0.04983244463801384 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.2389039546251297, + "style_target": 0, + "support_probability": 0.8553013801574707, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4913614798250683, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23246221244335175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3921661078929901, + 0.5367425084114075, + 0.6504968404769897, + 0.5367425084114075, + 0.06222407519817352, + 0.08691249787807465, + 0.028548402711749077, + 0.0707884430885315, + 0.034425389021635056 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.833808422088623, + "style_target": 1, + "support_probability": 0.4778570830821991, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03346008328219509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24265560507774353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48366379737854004, + 0.542106032371521, + 0.5894188284873962, + 0.542106032371521, + 0.07187343388795853, + 0.1625967174768448, + 0.06812656670808792, + 0.10197463631629944, + 0.11666052043437958 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.11135508120059967, + "style_target": 0, + "support_probability": 0.6308895945549011, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5192622057021986, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2373604029417038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3525048494338989, + 0.6216898560523987, + 0.21617215871810913, + 0.3146510720252991, + 0.10640308260917664, + 0.06058808043599129, + 0.03864153102040291, + 0.14138630032539368, + 0.035293933004140854, + 0.04507549852132797, + 0.02390855737030506 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 1, + "style_probability": 0.7664379477500916, + "style_target": 1, + "support_probability": 0.6884973645210266, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33337268124382324, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.296430766582489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.37660109996795654, + 0.5323860049247742, + 0.3085876703262329, + 0.31854546070098877, + 0.11870405077934265, + 0.03394889086484909, + 0.06118010729551315, + 0.023459788411855698, + 0.03206534683704376, + 0.039572834968566895 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 1, + "style_probability": 0.42436856031417847, + "style_target": 0, + "support_probability": 0.7654112577438354, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9420408686261423, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.039468660950660706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9952097535133362, + 0.3381795287132263, + 0.1508333832025528, + 0.35760387778282166, + 0.03799358382821083 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 0, + "style_probability": 0.3657536506652832, + "style_target": 1, + "support_probability": 0.9512475728988647, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8879546805865771, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04878001660108566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9990134239196777, + 0.30393919348716736, + 0.17074087262153625, + 0.42476725578308105, + 0.04523482173681259 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 1, + "style_probability": 0.36645346879959106, + "style_target": 0, + "support_probability": 0.9150866270065308, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.023840953112668244, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7348522543907166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18798723816871643, + 0.38045990467071533, + 0.36578044295310974, + 0.6400924921035767, + 0.060097746551036835, + 0.0774570181965828 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.6186145544052124, + "style_target": 1, + "support_probability": 0.02928265370428562, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019666061890960554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7629565000534058, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1854330450296402, + 0.3927074670791626, + 0.36472707986831665, + 0.7140886783599854, + 0.1691846400499344, + 0.06100377067923546, + 0.1434519737958908, + 0.09374792873859406, + 0.07046454399824142, + 0.08866109699010849 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.39544758200645447, + "style_target": 0, + "support_probability": 0.02919962629675865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13610519105913227, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19159899652004242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14701241254806519, + 0.10113585740327835, + 0.13022221624851227, + 0.8374003767967224, + 0.051157306879758835, + 0.08348187804222107, + 0.041565705090761185, + 0.03384855017066002, + 0.044242553412914276 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 0, + "style_probability": 0.38382622599601746, + "style_target": 1, + "support_probability": 0.2891198396682739, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.055837929594031756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20254573225975037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19598886370658875, + 0.19983866810798645, + 0.14667777717113495, + 0.956892728805542, + 0.10504943877458572, + 0.27552542090415955, + 0.12020697444677353, + 0.09470128268003464, + 0.10017600655555725 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 1, + "style_probability": 0.1588865965604782, + "style_target": 0, + "support_probability": 0.39109551906585693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10119044551888047, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5744614005088806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7019261121749878, + 0.2557274103164673, + 0.3656133711338043, + 0.609028697013855, + 0.18301863968372345, + 0.13801279664039612, + 0.1797471046447754, + 0.10105567425489426, + 0.13748794794082642, + 0.0835607722401619 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 1, + "style_probability": 0.5893970131874084, + "style_target": 1, + "support_probability": 0.08424990624189377, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3065634410358234, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33004122972488403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7599638104438782, + 0.20851735770702362, + 0.3641977608203888, + 0.6872350573539734, + 0.04752499237656593, + 0.08451367169618607, + 0.07769343256950378, + 0.1142115518450737, + 0.09848997741937637, + 0.09310133755207062, + 0.13378669321537018 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 0, + "style_probability": 0.7478071451187134, + "style_target": 0, + "support_probability": 0.14892540872097015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07083544967311757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9490857124328613, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7021986246109009, + 0.30718427896499634, + 0.41878294944763184, + 0.196971595287323, + 0.08824606239795685 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.5824407339096069, + "style_target": 1, + "support_probability": 0.07385814934968948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01999099822248061, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9876256585121155, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.653863251209259, + 0.28145670890808105, + 0.3955289125442505, + 0.20354126393795013, + 0.0757933259010315, + 0.09899801760911942, + 0.1068408191204071, + 0.08437778055667877, + 0.0798458382487297 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.34779679775238037, + "style_target": 0, + "support_probability": 0.009440422058105469, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17429861567588376, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3033991754055023, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6772560477256775, + 0.19990262389183044, + 0.5688258409500122, + 0.1784657984972, + 0.06319669634103775, + 0.11776285618543625 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.5635976791381836, + "style_target": 1, + "support_probability": 0.17463690042495728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03753877957132533, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6082471013069153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7731497287750244, + 0.2124554067850113, + 0.8167728185653687, + 0.16185155510902405, + 0.10502847284078598, + 0.13634760677814484 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.4205690920352936, + "style_target": 0, + "support_probability": 0.0872238427400589, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.38511989885119446, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.127410426735878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11083369702100754, + 0.8688398599624634, + 0.20319223403930664, + 0.2076977640390396, + 0.04090455546975136, + 0.06292840093374252, + 0.09367980062961578, + 0.06246405094861984, + 0.08051865547895432 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 0, + "style_probability": 0.4312310516834259, + "style_target": 1, + "support_probability": 0.3136293888092041, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5807646683024746, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10793661326169968, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08771709352731705, + 0.9255968332290649, + 0.2381221204996109, + 0.22334063053131104, + 0.10752091556787491, + 0.0490247942507267, + 0.31387782096862793, + 0.09040359407663345, + 0.13973718881607056 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 0, + "style_probability": 0.5972723960876465, + "style_target": 0, + "support_probability": 0.4231286346912384, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5473743533648947, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21385325491428375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17032845318317413, + 0.9711344838142395, + 0.6809656620025635, + 0.17032845318317413, + 0.03425488993525505, + 0.03609718009829521, + 0.03239299729466438, + 0.06717333197593689, + 0.06404823809862137, + 0.08680850267410278, + 0.08245554566383362 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 1, + "style_probability": 0.7292486429214478, + "style_target": 1, + "support_probability": 0.4831673204898834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.471105152345745, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23293530941009521, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1601761281490326, + 0.9750028848648071, + 0.7042258977890015, + 0.1601761281490326, + 0.0500069186091423, + 0.07279684394598007 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 0, + "style_probability": 0.5905287265777588, + "style_target": 0, + "support_probability": 0.4846169054508209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.45332632753837987, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38550207018852234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5473193526268005, + 0.6062759160995483, + 0.26248010993003845, + 0.32503005862236023, + 0.044833943247795105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.7361172437667847, + "style_target": 1, + "support_probability": 0.619111955165863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35806656443148077, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41326677799224854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4337763488292694, + 0.5993489623069763, + 0.2739320993423462, + 0.3447265625, + 0.38441532850265503 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.472684383392334, + "style_target": 0, + "support_probability": 0.6858299374580383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0020144183502358564, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9822589159011841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1736312359571457, + 0.3767773509025574, + 0.3517245054244995, + 0.1736312359571457, + 0.050759755074977875, + 0.0687248557806015 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 1, + "style_probability": 0.5177940130233765, + "style_target": 1, + "support_probability": 0.002157624112442136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009584095391978324, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9245999455451965, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1820620447397232, + 0.37316715717315674, + 0.357860803604126, + 0.1820620447397232, + 0.049721717834472656, + 0.08414632827043533, + 0.15976658463478088, + 0.06906919926404953, + 0.045776743441820145, + 0.28474193811416626, + 0.15823063254356384 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 0, + "style_probability": 0.5667529702186584, + "style_target": 0, + "support_probability": 0.010783086530864239, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5200545152545301, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5586215257644653, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6624226570129395, + 0.3885026276111603, + 0.5001922249794006, + 0.24223914742469788, + 0.05006612464785576, + 0.052606355398893356, + 0.08188264071941376, + 0.03558814898133278, + 0.05239489674568176 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.9386336207389832, + "style_target": 1, + "support_probability": 0.34553515911102295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027818794316333207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4053473472595215, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7554417252540588, + 0.4609684646129608, + 0.4501240849494934, + 0.2988528311252594, + 0.08694858103990555, + 0.11004806309938431, + 0.0903053730726242, + 0.0420105904340744, + 0.10784313827753067 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.09573069214820862, + "style_target": 0, + "support_probability": 0.5970535278320312, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9250157110501149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16333846747875214, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9541:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2822089195251465, + 0.9940734505653381, + 0.18431155383586884, + 0.6673929691314697, + 0.04732915759086609, + 0.11838310956954956, + 0.06096341460943222, + 0.090174101293087, + 0.1140943095088005, + 0.05711602792143822, + 0.08453841507434845 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 1, + "style_probability": 0.657916247844696, + "style_target": 1, + "support_probability": 0.956618070602417, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8498958861848326, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16036109626293182, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9541:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2643398940563202, + 0.984630286693573, + 0.1950131207704544, + 0.6680074334144592, + 0.13315722346305847, + 0.23984523117542267, + 0.07205857336521149, + 0.24312599003314972, + 0.0986749604344368, + 0.10260876268148422, + 0.3797130584716797 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 1, + "style_probability": 0.4541950225830078, + "style_target": 0, + "support_probability": 0.9518982172012329, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.024085461958591, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9732655882835388, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34565621614456177, + 0.43371182680130005, + 0.36985236406326294, + 0.26055312156677246, + 0.07890409976243973 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.8593522310256958, + "style_target": 1, + "support_probability": 0.030588621273636818, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012211770285814596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9814730286598206, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31084540486335754, + 0.40194597840309143, + 0.4164920151233673, + 0.2705176770687103, + 0.12970155477523804, + 0.1789400279521942, + 0.11963606625795364, + 0.25742900371551514, + 0.1770319640636444 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.2732403874397278, + "style_target": 0, + "support_probability": 0.056590430438518524, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8536441154628776, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14032790064811707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19266808032989502, + 0.9844310879707336, + 0.32231640815734863, + 0.25976645946502686, + 0.1582745760679245, + 0.10007965564727783 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6186914443969727, + "style_target": 1, + "support_probability": 0.9104745388031006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3119836342613222, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19938525557518005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22550147771835327, + 0.9888366460800171, + 0.3551296889781952, + 0.342732310295105, + 0.1064247265458107, + 0.11358335614204407 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.1154586672782898, + "style_target": 0, + "support_probability": 0.8730844855308533, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21108079468372146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8871016502380371, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6935773491859436, + 0.39473608136177063, + 0.35207659006118774, + 0.5937676429748535, + 0.14728689193725586, + 0.06623059511184692, + 0.09477050602436066, + 0.10184124857187271, + 0.0384405218064785 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 1, + "style_probability": 0.9235025644302368, + "style_target": 1, + "support_probability": 0.06195106357336044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06054750481470883, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7876282334327698, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6228168606758118, + 0.3983527719974518, + 0.3041899800300598, + 0.5543203949928284, + 0.0759972557425499, + 0.057002052664756775, + 0.06661250442266464, + 0.22176465392112732, + 0.10869699716567993 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 1, + "style_probability": 0.4814426302909851, + "style_target": 0, + "support_probability": 0.1146262064576149, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0229300346363425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.966179609298706, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7636393904685974, + 0.43947356939315796, + 0.2939766049385071, + 0.3236297070980072, + 0.10615108162164688, + 0.1603826880455017, + 0.10339903086423874, + 0.22957004606723785, + 0.1971614509820938, + 0.06508443504571915 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.7811926007270813, + "style_target": 1, + "support_probability": 0.022917676717042923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03809116989737152, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9125584959983826, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.731984555721283, + 0.4153980612754822, + 0.31074976921081543, + 0.31783103942871094, + 0.0502556636929512, + 0.1195802316069603 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.36671754717826843, + "style_target": 0, + "support_probability": 0.06378813832998276, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6761671959214407, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09068257361650467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5120882987976074, + 0.7770960330963135, + 0.2017832249403, + 0.533074140548706, + 0.03249746188521385 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.5830439925193787, + "style_target": 1, + "support_probability": 0.950420081615448, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13593971350603543, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2291845977306366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5593461990356445, + 0.8273952007293701, + 0.17828892171382904, + 0.6591046452522278, + 0.06281276047229767 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.09158174693584442, + "style_target": 0, + "support_probability": 0.9154892563819885, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8921660954758568, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04528431594371796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9954494833946228, + 0.4112257957458496, + 0.1941356062889099, + 0.4417591691017151, + 0.06400223076343536, + 0.056656233966350555 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 1, + "style_probability": 0.6323081851005554, + "style_target": 1, + "support_probability": 0.9158939123153687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8596317550596994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05941453576087952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.982585072517395, + 0.40565383434295654, + 0.2217322438955307, + 0.4265226125717163, + 0.09263536334037781, + 0.07248850166797638 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 1, + "style_probability": 0.47853371500968933, + "style_target": 0, + "support_probability": 0.9378088116645813, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6064069940253309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19322806596755981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5722105503082275, + 0.5723153948783875, + 0.9935199618339539, + 0.5689488649368286, + 0.08590960502624512, + 0.11131252348423004, + 0.19160650670528412, + 0.09604765474796295, + 0.09488629549741745 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 1, + "style_probability": 0.8060250878334045, + "style_target": 1, + "support_probability": 0.46498167514801025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30448383036912763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3271821141242981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5748991966247559, + 0.5860573649406433, + 0.9858031272888184, + 0.535489022731781, + 0.11244037747383118 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 0, + "style_probability": 0.5007520914077759, + "style_target": 0, + "support_probability": 0.33534902334213257, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8978492604214612, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18369777500629425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5914377570152283, + 0.5663190484046936, + 0.4584735929965973, + 0.9801729321479797, + 0.09964588284492493, + 0.05711747705936432, + 0.11005742847919464, + 0.1356683075428009, + 0.1580984890460968, + 0.15535788238048553 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.8111198544502258, + "style_target": 1, + "support_probability": 0.8568565845489502, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.556645838985206, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1545383334159851, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7634528875350952, + 0.6809915900230408, + 0.6253721117973328, + 0.9965534210205078, + 0.1104101836681366, + 0.09163862466812134, + 0.28172245621681213, + 0.1542806178331375, + 0.18265321850776672, + 0.14172227680683136 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.10718521475791931, + "style_target": 0, + "support_probability": 0.9325190186500549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5884975107392597, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17377927899360657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9412353038787842, + 0.414442241191864, + 0.22885066270828247, + 0.2852606475353241, + 0.21336524188518524 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 1, + "style_probability": 0.8447588086128235, + "style_target": 1, + "support_probability": 0.4621366858482361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4300760475998762, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21525253355503082, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9071438908576965, + 0.3724609911441803, + 0.22907096147537231, + 0.2318323850631714, + 0.19074666500091553 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 0, + "style_probability": 0.5909555554389954, + "style_target": 0, + "support_probability": 0.4147126376628876, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8748055894386187, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05870179086923599, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9617:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42355746030807495, + 0.9494203329086304, + 0.3840843737125397, + 0.42355746030807495, + 0.12178599834442139, + 0.11496405303478241 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.7747345566749573, + "style_target": 1, + "support_probability": 0.9579574465751648, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7450347741965967, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10564284026622772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9617:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42483386397361755, + 0.9220252633094788, + 0.31630122661590576, + 0.42483386397361755, + 0.10568132251501083, + 0.05555027723312378, + 0.09648814052343369, + 0.10040979087352753, + 0.04738416522741318, + 0.10631467401981354, + 0.1113676130771637 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.4437124729156494, + "style_target": 0, + "support_probability": 0.9085242748260498, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9456840174722091, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03784440457820892, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9620:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9803782105445862, + 0.13677023351192474, + 0.554553747177124, + 0.1319541484117508, + 0.028819920495152473, + 0.04895395040512085, + 0.040314339101314545, + 0.06515267491340637, + 0.09658388048410416 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.873321533203125, + "style_target": 1, + "support_probability": 0.9702624678611755, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6634880251242739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04130211099982262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9620:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9956640601158142, + 0.1748022884130478, + 0.6196053624153137, + 0.18482938408851624, + 0.100094735622406, + 0.1394164264202118, + 0.045227669179439545, + 0.14000758528709412, + 0.13524115085601807 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.12396325170993805, + "style_target": 0, + "support_probability": 0.9781620502471924, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.038734646418321486, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7118704319000244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31674718856811523, + 0.18746550381183624, + 0.10812272131443024, + 0.1976231187582016, + 0.05814880505204201, + 0.07663535326719284, + 0.025742769241333008, + 0.06482862681150436, + 0.04971734434366226, + 0.0704183354973793, + 0.10140165686607361 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.6371235251426697, + "style_target": 1, + "support_probability": 0.07617998123168945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014747341885217279, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6144707798957825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3293578326702118, + 0.22995677590370178, + 0.10840655118227005, + 0.1447310596704483, + 0.05166903883218765, + 0.04540335386991501, + 0.08552725613117218, + 0.09079176187515259, + 0.04549674689769745, + 0.04710942879319191, + 0.1549011766910553 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.3235416114330292, + "style_target": 0, + "support_probability": 0.07284621149301529, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8013617149141478, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.021436281502246857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9643:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31417420506477356, + 0.3751979470252991, + 0.31417420506477356, + 0.9544895887374878, + 0.08091798424720764 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 1, + "style_probability": 0.6478895545005798, + "style_target": 1, + "support_probability": 0.8653013706207275, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8939871525212402, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.03217422962188721, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9643:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3056005537509918, + 0.36279943585395813, + 0.3056005537509918, + 0.9643911719322205, + 0.06686300784349442 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 0, + "style_probability": 0.9080309867858887, + "style_target": 0, + "support_probability": 0.8497399091720581, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.254268775546271, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.755753219127655, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16637133061885834, + 0.3449517786502838, + 0.4463410973548889, + 0.6076323986053467, + 0.12620000541210175, + 0.2297256886959076 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 1, + "style_probability": 0.5833181142807007, + "style_target": 1, + "support_probability": 0.21326573193073273, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08362982864750394, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8368871808052063, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16189931333065033, + 0.35185515880584717, + 0.4238700270652771, + 0.6872952580451965, + 0.11199332028627396, + 0.09415091574192047, + 0.10789019614458084, + 0.06944575905799866, + 0.09888963401317596, + 0.1272837221622467, + 0.11132881045341492 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 1, + "style_probability": 0.37383735179901123, + "style_target": 0, + "support_probability": 0.08646971732378006, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6113468503471464, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1541195958852768, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4964642822742462, + 0.9061837196350098, + 0.48831117153167725, + 0.47687163949012756, + 0.04902356117963791, + 0.0905483067035675, + 0.023658523336052895, + 0.04106602072715759, + 0.07690414786338806 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 0, + "style_probability": 0.41076192259788513, + "style_target": 1, + "support_probability": 0.816009521484375, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1139604776901974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1910008043050766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5879512429237366, + 0.9426296949386597, + 0.5480473637580872, + 0.5085345506668091, + 0.09286810457706451, + 0.2095530927181244, + 0.0776459276676178, + 0.09608927369117737, + 0.07208479940891266 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 1, + "style_probability": 0.0871628150343895, + "style_target": 0, + "support_probability": 0.7747285962104797, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029797024564905794, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8493313789367676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32684698700904846, + 0.4360319674015045, + 0.5810268521308899, + 0.27873143553733826, + 0.12743589282035828, + 0.13380256295204163, + 0.1315912902355194, + 0.04694284498691559, + 0.0547906793653965, + 0.15802021324634552, + 0.17853862047195435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 1, + "style_probability": 0.5613483190536499, + "style_target": 1, + "support_probability": 0.029277734458446503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0886071259644621, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8110406398773193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3418058753013611, + 0.39702731370925903, + 0.5765497088432312, + 0.2797650098800659, + 0.09857102483510971, + 0.17701834440231323, + 0.11068075895309448, + 0.11072640866041183, + 0.382077157497406, + 0.22385333478450775, + 0.1892956793308258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 0, + "style_probability": 0.8483676314353943, + "style_target": 0, + "support_probability": 0.05464506149291992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09177049627606196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7101936936378479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16071905195713043, + 0.3770864009857178, + 0.8336960673332214, + 0.16071905195713043, + 0.10947854816913605 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.8252997994422913, + "style_target": 1, + "support_probability": 0.027371017262339592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013008913626777802, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7417349219322205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16693124175071716, + 0.39899489283561707, + 0.910112202167511, + 0.16693124175071716, + 0.1014629378914833 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.2820129990577698, + "style_target": 0, + "support_probability": 0.030594322830438614, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7092440893365125, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18738362193107605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23650550842285156, + 0.768795907497406, + 0.3652455508708954, + 0.5408438444137573, + 0.08839721232652664, + 0.09945022314786911 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 1, + "style_probability": 0.7836371660232544, + "style_target": 1, + "support_probability": 0.938929557800293, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.589675258184445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3701350688934326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20202398300170898, + 0.7192395329475403, + 0.27705782651901245, + 0.48138394951820374, + 0.24891521036624908, + 0.05341263487935066 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 1, + "style_probability": 0.4603258967399597, + "style_target": 0, + "support_probability": 0.8937850594520569, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7356440362311873, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06905395537614822, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25505656003952026, + 0.8280339241027832, + 0.41542309522628784, + 0.17450110614299774, + 0.03928850218653679, + 0.05188067629933357, + 0.06960641592741013, + 0.10396892577409744, + 0.05860844999551773 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.7451098561286926, + "style_target": 1, + "support_probability": 0.9663332104682922, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5503441065190213, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11355479806661606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2654661238193512, + 0.7869246602058411, + 0.40275582671165466, + 0.1648796647787094, + 0.044922567903995514 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.4003196358680725, + "style_target": 0, + "support_probability": 0.9244818687438965, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1530184061973117, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3596324920654297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2872720956802368, + 0.2592574656009674, + 0.2729320824146271, + 0.7676669359207153, + 0.05061358958482742, + 0.06075379624962807, + 0.05602891370654106, + 0.05536207556724548, + 0.05456063523888588, + 0.08431313186883926, + 0.16134485602378845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 1, + "style_probability": 0.8226085901260376, + "style_target": 1, + "support_probability": 0.05393785238265991, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04353614775107039, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2705683708190918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3333229422569275, + 0.249614417552948, + 0.32149744033813477, + 0.8266339898109436, + 0.06466645002365112, + 0.05926487222313881, + 0.14776870608329773, + 0.09213200211524963, + 0.08213619142770767, + 0.07820425927639008 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 1, + "style_probability": 0.41277778148651123, + "style_target": 0, + "support_probability": 0.09857741743326187, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6379773875082202, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.382864385843277, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9024602174758911, + 0.28877514600753784, + 0.4287532866001129, + 0.1745016723871231, + 0.08494213223457336 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 1, + "style_probability": 0.8241866230964661, + "style_target": 1, + "support_probability": 0.6860238909721375, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29349298981636407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.528293788433075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8400481343269348, + 0.2742939293384552, + 0.44919833540916443, + 0.22630718350410461, + 0.04151085391640663 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 0, + "style_probability": 0.6685258150100708, + "style_target": 0, + "support_probability": 0.4506379961967468, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4884934372395831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5570905804634094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30336669087409973, + 0.39184585213661194, + 0.2846425771713257, + 0.8138008713722229, + 0.35214468836784363, + 0.1469723880290985 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.9160385131835938, + "style_target": 1, + "support_probability": 0.3676256537437439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08199247196373606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8170847296714783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3255235254764557, + 0.3216501772403717, + 0.24333657324314117, + 0.7941807508468628, + 0.1123531237244606, + 0.06997120380401611, + 0.05685040354728699, + 0.09265320003032684, + 0.05877187103033066, + 0.08913274109363556, + 0.1204349473118782 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.47111189365386963, + "style_target": 0, + "support_probability": 0.13119463622570038, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8077828389037565, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3660821318626404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9425133466720581, + 0.429050475358963, + 0.4885534346103668, + 0.35104435682296753, + 0.05322397127747536, + 0.07691285014152527, + 0.055522579699754715, + 0.11594082415103912, + 0.07292622327804565 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.8892549872398376, + "style_target": 1, + "support_probability": 0.8713341355323792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13915962394999404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6266188621520996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9743062257766724, + 0.3594055771827698, + 0.6472227573394775, + 0.3805060088634491, + 0.08677137643098831, + 0.30587488412857056, + 0.1686461716890335, + 0.0745781883597374, + 0.15131236612796783 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.1894301176071167, + "style_target": 0, + "support_probability": 0.7292056083679199, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11337528081497836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7410256266593933, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3285287022590637, + 0.5455312728881836, + 0.3285287022590637, + 0.8260456323623657, + 0.05195198953151703, + 0.040287137031555176, + 0.09681715071201324, + 0.0659014880657196, + 0.09277557581663132, + 0.16865642368793488, + 0.08498096466064453 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 1, + "style_probability": 0.6726440787315369, + "style_target": 1, + "support_probability": 0.09391129016876221, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0474826303869737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.795134961605072, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34665653109550476, + 0.5532727241516113, + 0.34665653109550476, + 0.7749983668327332, + 0.05024014785885811, + 0.06591806560754776, + 0.04581042006611824, + 0.0477248951792717, + 0.0750260129570961, + 0.06611218303442001, + 0.04772592708468437 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 1, + "style_probability": 0.49485307931900024, + "style_target": 0, + "support_probability": 0.04096117243170738, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5627533146788638, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23704320192337036, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28802651166915894, + 0.09936612844467163, + 0.3352273106575012, + 0.9556329250335693, + 0.05296696722507477 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.352481871843338, + "style_target": 1, + "support_probability": 0.8418753147125244, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6129024261568737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15264660120010376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33204761147499084, + 0.08611953258514404, + 0.33691152930259705, + 0.9740740656852722, + 0.08640038222074509 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.6075887084007263, + "style_target": 0, + "support_probability": 0.8640657663345337, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17095962360650735, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3281131386756897, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18904899060726166, + 0.6420755386352539, + 0.3328936696052551, + 0.2242681384086609, + 0.11579366028308868, + 0.054422564804553986 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 0, + "style_probability": 0.42018505930900574, + "style_target": 1, + "support_probability": 0.2294294238090515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11414162038552188, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18493996560573578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18807397782802582, + 0.7054194808006287, + 0.45064467191696167, + 0.2195858508348465, + 0.11376854032278061, + 0.24809688329696655 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 1, + "style_probability": 0.26926887035369873, + "style_target": 0, + "support_probability": 0.5319153070449829, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028952474003424563, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9733315110206604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16878218948841095, + 0.5967589020729065, + 0.32035166025161743, + 0.3298451900482178, + 0.13056552410125732, + 0.2150658220052719, + 0.09755983203649521, + 0.13312935829162598, + 0.11812782287597656 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 0, + "style_probability": 0.1576959192752838, + "style_target": 1, + "support_probability": 0.005208397284150124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033438756920916688, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9675760865211487, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16703717410564423, + 0.5256580710411072, + 0.38257524371147156, + 0.36522310972213745, + 0.17408616840839386, + 0.16212141513824463, + 0.1140773817896843, + 0.13706476986408234, + 0.11934680491685867 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 1, + "style_probability": 0.49954360723495483, + "style_target": 0, + "support_probability": 0.005529484711587429, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15414049971607166, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.634738028049469, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3848634362220764, + 0.37019112706184387, + 0.7660108208656311, + 0.1635398417711258, + 0.1213005855679512, + 0.10720302909612656, + 0.08389963954687119, + 0.06417465209960938, + 0.11454028636217117, + 0.19344298541545868, + 0.0907978042960167 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 1, + "style_probability": 0.6261146068572998, + "style_target": 1, + "support_probability": 0.08822411298751831, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4840328863748697, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5212029814720154, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41090789437294006, + 0.37155526876449585, + 0.8180442452430725, + 0.1783655732870102, + 0.05971570312976837, + 0.05558362230658531, + 0.08162912726402283, + 0.06906758248806, + 0.05289088189601898, + 0.09798084944486618, + 0.03665248304605484 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 0, + "style_probability": 0.9187606573104858, + "style_target": 0, + "support_probability": 0.25584661960601807, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24527771346917068, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8251134753227234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26817256212234497, + 0.27972516417503357, + 0.22333607077598572, + 0.6923399567604065, + 0.24526354670524597 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 1, + "style_probability": 0.5148987770080566, + "style_target": 1, + "support_probability": 0.5282000303268433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07187790490585098, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.936202883720398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23135346174240112, + 0.2242414951324463, + 0.18488594889640808, + 0.6434226036071777, + 0.06964301317930222, + 0.08576210588216782 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 1, + "style_probability": 0.28970807790756226, + "style_target": 0, + "support_probability": 0.0957406610250473, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.24209416494839947, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12469621747732162, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17181524634361267, + 0.25870609283447266, + 0.1976800411939621, + 0.9123765230178833, + 0.09953883290290833, + 0.2520079016685486 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 0, + "style_probability": 0.3018164336681366, + "style_target": 1, + "support_probability": 0.5555616021156311, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03701076865482434, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08923489600419998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1269206404685974, + 0.2507754862308502, + 0.20945371687412262, + 0.9735090732574463, + 0.08480706065893173, + 0.11756667494773865 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 1, + "style_probability": 0.08446557819843292, + "style_target": 0, + "support_probability": 0.4630741775035858, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02764704550359938, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7731046676635742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14838965237140656, + 0.14838965237140656, + 0.7513986229896545, + 0.3596802353858948, + 0.06772290915250778, + 0.19767148792743683, + 0.12162382155656815, + 0.0655713900923729, + 0.17148073017597198 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 1, + "style_probability": 0.5952836275100708, + "style_target": 1, + "support_probability": 0.03044722229242325, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09332560746366476, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3510544002056122, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13043421506881714, + 0.13043421506881714, + 0.7579656839370728, + 0.3074035048484802, + 0.09733431041240692, + 0.13483652472496033, + 0.06308550387620926, + 0.046676430851221085, + 0.09803367406129837 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 0, + "style_probability": 0.7009677886962891, + "style_target": 0, + "support_probability": 0.10780631005764008, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7674125623486735, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19054336845874786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9539917707443237, + 0.40442344546318054, + 0.12758611142635345, + 0.20092490315437317, + 0.05157407373189926, + 0.1693524569272995, + 0.07663930207490921, + 0.35441944003105164, + 0.16982042789459229, + 0.06080633029341698, + 0.3156796395778656 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 1, + "style_probability": 0.6448259949684143, + "style_target": 1, + "support_probability": 0.7951866984367371, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7533858887462431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16713543236255646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9531524777412415, + 0.3332023322582245, + 0.1263805329799652, + 0.18221798539161682, + 0.048078883439302444 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 0, + "style_probability": 0.5595372915267944, + "style_target": 0, + "support_probability": 0.8252077102661133, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018685263936526114, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9689772725105286, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.327345073223114, + 0.5768148303031921, + 0.5925673842430115, + 0.30687353014945984, + 0.10636427253484726 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 1, + "style_probability": 0.581413209438324, + "style_target": 1, + "support_probability": 0.03369962424039841, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014294509754502979, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9942466616630554, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2971184551715851, + 0.5467756390571594, + 0.6366140842437744, + 0.3108848035335541, + 0.18986628949642181 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 1, + "style_probability": 0.1337822675704956, + "style_target": 0, + "support_probability": 0.013987715356051922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.161099655010722, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49061834812164307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28858980536460876, + 0.8489128351211548, + 0.36154064536094666, + 0.4791882336139679, + 0.2935166656970978, + 0.09750178456306458 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 1, + "style_probability": 0.5228509306907654, + "style_target": 1, + "support_probability": 0.1761149764060974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08844717163683798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8303725719451904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32306939363479614, + 0.8764776587486267, + 0.436862975358963, + 0.508008599281311, + 0.07493560761213303, + 0.08437769114971161 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 0, + "style_probability": 0.7994630932807922, + "style_target": 0, + "support_probability": 0.047320686280727386, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5207079913604452, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47701922059059143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3223842978477478, + 0.18750602006912231, + 0.8539832830429077, + 0.45032504200935364, + 0.09818338602781296, + 0.06390976905822754, + 0.09465689957141876, + 0.1532246172428131, + 0.11467579752206802 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 0, + "style_probability": 0.4864121675491333, + "style_target": 1, + "support_probability": 0.5614820718765259, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3000546888864632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7108195424079895, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3370227515697479, + 0.15732119977474213, + 0.8573569059371948, + 0.4793158769607544, + 0.09476755559444427, + 0.057059865444898605, + 0.19296754896640778, + 0.13381317257881165, + 0.13185690343379974, + 0.09209791570901871, + 0.0938706248998642 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 1, + "style_probability": 0.46279197931289673, + "style_target": 0, + "support_probability": 0.200319305062294, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7176219937254257, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08181937783956528, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4630082845687866, + 0.9606486558914185, + 0.34155091643333435, + 0.5095241069793701, + 0.06502722203731537, + 0.06830141693353653, + 0.08029751479625702, + 0.0726141482591629, + 0.12182474136352539, + 0.14690163731575012, + 0.0905868336558342 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.5997527241706848, + "style_target": 1, + "support_probability": 0.793357253074646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5060481560642696, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03495488315820694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6933842897415161, + 0.997470498085022, + 0.46419721841812134, + 0.5912306904792786, + 0.10830865055322647, + 0.03495902568101883, + 0.15695764124393463, + 0.10965722799301147, + 0.05582623556256294, + 0.15934500098228455 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.1421527862548828, + "style_target": 0, + "support_probability": 0.9591047167778015, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.054726798982345976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6098118424415588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4316082298755646, + 0.5205734372138977, + 0.19257664680480957, + 0.7081059813499451, + 0.09009035676717758 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 1, + "style_probability": 0.5981212258338928, + "style_target": 1, + "support_probability": 0.04045054316520691, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04315193417542717, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.690893828868866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.415366530418396, + 0.5545225143432617, + 0.164756178855896, + 0.7513133883476257, + 0.14080660045146942 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 0, + "style_probability": 0.7869912981987, + "style_target": 0, + "support_probability": 0.023822307586669922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.591597669625192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17275843024253845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33287063241004944, + 0.49632564187049866, + 0.9815478324890137, + 0.33287063241004944, + 0.09815257042646408, + 0.07912439107894897 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 0, + "style_probability": 0.41647645831108093, + "style_target": 1, + "support_probability": 0.7624273300170898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42285013357391654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1525510847568512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3591669499874115, + 0.4840993881225586, + 0.9714411497116089, + 0.3591669499874115, + 0.28600046038627625, + 0.07074373960494995, + 0.08400245755910873, + 0.16222348809242249, + 0.06819304823875427 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 1, + "style_probability": 0.3106275796890259, + "style_target": 0, + "support_probability": 0.603354275226593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16557132672764954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5730745792388916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8767774105072021, + 0.27273237705230713, + 0.4296243488788605, + 0.26862266659736633, + 0.09058263152837753, + 0.12766745686531067, + 0.06428614258766174, + 0.039759598672389984, + 0.06287605315446854 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.7219046354293823, + "style_target": 1, + "support_probability": 0.13677579164505005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10826212409397794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18814635276794434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9613074064254761, + 0.21075917780399323, + 0.41884034872055054, + 0.29083290696144104, + 0.04473956674337387, + 0.08865875005722046, + 0.08601956814527512, + 0.05549941584467888, + 0.15226402878761292 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.2868098318576813, + "style_target": 0, + "support_probability": 0.45560359954833984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08626479306899999, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8566462397575378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6492356061935425, + 0.277211993932724, + 0.6492356061935425, + 0.7133370041847229, + 0.0974615067243576, + 0.09392020106315613, + 0.09770906716585159, + 0.096439890563488, + 0.18502354621887207, + 0.09574069827795029, + 0.06600652635097504 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 1, + "style_probability": 0.6788482666015625, + "style_target": 1, + "support_probability": 0.046286776661872864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5707900471841799, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3396865725517273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7321202158927917, + 0.24335846304893494, + 0.7321202158927917, + 0.8444173336029053, + 0.04197772964835167, + 0.06031235307455063, + 0.10734791308641434, + 0.07999168336391449, + 0.06252705305814743, + 0.08581138402223587, + 0.12927095592021942 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 0, + "style_probability": 0.9087952375411987, + "style_target": 0, + "support_probability": 0.3613693416118622, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8641383472371944, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04265814274549484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9579074382781982, + 0.3227744698524475, + 0.3809097707271576, + 0.34655895829200745, + 0.10209698975086212 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.857551097869873, + "style_target": 1, + "support_probability": 0.9341698288917542, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07458462169582612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10719490796327591, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9791487455368042, + 0.46663081645965576, + 0.3585951328277588, + 0.4800093472003937, + 0.27827146649360657 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.0668269693851471, + "style_target": 0, + "support_probability": 0.8871940970420837, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6377687302625736, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0983521118760109, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7808626294136047, + 0.5356244444847107, + 0.22312985360622406, + 0.4659578204154968, + 0.04955710843205452, + 0.09684129059314728 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 1, + "style_probability": 0.7242608666419983, + "style_target": 1, + "support_probability": 0.7654867768287659, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33081394420003657, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16039156913757324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.67470383644104, + 0.5549953579902649, + 0.21781174838542938, + 0.4968242943286896, + 0.08775752782821655, + 0.07940974831581116 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 1, + "style_probability": 0.4547181725502014, + "style_target": 0, + "support_probability": 0.551604151725769, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.1742102550214454, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32205700874328613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5229551792144775, + 0.50837242603302, + 0.6233922243118286, + 0.9229460954666138, + 0.05154755711555481, + 0.0314418263733387, + 0.06434383243322372, + 0.03469491004943848, + 0.0565577894449234 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.6183391809463501, + "style_target": 1, + "support_probability": 0.061816342175006866, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09581959038634964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3527315557003021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5167152881622314, + 0.4655163288116455, + 0.5740691423416138, + 0.8903780579566956, + 0.05821250379085541, + 0.07266552001237869, + 0.07383845001459122, + 0.06384309381246567, + 0.04333881288766861, + 0.08784733712673187, + 0.04894937574863434 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.3434155583381653, + "style_target": 0, + "support_probability": 0.06251360476016998, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3220775187067935, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18107686936855316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35322701930999756, + 0.8277606964111328, + 0.31957659125328064, + 0.28411775827407837, + 0.150022953748703, + 0.14845643937587738, + 0.07093513756990433, + 0.10533114522695541, + 0.1066545769572258, + 0.044396672397851944, + 0.15628619492053986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.6640976667404175, + "style_target": 1, + "support_probability": 0.39529770612716675, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01923238078502829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3581624925136566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3864719867706299, + 0.8627389073371887, + 0.4235440790653229, + 0.38058075308799744, + 0.1965063512325287, + 0.17209185659885406, + 0.15992844104766846, + 0.1408439725637436, + 0.2113981395959854, + 0.17078480124473572 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.11111021786928177, + "style_target": 0, + "support_probability": 0.2611888349056244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6833284561558497, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.44499316811561584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8440634608268738, + 0.10665372014045715, + 0.9479350447654724, + 0.17976222932338715, + 0.05025596171617508 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 1, + "style_probability": 0.887021541595459, + "style_target": 1, + "support_probability": 0.48539498448371887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30628478481813287, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4073777198791504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7714871168136597, + 0.12678438425064087, + 0.9140051603317261, + 0.2120041847229004, + 0.05551671236753464 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 1, + "style_probability": 0.49096164107322693, + "style_target": 0, + "support_probability": 0.40118905901908875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3345947660463658, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8280948996543884, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.950164794921875, + 0.3031753599643707, + 0.789641797542572, + 0.46762123703956604, + 0.12763701379299164, + 0.07931353896856308 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 1, + "style_probability": 0.6837689876556396, + "style_target": 1, + "support_probability": 0.3346521556377411, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35624987040224415, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7049791812896729, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9316158294677734, + 0.3223493993282318, + 0.7462558746337891, + 0.4696267247200012, + 0.135334774851799, + 0.17775267362594604, + 0.15744024515151978, + 0.15491552650928497, + 0.09926678240299225 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 1, + "style_probability": 0.37677326798439026, + "style_target": 0, + "support_probability": 0.43049025535583496, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7517213963928868, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09494070708751678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5180981755256653, + 0.11318150162696838, + 0.9800930023193359, + 0.3845498859882355, + 0.0867561474442482, + 0.05166143178939819, + 0.05254516005516052, + 0.055246904492378235, + 0.07084731757640839 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.669357419013977, + "style_target": 1, + "support_probability": 0.8305472135543823, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11979724839629995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14298470318317413, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5294777154922485, + 0.158109650015831, + 0.9913495779037476, + 0.42453765869140625, + 0.19526182115077972, + 0.11902295053005219, + 0.11228002607822418, + 0.09514491260051727, + 0.09736770391464233 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.1156955435872078, + "style_target": 0, + "support_probability": 0.7735913395881653, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8472284186327176, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11226873844861984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9816961288452148, + 0.32526832818984985, + 0.24011726677417755, + 0.7061808705329895, + 0.12142935395240784, + 0.0923486277461052, + 0.3054197132587433, + 0.08929791301488876, + 0.08513642102479935, + 0.06525231152772903, + 0.03800571337342262 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 1, + "style_probability": 0.7450124621391296, + "style_target": 1, + "support_probability": 0.8560869693756104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7555379786178276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2388189136981964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9647449851036072, + 0.29299941658973694, + 0.24770280718803406, + 0.59300297498703, + 0.13100704550743103, + 0.0621739886701107, + 0.08426488190889359, + 0.05173439159989357, + 0.0805244967341423, + 0.077660471200943 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 0, + "style_probability": 0.72934889793396, + "style_target": 0, + "support_probability": 0.7556605935096741, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24802187654666832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6895213723182678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40840214490890503, + 0.7989595532417297, + 0.35186997056007385, + 0.3156561553478241, + 0.12522074580192566 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.8752198219299316, + "style_target": 1, + "support_probability": 0.12372215837240219, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025611981891344195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.956598699092865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37186649441719055, + 0.7755410075187683, + 0.32215455174446106, + 0.30970779061317444, + 0.2214440107345581, + 0.1194939911365509 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.3307580053806305, + "style_target": 0, + "support_probability": 0.019027499482035637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029450458040524957, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9399923086166382, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5785793662071228, + 0.34852710366249084, + 0.5298789739608765, + 0.5032315254211426, + 0.06206580623984337, + 0.23302268981933594 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.6068912744522095, + "style_target": 1, + "support_probability": 0.04472608491778374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00605142799217619, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9419124126434326, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.618657648563385, + 0.4045100808143616, + 0.6115524768829346, + 0.6395512819290161, + 0.1312284618616104, + 0.136598140001297 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.13062714040279388, + "style_target": 0, + "support_probability": 0.04437364637851715, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6561947371131439, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5452035069465637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7623027563095093, + 0.42599958181381226, + 0.5744858384132385, + 0.44532501697540283, + 0.04713469371199608, + 0.07352323085069656, + 0.07560841739177704, + 0.05739647150039673, + 0.07050726562738419 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8705961108207703, + "style_target": 1, + "support_probability": 0.5371516942977905, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40576966738286585, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42603981494903564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6701467037200928, + 0.36274033784866333, + 0.5122440457344055, + 0.399973601102829, + 0.0581192746758461, + 0.06212693825364113, + 0.09151846915483475, + 0.0666649267077446, + 0.07971884310245514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 1, + "style_probability": 0.40231695771217346, + "style_target": 0, + "support_probability": 0.5534378290176392, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1519258603764105, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8178784847259521, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30035659670829773, + 0.5057423710823059, + 0.40037602186203003, + 0.4446442723274231, + 0.10268905013799667, + 0.056939106434583664, + 0.08084961771965027, + 0.09575886279344559, + 0.06411082297563553, + 0.09639976173639297 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.8159245252609253, + "style_target": 1, + "support_probability": 0.09282493591308594, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.035815446608189416, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7518742084503174, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26342886686325073, + 0.457601398229599, + 0.3428298532962799, + 0.4629440903663635, + 0.11092950403690338 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.3133966028690338, + "style_target": 0, + "support_probability": 0.0881161019206047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036312123503433456, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8972371816635132, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5720462799072266, + 0.05887756496667862, + 0.08231530338525772, + 0.05887756496667862, + 0.061336763203144073 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 0, + "style_probability": 0.36204302310943604, + "style_target": 1, + "support_probability": 0.08658801019191742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028908321324910242, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.960390567779541, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6392669677734375, + 0.06528031080961227, + 0.1146160438656807, + 0.06528031080961227, + 0.028491804376244545, + 0.09866683930158615 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 1, + "style_probability": 0.3936799466609955, + "style_target": 0, + "support_probability": 0.04484089836478233, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8975994179843396, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.024898497387766838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3445727229118347, + 0.9843332171440125, + 0.1879803091287613, + 0.3445727229118347, + 0.07179807126522064, + 0.1091797798871994 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 0, + "style_probability": 0.46530500054359436, + "style_target": 1, + "support_probability": 0.8933738470077515, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4663841583734616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027960436418652534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4003012776374817, + 0.995688259601593, + 0.2334468960762024, + 0.4003012776374817, + 0.18433606624603271, + 0.11294621974229813 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 1, + "style_probability": 0.14973348379135132, + "style_target": 0, + "support_probability": 0.8722990155220032, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24841153496133847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3372337818145752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2949763238430023, + 0.2680242955684662, + 0.19022251665592194, + 0.92951500415802, + 0.05456598475575447, + 0.1008000448346138, + 0.05174693092703819, + 0.04673512279987335, + 0.07602375000715256 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 0, + "style_probability": 0.277158260345459, + "style_target": 1, + "support_probability": 0.5298305153846741, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24066837350026477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.334085077047348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2736622095108032, + 0.2569981813430786, + 0.17967620491981506, + 0.9392920732498169, + 0.09955964982509613, + 0.05499905347824097, + 0.06753358244895935, + 0.052887920290231705, + 0.09200672805309296 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 1, + "style_probability": 0.4642980396747589, + "style_target": 0, + "support_probability": 0.5240429043769836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008707379376620122, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9339296817779541, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36175641417503357, + 0.6948668956756592, + 0.5371975898742676, + 0.7038694024085999, + 0.12472570687532425, + 0.13278095424175262, + 0.08797243237495422, + 0.27046921849250793, + 0.09615473449230194, + 0.043882694095373154, + 0.13763046264648438 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 0, + "style_probability": 0.37620335817337036, + "style_target": 1, + "support_probability": 0.021899843588471413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01416928303844646, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9239391684532166, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35514822602272034, + 0.6704370379447937, + 0.5551785826683044, + 0.7181523442268372, + 0.10516845434904099 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 1, + "style_probability": 0.2532786726951599, + "style_target": 0, + "support_probability": 0.02603195421397686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6248264912497808, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19472356140613556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2738191485404968, + 0.8901903033256531, + 0.4734492599964142, + 0.28840526938438416, + 0.039178166538476944 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 1, + "style_probability": 0.7862384915351868, + "style_target": 1, + "support_probability": 0.549809992313385, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.611975989609558, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14166440069675446, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2884581983089447, + 0.8540338277816772, + 0.4587915539741516, + 0.30705374479293823, + 0.08563289791345596 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 0, + "style_probability": 0.6342202425003052, + "style_target": 0, + "support_probability": 0.6591057181358337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2050559450655948, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7077404260635376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5046743154525757, + 0.28758466243743896, + 0.6252914071083069, + 0.30242279171943665, + 0.11788857728242874, + 0.0610940158367157 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 1, + "style_probability": 0.804180383682251, + "style_target": 1, + "support_probability": 0.14145216345787048, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09344255735082747, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8165793418884277, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5009281039237976, + 0.30564436316490173, + 0.6254007816314697, + 0.3308590352535248, + 0.12025130540132523, + 0.07421836256980896, + 0.06005201116204262, + 0.0693747028708458, + 0.08397112041711807 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 0, + "style_probability": 0.6610521674156189, + "style_target": 0, + "support_probability": 0.0587390661239624, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12099707163541754, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6688506603240967, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2419951856136322, + 0.3014968931674957, + 0.27946633100509644, + 0.320234477519989, + 0.14153185486793518, + 0.15359282493591309, + 0.09675014019012451, + 0.07357125729322433, + 0.036298613995313644 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.7064881920814514, + "style_target": 1, + "support_probability": 0.2087641954421997, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011138567629441206, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7718527317047119, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2445693463087082, + 0.3235008418560028, + 0.32066500186920166, + 0.33892276883125305, + 0.06254661828279495, + 0.07886422425508499, + 0.12033286690711975, + 0.06649589538574219, + 0.14529024064540863 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.1764906495809555, + "style_target": 0, + "support_probability": 0.20869925618171692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14726236394097436, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7142321467399597, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08197975158691406, + 0.28963762521743774, + 0.6568887233734131, + 0.3965410590171814, + 0.19080741703510284, + 0.07127436250448227, + 0.04736781120300293, + 0.13905654847621918, + 0.08900599926710129, + 0.07227472960948944, + 0.051429737359285355 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 1, + "style_probability": 0.8135142922401428, + "style_target": 1, + "support_probability": 0.13332876563072205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17216508173538678, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7255522608757019, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.11448150128126144, + 0.3449895679950714, + 0.6507022976875305, + 0.5300000905990601, + 0.06931614130735397, + 0.11119718849658966, + 0.062342770397663116, + 0.19170093536376953, + 0.12114851176738739, + 0.14687830209732056, + 0.3452426493167877 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 0, + "style_probability": 0.5104021430015564, + "style_target": 0, + "support_probability": 0.1700567901134491, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06552085831326575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8315463662147522, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5207180380821228, + 0.3360796272754669, + 0.2096080482006073, + 0.817020833492279, + 0.1003679484128952 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.7669947743415833, + "style_target": 1, + "support_probability": 0.013577633537352085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023998515068654854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8709759712219238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4801928997039795, + 0.3341057598590851, + 0.21402226388454437, + 0.8080968856811523, + 0.1348726749420166, + 0.13410229980945587 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.21361656486988068, + "style_target": 0, + "support_probability": 0.01237452682107687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24608210131555738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6697835326194763, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.94473797082901, + 0.27838897705078125, + 0.28512874245643616, + 0.47389453649520874, + 0.27522361278533936, + 0.07324116677045822 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.7497760057449341, + "style_target": 1, + "support_probability": 0.10739678144454956, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027204381144536428, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7661979794502258, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9544811248779297, + 0.3406423330307007, + 0.3307252526283264, + 0.5829370617866516, + 0.12406852096319199, + 0.11450531333684921 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.22880567610263824, + "style_target": 0, + "support_probability": 0.07346270233392715, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.846941879984266, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07995568215847015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2363119125366211, + 0.23315748572349548, + 0.9873791337013245, + 0.1361478865146637, + 0.04478172957897186, + 0.06729743629693985, + 0.06065664440393448, + 0.06291495263576508, + 0.051550354808568954 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.6689233183860779, + "style_target": 1, + "support_probability": 0.8456040024757385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6698177821687908, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.039557237178087234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2196127027273178, + 0.2346125692129135, + 0.9841665029525757, + 0.13700461387634277, + 0.0541008897125721, + 0.04908011481165886, + 0.06612998247146606, + 0.04712902009487152, + 0.09613156318664551 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.27744242548942566, + "style_target": 0, + "support_probability": 0.8513179421424866, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5344897048095697, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11486218869686127, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9554747939109802, + 0.4710931181907654, + 0.5574470162391663, + 0.629969596862793, + 0.04341122880578041, + 0.05237051099538803, + 0.03234746679663658, + 0.05049556866288185, + 0.06895509362220764, + 0.06622598320245743, + 0.04232504591345787 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 1, + "style_probability": 0.8077633380889893, + "style_target": 1, + "support_probability": 0.33058300614356995, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4701429041203141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1338890790939331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9487112164497375, + 0.45381712913513184, + 0.5254971981048584, + 0.6065744757652283, + 0.06740433722734451 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 0, + "style_probability": 0.5737596154212952, + "style_target": 0, + "support_probability": 0.36692336201667786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2384797162065129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4463684558868408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5704437494277954, + 0.47184428572654724, + 0.8945052623748779, + 0.3155633509159088, + 0.11846049129962921 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 1, + "style_probability": 0.8558587431907654, + "style_target": 1, + "support_probability": 0.053384505212306976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09026398860970052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5601173043251038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6170905232429504, + 0.5108743906021118, + 0.9089803695678711, + 0.329186350107193, + 0.08239023387432098 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 0, + "style_probability": 0.7350287437438965, + "style_target": 0, + "support_probability": 0.03821207582950592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11651592835239374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7996067404747009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3079007863998413, + 0.27123627066612244, + 0.8730019927024841, + 0.24101871252059937, + 0.05184818431735039, + 0.15855908393859863 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 1, + "style_probability": 0.8823444843292236, + "style_target": 1, + "support_probability": 0.04622809216380119, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08372263368970126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7173299193382263, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3084774315357208, + 0.26809874176979065, + 0.8502752184867859, + 0.22874420881271362, + 0.045806996524333954, + 0.10973309725522995 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 0, + "style_probability": 0.677670955657959, + "style_target": 0, + "support_probability": 0.06768868118524551, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32790113300330515, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35583215951919556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22309871017932892, + 0.4072660207748413, + 0.23325465619564056, + 0.5416219234466553, + 0.11585967242717743, + 0.04601668193936348, + 0.07863151282072067, + 0.058527760207653046, + 0.0964045599102974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.875618577003479, + "style_target": 1, + "support_probability": 0.15910294651985168, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07292751766035026, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49206674098968506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22127491235733032, + 0.36181125044822693, + 0.2228117436170578, + 0.5218328833580017, + 0.08327211439609528, + 0.060780927538871765, + 0.32949554920196533, + 0.07906082272529602, + 0.11424904316663742, + 0.09905792027711868, + 0.060233697295188904 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.36670204997062683, + "style_target": 0, + "support_probability": 0.07060237228870392, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7409889497405012, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17662259936332703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4509548544883728, + 0.8301584720611572, + 0.45953524112701416, + 0.3213473856449127, + 0.051612161099910736, + 0.0864175483584404, + 0.24164029955863953, + 0.02580494061112404, + 0.038141511380672455, + 0.04729107394814491, + 0.1690881997346878 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.8091198801994324, + "style_target": 1, + "support_probability": 0.899490237236023, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17922900787291077, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21027351915836334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6124860048294067, + 0.9211281538009644, + 0.5763068795204163, + 0.37317895889282227, + 0.07569162547588348, + 0.09466997534036636, + 0.20861373841762543, + 0.12899792194366455, + 0.10424228757619858, + 0.10893236845731735 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.16375264525413513, + "style_target": 0, + "support_probability": 0.8751931190490723, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06280808620773419, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2790431082248688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43707358837127686, + 0.36347100138664246, + 0.8200975656509399, + 0.3140920102596283, + 0.15264016389846802 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 0, + "style_probability": 0.387805700302124, + "style_target": 1, + "support_probability": 0.030940188094973564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05871566031575337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27821025252342224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4347072243690491, + 0.39166438579559326, + 0.7959935069084167, + 0.355848103761673, + 0.08873338252305984 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 0, + "style_probability": 0.5581459403038025, + "style_target": 0, + "support_probability": 0.026720069348812103, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7380356711942432, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.047606050968170166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5259135961532593, + 0.37532156705856323, + 0.9774277210235596, + 0.5259135961532593, + 0.06737525016069412, + 0.10753709077835083 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 1, + "style_probability": 0.7468898892402649, + "style_target": 1, + "support_probability": 0.4875595271587372, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8939264802968621, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0411929115653038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4760626256465912, + 0.39511802792549133, + 0.9882513880729675, + 0.4760626256465912, + 0.110029436647892, + 0.0721028745174408 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 0, + "style_probability": 0.8934102654457092, + "style_target": 0, + "support_probability": 0.5814793109893799, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5694773568893488, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08356434851884842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9444497227668762, + 0.39387574791908264, + 0.20201939344406128, + 0.11998653411865234, + 0.039669182151556015, + 0.09977329522371292, + 0.054709725081920624, + 0.08758355677127838, + 0.11977270245552063 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 0, + "style_probability": 0.2977406084537506, + "style_target": 1, + "support_probability": 0.8733661770820618, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23752987325648325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0968603566288948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9709755182266235, + 0.42838090658187866, + 0.2424517124891281, + 0.12446295469999313, + 0.15584976971149445, + 0.07425899803638458, + 0.07627099007368088, + 0.07475320249795914, + 0.09387575089931488 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 1, + "style_probability": 0.15151022374629974, + "style_target": 0, + "support_probability": 0.886669933795929, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.528304023938849, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2821364402770996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22964227199554443, + 0.31470999121665955, + 0.936564028263092, + 0.22964227199554443, + 0.1201455295085907, + 0.21511898934841156, + 0.03486405313014984, + 0.037617869675159454, + 0.08560693264007568, + 0.15428747236728668, + 0.15221892297267914 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.48033055663108826, + "style_target": 1, + "support_probability": 0.8465471863746643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6763831500890479, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18152590095996857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22731631994247437, + 0.31105202436447144, + 0.9289668798446655, + 0.22731631994247437, + 0.04402018338441849, + 0.05967972055077553, + 0.08490350842475891, + 0.18774010241031647, + 0.14240674674510956, + 0.061345189809799194, + 0.09271489083766937 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.5222688913345337, + "style_target": 0, + "support_probability": 0.908439040184021, + "support_target": 1 + } + ], + "schema_version": "personavoice-language-evaluation-v1", + "status": "measured_evaluation_run", + "summary": { + "accept_accuracy": { + "lower": 0.823, + "mean": 0.8353333333333334, + "speaker_count": 1500.0, + "upper": 0.8476666666666667 + }, + "accept_brier": 0.1418294243209795, + "accept_ece": 0.16699149165654636, + "area_under_risk_coverage": 0.34340379288862394, + "budget_breakdown": { + "10": { + "acceptance_rate": 0.0, + "count": 277, + "f1_macro": 0.7700437102106037, + "false_abstention": 1.0, + "false_acceptance": 0.0, + "precision_macro": 0.7472924187725631, + "recall_macro": 0.7942238267148014 + }, + "11": { + "acceptance_rate": 0.0064794816414686825, + "count": 463, + "f1_macro": 0.7277592273330301, + "false_abstention": 0.9789473684210527, + "false_acceptance": 0.3333333333333333, + "precision_macro": 0.7192224622030238, + "recall_macro": 0.7365010799136069 + }, + "12": { + "acceptance_rate": 0.0, + "count": 4, + "f1_macro": 0.25, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.25, + "recall_macro": 0.25 + }, + "14": { + "acceptance_rate": 0.0, + "count": 2, + "f1_macro": 0.5, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.5, + "recall_macro": 0.5 + }, + "5": { + "acceptance_rate": 0.0013386880856760374, + "count": 747, + "f1_macro": 0.7476324387782299, + "false_abstention": 0.9915966386554622, + "false_acceptance": 0.0, + "precision_macro": 0.7315930388219545, + "recall_macro": 0.7643908969210174 + }, + "6": { + "acceptance_rate": 0.00392156862745098, + "count": 765, + "f1_macro": 0.7211684190584446, + "false_abstention": 0.984251968503937, + "false_acceptance": 0.3333333333333333, + "precision_macro": 0.7045751633986929, + "recall_macro": 0.738562091503268 + }, + "9": { + "acceptance_rate": 0.006738544474393531, + "count": 742, + "f1_macro": 0.7302990819665609, + "false_abstention": 0.9682539682539683, + "false_acceptance": 0.2, + "precision_macro": 0.7196765498652291, + "recall_macro": 0.7412398921832885 + } + }, + "citation_macro": { + "f1_macro": 0.7347940142840949, + "precision_macro": 0.7205, + "recall_macro": 0.7496666666666667 + }, + "citation_precision_mean": 0.7205, + "citation_precision_micro": 0.9003202562049639, + "citation_recall_mean": 0.7496666666666667, + "citation_recall_micro": 0.7496666666666667, + "conflict": { + "accuracy": 0.7783333333333333, + "brier": 0.15689609150951211, + "count": 3000, + "ece": 0.09677436601743102, + "f1": 0.6936895439889452, + "precision": 0.6430401366353544, + "recall": 0.753 + }, + "false_abstention": 0.982, + "false_acceptance": 0.25, + "positive_coverage": 0.018, + "risk_coverage": [ + { + "coverage": 0.0003333333333333333, + "risk": 0.0, + "threshold": 0.9733826879763859 + }, + { + "coverage": 0.0006666666666666666, + "risk": 0.0, + "threshold": 0.9718894354213582 + }, + { + "coverage": 0.001, + "risk": 0.0, + "threshold": 0.9531002266159541 + }, + { + "coverage": 0.0013333333333333333, + "risk": 0.0, + "threshold": 0.9529007943801765 + }, + { + "coverage": 0.0016666666666666668, + "risk": 0.2, + "threshold": 0.9458073330461055 + }, + { + "coverage": 0.002, + "risk": 0.16666666666666666, + "threshold": 0.9457130028936435 + }, + { + "coverage": 0.0023333333333333335, + "risk": 0.14285714285714285, + "threshold": 0.9456840174722091 + }, + { + "coverage": 0.0026666666666666666, + "risk": 0.125, + "threshold": 0.944958704698099 + }, + { + "coverage": 0.003, + "risk": 0.2222222222222222, + "threshold": 0.9430234536209753 + }, + { + "coverage": 0.0033333333333333335, + "risk": 0.3, + "threshold": 0.9420408686261423 + }, + { + "coverage": 0.0036666666666666666, + "risk": 0.2727272727272727, + "threshold": 0.9399806003968447 + }, + { + "coverage": 0.004, + "risk": 0.25, + "threshold": 0.93725267809188 + }, + { + "coverage": 0.004333333333333333, + "risk": 0.3076923076923077, + "threshold": 0.9366927066188065 + }, + { + "coverage": 0.004666666666666667, + "risk": 0.35714285714285715, + "threshold": 0.9325469612432336 + }, + { + "coverage": 0.005, + "risk": 0.4, + "threshold": 0.9301331861366363 + }, + { + "coverage": 0.005333333333333333, + "risk": 0.4375, + "threshold": 0.9250157110501149 + }, + { + "coverage": 0.005666666666666667, + "risk": 0.47058823529411764, + "threshold": 0.9243638193085285 + }, + { + "coverage": 0.006, + "risk": 0.5, + "threshold": 0.9222163398538663 + }, + { + "coverage": 0.006333333333333333, + "risk": 0.5263157894736842, + "threshold": 0.9220769949220147 + }, + { + "coverage": 0.006666666666666667, + "risk": 0.55, + "threshold": 0.9208263244396967 + }, + { + "coverage": 0.007, + "risk": 0.5714285714285714, + "threshold": 0.9201613112421213 + }, + { + "coverage": 0.007333333333333333, + "risk": 0.5454545454545454, + "threshold": 0.9195391070831268 + }, + { + "coverage": 0.007666666666666666, + "risk": 0.5652173913043478, + "threshold": 0.9188400231987766 + }, + { + "coverage": 0.008, + "risk": 0.5833333333333334, + "threshold": 0.9159871631479822 + }, + { + "coverage": 0.008333333333333333, + "risk": 0.6, + "threshold": 0.9150564255769958 + }, + { + "coverage": 0.008666666666666666, + "risk": 0.6153846153846154, + "threshold": 0.9133026871985797 + }, + { + "coverage": 0.009, + "risk": 0.6296296296296297, + "threshold": 0.9130786040452771 + }, + { + "coverage": 0.009333333333333334, + "risk": 0.6428571428571429, + "threshold": 0.9118906286820538 + }, + { + "coverage": 0.009666666666666667, + "risk": 0.6551724137931034, + "threshold": 0.9114693338749282 + }, + { + "coverage": 0.01, + "risk": 0.6666666666666666, + "threshold": 0.9102466722874961 + }, + { + "coverage": 0.010333333333333333, + "risk": 0.6774193548387096, + "threshold": 0.9093772801446084 + }, + { + "coverage": 0.010666666666666666, + "risk": 0.6875, + "threshold": 0.9069170647592399 + }, + { + "coverage": 0.011, + "risk": 0.6666666666666666, + "threshold": 0.905905409910487 + }, + { + "coverage": 0.011333333333333334, + "risk": 0.6764705882352942, + "threshold": 0.9058887740018307 + }, + { + "coverage": 0.011666666666666667, + "risk": 0.6857142857142857, + "threshold": 0.9058827651829366 + }, + { + "coverage": 0.012, + "risk": 0.6944444444444444, + "threshold": 0.9050818401003432 + }, + { + "coverage": 0.012333333333333333, + "risk": 0.7027027027027027, + "threshold": 0.9032039231521773 + }, + { + "coverage": 0.012666666666666666, + "risk": 0.7105263157894737, + "threshold": 0.9027303639183754 + }, + { + "coverage": 0.013, + "risk": 0.717948717948718, + "threshold": 0.9006859465418081 + }, + { + "coverage": 0.013333333333333334, + "risk": 0.725, + "threshold": 0.9006156641488237 + }, + { + "coverage": 0.013666666666666667, + "risk": 0.7317073170731707, + "threshold": 0.8993834545489118 + }, + { + "coverage": 0.014, + "risk": 0.7380952380952381, + "threshold": 0.8978492604214612 + }, + { + "coverage": 0.014333333333333333, + "risk": 0.7441860465116279, + "threshold": 0.8975994179843396 + }, + { + "coverage": 0.014666666666666666, + "risk": 0.7272727272727273, + "threshold": 0.895672670675534 + }, + { + "coverage": 0.015, + "risk": 0.7111111111111111, + "threshold": 0.8951843246585062 + }, + { + "coverage": 0.015333333333333332, + "risk": 0.6956521739130435, + "threshold": 0.8939871525212402 + }, + { + "coverage": 0.015666666666666666, + "risk": 0.6808510638297872, + "threshold": 0.8939264802968621 + }, + { + "coverage": 0.016, + "risk": 0.6875, + "threshold": 0.8929357683306208 + }, + { + "coverage": 0.01633333333333333, + "risk": 0.6938775510204082, + "threshold": 0.8922032589202666 + }, + { + "coverage": 0.016666666666666666, + "risk": 0.7, + "threshold": 0.8921660954758568 + }, + { + "coverage": 0.017, + "risk": 0.7058823529411765, + "threshold": 0.8919131658057802 + }, + { + "coverage": 0.017333333333333333, + "risk": 0.6923076923076923, + "threshold": 0.890539130948832 + }, + { + "coverage": 0.017666666666666667, + "risk": 0.6792452830188679, + "threshold": 0.8903641825527753 + }, + { + "coverage": 0.018, + "risk": 0.6666666666666666, + "threshold": 0.8896409437584376 + }, + { + "coverage": 0.018333333333333333, + "risk": 0.6727272727272727, + "threshold": 0.8880745793471662 + }, + { + "coverage": 0.018666666666666668, + "risk": 0.6607142857142857, + "threshold": 0.8880023290776142 + }, + { + "coverage": 0.019, + "risk": 0.6491228070175439, + "threshold": 0.8879546805865771 + }, + { + "coverage": 0.019333333333333334, + "risk": 0.6551724137931034, + "threshold": 0.8868768633243046 + }, + { + "coverage": 0.019666666666666666, + "risk": 0.6610169491525424, + "threshold": 0.8846002075520595 + }, + { + "coverage": 0.02, + "risk": 0.6666666666666666, + "threshold": 0.8822962781425082 + }, + { + "coverage": 0.02033333333333333, + "risk": 0.6721311475409836, + "threshold": 0.8805180753360974 + }, + { + "coverage": 0.020666666666666667, + "risk": 0.6612903225806451, + "threshold": 0.8799820227990205 + }, + { + "coverage": 0.021, + "risk": 0.6507936507936508, + "threshold": 0.8789896844097516 + }, + { + "coverage": 0.021333333333333333, + "risk": 0.65625, + "threshold": 0.8788577185549881 + }, + { + "coverage": 0.021666666666666667, + "risk": 0.6461538461538462, + "threshold": 0.8783856351170063 + }, + { + "coverage": 0.022, + "risk": 0.6515151515151515, + "threshold": 0.8780960954633116 + }, + { + "coverage": 0.022333333333333334, + "risk": 0.6567164179104478, + "threshold": 0.87667370562982 + }, + { + "coverage": 0.02266666666666667, + "risk": 0.6617647058823529, + "threshold": 0.8765304158452848 + }, + { + "coverage": 0.023, + "risk": 0.6666666666666666, + "threshold": 0.8763632797797644 + }, + { + "coverage": 0.023333333333333334, + "risk": 0.6571428571428571, + "threshold": 0.8759020066715724 + }, + { + "coverage": 0.023666666666666666, + "risk": 0.6619718309859155, + "threshold": 0.875277026107786 + }, + { + "coverage": 0.024, + "risk": 0.6666666666666666, + "threshold": 0.8748055894386187 + }, + { + "coverage": 0.024333333333333332, + "risk": 0.6712328767123288, + "threshold": 0.8736460943656303 + }, + { + "coverage": 0.024666666666666667, + "risk": 0.6756756756756757, + "threshold": 0.8732528594962062 + }, + { + "coverage": 0.025, + "risk": 0.68, + "threshold": 0.8732158006039532 + }, + { + "coverage": 0.025333333333333333, + "risk": 0.6842105263157895, + "threshold": 0.8731550026557752 + }, + { + "coverage": 0.025666666666666667, + "risk": 0.6883116883116883, + "threshold": 0.8725623685747027 + }, + { + "coverage": 0.026, + "risk": 0.6794871794871795, + "threshold": 0.8721630760940369 + }, + { + "coverage": 0.026333333333333334, + "risk": 0.6708860759493671, + "threshold": 0.8721407605980582 + }, + { + "coverage": 0.02666666666666667, + "risk": 0.675, + "threshold": 0.8712770989297794 + }, + { + "coverage": 0.027, + "risk": 0.6666666666666666, + "threshold": 0.8704936563323153 + }, + { + "coverage": 0.027333333333333334, + "risk": 0.6585365853658537, + "threshold": 0.8700546722945048 + }, + { + "coverage": 0.027666666666666666, + "risk": 0.6626506024096386, + "threshold": 0.869167128487579 + }, + { + "coverage": 0.028, + "risk": 0.6666666666666666, + "threshold": 0.8687861960707316 + }, + { + "coverage": 0.028333333333333332, + "risk": 0.6705882352941176, + "threshold": 0.8681561748441026 + }, + { + "coverage": 0.028666666666666667, + "risk": 0.6744186046511628, + "threshold": 0.8676651940887686 + }, + { + "coverage": 0.029, + "risk": 0.6781609195402298, + "threshold": 0.8671794242281491 + }, + { + "coverage": 0.029333333333333333, + "risk": 0.6818181818181818, + "threshold": 0.8666194859776551 + }, + { + "coverage": 0.029666666666666668, + "risk": 0.6741573033707865, + "threshold": 0.8658639320477104 + }, + { + "coverage": 0.03, + "risk": 0.6777777777777778, + "threshold": 0.864458355538514 + }, + { + "coverage": 0.030333333333333334, + "risk": 0.6813186813186813, + "threshold": 0.8641383472371944 + }, + { + "coverage": 0.030666666666666665, + "risk": 0.6847826086956522, + "threshold": 0.8636097575786792 + }, + { + "coverage": 0.031, + "risk": 0.6881720430107527, + "threshold": 0.8633117619412901 + }, + { + "coverage": 0.03133333333333333, + "risk": 0.6914893617021277, + "threshold": 0.8630376796848126 + }, + { + "coverage": 0.03166666666666667, + "risk": 0.6947368421052632, + "threshold": 0.8624922595946587 + }, + { + "coverage": 0.032, + "risk": 0.6979166666666666, + "threshold": 0.8614936857741091 + }, + { + "coverage": 0.03233333333333333, + "risk": 0.7010309278350515, + "threshold": 0.8612875593924985 + }, + { + "coverage": 0.03266666666666666, + "risk": 0.7040816326530612, + "threshold": 0.8606202288790727 + }, + { + "coverage": 0.033, + "risk": 0.696969696969697, + "threshold": 0.8596317550596994 + }, + { + "coverage": 0.03333333333333333, + "risk": 0.7, + "threshold": 0.8590443216014458 + }, + { + "coverage": 0.033666666666666664, + "risk": 0.7029702970297029, + "threshold": 0.8588949411265624 + }, + { + "coverage": 0.034, + "risk": 0.696078431372549, + "threshold": 0.8587455466356906 + }, + { + "coverage": 0.034333333333333334, + "risk": 0.6893203883495146, + "threshold": 0.8579095502532922 + }, + { + "coverage": 0.034666666666666665, + "risk": 0.6923076923076923, + "threshold": 0.8554350432606096 + }, + { + "coverage": 0.035, + "risk": 0.6857142857142857, + "threshold": 0.8554115892602251 + }, + { + "coverage": 0.035333333333333335, + "risk": 0.6792452830188679, + "threshold": 0.8552135845640834 + }, + { + "coverage": 0.035666666666666666, + "risk": 0.6822429906542056, + "threshold": 0.8542388224977436 + }, + { + "coverage": 0.036, + "risk": 0.6851851851851852, + "threshold": 0.8540194083734232 + }, + { + "coverage": 0.036333333333333336, + "risk": 0.6880733944954128, + "threshold": 0.8536441154628776 + }, + { + "coverage": 0.03666666666666667, + "risk": 0.6909090909090909, + "threshold": 0.8532624469839618 + }, + { + "coverage": 0.037, + "risk": 0.6846846846846847, + "threshold": 0.8530912962124134 + }, + { + "coverage": 0.037333333333333336, + "risk": 0.6785714285714286, + "threshold": 0.8530341222523629 + }, + { + "coverage": 0.03766666666666667, + "risk": 0.672566371681416, + "threshold": 0.8524633912226678 + }, + { + "coverage": 0.038, + "risk": 0.6666666666666666, + "threshold": 0.8505841455852826 + }, + { + "coverage": 0.03833333333333333, + "risk": 0.6608695652173913, + "threshold": 0.8498958861848326 + }, + { + "coverage": 0.03866666666666667, + "risk": 0.6551724137931034, + "threshold": 0.8497390390356808 + }, + { + "coverage": 0.039, + "risk": 0.6581196581196581, + "threshold": 0.8495939455063792 + }, + { + "coverage": 0.03933333333333333, + "risk": 0.6610169491525424, + "threshold": 0.8495577719854893 + }, + { + "coverage": 0.03966666666666667, + "risk": 0.6554621848739496, + "threshold": 0.8487040973337437 + }, + { + "coverage": 0.04, + "risk": 0.6583333333333333, + "threshold": 0.8484964684506682 + }, + { + "coverage": 0.04033333333333333, + "risk": 0.6611570247933884, + "threshold": 0.8484847324772256 + }, + { + "coverage": 0.04066666666666666, + "risk": 0.6557377049180327, + "threshold": 0.8484072199748827 + }, + { + "coverage": 0.041, + "risk": 0.6504065040650406, + "threshold": 0.847781008862587 + }, + { + "coverage": 0.04133333333333333, + "risk": 0.6532258064516129, + "threshold": 0.8472284186327176 + }, + { + "coverage": 0.041666666666666664, + "risk": 0.656, + "threshold": 0.8470767487883062 + }, + { + "coverage": 0.042, + "risk": 0.6587301587301587, + "threshold": 0.8469667474287728 + }, + { + "coverage": 0.042333333333333334, + "risk": 0.6614173228346457, + "threshold": 0.846941879984266 + }, + { + "coverage": 0.042666666666666665, + "risk": 0.65625, + "threshold": 0.846917187272946 + }, + { + "coverage": 0.043, + "risk": 0.6511627906976745, + "threshold": 0.8468810897730056 + }, + { + "coverage": 0.043333333333333335, + "risk": 0.6461538461538462, + "threshold": 0.8468510690901455 + }, + { + "coverage": 0.043666666666666666, + "risk": 0.648854961832061, + "threshold": 0.8468250034224323 + }, + { + "coverage": 0.044, + "risk": 0.6439393939393939, + "threshold": 0.8467960764110667 + }, + { + "coverage": 0.044333333333333336, + "risk": 0.6466165413533834, + "threshold": 0.845937333796648 + }, + { + "coverage": 0.04466666666666667, + "risk": 0.6417910447761194, + "threshold": 0.845836490077732 + }, + { + "coverage": 0.045, + "risk": 0.6444444444444445, + "threshold": 0.8452674184143995 + }, + { + "coverage": 0.04533333333333334, + "risk": 0.6470588235294118, + "threshold": 0.8450566875224504 + }, + { + "coverage": 0.04566666666666667, + "risk": 0.6423357664233577, + "threshold": 0.8445128065330691 + }, + { + "coverage": 0.046, + "risk": 0.644927536231884, + "threshold": 0.8441393743155667 + }, + { + "coverage": 0.04633333333333333, + "risk": 0.6474820143884892, + "threshold": 0.8438946263042376 + }, + { + "coverage": 0.04666666666666667, + "risk": 0.65, + "threshold": 0.843641957166833 + }, + { + "coverage": 0.047, + "risk": 0.6453900709219859, + "threshold": 0.8433214619240224 + }, + { + "coverage": 0.04733333333333333, + "risk": 0.647887323943662, + "threshold": 0.8428544920845015 + }, + { + "coverage": 0.04766666666666667, + "risk": 0.6433566433566433, + "threshold": 0.8427189339279362 + }, + { + "coverage": 0.048, + "risk": 0.6458333333333334, + "threshold": 0.8423645679192574 + }, + { + "coverage": 0.04833333333333333, + "risk": 0.6482758620689655, + "threshold": 0.8419112345617694 + }, + { + "coverage": 0.048666666666666664, + "risk": 0.6506849315068494, + "threshold": 0.8417584055751606 + }, + { + "coverage": 0.049, + "risk": 0.6530612244897959, + "threshold": 0.8414411192811247 + }, + { + "coverage": 0.04933333333333333, + "risk": 0.6554054054054054, + "threshold": 0.8409478988056658 + }, + { + "coverage": 0.049666666666666665, + "risk": 0.6577181208053692, + "threshold": 0.8408628135295515 + }, + { + "coverage": 0.05, + "risk": 0.66, + "threshold": 0.8400702892832058 + }, + { + "coverage": 0.050333333333333334, + "risk": 0.6556291390728477, + "threshold": 0.8386945606721664 + }, + { + "coverage": 0.050666666666666665, + "risk": 0.6578947368421053, + "threshold": 0.8378385801934775 + }, + { + "coverage": 0.051, + "risk": 0.6535947712418301, + "threshold": 0.8375042463444143 + }, + { + "coverage": 0.051333333333333335, + "risk": 0.6558441558441559, + "threshold": 0.8372899688749057 + }, + { + "coverage": 0.051666666666666666, + "risk": 0.6580645161290323, + "threshold": 0.8371850574252954 + }, + { + "coverage": 0.052, + "risk": 0.6602564102564102, + "threshold": 0.836904798278459 + }, + { + "coverage": 0.052333333333333336, + "risk": 0.6560509554140127, + "threshold": 0.8354856401225915 + }, + { + "coverage": 0.05266666666666667, + "risk": 0.6582278481012658, + "threshold": 0.8350436460717271 + }, + { + "coverage": 0.053, + "risk": 0.660377358490566, + "threshold": 0.8348723017883658 + }, + { + "coverage": 0.05333333333333334, + "risk": 0.6625, + "threshold": 0.8342155777740506 + }, + { + "coverage": 0.05366666666666667, + "risk": 0.6645962732919255, + "threshold": 0.8334024574211178 + }, + { + "coverage": 0.054, + "risk": 0.6604938271604939, + "threshold": 0.832997981441515 + }, + { + "coverage": 0.05433333333333333, + "risk": 0.6625766871165644, + "threshold": 0.8329941825410514 + }, + { + "coverage": 0.05466666666666667, + "risk": 0.6585365853658537, + "threshold": 0.8313684982435042 + }, + { + "coverage": 0.055, + "risk": 0.6606060606060606, + "threshold": 0.8310445159349644 + }, + { + "coverage": 0.05533333333333333, + "risk": 0.6566265060240963, + "threshold": 0.8301144089082388 + }, + { + "coverage": 0.05566666666666667, + "risk": 0.6586826347305389, + "threshold": 0.8288965852654337 + }, + { + "coverage": 0.056, + "risk": 0.6547619047619048, + "threshold": 0.82768387820917 + }, + { + "coverage": 0.05633333333333333, + "risk": 0.650887573964497, + "threshold": 0.8273865221519842 + }, + { + "coverage": 0.056666666666666664, + "risk": 0.6470588235294118, + "threshold": 0.8270570305565016 + }, + { + "coverage": 0.057, + "risk": 0.6432748538011696, + "threshold": 0.8264323907734225 + }, + { + "coverage": 0.05733333333333333, + "risk": 0.6395348837209303, + "threshold": 0.8242564216487932 + }, + { + "coverage": 0.057666666666666665, + "risk": 0.6358381502890174, + "threshold": 0.82403123285283 + }, + { + "coverage": 0.058, + "risk": 0.6379310344827587, + "threshold": 0.8235936058341959 + }, + { + "coverage": 0.058333333333333334, + "risk": 0.64, + "threshold": 0.8223750764708916 + }, + { + "coverage": 0.058666666666666666, + "risk": 0.6363636363636364, + "threshold": 0.8220562757755139 + }, + { + "coverage": 0.059, + "risk": 0.6384180790960452, + "threshold": 0.8218579960494334 + }, + { + "coverage": 0.059333333333333335, + "risk": 0.6404494382022472, + "threshold": 0.820583037645356 + }, + { + "coverage": 0.059666666666666666, + "risk": 0.6424581005586593, + "threshold": 0.8205185655276555 + }, + { + "coverage": 0.06, + "risk": 0.6388888888888888, + "threshold": 0.8204204468716708 + }, + { + "coverage": 0.060333333333333336, + "risk": 0.6353591160220995, + "threshold": 0.8189705930617454 + }, + { + "coverage": 0.06066666666666667, + "risk": 0.6318681318681318, + "threshold": 0.8177305651350342 + }, + { + "coverage": 0.061, + "risk": 0.6338797814207651, + "threshold": 0.8177031450732842 + }, + { + "coverage": 0.06133333333333333, + "risk": 0.6304347826086957, + "threshold": 0.8165279315538442 + }, + { + "coverage": 0.06166666666666667, + "risk": 0.6270270270270271, + "threshold": 0.8157318156073152 + }, + { + "coverage": 0.062, + "risk": 0.6290322580645161, + "threshold": 0.8154015540773413 + }, + { + "coverage": 0.06233333333333333, + "risk": 0.6256684491978609, + "threshold": 0.8151020857551246 + }, + { + "coverage": 0.06266666666666666, + "risk": 0.6276595744680851, + "threshold": 0.8147617096017877 + }, + { + "coverage": 0.063, + "risk": 0.6296296296296297, + "threshold": 0.8143833408079768 + }, + { + "coverage": 0.06333333333333334, + "risk": 0.6263157894736842, + "threshold": 0.8140524785039776 + }, + { + "coverage": 0.06366666666666666, + "risk": 0.6282722513089005, + "threshold": 0.8131339242118771 + }, + { + "coverage": 0.064, + "risk": 0.625, + "threshold": 0.8126324285577572 + }, + { + "coverage": 0.06433333333333334, + "risk": 0.6269430051813472, + "threshold": 0.8116393542627234 + }, + { + "coverage": 0.06466666666666666, + "risk": 0.6237113402061856, + "threshold": 0.8113085908351154 + }, + { + "coverage": 0.065, + "risk": 0.6205128205128205, + "threshold": 0.8111139117503275 + }, + { + "coverage": 0.06533333333333333, + "risk": 0.6224489795918368, + "threshold": 0.8108350799407383 + }, + { + "coverage": 0.06566666666666666, + "risk": 0.6243654822335025, + "threshold": 0.8102185684293439 + }, + { + "coverage": 0.066, + "risk": 0.6212121212121212, + "threshold": 0.8100225035945348 + }, + { + "coverage": 0.06633333333333333, + "risk": 0.6231155778894473, + "threshold": 0.809109152636132 + }, + { + "coverage": 0.06666666666666667, + "risk": 0.625, + "threshold": 0.8088607538781462 + }, + { + "coverage": 0.067, + "risk": 0.6218905472636815, + "threshold": 0.8085339428053068 + }, + { + "coverage": 0.06733333333333333, + "risk": 0.6237623762376238, + "threshold": 0.8077828389037565 + }, + { + "coverage": 0.06766666666666667, + "risk": 0.6206896551724138, + "threshold": 0.8075699699891674 + }, + { + "coverage": 0.068, + "risk": 0.6225490196078431, + "threshold": 0.8067810618879986 + }, + { + "coverage": 0.06833333333333333, + "risk": 0.6195121951219512, + "threshold": 0.8060881963666995 + }, + { + "coverage": 0.06866666666666667, + "risk": 0.616504854368932, + "threshold": 0.8051320514156981 + }, + { + "coverage": 0.069, + "risk": 0.6135265700483091, + "threshold": 0.804925709203957 + }, + { + "coverage": 0.06933333333333333, + "risk": 0.6153846153846154, + "threshold": 0.8047436270070587 + }, + { + "coverage": 0.06966666666666667, + "risk": 0.6124401913875598, + "threshold": 0.8041340277692611 + }, + { + "coverage": 0.07, + "risk": 0.6095238095238096, + "threshold": 0.8037975288075812 + }, + { + "coverage": 0.07033333333333333, + "risk": 0.6066350710900474, + "threshold": 0.8034736323316392 + }, + { + "coverage": 0.07066666666666667, + "risk": 0.6037735849056604, + "threshold": 0.8030034375453984 + }, + { + "coverage": 0.071, + "risk": 0.6056338028169014, + "threshold": 0.8027104628063535 + }, + { + "coverage": 0.07133333333333333, + "risk": 0.6074766355140186, + "threshold": 0.8022251932120413 + }, + { + "coverage": 0.07166666666666667, + "risk": 0.6093023255813953, + "threshold": 0.8018000867228547 + }, + { + "coverage": 0.072, + "risk": 0.6111111111111112, + "threshold": 0.8017527507591382 + }, + { + "coverage": 0.07233333333333333, + "risk": 0.6129032258064516, + "threshold": 0.8017041651349044 + }, + { + "coverage": 0.07266666666666667, + "risk": 0.6100917431192661, + "threshold": 0.8016825932585486 + }, + { + "coverage": 0.073, + "risk": 0.6118721461187214, + "threshold": 0.8013617149141478 + }, + { + "coverage": 0.07333333333333333, + "risk": 0.6090909090909091, + "threshold": 0.8012584515791142 + }, + { + "coverage": 0.07366666666666667, + "risk": 0.6108597285067874, + "threshold": 0.8011509911116399 + }, + { + "coverage": 0.074, + "risk": 0.6126126126126126, + "threshold": 0.8006335157928309 + }, + { + "coverage": 0.07433333333333333, + "risk": 0.6143497757847534, + "threshold": 0.8002399290267149 + }, + { + "coverage": 0.07466666666666667, + "risk": 0.6116071428571429, + "threshold": 0.800027473012421 + }, + { + "coverage": 0.075, + "risk": 0.6133333333333333, + "threshold": 0.7994657177303336 + }, + { + "coverage": 0.07533333333333334, + "risk": 0.6106194690265486, + "threshold": 0.7993555142534015 + }, + { + "coverage": 0.07566666666666666, + "risk": 0.6079295154185022, + "threshold": 0.7992288325749063 + }, + { + "coverage": 0.076, + "risk": 0.6052631578947368, + "threshold": 0.7989489730089966 + }, + { + "coverage": 0.07633333333333334, + "risk": 0.6026200873362445, + "threshold": 0.7985806583844729 + }, + { + "coverage": 0.07666666666666666, + "risk": 0.6043478260869565, + "threshold": 0.7984668637882695 + }, + { + "coverage": 0.077, + "risk": 0.6017316017316018, + "threshold": 0.7972972146470738 + }, + { + "coverage": 0.07733333333333334, + "risk": 0.5991379310344828, + "threshold": 0.7966762493212249 + }, + { + "coverage": 0.07766666666666666, + "risk": 0.6008583690987125, + "threshold": 0.7956245997338519 + }, + { + "coverage": 0.078, + "risk": 0.6025641025641025, + "threshold": 0.7953983769661676 + }, + { + "coverage": 0.07833333333333334, + "risk": 0.6042553191489362, + "threshold": 0.7945755177941775 + }, + { + "coverage": 0.07866666666666666, + "risk": 0.6059322033898306, + "threshold": 0.79454942933675 + }, + { + "coverage": 0.079, + "risk": 0.6075949367088608, + "threshold": 0.7943774421426696 + }, + { + "coverage": 0.07933333333333334, + "risk": 0.6050420168067226, + "threshold": 0.7943065164583206 + }, + { + "coverage": 0.07966666666666666, + "risk": 0.602510460251046, + "threshold": 0.7926878052722778 + }, + { + "coverage": 0.08, + "risk": 0.6041666666666666, + "threshold": 0.792349527403412 + }, + { + "coverage": 0.08033333333333334, + "risk": 0.6016597510373444, + "threshold": 0.7917990840608518 + }, + { + "coverage": 0.08066666666666666, + "risk": 0.6033057851239669, + "threshold": 0.7911457852705556 + }, + { + "coverage": 0.081, + "risk": 0.6049382716049383, + "threshold": 0.7906037523701679 + }, + { + "coverage": 0.08133333333333333, + "risk": 0.6024590163934426, + "threshold": 0.7896746579547527 + }, + { + "coverage": 0.08166666666666667, + "risk": 0.6040816326530613, + "threshold": 0.7895274644686433 + }, + { + "coverage": 0.082, + "risk": 0.6016260162601627, + "threshold": 0.7891642730530619 + }, + { + "coverage": 0.08233333333333333, + "risk": 0.6032388663967612, + "threshold": 0.7879915026485229 + }, + { + "coverage": 0.08266666666666667, + "risk": 0.6048387096774194, + "threshold": 0.7879498441421231 + }, + { + "coverage": 0.083, + "risk": 0.606425702811245, + "threshold": 0.7875469916614075 + }, + { + "coverage": 0.08333333333333333, + "risk": 0.604, + "threshold": 0.7875351514826078 + }, + { + "coverage": 0.08366666666666667, + "risk": 0.601593625498008, + "threshold": 0.7870574577470163 + }, + { + "coverage": 0.084, + "risk": 0.6031746031746031, + "threshold": 0.7866287534609339 + }, + { + "coverage": 0.08433333333333333, + "risk": 0.6047430830039525, + "threshold": 0.7862120229482885 + }, + { + "coverage": 0.08466666666666667, + "risk": 0.6023622047244095, + "threshold": 0.7857415637210061 + }, + { + "coverage": 0.085, + "risk": 0.6, + "threshold": 0.7856578442129387 + }, + { + "coverage": 0.08533333333333333, + "risk": 0.6015625, + "threshold": 0.785541904065525 + }, + { + "coverage": 0.08566666666666667, + "risk": 0.5992217898832685, + "threshold": 0.7855056355209376 + }, + { + "coverage": 0.086, + "risk": 0.5968992248062015, + "threshold": 0.7853396879431559 + }, + { + "coverage": 0.08633333333333333, + "risk": 0.5945945945945946, + "threshold": 0.7851816281351028 + }, + { + "coverage": 0.08666666666666667, + "risk": 0.5923076923076923, + "threshold": 0.7840528305688175 + }, + { + "coverage": 0.087, + "risk": 0.5938697318007663, + "threshold": 0.7838089519944985 + }, + { + "coverage": 0.08733333333333333, + "risk": 0.5954198473282443, + "threshold": 0.7836661843089757 + }, + { + "coverage": 0.08766666666666667, + "risk": 0.596958174904943, + "threshold": 0.782822579419933 + }, + { + "coverage": 0.088, + "risk": 0.5946969696969697, + "threshold": 0.7826765263722829 + }, + { + "coverage": 0.08833333333333333, + "risk": 0.5962264150943396, + "threshold": 0.7813304060029291 + }, + { + "coverage": 0.08866666666666667, + "risk": 0.5977443609022557, + "threshold": 0.7810714675804633 + }, + { + "coverage": 0.089, + "risk": 0.5955056179775281, + "threshold": 0.7801749218078309 + }, + { + "coverage": 0.08933333333333333, + "risk": 0.5970149253731343, + "threshold": 0.7799601585620377 + }, + { + "coverage": 0.08966666666666667, + "risk": 0.5947955390334573, + "threshold": 0.7790344867699481 + }, + { + "coverage": 0.09, + "risk": 0.5925925925925926, + "threshold": 0.7780356017536058 + }, + { + "coverage": 0.09033333333333333, + "risk": 0.5940959409594095, + "threshold": 0.7778404254697329 + }, + { + "coverage": 0.09066666666666667, + "risk": 0.5955882352941176, + "threshold": 0.7776412235709447 + }, + { + "coverage": 0.091, + "risk": 0.5934065934065934, + "threshold": 0.7774573611606768 + }, + { + "coverage": 0.09133333333333334, + "risk": 0.5912408759124088, + "threshold": 0.7774002024151142 + }, + { + "coverage": 0.09166666666666666, + "risk": 0.5927272727272728, + "threshold": 0.7773125226910165 + }, + { + "coverage": 0.092, + "risk": 0.5942028985507246, + "threshold": 0.7771736083673269 + }, + { + "coverage": 0.09233333333333334, + "risk": 0.592057761732852, + "threshold": 0.7767216617046966 + }, + { + "coverage": 0.09266666666666666, + "risk": 0.5899280575539568, + "threshold": 0.7766304696386674 + }, + { + "coverage": 0.093, + "risk": 0.5913978494623656, + "threshold": 0.7765223944440695 + }, + { + "coverage": 0.09333333333333334, + "risk": 0.5928571428571429, + "threshold": 0.7762710956572468 + }, + { + "coverage": 0.09366666666666666, + "risk": 0.594306049822064, + "threshold": 0.7762264657286516 + }, + { + "coverage": 0.094, + "risk": 0.5957446808510638, + "threshold": 0.775820650902795 + }, + { + "coverage": 0.09433333333333334, + "risk": 0.5936395759717314, + "threshold": 0.7757355538606701 + }, + { + "coverage": 0.09466666666666666, + "risk": 0.5915492957746479, + "threshold": 0.7755923254087325 + }, + { + "coverage": 0.095, + "risk": 0.5929824561403508, + "threshold": 0.7754653748189047 + }, + { + "coverage": 0.09533333333333334, + "risk": 0.5944055944055944, + "threshold": 0.7753036576950112 + }, + { + "coverage": 0.09566666666666666, + "risk": 0.5958188153310104, + "threshold": 0.7745941414086096 + }, + { + "coverage": 0.096, + "risk": 0.59375, + "threshold": 0.7744867257365995 + }, + { + "coverage": 0.09633333333333334, + "risk": 0.5916955017301038, + "threshold": 0.7740907251304547 + }, + { + "coverage": 0.09666666666666666, + "risk": 0.5896551724137931, + "threshold": 0.7739379753304916 + }, + { + "coverage": 0.097, + "risk": 0.5876288659793815, + "threshold": 0.7737283865702674 + }, + { + "coverage": 0.09733333333333333, + "risk": 0.5856164383561644, + "threshold": 0.7734305266330996 + }, + { + "coverage": 0.09766666666666667, + "risk": 0.5836177474402731, + "threshold": 0.7727560055610231 + }, + { + "coverage": 0.098, + "risk": 0.5850340136054422, + "threshold": 0.7726344091109753 + }, + { + "coverage": 0.09833333333333333, + "risk": 0.5830508474576271, + "threshold": 0.7723924476699483 + }, + { + "coverage": 0.09866666666666667, + "risk": 0.581081081081081, + "threshold": 0.7721014546298423 + }, + { + "coverage": 0.099, + "risk": 0.5791245791245792, + "threshold": 0.7712995263999456 + }, + { + "coverage": 0.09933333333333333, + "risk": 0.5805369127516778, + "threshold": 0.7711826317882372 + }, + { + "coverage": 0.09966666666666667, + "risk": 0.5819397993311036, + "threshold": 0.7708403185739855 + }, + { + "coverage": 0.1, + "risk": 0.5833333333333334, + "threshold": 0.7708138360166118 + }, + { + "coverage": 0.10033333333333333, + "risk": 0.5813953488372093, + "threshold": 0.7702144458553817 + }, + { + "coverage": 0.10066666666666667, + "risk": 0.5827814569536424, + "threshold": 0.7696964874552713 + }, + { + "coverage": 0.101, + "risk": 0.5841584158415841, + "threshold": 0.7696642187761873 + }, + { + "coverage": 0.10133333333333333, + "risk": 0.5822368421052632, + "threshold": 0.7693162510051221 + }, + { + "coverage": 0.10166666666666667, + "risk": 0.580327868852459, + "threshold": 0.7689268471074047 + }, + { + "coverage": 0.102, + "risk": 0.5816993464052288, + "threshold": 0.7687580741977624 + }, + { + "coverage": 0.10233333333333333, + "risk": 0.5830618892508144, + "threshold": 0.7683531109908063 + }, + { + "coverage": 0.10266666666666667, + "risk": 0.5811688311688312, + "threshold": 0.7682188904025737 + }, + { + "coverage": 0.103, + "risk": 0.5825242718446602, + "threshold": 0.7674125623486735 + }, + { + "coverage": 0.10333333333333333, + "risk": 0.5806451612903226, + "threshold": 0.7670234110580697 + }, + { + "coverage": 0.10366666666666667, + "risk": 0.5819935691318328, + "threshold": 0.7667996460549439 + }, + { + "coverage": 0.104, + "risk": 0.5801282051282052, + "threshold": 0.7664555532248202 + }, + { + "coverage": 0.10433333333333333, + "risk": 0.5782747603833865, + "threshold": 0.7658437802136003 + }, + { + "coverage": 0.10466666666666667, + "risk": 0.5764331210191083, + "threshold": 0.7655873861798482 + }, + { + "coverage": 0.105, + "risk": 0.5777777777777777, + "threshold": 0.7654925549973649 + }, + { + "coverage": 0.10533333333333333, + "risk": 0.5759493670886076, + "threshold": 0.7650902505548032 + }, + { + "coverage": 0.10566666666666667, + "risk": 0.5772870662460567, + "threshold": 0.7641372310246415 + }, + { + "coverage": 0.106, + "risk": 0.5786163522012578, + "threshold": 0.7640020578806488 + }, + { + "coverage": 0.10633333333333334, + "risk": 0.5799373040752351, + "threshold": 0.7639645585284548 + }, + { + "coverage": 0.10666666666666667, + "risk": 0.58125, + "threshold": 0.763374322311578 + }, + { + "coverage": 0.107, + "risk": 0.5825545171339563, + "threshold": 0.762536108165507 + }, + { + "coverage": 0.10733333333333334, + "risk": 0.5807453416149069, + "threshold": 0.7617538526732102 + }, + { + "coverage": 0.10766666666666666, + "risk": 0.5820433436532507, + "threshold": 0.7617337914775032 + }, + { + "coverage": 0.108, + "risk": 0.5802469135802469, + "threshold": 0.7615845940012015 + }, + { + "coverage": 0.10833333333333334, + "risk": 0.5815384615384616, + "threshold": 0.7615422415394029 + }, + { + "coverage": 0.10866666666666666, + "risk": 0.5828220858895705, + "threshold": 0.7614346077133448 + }, + { + "coverage": 0.109, + "risk": 0.581039755351682, + "threshold": 0.7612876681224388 + }, + { + "coverage": 0.10933333333333334, + "risk": 0.5792682926829268, + "threshold": 0.7607847328968627 + }, + { + "coverage": 0.10966666666666666, + "risk": 0.5775075987841946, + "threshold": 0.760150295678613 + }, + { + "coverage": 0.11, + "risk": 0.5757575757575758, + "threshold": 0.7589508146072926 + }, + { + "coverage": 0.11033333333333334, + "risk": 0.5740181268882175, + "threshold": 0.7588446305932476 + }, + { + "coverage": 0.11066666666666666, + "risk": 0.5753012048192772, + "threshold": 0.7586764239657393 + }, + { + "coverage": 0.111, + "risk": 0.5765765765765766, + "threshold": 0.7582464931343746 + }, + { + "coverage": 0.11133333333333334, + "risk": 0.5778443113772455, + "threshold": 0.7578742751853891 + }, + { + "coverage": 0.11166666666666666, + "risk": 0.5791044776119403, + "threshold": 0.75772938719404 + }, + { + "coverage": 0.112, + "risk": 0.5773809523809523, + "threshold": 0.7567783852583645 + }, + { + "coverage": 0.11233333333333333, + "risk": 0.5756676557863502, + "threshold": 0.7566091276250712 + }, + { + "coverage": 0.11266666666666666, + "risk": 0.5739644970414202, + "threshold": 0.75644700550695 + }, + { + "coverage": 0.113, + "risk": 0.5722713864306784, + "threshold": 0.7562764133138761 + }, + { + "coverage": 0.11333333333333333, + "risk": 0.5705882352941176, + "threshold": 0.7555379786178276 + }, + { + "coverage": 0.11366666666666667, + "risk": 0.5718475073313783, + "threshold": 0.7548299613918654 + }, + { + "coverage": 0.114, + "risk": 0.5730994152046783, + "threshold": 0.7537851834734445 + }, + { + "coverage": 0.11433333333333333, + "risk": 0.5714285714285714, + "threshold": 0.7533858887462431 + }, + { + "coverage": 0.11466666666666667, + "risk": 0.5726744186046512, + "threshold": 0.7528578794483151 + }, + { + "coverage": 0.115, + "risk": 0.5739130434782609, + "threshold": 0.7522601114856126 + }, + { + "coverage": 0.11533333333333333, + "risk": 0.5722543352601156, + "threshold": 0.7521609299846154 + }, + { + "coverage": 0.11566666666666667, + "risk": 0.5734870317002881, + "threshold": 0.7517834848998318 + }, + { + "coverage": 0.116, + "risk": 0.5747126436781609, + "threshold": 0.7517213963928868 + }, + { + "coverage": 0.11633333333333333, + "risk": 0.5759312320916905, + "threshold": 0.7515836077823719 + }, + { + "coverage": 0.11666666666666667, + "risk": 0.5742857142857143, + "threshold": 0.7509551038202973 + }, + { + "coverage": 0.117, + "risk": 0.5726495726495726, + "threshold": 0.7509209718063437 + }, + { + "coverage": 0.11733333333333333, + "risk": 0.5710227272727273, + "threshold": 0.7509031718929413 + }, + { + "coverage": 0.11766666666666667, + "risk": 0.5694050991501416, + "threshold": 0.7507931220208205 + }, + { + "coverage": 0.118, + "risk": 0.5677966101694916, + "threshold": 0.7487151292407465 + }, + { + "coverage": 0.11833333333333333, + "risk": 0.5661971830985916, + "threshold": 0.7472092343173928 + }, + { + "coverage": 0.11866666666666667, + "risk": 0.5674157303370787, + "threshold": 0.7471336134270148 + }, + { + "coverage": 0.119, + "risk": 0.5658263305322129, + "threshold": 0.7462307217596654 + }, + { + "coverage": 0.11933333333333333, + "risk": 0.5642458100558659, + "threshold": 0.745853767290099 + }, + { + "coverage": 0.11966666666666667, + "risk": 0.5626740947075209, + "threshold": 0.7450347741965967 + }, + { + "coverage": 0.12, + "risk": 0.5611111111111111, + "threshold": 0.7449100434273396 + }, + { + "coverage": 0.12033333333333333, + "risk": 0.5623268698060941, + "threshold": 0.7448857824593837 + }, + { + "coverage": 0.12066666666666667, + "risk": 0.56353591160221, + "threshold": 0.7443596033478339 + }, + { + "coverage": 0.121, + "risk": 0.5647382920110193, + "threshold": 0.7440191736382001 + }, + { + "coverage": 0.12133333333333333, + "risk": 0.5659340659340659, + "threshold": 0.7439942539832074 + }, + { + "coverage": 0.12166666666666667, + "risk": 0.5643835616438356, + "threshold": 0.7438316129095263 + }, + { + "coverage": 0.122, + "risk": 0.5628415300546448, + "threshold": 0.7437945729461481 + }, + { + "coverage": 0.12233333333333334, + "risk": 0.5640326975476839, + "threshold": 0.7437003142363617 + }, + { + "coverage": 0.12266666666666666, + "risk": 0.5625, + "threshold": 0.742604785382724 + }, + { + "coverage": 0.123, + "risk": 0.5609756097560976, + "threshold": 0.7419561629361656 + }, + { + "coverage": 0.12333333333333334, + "risk": 0.5594594594594594, + "threshold": 0.7412538847803217 + }, + { + "coverage": 0.12366666666666666, + "risk": 0.5606469002695418, + "threshold": 0.7409889497405012 + }, + { + "coverage": 0.124, + "risk": 0.5618279569892473, + "threshold": 0.7399826883064982 + }, + { + "coverage": 0.12433333333333334, + "risk": 0.5630026809651475, + "threshold": 0.7398140528632923 + }, + { + "coverage": 0.12466666666666666, + "risk": 0.5614973262032086, + "threshold": 0.7397583438987202 + }, + { + "coverage": 0.125, + "risk": 0.56, + "threshold": 0.7397188291448271 + }, + { + "coverage": 0.12533333333333332, + "risk": 0.5585106382978723, + "threshold": 0.7390824545984999 + }, + { + "coverage": 0.12566666666666668, + "risk": 0.5570291777188329, + "threshold": 0.7388518626210303 + }, + { + "coverage": 0.126, + "risk": 0.5582010582010583, + "threshold": 0.7382970988813198 + }, + { + "coverage": 0.12633333333333333, + "risk": 0.5593667546174143, + "threshold": 0.7380356711942432 + }, + { + "coverage": 0.12666666666666668, + "risk": 0.5605263157894737, + "threshold": 0.7372797068739652 + }, + { + "coverage": 0.127, + "risk": 0.5616797900262467, + "threshold": 0.7370768401013089 + }, + { + "coverage": 0.12733333333333333, + "risk": 0.5602094240837696, + "threshold": 0.7363984321805679 + }, + { + "coverage": 0.12766666666666668, + "risk": 0.5613577023498695, + "threshold": 0.7356440362311873 + }, + { + "coverage": 0.128, + "risk": 0.5598958333333334, + "threshold": 0.735404081873614 + }, + { + "coverage": 0.12833333333333333, + "risk": 0.5584415584415584, + "threshold": 0.7352533688803351 + }, + { + "coverage": 0.12866666666666668, + "risk": 0.5569948186528497, + "threshold": 0.7342314933747399 + }, + { + "coverage": 0.129, + "risk": 0.5581395348837209, + "threshold": 0.7339330695112182 + }, + { + "coverage": 0.12933333333333333, + "risk": 0.5592783505154639, + "threshold": 0.7338015797819452 + }, + { + "coverage": 0.12966666666666668, + "risk": 0.5578406169665809, + "threshold": 0.7335814795845552 + }, + { + "coverage": 0.13, + "risk": 0.5564102564102564, + "threshold": 0.7335002871436156 + }, + { + "coverage": 0.13033333333333333, + "risk": 0.5549872122762148, + "threshold": 0.7320083169829021 + }, + { + "coverage": 0.13066666666666665, + "risk": 0.5561224489795918, + "threshold": 0.7313982062030177 + }, + { + "coverage": 0.131, + "risk": 0.5572519083969466, + "threshold": 0.7310391804030308 + }, + { + "coverage": 0.13133333333333333, + "risk": 0.5558375634517766, + "threshold": 0.7306095613002795 + }, + { + "coverage": 0.13166666666666665, + "risk": 0.5569620253164557, + "threshold": 0.7305547426718123 + }, + { + "coverage": 0.132, + "risk": 0.5555555555555556, + "threshold": 0.7301970652419751 + }, + { + "coverage": 0.13233333333333333, + "risk": 0.5566750629722922, + "threshold": 0.7297349127817787 + }, + { + "coverage": 0.13266666666666665, + "risk": 0.5552763819095478, + "threshold": 0.7286233009178607 + }, + { + "coverage": 0.133, + "risk": 0.5538847117794486, + "threshold": 0.7279896438183089 + }, + { + "coverage": 0.13333333333333333, + "risk": 0.5525, + "threshold": 0.7278226417242328 + }, + { + "coverage": 0.13366666666666666, + "risk": 0.5536159600997507, + "threshold": 0.7275105072169935 + }, + { + "coverage": 0.134, + "risk": 0.554726368159204, + "threshold": 0.7274120615691915 + }, + { + "coverage": 0.13433333333333333, + "risk": 0.5533498759305211, + "threshold": 0.7265776819236632 + }, + { + "coverage": 0.13466666666666666, + "risk": 0.551980198019802, + "threshold": 0.7260892893999891 + }, + { + "coverage": 0.135, + "risk": 0.5506172839506173, + "threshold": 0.72594080178158 + }, + { + "coverage": 0.13533333333333333, + "risk": 0.5517241379310345, + "threshold": 0.7253912605193052 + }, + { + "coverage": 0.13566666666666666, + "risk": 0.5503685503685504, + "threshold": 0.7253333965823687 + }, + { + "coverage": 0.136, + "risk": 0.5514705882352942, + "threshold": 0.7248735339764205 + }, + { + "coverage": 0.13633333333333333, + "risk": 0.5525672371638142, + "threshold": 0.7246382977471547 + }, + { + "coverage": 0.13666666666666666, + "risk": 0.551219512195122, + "threshold": 0.7246079739384005 + }, + { + "coverage": 0.137, + "risk": 0.5523114355231143, + "threshold": 0.724493606871081 + }, + { + "coverage": 0.13733333333333334, + "risk": 0.5509708737864077, + "threshold": 0.7238132144120324 + }, + { + "coverage": 0.13766666666666666, + "risk": 0.549636803874092, + "threshold": 0.7233702298653707 + }, + { + "coverage": 0.138, + "risk": 0.5483091787439613, + "threshold": 0.7224490658023581 + }, + { + "coverage": 0.13833333333333334, + "risk": 0.5469879518072289, + "threshold": 0.7223370671666371 + }, + { + "coverage": 0.13866666666666666, + "risk": 0.5480769230769231, + "threshold": 0.7218013103452927 + }, + { + "coverage": 0.139, + "risk": 0.5491606714628298, + "threshold": 0.7200924727082639 + }, + { + "coverage": 0.13933333333333334, + "risk": 0.5478468899521531, + "threshold": 0.7192105523370538 + }, + { + "coverage": 0.13966666666666666, + "risk": 0.548926014319809, + "threshold": 0.7186177487281394 + }, + { + "coverage": 0.14, + "risk": 0.55, + "threshold": 0.7176219937254257 + }, + { + "coverage": 0.14033333333333334, + "risk": 0.5510688836104513, + "threshold": 0.7175129180971865 + }, + { + "coverage": 0.14066666666666666, + "risk": 0.5521327014218009, + "threshold": 0.7171193936490158 + }, + { + "coverage": 0.141, + "risk": 0.5531914893617021, + "threshold": 0.7168907520019632 + }, + { + "coverage": 0.14133333333333334, + "risk": 0.5542452830188679, + "threshold": 0.7164746618231898 + }, + { + "coverage": 0.14166666666666666, + "risk": 0.5552941176470588, + "threshold": 0.7163840854073169 + }, + { + "coverage": 0.142, + "risk": 0.5539906103286385, + "threshold": 0.7158729346980538 + }, + { + "coverage": 0.14233333333333334, + "risk": 0.5550351288056206, + "threshold": 0.7141406820982525 + }, + { + "coverage": 0.14266666666666666, + "risk": 0.5537383177570093, + "threshold": 0.7134702543056232 + }, + { + "coverage": 0.143, + "risk": 0.5524475524475524, + "threshold": 0.7126496754497299 + }, + { + "coverage": 0.14333333333333334, + "risk": 0.5511627906976744, + "threshold": 0.7118183781433383 + }, + { + "coverage": 0.14366666666666666, + "risk": 0.5522041763341067, + "threshold": 0.7116075236725686 + }, + { + "coverage": 0.144, + "risk": 0.5509259259259259, + "threshold": 0.7103248951187731 + }, + { + "coverage": 0.14433333333333334, + "risk": 0.5496535796766744, + "threshold": 0.7097063083853357 + }, + { + "coverage": 0.14466666666666667, + "risk": 0.5483870967741935, + "threshold": 0.7094343330894225 + }, + { + "coverage": 0.145, + "risk": 0.5494252873563218, + "threshold": 0.7092440893365125 + }, + { + "coverage": 0.14533333333333334, + "risk": 0.5504587155963303, + "threshold": 0.7090157476316779 + }, + { + "coverage": 0.14566666666666667, + "risk": 0.551487414187643, + "threshold": 0.7087443072088927 + }, + { + "coverage": 0.146, + "risk": 0.5502283105022832, + "threshold": 0.7077237668758231 + }, + { + "coverage": 0.14633333333333334, + "risk": 0.5489749430523918, + "threshold": 0.7071937466264634 + }, + { + "coverage": 0.14666666666666667, + "risk": 0.5477272727272727, + "threshold": 0.7065706268653634 + }, + { + "coverage": 0.147, + "risk": 0.546485260770975, + "threshold": 0.7057003832699963 + }, + { + "coverage": 0.14733333333333334, + "risk": 0.5475113122171946, + "threshold": 0.7055631371798425 + }, + { + "coverage": 0.14766666666666667, + "risk": 0.54627539503386, + "threshold": 0.7053781181586889 + }, + { + "coverage": 0.148, + "risk": 0.545045045045045, + "threshold": 0.7051893245041247 + }, + { + "coverage": 0.14833333333333334, + "risk": 0.5460674157303371, + "threshold": 0.7049201813232444 + }, + { + "coverage": 0.14866666666666667, + "risk": 0.547085201793722, + "threshold": 0.7048574313141808 + }, + { + "coverage": 0.149, + "risk": 0.5480984340044742, + "threshold": 0.7046877951524612 + }, + { + "coverage": 0.14933333333333335, + "risk": 0.5491071428571429, + "threshold": 0.7041537839623103 + }, + { + "coverage": 0.14966666666666667, + "risk": 0.5478841870824054, + "threshold": 0.7040953441582193 + }, + { + "coverage": 0.15, + "risk": 0.5488888888888889, + "threshold": 0.7038025858745253 + }, + { + "coverage": 0.15033333333333335, + "risk": 0.549889135254989, + "threshold": 0.7028940846271295 + }, + { + "coverage": 0.15066666666666667, + "risk": 0.5486725663716814, + "threshold": 0.7027746783171831 + }, + { + "coverage": 0.151, + "risk": 0.5474613686534217, + "threshold": 0.7023068858000271 + }, + { + "coverage": 0.15133333333333332, + "risk": 0.5462555066079295, + "threshold": 0.7022171114579521 + }, + { + "coverage": 0.15166666666666667, + "risk": 0.5472527472527473, + "threshold": 0.7015613571402858 + }, + { + "coverage": 0.152, + "risk": 0.5460526315789473, + "threshold": 0.7012212794115783 + }, + { + "coverage": 0.15233333333333332, + "risk": 0.5470459518599562, + "threshold": 0.7009784977630316 + }, + { + "coverage": 0.15266666666666667, + "risk": 0.5480349344978166, + "threshold": 0.7002255568983289 + }, + { + "coverage": 0.153, + "risk": 0.5468409586056645, + "threshold": 0.6998043210472998 + }, + { + "coverage": 0.15333333333333332, + "risk": 0.5456521739130434, + "threshold": 0.6996627940763891 + }, + { + "coverage": 0.15366666666666667, + "risk": 0.544468546637744, + "threshold": 0.6994789180637159 + }, + { + "coverage": 0.154, + "risk": 0.5432900432900433, + "threshold": 0.6986706759033167 + }, + { + "coverage": 0.15433333333333332, + "risk": 0.5442764578833693, + "threshold": 0.6981050015554457 + }, + { + "coverage": 0.15466666666666667, + "risk": 0.5452586206896551, + "threshold": 0.6977855036629167 + }, + { + "coverage": 0.155, + "risk": 0.5440860215053763, + "threshold": 0.6968651872149284 + }, + { + "coverage": 0.15533333333333332, + "risk": 0.5450643776824035, + "threshold": 0.6966439743334621 + }, + { + "coverage": 0.15566666666666668, + "risk": 0.5438972162740899, + "threshold": 0.6960032038069134 + }, + { + "coverage": 0.156, + "risk": 0.5427350427350427, + "threshold": 0.6959518475329645 + }, + { + "coverage": 0.15633333333333332, + "risk": 0.5437100213219617, + "threshold": 0.6956169438216738 + }, + { + "coverage": 0.15666666666666668, + "risk": 0.5446808510638298, + "threshold": 0.6948528803115452 + }, + { + "coverage": 0.157, + "risk": 0.5456475583864119, + "threshold": 0.6942116144216475 + }, + { + "coverage": 0.15733333333333333, + "risk": 0.5444915254237288, + "threshold": 0.6940285129349175 + }, + { + "coverage": 0.15766666666666668, + "risk": 0.5433403805496829, + "threshold": 0.6933591022131795 + }, + { + "coverage": 0.158, + "risk": 0.5421940928270043, + "threshold": 0.6926354471876541 + }, + { + "coverage": 0.15833333333333333, + "risk": 0.5410526315789473, + "threshold": 0.6916319854851354 + }, + { + "coverage": 0.15866666666666668, + "risk": 0.542016806722689, + "threshold": 0.691274363475921 + }, + { + "coverage": 0.159, + "risk": 0.5429769392033543, + "threshold": 0.6907336804120989 + }, + { + "coverage": 0.15933333333333333, + "risk": 0.5418410041841004, + "threshold": 0.6898549238958821 + }, + { + "coverage": 0.15966666666666668, + "risk": 0.5407098121085595, + "threshold": 0.6875837572886022 + }, + { + "coverage": 0.16, + "risk": 0.5395833333333333, + "threshold": 0.6875604523237825 + }, + { + "coverage": 0.16033333333333333, + "risk": 0.5384615384615384, + "threshold": 0.6856800309161827 + }, + { + "coverage": 0.16066666666666668, + "risk": 0.5373443983402489, + "threshold": 0.6851319094853032 + }, + { + "coverage": 0.161, + "risk": 0.5362318840579711, + "threshold": 0.6846430749740823 + }, + { + "coverage": 0.16133333333333333, + "risk": 0.5351239669421488, + "threshold": 0.6842960546783807 + }, + { + "coverage": 0.16166666666666665, + "risk": 0.534020618556701, + "threshold": 0.6833284561558497 + }, + { + "coverage": 0.162, + "risk": 0.5329218106995884, + "threshold": 0.6824528405366944 + }, + { + "coverage": 0.16233333333333333, + "risk": 0.5318275154004107, + "threshold": 0.6817678095166158 + }, + { + "coverage": 0.16266666666666665, + "risk": 0.5327868852459017, + "threshold": 0.6815802033186179 + }, + { + "coverage": 0.163, + "risk": 0.5337423312883436, + "threshold": 0.6813128204628988 + }, + { + "coverage": 0.16333333333333333, + "risk": 0.5326530612244897, + "threshold": 0.6809905118817209 + }, + { + "coverage": 0.16366666666666665, + "risk": 0.5315682281059063, + "threshold": 0.6798307986003973 + }, + { + "coverage": 0.164, + "risk": 0.5304878048780488, + "threshold": 0.6792039521302486 + }, + { + "coverage": 0.16433333333333333, + "risk": 0.5294117647058824, + "threshold": 0.6789379029717739 + }, + { + "coverage": 0.16466666666666666, + "risk": 0.52834008097166, + "threshold": 0.6787374252128451 + }, + { + "coverage": 0.165, + "risk": 0.5292929292929293, + "threshold": 0.678433390778265 + }, + { + "coverage": 0.16533333333333333, + "risk": 0.530241935483871, + "threshold": 0.6777883072830766 + }, + { + "coverage": 0.16566666666666666, + "risk": 0.5291750503018109, + "threshold": 0.6774169348828936 + }, + { + "coverage": 0.166, + "risk": 0.5301204819277109, + "threshold": 0.6773913925861089 + }, + { + "coverage": 0.16633333333333333, + "risk": 0.5290581162324649, + "threshold": 0.6763831500890479 + }, + { + "coverage": 0.16666666666666666, + "risk": 0.53, + "threshold": 0.6761671959214407 + }, + { + "coverage": 0.167, + "risk": 0.5289421157684631, + "threshold": 0.675289041603924 + }, + { + "coverage": 0.16733333333333333, + "risk": 0.5278884462151394, + "threshold": 0.6751686577330692 + }, + { + "coverage": 0.16766666666666666, + "risk": 0.5268389662027833, + "threshold": 0.6747624958527317 + }, + { + "coverage": 0.168, + "risk": 0.5257936507936508, + "threshold": 0.6746332661337635 + }, + { + "coverage": 0.16833333333333333, + "risk": 0.5247524752475248, + "threshold": 0.6745207352383605 + }, + { + "coverage": 0.16866666666666666, + "risk": 0.5237154150197628, + "threshold": 0.6744462465148525 + }, + { + "coverage": 0.169, + "risk": 0.52465483234714, + "threshold": 0.6743379256280659 + }, + { + "coverage": 0.16933333333333334, + "risk": 0.5236220472440944, + "threshold": 0.673878558880682 + }, + { + "coverage": 0.16966666666666666, + "risk": 0.5225933202357563, + "threshold": 0.6736664566868987 + }, + { + "coverage": 0.17, + "risk": 0.5215686274509804, + "threshold": 0.6729981531552595 + }, + { + "coverage": 0.17033333333333334, + "risk": 0.5225048923679061, + "threshold": 0.6729599870790174 + }, + { + "coverage": 0.17066666666666666, + "risk": 0.521484375, + "threshold": 0.6714532542390098 + }, + { + "coverage": 0.171, + "risk": 0.5224171539961013, + "threshold": 0.6712108085429223 + }, + { + "coverage": 0.17133333333333334, + "risk": 0.5214007782101168, + "threshold": 0.6706090721950259 + }, + { + "coverage": 0.17166666666666666, + "risk": 0.5223300970873787, + "threshold": 0.670256348146697 + }, + { + "coverage": 0.172, + "risk": 0.5213178294573644, + "threshold": 0.6698303079356885 + }, + { + "coverage": 0.17233333333333334, + "risk": 0.5203094777562862, + "threshold": 0.6698177821687908 + }, + { + "coverage": 0.17266666666666666, + "risk": 0.5212355212355212, + "threshold": 0.6697748815943311 + }, + { + "coverage": 0.173, + "risk": 0.5221579961464354, + "threshold": 0.6684672862708828 + }, + { + "coverage": 0.17333333333333334, + "risk": 0.5211538461538462, + "threshold": 0.6683066515920022 + }, + { + "coverage": 0.17366666666666666, + "risk": 0.5201535508637236, + "threshold": 0.6680083599971584 + }, + { + "coverage": 0.174, + "risk": 0.5191570881226054, + "threshold": 0.6678699313663115 + }, + { + "coverage": 0.17433333333333334, + "risk": 0.5181644359464627, + "threshold": 0.6671539723558438 + }, + { + "coverage": 0.17466666666666666, + "risk": 0.517175572519084, + "threshold": 0.6670247292401619 + }, + { + "coverage": 0.175, + "risk": 0.5161904761904762, + "threshold": 0.6669248956463923 + }, + { + "coverage": 0.17533333333333334, + "risk": 0.5152091254752852, + "threshold": 0.6667011513732267 + }, + { + "coverage": 0.17566666666666667, + "risk": 0.5142314990512334, + "threshold": 0.6664276722967089 + }, + { + "coverage": 0.176, + "risk": 0.5132575757575758, + "threshold": 0.6664050197340032 + }, + { + "coverage": 0.17633333333333334, + "risk": 0.5141776937618148, + "threshold": 0.6650874310288568 + }, + { + "coverage": 0.17666666666666667, + "risk": 0.5150943396226415, + "threshold": 0.6650759184251909 + }, + { + "coverage": 0.177, + "risk": 0.5141242937853108, + "threshold": 0.6647611354996531 + }, + { + "coverage": 0.17733333333333334, + "risk": 0.5150375939849624, + "threshold": 0.6647419528251756 + }, + { + "coverage": 0.17766666666666667, + "risk": 0.5140712945590994, + "threshold": 0.66466324971291 + }, + { + "coverage": 0.178, + "risk": 0.5149812734082397, + "threshold": 0.6639256391588887 + }, + { + "coverage": 0.17833333333333334, + "risk": 0.514018691588785, + "threshold": 0.6637895073410478 + }, + { + "coverage": 0.17866666666666667, + "risk": 0.5130597014925373, + "threshold": 0.6637401793083878 + }, + { + "coverage": 0.179, + "risk": 0.5121042830540037, + "threshold": 0.6634880251242739 + }, + { + "coverage": 0.17933333333333334, + "risk": 0.5111524163568774, + "threshold": 0.6626383443228978 + }, + { + "coverage": 0.17966666666666667, + "risk": 0.5102040816326531, + "threshold": 0.6623899611134089 + }, + { + "coverage": 0.18, + "risk": 0.5092592592592593, + "threshold": 0.662369665771225 + }, + { + "coverage": 0.18033333333333335, + "risk": 0.5101663585951941, + "threshold": 0.6621104451364458 + }, + { + "coverage": 0.18066666666666667, + "risk": 0.5092250922509225, + "threshold": 0.6620708912108599 + }, + { + "coverage": 0.181, + "risk": 0.5082872928176796, + "threshold": 0.6613496198760948 + }, + { + "coverage": 0.18133333333333335, + "risk": 0.5073529411764706, + "threshold": 0.6604973086952293 + }, + { + "coverage": 0.18166666666666667, + "risk": 0.5064220183486239, + "threshold": 0.660098087467337 + }, + { + "coverage": 0.182, + "risk": 0.5073260073260073, + "threshold": 0.6599419225973289 + }, + { + "coverage": 0.18233333333333332, + "risk": 0.5082266910420475, + "threshold": 0.6596795643663335 + }, + { + "coverage": 0.18266666666666667, + "risk": 0.5091240875912408, + "threshold": 0.6591528375224838 + }, + { + "coverage": 0.183, + "risk": 0.5100182149362478, + "threshold": 0.6583841750010676 + }, + { + "coverage": 0.18333333333333332, + "risk": 0.5109090909090909, + "threshold": 0.6568225703827892 + }, + { + "coverage": 0.18366666666666667, + "risk": 0.5117967332123412, + "threshold": 0.6561947371131439 + }, + { + "coverage": 0.184, + "risk": 0.5108695652173914, + "threshold": 0.6558865992375693 + }, + { + "coverage": 0.18433333333333332, + "risk": 0.5099457504520796, + "threshold": 0.6558707754324823 + }, + { + "coverage": 0.18466666666666667, + "risk": 0.5108303249097473, + "threshold": 0.6554378926021108 + }, + { + "coverage": 0.185, + "risk": 0.5099099099099099, + "threshold": 0.654930287290566 + }, + { + "coverage": 0.18533333333333332, + "risk": 0.5107913669064749, + "threshold": 0.6544423526557988 + }, + { + "coverage": 0.18566666666666667, + "risk": 0.5098743267504489, + "threshold": 0.6542155717454969 + }, + { + "coverage": 0.186, + "risk": 0.510752688172043, + "threshold": 0.6541794518194106 + }, + { + "coverage": 0.18633333333333332, + "risk": 0.5098389982110912, + "threshold": 0.6536568589468208 + }, + { + "coverage": 0.18666666666666668, + "risk": 0.5089285714285714, + "threshold": 0.6533979735773942 + }, + { + "coverage": 0.187, + "risk": 0.5080213903743316, + "threshold": 0.653081329366992 + }, + { + "coverage": 0.18733333333333332, + "risk": 0.5071174377224199, + "threshold": 0.6529793042631865 + }, + { + "coverage": 0.18766666666666668, + "risk": 0.5062166962699822, + "threshold": 0.6526381181851967 + }, + { + "coverage": 0.188, + "risk": 0.5053191489361702, + "threshold": 0.6526039325291078 + }, + { + "coverage": 0.18833333333333332, + "risk": 0.504424778761062, + "threshold": 0.6525238470024702 + }, + { + "coverage": 0.18866666666666668, + "risk": 0.5053003533568905, + "threshold": 0.6518170709619081 + }, + { + "coverage": 0.189, + "risk": 0.5061728395061729, + "threshold": 0.6516413728092524 + }, + { + "coverage": 0.18933333333333333, + "risk": 0.5070422535211268, + "threshold": 0.6515395951995209 + }, + { + "coverage": 0.18966666666666668, + "risk": 0.507908611599297, + "threshold": 0.6515119406839177 + }, + { + "coverage": 0.19, + "risk": 0.5070175438596491, + "threshold": 0.6490041678803178 + }, + { + "coverage": 0.19033333333333333, + "risk": 0.5078809106830122, + "threshold": 0.648742487831465 + }, + { + "coverage": 0.19066666666666668, + "risk": 0.506993006993007, + "threshold": 0.6469839687764671 + }, + { + "coverage": 0.191, + "risk": 0.506108202443281, + "threshold": 0.6469785531160896 + }, + { + "coverage": 0.19133333333333333, + "risk": 0.5052264808362369, + "threshold": 0.6464139337519265 + }, + { + "coverage": 0.19166666666666668, + "risk": 0.5043478260869565, + "threshold": 0.646058079617274 + }, + { + "coverage": 0.192, + "risk": 0.5034722222222222, + "threshold": 0.6459288806024778 + }, + { + "coverage": 0.19233333333333333, + "risk": 0.5025996533795494, + "threshold": 0.6457137000753554 + }, + { + "coverage": 0.19266666666666668, + "risk": 0.5034602076124568, + "threshold": 0.6454814076237174 + }, + { + "coverage": 0.193, + "risk": 0.5043177892918825, + "threshold": 0.6442865047876956 + }, + { + "coverage": 0.19333333333333333, + "risk": 0.5051724137931034, + "threshold": 0.6441047161498488 + }, + { + "coverage": 0.19366666666666665, + "risk": 0.504302925989673, + "threshold": 0.6436700332153587 + }, + { + "coverage": 0.194, + "risk": 0.5034364261168385, + "threshold": 0.6431381718456007 + }, + { + "coverage": 0.19433333333333333, + "risk": 0.5025728987993139, + "threshold": 0.6429332977113569 + }, + { + "coverage": 0.19466666666666665, + "risk": 0.5017123287671232, + "threshold": 0.6424328346717909 + }, + { + "coverage": 0.195, + "risk": 0.5008547008547009, + "threshold": 0.6418876661249903 + }, + { + "coverage": 0.19533333333333333, + "risk": 0.5, + "threshold": 0.6417599349152859 + }, + { + "coverage": 0.19566666666666666, + "risk": 0.4991482112436116, + "threshold": 0.6416980241260908 + }, + { + "coverage": 0.196, + "risk": 0.49829931972789115, + "threshold": 0.6416044626270953 + }, + { + "coverage": 0.19633333333333333, + "risk": 0.49745331069609505, + "threshold": 0.6408760502060964 + }, + { + "coverage": 0.19666666666666666, + "risk": 0.49830508474576274, + "threshold": 0.6406562385863672 + }, + { + "coverage": 0.197, + "risk": 0.49746192893401014, + "threshold": 0.6406454152901196 + }, + { + "coverage": 0.19733333333333333, + "risk": 0.4966216216216216, + "threshold": 0.6404912188637849 + }, + { + "coverage": 0.19766666666666666, + "risk": 0.4957841483979764, + "threshold": 0.6398938262634376 + }, + { + "coverage": 0.198, + "risk": 0.494949494949495, + "threshold": 0.6392927054388706 + }, + { + "coverage": 0.19833333333333333, + "risk": 0.49411764705882355, + "threshold": 0.6383476138684603 + }, + { + "coverage": 0.19866666666666666, + "risk": 0.4949664429530201, + "threshold": 0.638249315392807 + }, + { + "coverage": 0.199, + "risk": 0.49581239530988275, + "threshold": 0.6379773875082202 + }, + { + "coverage": 0.19933333333333333, + "risk": 0.49665551839464883, + "threshold": 0.6377687302625736 + }, + { + "coverage": 0.19966666666666666, + "risk": 0.4958263772954925, + "threshold": 0.637355454491252 + }, + { + "coverage": 0.2, + "risk": 0.49666666666666665, + "threshold": 0.6372757304465088 + }, + { + "coverage": 0.20033333333333334, + "risk": 0.49584026622296173, + "threshold": 0.6366701084973936 + }, + { + "coverage": 0.20066666666666666, + "risk": 0.4950166112956811, + "threshold": 0.6365465252605044 + }, + { + "coverage": 0.201, + "risk": 0.49585406301824214, + "threshold": 0.6361969100006164 + }, + { + "coverage": 0.20133333333333334, + "risk": 0.4966887417218543, + "threshold": 0.6357967946588059 + }, + { + "coverage": 0.20166666666666666, + "risk": 0.4975206611570248, + "threshold": 0.6356841184564072 + }, + { + "coverage": 0.202, + "risk": 0.49834983498349833, + "threshold": 0.6355431625723601 + }, + { + "coverage": 0.20233333333333334, + "risk": 0.4975288303130148, + "threshold": 0.6346591531071182 + }, + { + "coverage": 0.20266666666666666, + "risk": 0.49835526315789475, + "threshold": 0.6343651318804112 + }, + { + "coverage": 0.203, + "risk": 0.4975369458128079, + "threshold": 0.6340591868862699 + }, + { + "coverage": 0.20333333333333334, + "risk": 0.4967213114754098, + "threshold": 0.6336875566401041 + }, + { + "coverage": 0.20366666666666666, + "risk": 0.4959083469721768, + "threshold": 0.6324050557278338 + }, + { + "coverage": 0.204, + "risk": 0.4950980392156863, + "threshold": 0.6298780378643606 + }, + { + "coverage": 0.20433333333333334, + "risk": 0.49429037520391517, + "threshold": 0.6296926171647325 + }, + { + "coverage": 0.20466666666666666, + "risk": 0.49348534201954397, + "threshold": 0.6288683820019709 + }, + { + "coverage": 0.205, + "risk": 0.4926829268292683, + "threshold": 0.628831998184455 + }, + { + "coverage": 0.20533333333333334, + "risk": 0.49188311688311687, + "threshold": 0.6285984906131041 + }, + { + "coverage": 0.20566666666666666, + "risk": 0.49108589951377635, + "threshold": 0.6284911141478391 + }, + { + "coverage": 0.206, + "risk": 0.49029126213592233, + "threshold": 0.6280717133333427 + }, + { + "coverage": 0.20633333333333334, + "risk": 0.4894991922455573, + "threshold": 0.6280620297678854 + }, + { + "coverage": 0.20666666666666667, + "risk": 0.49032258064516127, + "threshold": 0.6279602978777924 + }, + { + "coverage": 0.207, + "risk": 0.4895330112721417, + "threshold": 0.6278530187752897 + }, + { + "coverage": 0.20733333333333334, + "risk": 0.4887459807073955, + "threshold": 0.6278470061418678 + }, + { + "coverage": 0.20766666666666667, + "risk": 0.4895666131621188, + "threshold": 0.6278312203576752 + }, + { + "coverage": 0.208, + "risk": 0.48878205128205127, + "threshold": 0.6274074309124678 + }, + { + "coverage": 0.20833333333333334, + "risk": 0.4896, + "threshold": 0.6272361453239333 + }, + { + "coverage": 0.20866666666666667, + "risk": 0.4904153354632588, + "threshold": 0.6260865676210964 + }, + { + "coverage": 0.209, + "risk": 0.48963317384370014, + "threshold": 0.6256957203525987 + }, + { + "coverage": 0.20933333333333334, + "risk": 0.4888535031847134, + "threshold": 0.6256633148870357 + }, + { + "coverage": 0.20966666666666667, + "risk": 0.48807631160572335, + "threshold": 0.6256150133701291 + }, + { + "coverage": 0.21, + "risk": 0.4873015873015873, + "threshold": 0.6254166232695569 + }, + { + "coverage": 0.21033333333333334, + "risk": 0.4865293185419968, + "threshold": 0.6248264912497808 + }, + { + "coverage": 0.21066666666666667, + "risk": 0.4873417721518987, + "threshold": 0.6247696050534375 + }, + { + "coverage": 0.211, + "risk": 0.48657187993680884, + "threshold": 0.6234869450099885 + }, + { + "coverage": 0.21133333333333335, + "risk": 0.48580441640378547, + "threshold": 0.6233577901458911 + }, + { + "coverage": 0.21166666666666667, + "risk": 0.48503937007874015, + "threshold": 0.6233426671180315 + }, + { + "coverage": 0.212, + "risk": 0.48427672955974843, + "threshold": 0.6229684479776766 + }, + { + "coverage": 0.21233333333333335, + "risk": 0.4850863422291994, + "threshold": 0.622930476524515 + }, + { + "coverage": 0.21266666666666667, + "risk": 0.4843260188087774, + "threshold": 0.6217597346441454 + }, + { + "coverage": 0.213, + "risk": 0.4835680751173709, + "threshold": 0.6212227668789791 + }, + { + "coverage": 0.21333333333333335, + "risk": 0.4828125, + "threshold": 0.620573102033351 + }, + { + "coverage": 0.21366666666666667, + "risk": 0.48205928237129486, + "threshold": 0.6202656463467003 + }, + { + "coverage": 0.214, + "risk": 0.48130841121495327, + "threshold": 0.6191216873847333 + }, + { + "coverage": 0.21433333333333332, + "risk": 0.4821150855365474, + "threshold": 0.6173518411838647 + }, + { + "coverage": 0.21466666666666667, + "risk": 0.4813664596273292, + "threshold": 0.6171320943522165 + }, + { + "coverage": 0.215, + "risk": 0.4806201550387597, + "threshold": 0.6170985044312141 + }, + { + "coverage": 0.21533333333333332, + "risk": 0.48142414860681115, + "threshold": 0.6170929176042922 + }, + { + "coverage": 0.21566666666666667, + "risk": 0.482225656877898, + "threshold": 0.6170793272183432 + }, + { + "coverage": 0.216, + "risk": 0.48148148148148145, + "threshold": 0.6168171867070195 + }, + { + "coverage": 0.21633333333333332, + "risk": 0.48228043143297383, + "threshold": 0.6159332345178514 + }, + { + "coverage": 0.21666666666666667, + "risk": 0.4815384615384615, + "threshold": 0.615752799083786 + }, + { + "coverage": 0.217, + "risk": 0.4807987711213518, + "threshold": 0.6153364729883926 + }, + { + "coverage": 0.21733333333333332, + "risk": 0.48006134969325154, + "threshold": 0.6149591092524371 + }, + { + "coverage": 0.21766666666666667, + "risk": 0.4793261868300153, + "threshold": 0.6139308683985858 + }, + { + "coverage": 0.218, + "risk": 0.4785932721712538, + "threshold": 0.6137127881651984 + }, + { + "coverage": 0.21833333333333332, + "risk": 0.4778625954198473, + "threshold": 0.613437107213308 + }, + { + "coverage": 0.21866666666666668, + "risk": 0.47865853658536583, + "threshold": 0.6133447653697948 + }, + { + "coverage": 0.219, + "risk": 0.4779299847792998, + "threshold": 0.6129024261568737 + }, + { + "coverage": 0.21933333333333332, + "risk": 0.4787234042553192, + "threshold": 0.612535292536486 + }, + { + "coverage": 0.21966666666666668, + "risk": 0.4795144157814871, + "threshold": 0.6122420518878025 + }, + { + "coverage": 0.22, + "risk": 0.47878787878787876, + "threshold": 0.6121535559675415 + }, + { + "coverage": 0.22033333333333333, + "risk": 0.47806354009077157, + "threshold": 0.6119937000898267 + }, + { + "coverage": 0.22066666666666668, + "risk": 0.4773413897280967, + "threshold": 0.611975989609558 + }, + { + "coverage": 0.221, + "risk": 0.4766214177978884, + "threshold": 0.611422202337824 + }, + { + "coverage": 0.22133333333333333, + "risk": 0.47740963855421686, + "threshold": 0.6113468503471464 + }, + { + "coverage": 0.22166666666666668, + "risk": 0.4781954887218045, + "threshold": 0.6112912918554559 + }, + { + "coverage": 0.222, + "risk": 0.47897897897897895, + "threshold": 0.6110861565834144 + }, + { + "coverage": 0.22233333333333333, + "risk": 0.4782608695652174, + "threshold": 0.6105832538753054 + }, + { + "coverage": 0.22266666666666668, + "risk": 0.47754491017964074, + "threshold": 0.6105728283721716 + }, + { + "coverage": 0.223, + "risk": 0.476831091180867, + "threshold": 0.6104460018221317 + }, + { + "coverage": 0.22333333333333333, + "risk": 0.4761194029850746, + "threshold": 0.6103443107122135 + }, + { + "coverage": 0.22366666666666668, + "risk": 0.47540983606557374, + "threshold": 0.609876103231471 + }, + { + "coverage": 0.224, + "risk": 0.47470238095238093, + "threshold": 0.6093935941760367 + }, + { + "coverage": 0.22433333333333333, + "risk": 0.4739970282317979, + "threshold": 0.6090904103035456 + }, + { + "coverage": 0.22466666666666665, + "risk": 0.4732937685459941, + "threshold": 0.6078805760087709 + }, + { + "coverage": 0.225, + "risk": 0.4740740740740741, + "threshold": 0.6075930490357304 + }, + { + "coverage": 0.22533333333333333, + "risk": 0.47337278106508873, + "threshold": 0.6064069940253309 + }, + { + "coverage": 0.22566666666666665, + "risk": 0.4726735598227474, + "threshold": 0.6059898117552309 + }, + { + "coverage": 0.226, + "risk": 0.471976401179941, + "threshold": 0.605744892162778 + }, + { + "coverage": 0.22633333333333333, + "risk": 0.47128129602356406, + "threshold": 0.6052251631379141 + }, + { + "coverage": 0.22666666666666666, + "risk": 0.47058823529411764, + "threshold": 0.6051864340479375 + }, + { + "coverage": 0.227, + "risk": 0.4698972099853157, + "threshold": 0.6045597908440397 + }, + { + "coverage": 0.22733333333333333, + "risk": 0.4706744868035191, + "threshold": 0.6040018760959361 + }, + { + "coverage": 0.22766666666666666, + "risk": 0.4699853587115666, + "threshold": 0.6030596508769932 + }, + { + "coverage": 0.228, + "risk": 0.47076023391812866, + "threshold": 0.6022156157804288 + }, + { + "coverage": 0.22833333333333333, + "risk": 0.47153284671532847, + "threshold": 0.602109036951884 + }, + { + "coverage": 0.22866666666666666, + "risk": 0.4708454810495627, + "threshold": 0.6019484687386014 + }, + { + "coverage": 0.229, + "risk": 0.47016011644832606, + "threshold": 0.601891233270859 + }, + { + "coverage": 0.22933333333333333, + "risk": 0.47093023255813954, + "threshold": 0.6015095221849546 + }, + { + "coverage": 0.22966666666666666, + "risk": 0.4716981132075472, + "threshold": 0.6011110568185366 + }, + { + "coverage": 0.23, + "risk": 0.47246376811594204, + "threshold": 0.6001472108106896 + }, + { + "coverage": 0.23033333333333333, + "risk": 0.4717800289435601, + "threshold": 0.5989777061604461 + }, + { + "coverage": 0.23066666666666666, + "risk": 0.4725433526011561, + "threshold": 0.5985027804768971 + }, + { + "coverage": 0.231, + "risk": 0.47186147186147187, + "threshold": 0.5983503962584368 + }, + { + "coverage": 0.23133333333333334, + "risk": 0.4711815561959654, + "threshold": 0.5975180727509983 + }, + { + "coverage": 0.23166666666666666, + "risk": 0.47050359712230216, + "threshold": 0.597481538256208 + }, + { + "coverage": 0.232, + "risk": 0.4698275862068966, + "threshold": 0.5971135036538952 + }, + { + "coverage": 0.23233333333333334, + "risk": 0.46915351506456243, + "threshold": 0.5960011453190517 + }, + { + "coverage": 0.23266666666666666, + "risk": 0.4684813753581662, + "threshold": 0.5958740805906643 + }, + { + "coverage": 0.233, + "risk": 0.4678111587982833, + "threshold": 0.5953487830645577 + }, + { + "coverage": 0.23333333333333334, + "risk": 0.46714285714285714, + "threshold": 0.5950766449110994 + }, + { + "coverage": 0.23366666666666666, + "risk": 0.46790299572039945, + "threshold": 0.5949123919099678 + }, + { + "coverage": 0.234, + "risk": 0.4672364672364672, + "threshold": 0.5948761879509669 + }, + { + "coverage": 0.23433333333333334, + "risk": 0.46799431009957326, + "threshold": 0.5948331442780288 + }, + { + "coverage": 0.23466666666666666, + "risk": 0.46732954545454547, + "threshold": 0.5947280915068006 + }, + { + "coverage": 0.235, + "risk": 0.4666666666666667, + "threshold": 0.5947020864605138 + }, + { + "coverage": 0.23533333333333334, + "risk": 0.4660056657223796, + "threshold": 0.5943933675732599 + }, + { + "coverage": 0.23566666666666666, + "risk": 0.46534653465346537, + "threshold": 0.5943889467142924 + }, + { + "coverage": 0.236, + "risk": 0.4646892655367232, + "threshold": 0.5939637285946954 + }, + { + "coverage": 0.23633333333333334, + "risk": 0.46544428772919605, + "threshold": 0.593957128502683 + }, + { + "coverage": 0.23666666666666666, + "risk": 0.4647887323943662, + "threshold": 0.5936518342467619 + }, + { + "coverage": 0.237, + "risk": 0.46554149085794655, + "threshold": 0.5935834557735742 + }, + { + "coverage": 0.23733333333333334, + "risk": 0.4648876404494382, + "threshold": 0.5929962164483915 + }, + { + "coverage": 0.23766666666666666, + "risk": 0.4642356241234222, + "threshold": 0.5927181316376487 + }, + { + "coverage": 0.238, + "risk": 0.4635854341736695, + "threshold": 0.5927066864670303 + }, + { + "coverage": 0.23833333333333334, + "risk": 0.46293706293706294, + "threshold": 0.5920168154167245 + }, + { + "coverage": 0.23866666666666667, + "risk": 0.4622905027932961, + "threshold": 0.5918591801124986 + }, + { + "coverage": 0.239, + "risk": 0.4630404463040446, + "threshold": 0.5917250393204215 + }, + { + "coverage": 0.23933333333333334, + "risk": 0.4623955431754875, + "threshold": 0.591597669625192 + }, + { + "coverage": 0.23966666666666667, + "risk": 0.4617524339360223, + "threshold": 0.5915958096745655 + }, + { + "coverage": 0.24, + "risk": 0.46111111111111114, + "threshold": 0.5907921805753575 + }, + { + "coverage": 0.24033333333333334, + "risk": 0.4604715672676838, + "threshold": 0.5907410316014555 + }, + { + "coverage": 0.24066666666666667, + "risk": 0.4598337950138504, + "threshold": 0.5898367666717828 + }, + { + "coverage": 0.241, + "risk": 0.45919778699861685, + "threshold": 0.589675258184445 + }, + { + "coverage": 0.24133333333333334, + "risk": 0.4585635359116022, + "threshold": 0.5888122418959518 + }, + { + "coverage": 0.24166666666666667, + "risk": 0.4579310344827586, + "threshold": 0.5886263979041644 + }, + { + "coverage": 0.242, + "risk": 0.4573002754820937, + "threshold": 0.5884975107392597 + }, + { + "coverage": 0.24233333333333335, + "risk": 0.4580467675378267, + "threshold": 0.5880526870532033 + }, + { + "coverage": 0.24266666666666667, + "risk": 0.4574175824175824, + "threshold": 0.5875979745413116 + }, + { + "coverage": 0.243, + "risk": 0.4567901234567901, + "threshold": 0.5874097112671482 + }, + { + "coverage": 0.24333333333333335, + "risk": 0.45616438356164385, + "threshold": 0.5870029886328101 + }, + { + "coverage": 0.24366666666666667, + "risk": 0.45554035567715456, + "threshold": 0.586769641811536 + }, + { + "coverage": 0.244, + "risk": 0.45491803278688525, + "threshold": 0.5866005064421744 + }, + { + "coverage": 0.24433333333333335, + "risk": 0.4542974079126876, + "threshold": 0.5857146478896426 + }, + { + "coverage": 0.24466666666666667, + "risk": 0.4550408719346049, + "threshold": 0.5854829286951632 + }, + { + "coverage": 0.245, + "risk": 0.454421768707483, + "threshold": 0.5850714734633125 + }, + { + "coverage": 0.24533333333333332, + "risk": 0.45516304347826086, + "threshold": 0.584686750387768 + }, + { + "coverage": 0.24566666666666667, + "risk": 0.45454545454545453, + "threshold": 0.583933999477932 + }, + { + "coverage": 0.246, + "risk": 0.453929539295393, + "threshold": 0.5835719411249631 + }, + { + "coverage": 0.24633333333333332, + "risk": 0.45331529093369416, + "threshold": 0.5830048929909415 + }, + { + "coverage": 0.24666666666666667, + "risk": 0.4527027027027027, + "threshold": 0.5822551549644239 + }, + { + "coverage": 0.247, + "risk": 0.45209176788124156, + "threshold": 0.5820585069064664 + }, + { + "coverage": 0.24733333333333332, + "risk": 0.45148247978436656, + "threshold": 0.5813139971320689 + }, + { + "coverage": 0.24766666666666667, + "risk": 0.4508748317631225, + "threshold": 0.5807646683024746 + }, + { + "coverage": 0.248, + "risk": 0.45026881720430106, + "threshold": 0.5802903024117951 + }, + { + "coverage": 0.24833333333333332, + "risk": 0.45100671140939597, + "threshold": 0.580243190342143 + }, + { + "coverage": 0.24866666666666667, + "risk": 0.450402144772118, + "threshold": 0.5800548223165833 + }, + { + "coverage": 0.249, + "risk": 0.45113788487282463, + "threshold": 0.5798649803285768 + }, + { + "coverage": 0.24933333333333332, + "risk": 0.45187165775401067, + "threshold": 0.5788757242043848 + }, + { + "coverage": 0.24966666666666668, + "risk": 0.4512683578104139, + "threshold": 0.5788738976383279 + }, + { + "coverage": 0.25, + "risk": 0.45066666666666666, + "threshold": 0.5787401322289586 + }, + { + "coverage": 0.25033333333333335, + "risk": 0.45006657789613846, + "threshold": 0.5784887917229737 + }, + { + "coverage": 0.25066666666666665, + "risk": 0.4507978723404255, + "threshold": 0.5784658883388538 + }, + { + "coverage": 0.251, + "risk": 0.450199203187251, + "threshold": 0.5781179662134004 + }, + { + "coverage": 0.25133333333333335, + "risk": 0.4509283819628647, + "threshold": 0.5780428172672004 + }, + { + "coverage": 0.25166666666666665, + "risk": 0.4503311258278146, + "threshold": 0.5770547508292841 + }, + { + "coverage": 0.252, + "risk": 0.4497354497354497, + "threshold": 0.5758594750329907 + }, + { + "coverage": 0.25233333333333335, + "risk": 0.44914134742404227, + "threshold": 0.5746981143547482 + }, + { + "coverage": 0.25266666666666665, + "risk": 0.44854881266490765, + "threshold": 0.5743631026860165 + }, + { + "coverage": 0.253, + "risk": 0.4479578392621871, + "threshold": 0.5740849531428022 + }, + { + "coverage": 0.25333333333333335, + "risk": 0.4486842105263158, + "threshold": 0.5731443028392953 + }, + { + "coverage": 0.25366666666666665, + "risk": 0.4480946123521682, + "threshold": 0.5717547129587779 + }, + { + "coverage": 0.254, + "risk": 0.44750656167979, + "threshold": 0.5716170225736722 + }, + { + "coverage": 0.25433333333333336, + "risk": 0.44823066841415465, + "threshold": 0.5711171640171374 + }, + { + "coverage": 0.25466666666666665, + "risk": 0.4476439790575916, + "threshold": 0.5707900471841799 + }, + { + "coverage": 0.255, + "risk": 0.44836601307189544, + "threshold": 0.570044668909934 + }, + { + "coverage": 0.25533333333333336, + "risk": 0.44778067885117495, + "threshold": 0.5697439563352132 + }, + { + "coverage": 0.25566666666666665, + "risk": 0.4485006518904824, + "threshold": 0.5694773568893488 + }, + { + "coverage": 0.256, + "risk": 0.4479166666666667, + "threshold": 0.5694192070718344 + }, + { + "coverage": 0.25633333333333336, + "risk": 0.4486345903771131, + "threshold": 0.569067178294863 + }, + { + "coverage": 0.25666666666666665, + "risk": 0.44805194805194803, + "threshold": 0.5685501725826767 + }, + { + "coverage": 0.257, + "risk": 0.4474708171206226, + "threshold": 0.568110069583704 + }, + { + "coverage": 0.25733333333333336, + "risk": 0.44689119170984454, + "threshold": 0.5673253088181542 + }, + { + "coverage": 0.25766666666666665, + "risk": 0.4463130659767141, + "threshold": 0.5663589451938319 + }, + { + "coverage": 0.258, + "risk": 0.44573643410852715, + "threshold": 0.566192666654239 + }, + { + "coverage": 0.25833333333333336, + "risk": 0.44516129032258067, + "threshold": 0.5659720194440927 + }, + { + "coverage": 0.25866666666666666, + "risk": 0.44458762886597936, + "threshold": 0.5647675556137091 + }, + { + "coverage": 0.259, + "risk": 0.444015444015444, + "threshold": 0.5644880996167683 + }, + { + "coverage": 0.25933333333333336, + "risk": 0.4434447300771208, + "threshold": 0.5634501344687 + }, + { + "coverage": 0.25966666666666666, + "risk": 0.4428754813863928, + "threshold": 0.5634035383569337 + }, + { + "coverage": 0.26, + "risk": 0.4423076923076923, + "threshold": 0.5630688983978036 + }, + { + "coverage": 0.26033333333333336, + "risk": 0.44302176696542894, + "threshold": 0.5627533146788638 + }, + { + "coverage": 0.26066666666666666, + "risk": 0.4437340153452685, + "threshold": 0.5624535142919453 + }, + { + "coverage": 0.261, + "risk": 0.44316730523627074, + "threshold": 0.5613279143185477 + }, + { + "coverage": 0.2613333333333333, + "risk": 0.44387755102040816, + "threshold": 0.5610023518196107 + }, + { + "coverage": 0.26166666666666666, + "risk": 0.4445859872611465, + "threshold": 0.5589713160240447 + }, + { + "coverage": 0.262, + "risk": 0.4440203562340967, + "threshold": 0.5586034288322922 + }, + { + "coverage": 0.2623333333333333, + "risk": 0.4434561626429479, + "threshold": 0.5585049764609161 + }, + { + "coverage": 0.26266666666666666, + "risk": 0.44289340101522845, + "threshold": 0.557962679999564 + }, + { + "coverage": 0.263, + "risk": 0.4423320659062104, + "threshold": 0.557437652832193 + }, + { + "coverage": 0.2633333333333333, + "risk": 0.4417721518987342, + "threshold": 0.557013659525844 + }, + { + "coverage": 0.26366666666666666, + "risk": 0.44121365360303416, + "threshold": 0.5566551928305997 + }, + { + "coverage": 0.264, + "risk": 0.44065656565656564, + "threshold": 0.556645838985206 + }, + { + "coverage": 0.2643333333333333, + "risk": 0.4401008827238335, + "threshold": 0.5556711059499156 + }, + { + "coverage": 0.26466666666666666, + "risk": 0.43954659949622166, + "threshold": 0.5555303522586507 + }, + { + "coverage": 0.265, + "risk": 0.4389937106918239, + "threshold": 0.5554653612579414 + }, + { + "coverage": 0.2653333333333333, + "risk": 0.43844221105527637, + "threshold": 0.5546733923065813 + }, + { + "coverage": 0.26566666666666666, + "risk": 0.437892095357591, + "threshold": 0.5538054704594081 + }, + { + "coverage": 0.266, + "risk": 0.43734335839599, + "threshold": 0.5537788134071896 + }, + { + "coverage": 0.2663333333333333, + "risk": 0.4380475594493116, + "threshold": 0.5530027605385344 + }, + { + "coverage": 0.26666666666666666, + "risk": 0.4375, + "threshold": 0.5528896693812284 + }, + { + "coverage": 0.267, + "risk": 0.4369538077403246, + "threshold": 0.5526672430009125 + }, + { + "coverage": 0.2673333333333333, + "risk": 0.43640897755610975, + "threshold": 0.5523174850149801 + }, + { + "coverage": 0.26766666666666666, + "risk": 0.43586550435865506, + "threshold": 0.5512073006024577 + }, + { + "coverage": 0.268, + "risk": 0.43656716417910446, + "threshold": 0.5509241667307954 + }, + { + "coverage": 0.2683333333333333, + "risk": 0.4372670807453416, + "threshold": 0.5505324731478348 + }, + { + "coverage": 0.26866666666666666, + "risk": 0.43672456575682383, + "threshold": 0.5503441065190213 + }, + { + "coverage": 0.269, + "risk": 0.43742255266418834, + "threshold": 0.5502755458772947 + }, + { + "coverage": 0.2693333333333333, + "risk": 0.4368811881188119, + "threshold": 0.5501849433017235 + }, + { + "coverage": 0.26966666666666667, + "risk": 0.4363411619283066, + "threshold": 0.5501717310987473 + }, + { + "coverage": 0.27, + "risk": 0.43580246913580245, + "threshold": 0.5500426392122029 + }, + { + "coverage": 0.2703333333333333, + "risk": 0.4352651048088779, + "threshold": 0.5495337859119455 + }, + { + "coverage": 0.27066666666666667, + "risk": 0.43596059113300495, + "threshold": 0.5492995775275755 + }, + { + "coverage": 0.271, + "risk": 0.43665436654366546, + "threshold": 0.5480517578496565 + }, + { + "coverage": 0.2713333333333333, + "risk": 0.43611793611793614, + "threshold": 0.5477859166285252 + }, + { + "coverage": 0.27166666666666667, + "risk": 0.43558282208588955, + "threshold": 0.5473743533648947 + }, + { + "coverage": 0.272, + "risk": 0.43504901960784315, + "threshold": 0.5468729165553577 + }, + { + "coverage": 0.2723333333333333, + "risk": 0.43451652386780903, + "threshold": 0.5468399741729248 + }, + { + "coverage": 0.27266666666666667, + "risk": 0.4339853300733496, + "threshold": 0.5465188619218692 + }, + { + "coverage": 0.273, + "risk": 0.43345543345543347, + "threshold": 0.546131331793401 + }, + { + "coverage": 0.2733333333333333, + "risk": 0.4329268292682927, + "threshold": 0.5454367421122924 + }, + { + "coverage": 0.27366666666666667, + "risk": 0.43239951278928135, + "threshold": 0.5449674503267076 + }, + { + "coverage": 0.274, + "risk": 0.4318734793187348, + "threshold": 0.5447028210931454 + }, + { + "coverage": 0.2743333333333333, + "risk": 0.4313487241798299, + "threshold": 0.5445614939954093 + }, + { + "coverage": 0.27466666666666667, + "risk": 0.4308252427184466, + "threshold": 0.5440287784267918 + }, + { + "coverage": 0.275, + "risk": 0.4303030303030303, + "threshold": 0.5439438128733374 + }, + { + "coverage": 0.2753333333333333, + "risk": 0.4297820823244552, + "threshold": 0.5434536486128589 + }, + { + "coverage": 0.27566666666666667, + "risk": 0.42926239419588874, + "threshold": 0.5424296314347999 + }, + { + "coverage": 0.276, + "risk": 0.428743961352657, + "threshold": 0.5417698724702695 + }, + { + "coverage": 0.2763333333333333, + "risk": 0.428226779252111, + "threshold": 0.5406827152647854 + }, + { + "coverage": 0.27666666666666667, + "risk": 0.42771084337349397, + "threshold": 0.5394554504553054 + }, + { + "coverage": 0.277, + "risk": 0.42839951865222625, + "threshold": 0.5392566310923054 + }, + { + "coverage": 0.2773333333333333, + "risk": 0.42788461538461536, + "threshold": 0.53922530216069 + }, + { + "coverage": 0.2776666666666667, + "risk": 0.42737094837935174, + "threshold": 0.5392015371891108 + }, + { + "coverage": 0.278, + "risk": 0.42805755395683454, + "threshold": 0.5386757059874941 + }, + { + "coverage": 0.2783333333333333, + "risk": 0.4275449101796407, + "threshold": 0.5377120285775794 + }, + { + "coverage": 0.2786666666666667, + "risk": 0.4270334928229665, + "threshold": 0.5370037326979176 + }, + { + "coverage": 0.279, + "risk": 0.4265232974910394, + "threshold": 0.5367812216214976 + }, + { + "coverage": 0.2793333333333333, + "risk": 0.42601431980906923, + "threshold": 0.5361509858143029 + }, + { + "coverage": 0.2796666666666667, + "risk": 0.42550655542312277, + "threshold": 0.5359835190124537 + }, + { + "coverage": 0.28, + "risk": 0.425, + "threshold": 0.5354853120574983 + }, + { + "coverage": 0.2803333333333333, + "risk": 0.4244946492271106, + "threshold": 0.5347070341201001 + }, + { + "coverage": 0.2806666666666667, + "risk": 0.42399049881235157, + "threshold": 0.534669750982939 + }, + { + "coverage": 0.281, + "risk": 0.4246737841043891, + "threshold": 0.5344897048095697 + }, + { + "coverage": 0.2813333333333333, + "risk": 0.4253554502369668, + "threshold": 0.5341310808341548 + }, + { + "coverage": 0.2816666666666667, + "risk": 0.4260355029585799, + "threshold": 0.5339891942905393 + }, + { + "coverage": 0.282, + "risk": 0.425531914893617, + "threshold": 0.5338474467857576 + }, + { + "coverage": 0.2823333333333333, + "risk": 0.42502951593860683, + "threshold": 0.5336812317457474 + }, + { + "coverage": 0.2826666666666667, + "risk": 0.42452830188679247, + "threshold": 0.5331848665251855 + }, + { + "coverage": 0.283, + "risk": 0.425206124852768, + "threshold": 0.532367127711895 + }, + { + "coverage": 0.2833333333333333, + "risk": 0.42470588235294116, + "threshold": 0.5323577737178055 + }, + { + "coverage": 0.2836666666666667, + "risk": 0.42420681551116335, + "threshold": 0.5321345237775493 + }, + { + "coverage": 0.284, + "risk": 0.42370892018779344, + "threshold": 0.5301352777370916 + }, + { + "coverage": 0.2843333333333333, + "risk": 0.4232121922626026, + "threshold": 0.529504685476973 + }, + { + "coverage": 0.2846666666666667, + "risk": 0.42271662763466045, + "threshold": 0.5291859504400065 + }, + { + "coverage": 0.285, + "risk": 0.4222222222222222, + "threshold": 0.5284423531115581 + }, + { + "coverage": 0.2853333333333333, + "risk": 0.42289719626168226, + "threshold": 0.528304023938849 + }, + { + "coverage": 0.2856666666666667, + "risk": 0.4224037339556593, + "threshold": 0.5276142505286564 + }, + { + "coverage": 0.286, + "risk": 0.4230769230769231, + "threshold": 0.5270335737616801 + }, + { + "coverage": 0.28633333333333333, + "risk": 0.42258440046565776, + "threshold": 0.5266559290824001 + }, + { + "coverage": 0.2866666666666667, + "risk": 0.42209302325581394, + "threshold": 0.5266055701659835 + }, + { + "coverage": 0.287, + "risk": 0.42276422764227645, + "threshold": 0.5263601140267719 + }, + { + "coverage": 0.28733333333333333, + "risk": 0.4222737819025522, + "threshold": 0.5263592806000463 + }, + { + "coverage": 0.2876666666666667, + "risk": 0.42178447276940906, + "threshold": 0.5259949129234138 + }, + { + "coverage": 0.288, + "risk": 0.4212962962962963, + "threshold": 0.5252852195435708 + }, + { + "coverage": 0.28833333333333333, + "risk": 0.4208092485549133, + "threshold": 0.5250609375420703 + }, + { + "coverage": 0.2886666666666667, + "risk": 0.42032332563510394, + "threshold": 0.5249593917092739 + }, + { + "coverage": 0.289, + "risk": 0.419838523644752, + "threshold": 0.5247162603606954 + }, + { + "coverage": 0.28933333333333333, + "risk": 0.4205069124423963, + "threshold": 0.5246988467584414 + }, + { + "coverage": 0.2896666666666667, + "risk": 0.42002301495972383, + "threshold": 0.5242736746002871 + }, + { + "coverage": 0.29, + "risk": 0.4206896551724138, + "threshold": 0.5241953489143961 + }, + { + "coverage": 0.29033333333333333, + "risk": 0.42020665901262916, + "threshold": 0.5233672060408908 + }, + { + "coverage": 0.2906666666666667, + "risk": 0.4197247706422018, + "threshold": 0.5230749523014344 + }, + { + "coverage": 0.291, + "risk": 0.41924398625429554, + "threshold": 0.5220883845011508 + }, + { + "coverage": 0.29133333333333333, + "risk": 0.41990846681922195, + "threshold": 0.5219016479694836 + }, + { + "coverage": 0.2916666666666667, + "risk": 0.4205714285714286, + "threshold": 0.5218060208893979 + }, + { + "coverage": 0.292, + "risk": 0.4200913242009132, + "threshold": 0.52163605661513 + }, + { + "coverage": 0.29233333333333333, + "risk": 0.41961231470923605, + "threshold": 0.5215827401363127 + }, + { + "coverage": 0.2926666666666667, + "risk": 0.4191343963553531, + "threshold": 0.5214826478374279 + }, + { + "coverage": 0.293, + "risk": 0.41865756541524457, + "threshold": 0.5213783030503026 + }, + { + "coverage": 0.29333333333333333, + "risk": 0.4193181818181818, + "threshold": 0.5209500882187199 + }, + { + "coverage": 0.2936666666666667, + "risk": 0.4188422247446084, + "threshold": 0.520808119299204 + }, + { + "coverage": 0.294, + "risk": 0.41950113378684806, + "threshold": 0.5207079913604452 + }, + { + "coverage": 0.29433333333333334, + "risk": 0.4190260475651189, + "threshold": 0.5200929946964882 + }, + { + "coverage": 0.2946666666666667, + "risk": 0.41855203619909503, + "threshold": 0.5200545152545301 + }, + { + "coverage": 0.295, + "risk": 0.4180790960451977, + "threshold": 0.519945694962102 + }, + { + "coverage": 0.29533333333333334, + "risk": 0.417607223476298, + "threshold": 0.5199278168701513 + }, + { + "coverage": 0.2956666666666667, + "risk": 0.41713641488162345, + "threshold": 0.5196408779287416 + }, + { + "coverage": 0.296, + "risk": 0.4177927927927928, + "threshold": 0.5192622057021986 + }, + { + "coverage": 0.29633333333333334, + "risk": 0.41732283464566927, + "threshold": 0.5185098546811883 + }, + { + "coverage": 0.2966666666666667, + "risk": 0.41685393258426967, + "threshold": 0.5184771129663026 + }, + { + "coverage": 0.297, + "risk": 0.4163860830527497, + "threshold": 0.518010237790635 + }, + { + "coverage": 0.29733333333333334, + "risk": 0.41591928251121074, + "threshold": 0.5172705899460901 + }, + { + "coverage": 0.2976666666666667, + "risk": 0.4154535274356103, + "threshold": 0.5165226036900856 + }, + { + "coverage": 0.298, + "risk": 0.41498881431767337, + "threshold": 0.5164457083182628 + }, + { + "coverage": 0.29833333333333334, + "risk": 0.4145251396648045, + "threshold": 0.5160385956433248 + }, + { + "coverage": 0.2986666666666667, + "risk": 0.4140625, + "threshold": 0.5155547639855125 + }, + { + "coverage": 0.299, + "risk": 0.4136008918617614, + "threshold": 0.5148434198323163 + }, + { + "coverage": 0.29933333333333334, + "risk": 0.4131403118040089, + "threshold": 0.5143967344523249 + }, + { + "coverage": 0.2996666666666667, + "risk": 0.41268075639599555, + "threshold": 0.5141034282436601 + }, + { + "coverage": 0.3, + "risk": 0.4122222222222222, + "threshold": 0.5137935697501138 + }, + { + "coverage": 0.30033333333333334, + "risk": 0.4117647058823529, + "threshold": 0.513587690394911 + }, + { + "coverage": 0.3006666666666667, + "risk": 0.4113082039911308, + "threshold": 0.5133321276168039 + }, + { + "coverage": 0.301, + "risk": 0.4108527131782946, + "threshold": 0.5109273473266109 + }, + { + "coverage": 0.30133333333333334, + "risk": 0.4103982300884956, + "threshold": 0.5107210729796581 + }, + { + "coverage": 0.3016666666666667, + "risk": 0.40994475138121544, + "threshold": 0.5102220810260396 + }, + { + "coverage": 0.302, + "risk": 0.4094922737306843, + "threshold": 0.5093726710394475 + }, + { + "coverage": 0.30233333333333334, + "risk": 0.40904079382579933, + "threshold": 0.5092868254737304 + }, + { + "coverage": 0.30266666666666664, + "risk": 0.40859030837004406, + "threshold": 0.5087967283964128 + }, + { + "coverage": 0.303, + "risk": 0.4081408140814081, + "threshold": 0.5087658272854751 + }, + { + "coverage": 0.30333333333333334, + "risk": 0.4076923076923077, + "threshold": 0.5081960675299803 + }, + { + "coverage": 0.30366666666666664, + "risk": 0.4083424807903403, + "threshold": 0.5081304274495789 + }, + { + "coverage": 0.304, + "risk": 0.40789473684210525, + "threshold": 0.5073221505964045 + }, + { + "coverage": 0.30433333333333334, + "risk": 0.40744797371303393, + "threshold": 0.5072018243515469 + }, + { + "coverage": 0.30466666666666664, + "risk": 0.40700218818380746, + "threshold": 0.5071658078911635 + }, + { + "coverage": 0.305, + "risk": 0.4065573770491803, + "threshold": 0.5071616643985518 + }, + { + "coverage": 0.30533333333333335, + "risk": 0.40611353711790393, + "threshold": 0.5068495766076225 + }, + { + "coverage": 0.30566666666666664, + "risk": 0.40676117775354415, + "threshold": 0.5060918809627882 + }, + { + "coverage": 0.306, + "risk": 0.40631808278867104, + "threshold": 0.5060481560642696 + }, + { + "coverage": 0.30633333333333335, + "risk": 0.4058759521218716, + "threshold": 0.5060437296154033 + }, + { + "coverage": 0.30666666666666664, + "risk": 0.40543478260869564, + "threshold": 0.505598590902518 + }, + { + "coverage": 0.307, + "risk": 0.40499457111834963, + "threshold": 0.5052692068683386 + }, + { + "coverage": 0.30733333333333335, + "risk": 0.40563991323210413, + "threshold": 0.5052221630366991 + }, + { + "coverage": 0.30766666666666664, + "risk": 0.40520043336944744, + "threshold": 0.5049356883914671 + }, + { + "coverage": 0.308, + "risk": 0.40476190476190477, + "threshold": 0.5048481969772525 + }, + { + "coverage": 0.30833333333333335, + "risk": 0.40540540540540543, + "threshold": 0.5047345518632005 + }, + { + "coverage": 0.30866666666666664, + "risk": 0.4060475161987041, + "threshold": 0.5045022389829547 + }, + { + "coverage": 0.309, + "risk": 0.40560949298813376, + "threshold": 0.504492891874254 + }, + { + "coverage": 0.30933333333333335, + "risk": 0.4051724137931034, + "threshold": 0.5038394377970146 + }, + { + "coverage": 0.30966666666666665, + "risk": 0.4058127018299246, + "threshold": 0.5035430329392686 + }, + { + "coverage": 0.31, + "risk": 0.4053763440860215, + "threshold": 0.5029832925181943 + }, + { + "coverage": 0.31033333333333335, + "risk": 0.40494092373791624, + "threshold": 0.502303428749251 + }, + { + "coverage": 0.31066666666666665, + "risk": 0.40450643776824036, + "threshold": 0.5021970424581916 + }, + { + "coverage": 0.311, + "risk": 0.4040728831725616, + "threshold": 0.501380914439066 + }, + { + "coverage": 0.31133333333333335, + "risk": 0.40471092077087795, + "threshold": 0.49997884257470415 + }, + { + "coverage": 0.31166666666666665, + "risk": 0.4053475935828877, + "threshold": 0.4997272702197246 + }, + { + "coverage": 0.312, + "risk": 0.4049145299145299, + "threshold": 0.4996013277656661 + }, + { + "coverage": 0.31233333333333335, + "risk": 0.4055496264674493, + "threshold": 0.49899132699377313 + }, + { + "coverage": 0.31266666666666665, + "risk": 0.4051172707889126, + "threshold": 0.49898640879997314 + }, + { + "coverage": 0.313, + "risk": 0.4057507987220447, + "threshold": 0.49893529259374975 + }, + { + "coverage": 0.31333333333333335, + "risk": 0.40638297872340423, + "threshold": 0.4984809112644491 + }, + { + "coverage": 0.31366666666666665, + "risk": 0.4059511158342189, + "threshold": 0.4974504420421068 + }, + { + "coverage": 0.314, + "risk": 0.40552016985138006, + "threshold": 0.4968024148433461 + }, + { + "coverage": 0.31433333333333335, + "risk": 0.4050901378579003, + "threshold": 0.4956664873904139 + }, + { + "coverage": 0.31466666666666665, + "risk": 0.4057203389830508, + "threshold": 0.4956298754091374 + }, + { + "coverage": 0.315, + "risk": 0.4052910052910053, + "threshold": 0.4955146709356617 + }, + { + "coverage": 0.31533333333333335, + "risk": 0.4059196617336152, + "threshold": 0.495209447748573 + }, + { + "coverage": 0.31566666666666665, + "risk": 0.4065469904963041, + "threshold": 0.494574655947071 + }, + { + "coverage": 0.316, + "risk": 0.4061181434599156, + "threshold": 0.4940022909700423 + }, + { + "coverage": 0.31633333333333336, + "risk": 0.4067439409905163, + "threshold": 0.4935541906437191 + }, + { + "coverage": 0.31666666666666665, + "risk": 0.4073684210526316, + "threshold": 0.4932822147930618 + }, + { + "coverage": 0.317, + "risk": 0.4069400630914827, + "threshold": 0.4926115083241339 + }, + { + "coverage": 0.31733333333333336, + "risk": 0.4065126050420168, + "threshold": 0.4916396558627869 + }, + { + "coverage": 0.31766666666666665, + "risk": 0.4060860440713536, + "threshold": 0.4913614798250683 + }, + { + "coverage": 0.318, + "risk": 0.4056603773584906, + "threshold": 0.49125880596805027 + }, + { + "coverage": 0.31833333333333336, + "risk": 0.40523560209424087, + "threshold": 0.49043270286221213 + }, + { + "coverage": 0.31866666666666665, + "risk": 0.40585774058577406, + "threshold": 0.4902721500940605 + }, + { + "coverage": 0.319, + "risk": 0.406478578892372, + "threshold": 0.4902237733626157 + }, + { + "coverage": 0.31933333333333336, + "risk": 0.40605427974947805, + "threshold": 0.4892392420686221 + }, + { + "coverage": 0.31966666666666665, + "risk": 0.40667361835245047, + "threshold": 0.4885250718834828 + }, + { + "coverage": 0.32, + "risk": 0.40625, + "threshold": 0.4884934372395831 + }, + { + "coverage": 0.32033333333333336, + "risk": 0.40582726326742974, + "threshold": 0.4881061682544016 + }, + { + "coverage": 0.32066666666666666, + "risk": 0.40540540540540543, + "threshold": 0.48807338172222986 + }, + { + "coverage": 0.321, + "risk": 0.40498442367601245, + "threshold": 0.4873145467895962 + }, + { + "coverage": 0.32133333333333336, + "risk": 0.4045643153526971, + "threshold": 0.4870060974744526 + }, + { + "coverage": 0.32166666666666666, + "risk": 0.40414507772020725, + "threshold": 0.48662494611734625 + }, + { + "coverage": 0.322, + "risk": 0.40372670807453415, + "threshold": 0.4864721317932527 + }, + { + "coverage": 0.32233333333333336, + "risk": 0.40330920372285417, + "threshold": 0.48595567070825063 + }, + { + "coverage": 0.32266666666666666, + "risk": 0.40392561983471076, + "threshold": 0.4857872340695805 + }, + { + "coverage": 0.323, + "risk": 0.40454076367389064, + "threshold": 0.48547594851995607 + }, + { + "coverage": 0.3233333333333333, + "risk": 0.4041237113402062, + "threshold": 0.48505800831937484 + }, + { + "coverage": 0.32366666666666666, + "risk": 0.40370751802265703, + "threshold": 0.4849233035186386 + }, + { + "coverage": 0.324, + "risk": 0.40329218106995884, + "threshold": 0.4846066173709819 + }, + { + "coverage": 0.3243333333333333, + "risk": 0.4028776978417266, + "threshold": 0.4845419421442284 + }, + { + "coverage": 0.32466666666666666, + "risk": 0.4024640657084189, + "threshold": 0.4840328863748697 + }, + { + "coverage": 0.325, + "risk": 0.40205128205128204, + "threshold": 0.48316253898438594 + }, + { + "coverage": 0.3253333333333333, + "risk": 0.4016393442622951, + "threshold": 0.48205852764183726 + }, + { + "coverage": 0.32566666666666666, + "risk": 0.40122824974411464, + "threshold": 0.48175309398461685 + }, + { + "coverage": 0.326, + "risk": 0.40081799591002043, + "threshold": 0.48112501666991997 + }, + { + "coverage": 0.3263333333333333, + "risk": 0.4004085801838611, + "threshold": 0.47950331364343035 + }, + { + "coverage": 0.32666666666666666, + "risk": 0.4, + "threshold": 0.47879783765341455 + }, + { + "coverage": 0.327, + "risk": 0.399592252803262, + "threshold": 0.47818530367146594 + }, + { + "coverage": 0.3273333333333333, + "risk": 0.39918533604887985, + "threshold": 0.47685627894601 + }, + { + "coverage": 0.32766666666666666, + "risk": 0.3987792472024415, + "threshold": 0.4765538005520123 + }, + { + "coverage": 0.328, + "risk": 0.3983739837398374, + "threshold": 0.4759585003519064 + }, + { + "coverage": 0.3283333333333333, + "risk": 0.3979695431472081, + "threshold": 0.47591991474485423 + }, + { + "coverage": 0.32866666666666666, + "risk": 0.3975659229208925, + "threshold": 0.4757826956596848 + }, + { + "coverage": 0.329, + "risk": 0.3971631205673759, + "threshold": 0.47568719623479383 + }, + { + "coverage": 0.3293333333333333, + "risk": 0.3967611336032389, + "threshold": 0.47552278092694844 + }, + { + "coverage": 0.32966666666666666, + "risk": 0.39737108190091003, + "threshold": 0.47378378588730735 + }, + { + "coverage": 0.33, + "risk": 0.396969696969697, + "threshold": 0.47319859604997677 + }, + { + "coverage": 0.3303333333333333, + "risk": 0.39656912209889, + "threshold": 0.4730894942067735 + }, + { + "coverage": 0.33066666666666666, + "risk": 0.3961693548387097, + "threshold": 0.471589616378926 + }, + { + "coverage": 0.331, + "risk": 0.3957703927492447, + "threshold": 0.47146965016716064 + }, + { + "coverage": 0.3313333333333333, + "risk": 0.3953722334004024, + "threshold": 0.471105152345745 + }, + { + "coverage": 0.33166666666666667, + "risk": 0.39597989949748746, + "threshold": 0.471066623906545 + }, + { + "coverage": 0.332, + "risk": 0.39558232931726905, + "threshold": 0.4702939469402405 + }, + { + "coverage": 0.3323333333333333, + "risk": 0.39518555667001004, + "threshold": 0.4702182458147914 + }, + { + "coverage": 0.33266666666666667, + "risk": 0.39478957915831664, + "threshold": 0.4701429041203141 + }, + { + "coverage": 0.333, + "risk": 0.3943943943943944, + "threshold": 0.46898157611802893 + }, + { + "coverage": 0.3333333333333333, + "risk": 0.395, + "threshold": 0.46884950238003237 + }, + { + "coverage": 0.33366666666666667, + "risk": 0.3946053946053946, + "threshold": 0.4681940248294925 + }, + { + "coverage": 0.334, + "risk": 0.39421157684630737, + "threshold": 0.46817963039096294 + }, + { + "coverage": 0.3343333333333333, + "risk": 0.3938185443668993, + "threshold": 0.4680437360669587 + }, + { + "coverage": 0.33466666666666667, + "risk": 0.3934262948207171, + "threshold": 0.4680130794010718 + }, + { + "coverage": 0.335, + "risk": 0.39303482587064675, + "threshold": 0.4679787189681107 + }, + { + "coverage": 0.3353333333333333, + "risk": 0.3926441351888668, + "threshold": 0.46739487615619063 + }, + { + "coverage": 0.33566666666666667, + "risk": 0.39225422045680236, + "threshold": 0.4672865393451643 + }, + { + "coverage": 0.336, + "risk": 0.39186507936507936, + "threshold": 0.4672589184390356 + }, + { + "coverage": 0.3363333333333333, + "risk": 0.3924677898909812, + "threshold": 0.4671731016990259 + }, + { + "coverage": 0.33666666666666667, + "risk": 0.3920792079207921, + "threshold": 0.4663841583734616 + }, + { + "coverage": 0.337, + "risk": 0.3916913946587537, + "threshold": 0.466140662474459 + }, + { + "coverage": 0.3373333333333333, + "risk": 0.391304347826087, + "threshold": 0.4655802369151126 + }, + { + "coverage": 0.33766666666666667, + "risk": 0.39091806515301086, + "threshold": 0.46504065842587844 + }, + { + "coverage": 0.338, + "risk": 0.3915187376725838, + "threshold": 0.4648183604996594 + }, + { + "coverage": 0.3383333333333333, + "risk": 0.39113300492610836, + "threshold": 0.46414229102045584 + }, + { + "coverage": 0.33866666666666667, + "risk": 0.390748031496063, + "threshold": 0.4640315867218282 + }, + { + "coverage": 0.339, + "risk": 0.39036381514257623, + "threshold": 0.46352527018370704 + }, + { + "coverage": 0.3393333333333333, + "risk": 0.39096267190569745, + "threshold": 0.4625302899412938 + }, + { + "coverage": 0.3396666666666667, + "risk": 0.39057899901864573, + "threshold": 0.46134039933584303 + }, + { + "coverage": 0.34, + "risk": 0.3911764705882353, + "threshold": 0.45984646338017454 + }, + { + "coverage": 0.3403333333333333, + "risk": 0.3917727717923604, + "threshold": 0.4580027546196785 + }, + { + "coverage": 0.3406666666666667, + "risk": 0.3913894324853229, + "threshold": 0.4571287892853647 + }, + { + "coverage": 0.341, + "risk": 0.39100684261974583, + "threshold": 0.4570331452762394 + }, + { + "coverage": 0.3413333333333333, + "risk": 0.390625, + "threshold": 0.45681816208889675 + }, + { + "coverage": 0.3416666666666667, + "risk": 0.3902439024390244, + "threshold": 0.4566128368224171 + }, + { + "coverage": 0.342, + "risk": 0.3898635477582846, + "threshold": 0.4566023701468615 + }, + { + "coverage": 0.3423333333333333, + "risk": 0.3904576436222006, + "threshold": 0.4561956834176207 + }, + { + "coverage": 0.3426666666666667, + "risk": 0.39105058365758755, + "threshold": 0.4560960535819609 + }, + { + "coverage": 0.343, + "risk": 0.391642371234208, + "threshold": 0.45607061590020825 + }, + { + "coverage": 0.3433333333333333, + "risk": 0.3912621359223301, + "threshold": 0.4552440964347579 + }, + { + "coverage": 0.3436666666666667, + "risk": 0.39088263821532493, + "threshold": 0.4551386964707768 + }, + { + "coverage": 0.344, + "risk": 0.3905038759689923, + "threshold": 0.45455372839474506 + }, + { + "coverage": 0.3443333333333333, + "risk": 0.39012584704743464, + "threshold": 0.454531616615627 + }, + { + "coverage": 0.3446666666666667, + "risk": 0.3897485493230174, + "threshold": 0.45332632753837987 + }, + { + "coverage": 0.345, + "risk": 0.3893719806763285, + "threshold": 0.4532602983971678 + }, + { + "coverage": 0.3453333333333333, + "risk": 0.388996138996139, + "threshold": 0.4529511212106362 + }, + { + "coverage": 0.3456666666666667, + "risk": 0.38862102217936356, + "threshold": 0.4528255304484709 + }, + { + "coverage": 0.346, + "risk": 0.3882466281310212, + "threshold": 0.452781949777279 + }, + { + "coverage": 0.3463333333333333, + "risk": 0.38787295476419636, + "threshold": 0.452406207304753 + }, + { + "coverage": 0.3466666666666667, + "risk": 0.3875, + "threshold": 0.45225454426975975 + }, + { + "coverage": 0.347, + "risk": 0.3871277617675312, + "threshold": 0.4521635179615753 + }, + { + "coverage": 0.3473333333333333, + "risk": 0.3867562380038388, + "threshold": 0.45207401252094725 + }, + { + "coverage": 0.3476666666666667, + "risk": 0.38734419942473636, + "threshold": 0.4519948533739598 + }, + { + "coverage": 0.348, + "risk": 0.38697318007662834, + "threshold": 0.451698529829411 + }, + { + "coverage": 0.34833333333333333, + "risk": 0.3866028708133971, + "threshold": 0.4497126957166422 + }, + { + "coverage": 0.3486666666666667, + "risk": 0.3862332695984704, + "threshold": 0.44969659184034566 + }, + { + "coverage": 0.349, + "risk": 0.3858643744030564, + "threshold": 0.4489952938178268 + }, + { + "coverage": 0.34933333333333333, + "risk": 0.38549618320610685, + "threshold": 0.44884384651971154 + }, + { + "coverage": 0.3496666666666667, + "risk": 0.3851286939942803, + "threshold": 0.44877270265786506 + }, + { + "coverage": 0.35, + "risk": 0.38476190476190475, + "threshold": 0.4480972084070836 + }, + { + "coverage": 0.35033333333333333, + "risk": 0.384395813510942, + "threshold": 0.44790318560583087 + }, + { + "coverage": 0.3506666666666667, + "risk": 0.3840304182509506, + "threshold": 0.44744940525636767 + }, + { + "coverage": 0.351, + "risk": 0.38366571699905033, + "threshold": 0.44719767826305096 + }, + { + "coverage": 0.35133333333333333, + "risk": 0.38330170777988615, + "threshold": 0.4469532018429291 + }, + { + "coverage": 0.3516666666666667, + "risk": 0.38293838862559243, + "threshold": 0.44659528645966423 + }, + { + "coverage": 0.352, + "risk": 0.38257575757575757, + "threshold": 0.4464214552936332 + }, + { + "coverage": 0.35233333333333333, + "risk": 0.38221381267738885, + "threshold": 0.4461084800000765 + }, + { + "coverage": 0.3526666666666667, + "risk": 0.3818525519848771, + "threshold": 0.4460697110100808 + }, + { + "coverage": 0.353, + "risk": 0.3814919735599622, + "threshold": 0.44587970794979825 + }, + { + "coverage": 0.35333333333333333, + "risk": 0.38207547169811323, + "threshold": 0.4456748012764417 + }, + { + "coverage": 0.3536666666666667, + "risk": 0.3817153628652215, + "threshold": 0.44527171031656465 + }, + { + "coverage": 0.354, + "risk": 0.3813559322033898, + "threshold": 0.44510342111398477 + }, + { + "coverage": 0.35433333333333333, + "risk": 0.3819379115710254, + "threshold": 0.4448719309164728 + }, + { + "coverage": 0.3546666666666667, + "risk": 0.3815789473684211, + "threshold": 0.44459495919349223 + }, + { + "coverage": 0.355, + "risk": 0.3812206572769953, + "threshold": 0.4445025516772885 + }, + { + "coverage": 0.35533333333333333, + "risk": 0.3808630393996248, + "threshold": 0.44414370892937016 + }, + { + "coverage": 0.3556666666666667, + "risk": 0.38144329896907214, + "threshold": 0.44379725790773994 + }, + { + "coverage": 0.356, + "risk": 0.38108614232209737, + "threshold": 0.44373688742842027 + }, + { + "coverage": 0.35633333333333334, + "risk": 0.3807296538821328, + "threshold": 0.4435778270280829 + }, + { + "coverage": 0.3566666666666667, + "risk": 0.38037383177570094, + "threshold": 0.442214451481251 + }, + { + "coverage": 0.357, + "risk": 0.3800186741363212, + "threshold": 0.44206291365442096 + }, + { + "coverage": 0.35733333333333334, + "risk": 0.37966417910447764, + "threshold": 0.4419166791170601 + }, + { + "coverage": 0.3576666666666667, + "risk": 0.3793103448275862, + "threshold": 0.4410565676869937 + }, + { + "coverage": 0.358, + "risk": 0.37988826815642457, + "threshold": 0.4408506346302387 + }, + { + "coverage": 0.35833333333333334, + "risk": 0.37953488372093025, + "threshold": 0.44019648420204405 + }, + { + "coverage": 0.3586666666666667, + "risk": 0.379182156133829, + "threshold": 0.43957244205806334 + }, + { + "coverage": 0.359, + "risk": 0.3788300835654596, + "threshold": 0.4395435288000105 + }, + { + "coverage": 0.35933333333333334, + "risk": 0.3784786641929499, + "threshold": 0.4391639171097688 + }, + { + "coverage": 0.3596666666666667, + "risk": 0.37812789620018533, + "threshold": 0.4390238812316432 + }, + { + "coverage": 0.36, + "risk": 0.37777777777777777, + "threshold": 0.4389827393887052 + }, + { + "coverage": 0.36033333333333334, + "risk": 0.3774283071230342, + "threshold": 0.43855489959370003 + }, + { + "coverage": 0.3606666666666667, + "risk": 0.37707948243992606, + "threshold": 0.4364885098831839 + }, + { + "coverage": 0.361, + "risk": 0.3767313019390582, + "threshold": 0.43636259726875737 + }, + { + "coverage": 0.36133333333333334, + "risk": 0.3763837638376384, + "threshold": 0.4360549893427084 + }, + { + "coverage": 0.3616666666666667, + "risk": 0.37695852534562213, + "threshold": 0.4360051525946869 + }, + { + "coverage": 0.362, + "risk": 0.3766114180478821, + "threshold": 0.435479316792897 + }, + { + "coverage": 0.36233333333333334, + "risk": 0.37626494940202393, + "threshold": 0.4350143862096919 + }, + { + "coverage": 0.3626666666666667, + "risk": 0.37683823529411764, + "threshold": 0.434725197353562 + }, + { + "coverage": 0.363, + "risk": 0.37649219467401285, + "threshold": 0.434486815813003 + }, + { + "coverage": 0.36333333333333334, + "risk": 0.37706422018348623, + "threshold": 0.4343786049040177 + }, + { + "coverage": 0.3636666666666667, + "risk": 0.3767186067827681, + "threshold": 0.4330848076091322 + }, + { + "coverage": 0.364, + "risk": 0.37637362637362637, + "threshold": 0.43251568594614714 + }, + { + "coverage": 0.36433333333333334, + "risk": 0.37602927721866425, + "threshold": 0.43232027803544054 + }, + { + "coverage": 0.36466666666666664, + "risk": 0.3756855575868373, + "threshold": 0.43143088825165965 + }, + { + "coverage": 0.365, + "risk": 0.37625570776255707, + "threshold": 0.4305881544621302 + }, + { + "coverage": 0.36533333333333334, + "risk": 0.3759124087591241, + "threshold": 0.4300760475998762 + }, + { + "coverage": 0.36566666666666664, + "risk": 0.37556973564266183, + "threshold": 0.4300369510317724 + }, + { + "coverage": 0.366, + "risk": 0.37522768670309653, + "threshold": 0.42728862806034407 + }, + { + "coverage": 0.36633333333333334, + "risk": 0.37579617834394907, + "threshold": 0.427272847534632 + }, + { + "coverage": 0.36666666666666664, + "risk": 0.37545454545454543, + "threshold": 0.4269924881781926 + }, + { + "coverage": 0.367, + "risk": 0.3760217983651226, + "threshold": 0.4266105897996614 + }, + { + "coverage": 0.36733333333333335, + "risk": 0.37568058076225047, + "threshold": 0.4265657923388194 + }, + { + "coverage": 0.36766666666666664, + "risk": 0.37533998186763373, + "threshold": 0.424395691524814 + }, + { + "coverage": 0.368, + "risk": 0.375, + "threshold": 0.42429099424812355 + }, + { + "coverage": 0.36833333333333335, + "risk": 0.3746606334841629, + "threshold": 0.4241881583837756 + }, + { + "coverage": 0.36866666666666664, + "risk": 0.3743218806509946, + "threshold": 0.4238730192992501 + }, + { + "coverage": 0.369, + "risk": 0.37398373983739835, + "threshold": 0.42350249625564806 + }, + { + "coverage": 0.36933333333333335, + "risk": 0.37364620938628157, + "threshold": 0.42318682578910854 + }, + { + "coverage": 0.36966666666666664, + "risk": 0.37330928764652843, + "threshold": 0.42301638212050746 + }, + { + "coverage": 0.37, + "risk": 0.372972972972973, + "threshold": 0.42295651344718316 + }, + { + "coverage": 0.37033333333333335, + "risk": 0.3726372637263726, + "threshold": 0.42285013357391654 + }, + { + "coverage": 0.37066666666666664, + "risk": 0.3723021582733813, + "threshold": 0.4223270548747978 + }, + { + "coverage": 0.371, + "risk": 0.3728661275831087, + "threshold": 0.4212152312619527 + }, + { + "coverage": 0.37133333333333335, + "risk": 0.3734290843806104, + "threshold": 0.42029685906041125 + }, + { + "coverage": 0.37166666666666665, + "risk": 0.37309417040358744, + "threshold": 0.42024742226641937 + }, + { + "coverage": 0.372, + "risk": 0.3727598566308244, + "threshold": 0.4195316165160941 + }, + { + "coverage": 0.37233333333333335, + "risk": 0.37242614145031333, + "threshold": 0.41850596269180595 + }, + { + "coverage": 0.37266666666666665, + "risk": 0.37209302325581395, + "threshold": 0.41813427588656205 + }, + { + "coverage": 0.373, + "risk": 0.3717605004468275, + "threshold": 0.41800754747432145 + }, + { + "coverage": 0.37333333333333335, + "risk": 0.37142857142857144, + "threshold": 0.41784556928180416 + }, + { + "coverage": 0.37366666666666665, + "risk": 0.3710972346119536, + "threshold": 0.41765422278801445 + }, + { + "coverage": 0.374, + "risk": 0.3716577540106952, + "threshold": 0.4162847614658802 + }, + { + "coverage": 0.37433333333333335, + "risk": 0.371326803205699, + "threshold": 0.41566078783340216 + }, + { + "coverage": 0.37466666666666665, + "risk": 0.3709964412811388, + "threshold": 0.4140359542525169 + }, + { + "coverage": 0.375, + "risk": 0.37066666666666664, + "threshold": 0.4135225152710254 + }, + { + "coverage": 0.37533333333333335, + "risk": 0.3703374777975133, + "threshold": 0.41335795020488686 + }, + { + "coverage": 0.37566666666666665, + "risk": 0.3700088731144632, + "threshold": 0.4128348742252318 + }, + { + "coverage": 0.376, + "risk": 0.3696808510638298, + "threshold": 0.4128308017353513 + }, + { + "coverage": 0.37633333333333335, + "risk": 0.36935341009743133, + "threshold": 0.41252055169830976 + }, + { + "coverage": 0.37666666666666665, + "risk": 0.36902654867256635, + "threshold": 0.4125200128307407 + }, + { + "coverage": 0.377, + "risk": 0.3687002652519894, + "threshold": 0.4122510116332735 + }, + { + "coverage": 0.37733333333333335, + "risk": 0.36925795053003535, + "threshold": 0.4119083651030854 + }, + { + "coverage": 0.37766666666666665, + "risk": 0.36893203883495146, + "threshold": 0.41118248024196075 + }, + { + "coverage": 0.378, + "risk": 0.36860670194003525, + "threshold": 0.4107467342024407 + }, + { + "coverage": 0.37833333333333335, + "risk": 0.3682819383259912, + "threshold": 0.4103595415486095 + }, + { + "coverage": 0.37866666666666665, + "risk": 0.36795774647887325, + "threshold": 0.4085917430742146 + }, + { + "coverage": 0.379, + "risk": 0.3676341248900616, + "threshold": 0.40836457157771583 + }, + { + "coverage": 0.37933333333333336, + "risk": 0.36731107205623903, + "threshold": 0.4071658733865481 + }, + { + "coverage": 0.37966666666666665, + "risk": 0.3669885864793679, + "threshold": 0.40705931779402343 + }, + { + "coverage": 0.38, + "risk": 0.36666666666666664, + "threshold": 0.4068419397679556 + }, + { + "coverage": 0.38033333333333336, + "risk": 0.3663453111305872, + "threshold": 0.40659606507448015 + }, + { + "coverage": 0.38066666666666665, + "risk": 0.36690017513134854, + "threshold": 0.4063730435587251 + }, + { + "coverage": 0.381, + "risk": 0.3665791776027997, + "threshold": 0.4060843516447611 + }, + { + "coverage": 0.38133333333333336, + "risk": 0.36625874125874125, + "threshold": 0.40576966738286585 + }, + { + "coverage": 0.38166666666666665, + "risk": 0.36593886462882097, + "threshold": 0.4051861163017864 + }, + { + "coverage": 0.382, + "risk": 0.3656195462478185, + "threshold": 0.4046887596701169 + }, + { + "coverage": 0.38233333333333336, + "risk": 0.3653007846556234, + "threshold": 0.40447755456742934 + }, + { + "coverage": 0.38266666666666665, + "risk": 0.36585365853658536, + "threshold": 0.40442281146061837 + }, + { + "coverage": 0.383, + "risk": 0.36553524804177545, + "threshold": 0.4041680340570111 + }, + { + "coverage": 0.38333333333333336, + "risk": 0.3652173913043478, + "threshold": 0.4030033824895539 + }, + { + "coverage": 0.38366666666666666, + "risk": 0.3649000868809731, + "threshold": 0.4029816144144351 + }, + { + "coverage": 0.384, + "risk": 0.3645833333333333, + "threshold": 0.4016814771350851 + }, + { + "coverage": 0.38433333333333336, + "risk": 0.3642671292281006, + "threshold": 0.4013805894376643 + }, + { + "coverage": 0.38466666666666666, + "risk": 0.36395147313691506, + "threshold": 0.40114833002993777 + }, + { + "coverage": 0.385, + "risk": 0.3645021645021645, + "threshold": 0.4010490919193241 + }, + { + "coverage": 0.38533333333333336, + "risk": 0.36418685121107264, + "threshold": 0.4010483494787885 + }, + { + "coverage": 0.38566666666666666, + "risk": 0.36387208297320656, + "threshold": 0.4008906034350538 + }, + { + "coverage": 0.386, + "risk": 0.3635578583765112, + "threshold": 0.400862942038875 + }, + { + "coverage": 0.3863333333333333, + "risk": 0.363244176013805, + "threshold": 0.400760081585198 + }, + { + "coverage": 0.38666666666666666, + "risk": 0.3629310344827586, + "threshold": 0.4003451694502651 + }, + { + "coverage": 0.387, + "risk": 0.36261843238587427, + "threshold": 0.3995123674062314 + }, + { + "coverage": 0.3873333333333333, + "risk": 0.3623063683304647, + "threshold": 0.3993026438267923 + }, + { + "coverage": 0.38766666666666666, + "risk": 0.3619948409286328, + "threshold": 0.399221860336186 + }, + { + "coverage": 0.388, + "risk": 0.36168384879725085, + "threshold": 0.3988987360861959 + }, + { + "coverage": 0.3883333333333333, + "risk": 0.3622317596566524, + "threshold": 0.3987803308472346 + }, + { + "coverage": 0.38866666666666666, + "risk": 0.36192109777015435, + "threshold": 0.39837077859759595 + }, + { + "coverage": 0.389, + "risk": 0.36246786632390743, + "threshold": 0.39811190046240835 + }, + { + "coverage": 0.3893333333333333, + "risk": 0.3621575342465753, + "threshold": 0.397189136819295 + }, + { + "coverage": 0.38966666666666666, + "risk": 0.3618477331052181, + "threshold": 0.39717467873471324 + }, + { + "coverage": 0.39, + "risk": 0.36153846153846153, + "threshold": 0.3971706875983068 + }, + { + "coverage": 0.3903333333333333, + "risk": 0.36122971818958155, + "threshold": 0.3967220282831545 + }, + { + "coverage": 0.39066666666666666, + "risk": 0.3609215017064846, + "threshold": 0.3961803300456149 + }, + { + "coverage": 0.391, + "risk": 0.36061381074168797, + "threshold": 0.39563315259092546 + }, + { + "coverage": 0.3913333333333333, + "risk": 0.36030664395229983, + "threshold": 0.39556839400747806 + }, + { + "coverage": 0.39166666666666666, + "risk": 0.36, + "threshold": 0.39509922234336026 + }, + { + "coverage": 0.392, + "risk": 0.3596938775510204, + "threshold": 0.3943495509432756 + }, + { + "coverage": 0.3923333333333333, + "risk": 0.35938827527612577, + "threshold": 0.3931131976297305 + }, + { + "coverage": 0.39266666666666666, + "risk": 0.35908319185059423, + "threshold": 0.3930761479137016 + }, + { + "coverage": 0.393, + "risk": 0.35877862595419846, + "threshold": 0.39275694991248306 + }, + { + "coverage": 0.3933333333333333, + "risk": 0.35847457627118645, + "threshold": 0.3923647100337377 + }, + { + "coverage": 0.39366666666666666, + "risk": 0.35817104149026247, + "threshold": 0.3922913209918408 + }, + { + "coverage": 0.394, + "risk": 0.35786802030456855, + "threshold": 0.39107702495955654 + }, + { + "coverage": 0.3943333333333333, + "risk": 0.35756551141166526, + "threshold": 0.3906371568176689 + }, + { + "coverage": 0.39466666666666667, + "risk": 0.3572635135135135, + "threshold": 0.3868497677459203 + }, + { + "coverage": 0.395, + "risk": 0.3569620253164557, + "threshold": 0.38679225031206926 + }, + { + "coverage": 0.3953333333333333, + "risk": 0.3566610455311973, + "threshold": 0.38634566657991865 + }, + { + "coverage": 0.39566666666666667, + "risk": 0.35636057287278855, + "threshold": 0.385938087290175 + }, + { + "coverage": 0.396, + "risk": 0.3560606060606061, + "threshold": 0.3857168570741163 + }, + { + "coverage": 0.3963333333333333, + "risk": 0.35576114381833474, + "threshold": 0.3856367346271604 + }, + { + "coverage": 0.39666666666666667, + "risk": 0.3563025210084034, + "threshold": 0.38545813155093367 + }, + { + "coverage": 0.397, + "risk": 0.35600335852225024, + "threshold": 0.38540148449298767 + }, + { + "coverage": 0.3973333333333333, + "risk": 0.35570469798657717, + "threshold": 0.3851937546365387 + }, + { + "coverage": 0.39766666666666667, + "risk": 0.3562447611064543, + "threshold": 0.38511989885119446 + }, + { + "coverage": 0.398, + "risk": 0.35678391959798994, + "threshold": 0.3850881350154915 + }, + { + "coverage": 0.3983333333333333, + "risk": 0.35648535564853556, + "threshold": 0.38508757650667286 + }, + { + "coverage": 0.39866666666666667, + "risk": 0.35618729096989965, + "threshold": 0.3850783255689109 + }, + { + "coverage": 0.399, + "risk": 0.3558897243107769, + "threshold": 0.3850680925430547 + }, + { + "coverage": 0.3993333333333333, + "risk": 0.3555926544240401, + "threshold": 0.3850245150009337 + }, + { + "coverage": 0.39966666666666667, + "risk": 0.35529608006672225, + "threshold": 0.38395279939130833 + }, + { + "coverage": 0.4, + "risk": 0.355, + "threshold": 0.38338684721426225 + }, + { + "coverage": 0.4003333333333333, + "risk": 0.3547044129891757, + "threshold": 0.38297665883431437 + }, + { + "coverage": 0.40066666666666667, + "risk": 0.3544093178036606, + "threshold": 0.3825110350607596 + }, + { + "coverage": 0.401, + "risk": 0.3541147132169576, + "threshold": 0.3824110396936946 + }, + { + "coverage": 0.4013333333333333, + "risk": 0.3538205980066445, + "threshold": 0.3816525863117464 + }, + { + "coverage": 0.40166666666666667, + "risk": 0.35352697095435687, + "threshold": 0.38153533023380604 + }, + { + "coverage": 0.402, + "risk": 0.35323383084577115, + "threshold": 0.3814538126197653 + }, + { + "coverage": 0.4023333333333333, + "risk": 0.35294117647058826, + "threshold": 0.3808678236705368 + }, + { + "coverage": 0.4026666666666667, + "risk": 0.3526490066225166, + "threshold": 0.38069254234240796 + }, + { + "coverage": 0.403, + "risk": 0.3523573200992556, + "threshold": 0.380276599378093 + }, + { + "coverage": 0.4033333333333333, + "risk": 0.35206611570247937, + "threshold": 0.37989381715416815 + }, + { + "coverage": 0.4036666666666667, + "risk": 0.35177539223782, + "threshold": 0.37968848107668035 + }, + { + "coverage": 0.404, + "risk": 0.35148514851485146, + "threshold": 0.37882736554601265 + }, + { + "coverage": 0.4043333333333333, + "risk": 0.3520197856553998, + "threshold": 0.3787789171082793 + }, + { + "coverage": 0.4046666666666667, + "risk": 0.3517298187808896, + "threshold": 0.3786794029566008 + }, + { + "coverage": 0.405, + "risk": 0.351440329218107, + "threshold": 0.3779715952274959 + }, + { + "coverage": 0.4053333333333333, + "risk": 0.3511513157894737, + "threshold": 0.3776822677721602 + }, + { + "coverage": 0.4056666666666667, + "risk": 0.35086277732128185, + "threshold": 0.3767669790763982 + }, + { + "coverage": 0.406, + "risk": 0.3505747126436782, + "threshold": 0.37666186511169264 + }, + { + "coverage": 0.4063333333333333, + "risk": 0.3502871205906481, + "threshold": 0.376502557514937 + }, + { + "coverage": 0.4066666666666667, + "risk": 0.35, + "threshold": 0.37554516723431985 + }, + { + "coverage": 0.407, + "risk": 0.3497133497133497, + "threshold": 0.375116629629604 + }, + { + "coverage": 0.4073333333333333, + "risk": 0.34942716857610473, + "threshold": 0.37475681569442193 + }, + { + "coverage": 0.4076666666666667, + "risk": 0.3491414554374489, + "threshold": 0.37387728233248313 + }, + { + "coverage": 0.408, + "risk": 0.3488562091503268, + "threshold": 0.37356167074072544 + }, + { + "coverage": 0.4083333333333333, + "risk": 0.3485714285714286, + "threshold": 0.3735240159200843 + }, + { + "coverage": 0.4086666666666667, + "risk": 0.34828711256117456, + "threshold": 0.37339175295985705 + }, + { + "coverage": 0.409, + "risk": 0.3480032599837001, + "threshold": 0.37319397669223986 + }, + { + "coverage": 0.4093333333333333, + "risk": 0.3477198697068404, + "threshold": 0.3725622095554774 + }, + { + "coverage": 0.4096666666666667, + "risk": 0.34743694060211555, + "threshold": 0.37155475516781394 + }, + { + "coverage": 0.41, + "risk": 0.34715447154471546, + "threshold": 0.37068303086548016 + }, + { + "coverage": 0.4103333333333333, + "risk": 0.346872461413485, + "threshold": 0.3692452281010462 + }, + { + "coverage": 0.4106666666666667, + "risk": 0.3465909090909091, + "threshold": 0.3690924551839784 + }, + { + "coverage": 0.411, + "risk": 0.34630981346309814, + "threshold": 0.3688661915813816 + }, + { + "coverage": 0.41133333333333333, + "risk": 0.3460291734197731, + "threshold": 0.3684754455778083 + }, + { + "coverage": 0.4116666666666667, + "risk": 0.345748987854251, + "threshold": 0.3671194414767046 + }, + { + "coverage": 0.412, + "risk": 0.3454692556634304, + "threshold": 0.3668056557927197 + }, + { + "coverage": 0.41233333333333333, + "risk": 0.3451899757477769, + "threshold": 0.3658367353089298 + }, + { + "coverage": 0.4126666666666667, + "risk": 0.345718901453958, + "threshold": 0.36531122858255094 + }, + { + "coverage": 0.413, + "risk": 0.3454398708635997, + "threshold": 0.3647403283070498 + }, + { + "coverage": 0.41333333333333333, + "risk": 0.34596774193548385, + "threshold": 0.3646234505036887 + }, + { + "coverage": 0.4136666666666667, + "risk": 0.34568896051571313, + "threshold": 0.36380524868707315 + }, + { + "coverage": 0.414, + "risk": 0.34541062801932365, + "threshold": 0.36360134610158906 + }, + { + "coverage": 0.41433333333333333, + "risk": 0.34513274336283184, + "threshold": 0.363520803910956 + }, + { + "coverage": 0.4146666666666667, + "risk": 0.34485530546623794, + "threshold": 0.36285411091013564 + }, + { + "coverage": 0.415, + "risk": 0.344578313253012, + "threshold": 0.36229247842880025 + }, + { + "coverage": 0.41533333333333333, + "risk": 0.34430176565008025, + "threshold": 0.3619843080778447 + }, + { + "coverage": 0.4156666666666667, + "risk": 0.3448275862068966, + "threshold": 0.361933252743002 + }, + { + "coverage": 0.416, + "risk": 0.34455128205128205, + "threshold": 0.36110714130458216 + }, + { + "coverage": 0.41633333333333333, + "risk": 0.3450760608486789, + "threshold": 0.360793213433982 + }, + { + "coverage": 0.4166666666666667, + "risk": 0.3448, + "threshold": 0.3607513085189345 + }, + { + "coverage": 0.417, + "risk": 0.3445243804956035, + "threshold": 0.3601678916963864 + }, + { + "coverage": 0.41733333333333333, + "risk": 0.3442492012779553, + "threshold": 0.3596356402276331 + }, + { + "coverage": 0.4176666666666667, + "risk": 0.34397446129289705, + "threshold": 0.35808368859991246 + }, + { + "coverage": 0.418, + "risk": 0.3437001594896332, + "threshold": 0.35806656443148077 + }, + { + "coverage": 0.41833333333333333, + "risk": 0.3434262948207171, + "threshold": 0.3566026092552871 + }, + { + "coverage": 0.4186666666666667, + "risk": 0.3431528662420382, + "threshold": 0.35624987040224415 + }, + { + "coverage": 0.419, + "risk": 0.3428798727128083, + "threshold": 0.3559340976568295 + }, + { + "coverage": 0.41933333333333334, + "risk": 0.3426073131955485, + "threshold": 0.35537310947100387 + }, + { + "coverage": 0.4196666666666667, + "risk": 0.34233518665607626, + "threshold": 0.354263909053234 + }, + { + "coverage": 0.42, + "risk": 0.34206349206349207, + "threshold": 0.35425412544543594 + }, + { + "coverage": 0.42033333333333334, + "risk": 0.34179222839016654, + "threshold": 0.35331569905512017 + }, + { + "coverage": 0.4206666666666667, + "risk": 0.3415213946117274, + "threshold": 0.3533086237799431 + }, + { + "coverage": 0.421, + "risk": 0.3412509897070467, + "threshold": 0.3505901992826319 + }, + { + "coverage": 0.42133333333333334, + "risk": 0.34098101265822783, + "threshold": 0.3501724118461752 + }, + { + "coverage": 0.4216666666666667, + "risk": 0.34071146245059286, + "threshold": 0.34940559723276227 + }, + { + "coverage": 0.422, + "risk": 0.3404423380726698, + "threshold": 0.3489389926553903 + }, + { + "coverage": 0.42233333333333334, + "risk": 0.34017363851617993, + "threshold": 0.348392355026268 + }, + { + "coverage": 0.4226666666666667, + "risk": 0.33990536277602523, + "threshold": 0.3479442524817977 + }, + { + "coverage": 0.423, + "risk": 0.3396375098502758, + "threshold": 0.34780421272953727 + }, + { + "coverage": 0.42333333333333334, + "risk": 0.33937007874015745, + "threshold": 0.34757347105455927 + }, + { + "coverage": 0.4236666666666667, + "risk": 0.33988985051140835, + "threshold": 0.34750946512336056 + }, + { + "coverage": 0.424, + "risk": 0.34040880503144655, + "threshold": 0.3472354057003388 + }, + { + "coverage": 0.42433333333333334, + "risk": 0.34092694422623726, + "threshold": 0.34694912995553034 + }, + { + "coverage": 0.4246666666666667, + "risk": 0.34065934065934067, + "threshold": 0.34684756405529693 + }, + { + "coverage": 0.425, + "risk": 0.3411764705882353, + "threshold": 0.34684292004433986 + }, + { + "coverage": 0.42533333333333334, + "risk": 0.3409090909090909, + "threshold": 0.3455634776270764 + }, + { + "coverage": 0.4256666666666667, + "risk": 0.34064212999216914, + "threshold": 0.34494363320244315 + }, + { + "coverage": 0.426, + "risk": 0.3403755868544601, + "threshold": 0.34490995222143195 + }, + { + "coverage": 0.42633333333333334, + "risk": 0.34010946051602814, + "threshold": 0.34437544496792255 + }, + { + "coverage": 0.4266666666666667, + "risk": 0.33984375, + "threshold": 0.3432184775263036 + }, + { + "coverage": 0.427, + "risk": 0.3395784543325527, + "threshold": 0.34162153196142103 + }, + { + "coverage": 0.42733333333333334, + "risk": 0.33931357254290173, + "threshold": 0.3412871053169141 + }, + { + "coverage": 0.42766666666666664, + "risk": 0.33982852689010135, + "threshold": 0.340600970509087 + }, + { + "coverage": 0.428, + "risk": 0.34034267912772587, + "threshold": 0.34008616770649597 + }, + { + "coverage": 0.42833333333333334, + "risk": 0.34007782101167316, + "threshold": 0.33914677036125396 + }, + { + "coverage": 0.42866666666666664, + "risk": 0.33981337480559876, + "threshold": 0.33864713789492484 + }, + { + "coverage": 0.429, + "risk": 0.34032634032634035, + "threshold": 0.33841144755669406 + }, + { + "coverage": 0.42933333333333334, + "risk": 0.34006211180124224, + "threshold": 0.3384047515471742 + }, + { + "coverage": 0.42966666666666664, + "risk": 0.33979829325058186, + "threshold": 0.338371723288873 + }, + { + "coverage": 0.43, + "risk": 0.3395348837209302, + "threshold": 0.33820743030052114 + }, + { + "coverage": 0.43033333333333335, + "risk": 0.33927188226181254, + "threshold": 0.33800480222437457 + }, + { + "coverage": 0.43066666666666664, + "risk": 0.33900928792569657, + "threshold": 0.33799022887390234 + }, + { + "coverage": 0.431, + "risk": 0.33874709976798145, + "threshold": 0.33778168693867955 + }, + { + "coverage": 0.43133333333333335, + "risk": 0.3384853168469861, + "threshold": 0.3374003470394275 + }, + { + "coverage": 0.43166666666666664, + "risk": 0.3382239382239382, + "threshold": 0.3365697718835956 + }, + { + "coverage": 0.432, + "risk": 0.33796296296296297, + "threshold": 0.3353531664688215 + }, + { + "coverage": 0.43233333333333335, + "risk": 0.33770239013107173, + "threshold": 0.3345947660463658 + }, + { + "coverage": 0.43266666666666664, + "risk": 0.337442218798151, + "threshold": 0.3336266863600014 + }, + { + "coverage": 0.433, + "risk": 0.3371824480369515, + "threshold": 0.33337268124382324 + }, + { + "coverage": 0.43333333333333335, + "risk": 0.33692307692307694, + "threshold": 0.33176853465593514 + }, + { + "coverage": 0.43366666666666664, + "risk": 0.3366641045349731, + "threshold": 0.3314633335671277 + }, + { + "coverage": 0.434, + "risk": 0.33640552995391704, + "threshold": 0.3314471270481763 + }, + { + "coverage": 0.43433333333333335, + "risk": 0.33614735226400616, + "threshold": 0.3313694269718309 + }, + { + "coverage": 0.43466666666666665, + "risk": 0.33588957055214724, + "threshold": 0.33081394420003657 + }, + { + "coverage": 0.435, + "risk": 0.335632183908046, + "threshold": 0.33029059669660227 + }, + { + "coverage": 0.43533333333333335, + "risk": 0.33537519142419603, + "threshold": 0.3300006583913802 + }, + { + "coverage": 0.43566666666666665, + "risk": 0.3351185921958684, + "threshold": 0.328815571751793 + }, + { + "coverage": 0.436, + "risk": 0.3348623853211009, + "threshold": 0.3288082675055062 + }, + { + "coverage": 0.43633333333333335, + "risk": 0.33460656990068754, + "threshold": 0.32790113300330515 + }, + { + "coverage": 0.43666666666666665, + "risk": 0.33435114503816793, + "threshold": 0.3278168293933007 + }, + { + "coverage": 0.437, + "risk": 0.3340961098398169, + "threshold": 0.3275648971144398 + }, + { + "coverage": 0.43733333333333335, + "risk": 0.33384146341463417, + "threshold": 0.32724283047245545 + }, + { + "coverage": 0.43766666666666665, + "risk": 0.3335872048743336, + "threshold": 0.32711855703290676 + }, + { + "coverage": 0.438, + "risk": 0.3333333333333333, + "threshold": 0.3270482297325782 + }, + { + "coverage": 0.43833333333333335, + "risk": 0.33307984790874523, + "threshold": 0.32643269997673613 + }, + { + "coverage": 0.43866666666666665, + "risk": 0.33282674772036475, + "threshold": 0.3263834754048825 + }, + { + "coverage": 0.439, + "risk": 0.3325740318906606, + "threshold": 0.3253351518276387 + }, + { + "coverage": 0.43933333333333335, + "risk": 0.3323216995447648, + "threshold": 0.32474670484529516 + }, + { + "coverage": 0.43966666666666665, + "risk": 0.33206974981046244, + "threshold": 0.32451955511432184 + }, + { + "coverage": 0.44, + "risk": 0.33181818181818185, + "threshold": 0.32449140596848874 + }, + { + "coverage": 0.44033333333333335, + "risk": 0.3315669947009841, + "threshold": 0.32433259615174515 + }, + { + "coverage": 0.44066666666666665, + "risk": 0.3313161875945537, + "threshold": 0.3240164017972202 + }, + { + "coverage": 0.441, + "risk": 0.3310657596371882, + "threshold": 0.32396843023819477 + }, + { + "coverage": 0.44133333333333336, + "risk": 0.33081570996978854, + "threshold": 0.3234603214556536 + }, + { + "coverage": 0.44166666666666665, + "risk": 0.3305660377358491, + "threshold": 0.3233406995101528 + }, + { + "coverage": 0.442, + "risk": 0.33031674208144796, + "threshold": 0.3230108015033075 + }, + { + "coverage": 0.44233333333333336, + "risk": 0.3300678221552374, + "threshold": 0.3229276750937931 + }, + { + "coverage": 0.44266666666666665, + "risk": 0.32981927710843373, + "threshold": 0.32286165138300227 + }, + { + "coverage": 0.443, + "risk": 0.3295711060948081, + "threshold": 0.3224102711559468 + }, + { + "coverage": 0.44333333333333336, + "risk": 0.3293233082706767, + "threshold": 0.3220775187067935 + }, + { + "coverage": 0.44366666666666665, + "risk": 0.3290758827948911, + "threshold": 0.3220378212421051 + }, + { + "coverage": 0.444, + "risk": 0.32882882882882886, + "threshold": 0.3214462270536897 + }, + { + "coverage": 0.44433333333333336, + "risk": 0.3285821455363841, + "threshold": 0.32079494672700437 + }, + { + "coverage": 0.44466666666666665, + "risk": 0.328335832083958, + "threshold": 0.3200196527464467 + }, + { + "coverage": 0.445, + "risk": 0.32808988764044944, + "threshold": 0.32000757490076603 + }, + { + "coverage": 0.44533333333333336, + "risk": 0.3285928143712575, + "threshold": 0.31941476865005713 + }, + { + "coverage": 0.44566666666666666, + "risk": 0.32834704562453254, + "threshold": 0.31827352305316237 + }, + { + "coverage": 0.446, + "risk": 0.328101644245142, + "threshold": 0.31715674091821067 + }, + { + "coverage": 0.44633333333333336, + "risk": 0.3278566094100075, + "threshold": 0.3169222097483965 + }, + { + "coverage": 0.44666666666666666, + "risk": 0.3276119402985075, + "threshold": 0.31658010790544544 + }, + { + "coverage": 0.447, + "risk": 0.32811334824757643, + "threshold": 0.3162368587973804 + }, + { + "coverage": 0.44733333333333336, + "risk": 0.32786885245901637, + "threshold": 0.3159310752376807 + }, + { + "coverage": 0.44766666666666666, + "risk": 0.3276247207743857, + "threshold": 0.3159024182474053 + }, + { + "coverage": 0.448, + "risk": 0.3273809523809524, + "threshold": 0.31566350735825743 + }, + { + "coverage": 0.4483333333333333, + "risk": 0.3271375464684015, + "threshold": 0.3154492553549784 + }, + { + "coverage": 0.44866666666666666, + "risk": 0.32689450222882616, + "threshold": 0.31530806034513353 + }, + { + "coverage": 0.449, + "risk": 0.3266518188567186, + "threshold": 0.3149139674487634 + }, + { + "coverage": 0.4493333333333333, + "risk": 0.3264094955489614, + "threshold": 0.3145241840591934 + }, + { + "coverage": 0.44966666666666666, + "risk": 0.32616753150481836, + "threshold": 0.31424828139474753 + }, + { + "coverage": 0.45, + "risk": 0.32592592592592595, + "threshold": 0.31366875247807435 + }, + { + "coverage": 0.4503333333333333, + "risk": 0.32568467801628426, + "threshold": 0.3134011539933861 + }, + { + "coverage": 0.45066666666666666, + "risk": 0.3254437869822485, + "threshold": 0.3131429759741914 + }, + { + "coverage": 0.451, + "risk": 0.3252032520325203, + "threshold": 0.3127777047011032 + }, + { + "coverage": 0.4513333333333333, + "risk": 0.3249630723781389, + "threshold": 0.3125507781340378 + }, + { + "coverage": 0.45166666666666666, + "risk": 0.3247232472324723, + "threshold": 0.31253491195043576 + }, + { + "coverage": 0.452, + "risk": 0.32448377581120946, + "threshold": 0.3119976791846383 + }, + { + "coverage": 0.4523333333333333, + "risk": 0.32424465733235075, + "threshold": 0.3119836342613222 + }, + { + "coverage": 0.45266666666666666, + "risk": 0.3240058910162003, + "threshold": 0.3119077347217954 + }, + { + "coverage": 0.453, + "risk": 0.3237674760853569, + "threshold": 0.31152980516816736 + }, + { + "coverage": 0.4533333333333333, + "risk": 0.3235294117647059, + "threshold": 0.3111680596540624 + }, + { + "coverage": 0.45366666666666666, + "risk": 0.3232916972814107, + "threshold": 0.3109476948624594 + }, + { + "coverage": 0.454, + "risk": 0.32305433186490456, + "threshold": 0.3109130916849447 + }, + { + "coverage": 0.4543333333333333, + "risk": 0.3235509904622157, + "threshold": 0.31051397224140115 + }, + { + "coverage": 0.45466666666666666, + "risk": 0.32331378299120234, + "threshold": 0.3098478638814953 + }, + { + "coverage": 0.455, + "risk": 0.3230769230769231, + "threshold": 0.3096156146397189 + }, + { + "coverage": 0.4553333333333333, + "risk": 0.32284040995607616, + "threshold": 0.3090055671318057 + }, + { + "coverage": 0.45566666666666666, + "risk": 0.32260424286759326, + "threshold": 0.3087457062382939 + }, + { + "coverage": 0.456, + "risk": 0.3223684210526316, + "threshold": 0.3080124109654501 + }, + { + "coverage": 0.4563333333333333, + "risk": 0.32213294375456536, + "threshold": 0.30792337730992736 + }, + { + "coverage": 0.45666666666666667, + "risk": 0.32262773722627736, + "threshold": 0.30697419137892334 + }, + { + "coverage": 0.457, + "risk": 0.3223924142961342, + "threshold": 0.30695388377513144 + }, + { + "coverage": 0.4573333333333333, + "risk": 0.32215743440233235, + "threshold": 0.3067067740218575 + }, + { + "coverage": 0.45766666666666667, + "risk": 0.3219227967953387, + "threshold": 0.3065634410358234 + }, + { + "coverage": 0.458, + "risk": 0.32168850072780203, + "threshold": 0.30650755707350014 + }, + { + "coverage": 0.4583333333333333, + "risk": 0.32145454545454544, + "threshold": 0.30633424766116574 + }, + { + "coverage": 0.45866666666666667, + "risk": 0.32122093023255816, + "threshold": 0.3063301206805698 + }, + { + "coverage": 0.459, + "risk": 0.32098765432098764, + "threshold": 0.30628478481813287 + }, + { + "coverage": 0.4593333333333333, + "risk": 0.32075471698113206, + "threshold": 0.3060033006039556 + }, + { + "coverage": 0.45966666666666667, + "risk": 0.3205221174764322, + "threshold": 0.3059486932037972 + }, + { + "coverage": 0.46, + "risk": 0.32028985507246377, + "threshold": 0.3057457852738136 + }, + { + "coverage": 0.4603333333333333, + "risk": 0.32005792903692976, + "threshold": 0.3055626477179648 + }, + { + "coverage": 0.46066666666666667, + "risk": 0.31982633863965265, + "threshold": 0.3055425504416571 + }, + { + "coverage": 0.461, + "risk": 0.3195950831525669, + "threshold": 0.3052947071843961 + }, + { + "coverage": 0.4613333333333333, + "risk": 0.319364161849711, + "threshold": 0.30448383036912763 + }, + { + "coverage": 0.46166666666666667, + "risk": 0.3191335740072202, + "threshold": 0.3039334637668967 + }, + { + "coverage": 0.462, + "risk": 0.3189033189033189, + "threshold": 0.303718995335932 + }, + { + "coverage": 0.4623333333333333, + "risk": 0.3186733958183129, + "threshold": 0.3034257497848066 + }, + { + "coverage": 0.46266666666666667, + "risk": 0.3184438040345821, + "threshold": 0.30324696767934256 + }, + { + "coverage": 0.463, + "risk": 0.3182145428365731, + "threshold": 0.30318704486506903 + }, + { + "coverage": 0.4633333333333333, + "risk": 0.31798561151079136, + "threshold": 0.30215631141069244 + }, + { + "coverage": 0.46366666666666667, + "risk": 0.3177570093457944, + "threshold": 0.3018989146100053 + }, + { + "coverage": 0.464, + "risk": 0.3175287356321839, + "threshold": 0.3018789159367936 + }, + { + "coverage": 0.4643333333333333, + "risk": 0.3173007896625987, + "threshold": 0.30182669309186616 + }, + { + "coverage": 0.4646666666666667, + "risk": 0.3170731707317073, + "threshold": 0.3016126465840397 + }, + { + "coverage": 0.465, + "risk": 0.31756272401433694, + "threshold": 0.30153484253380874 + }, + { + "coverage": 0.4653333333333333, + "risk": 0.3173352435530086, + "threshold": 0.3014769694384496 + }, + { + "coverage": 0.4656666666666667, + "risk": 0.31710808876163205, + "threshold": 0.30073594045654023 + }, + { + "coverage": 0.466, + "risk": 0.3168812589413448, + "threshold": 0.3002354920839604 + }, + { + "coverage": 0.4663333333333333, + "risk": 0.3166547533952823, + "threshold": 0.3000546888864632 + }, + { + "coverage": 0.4666666666666667, + "risk": 0.31642857142857145, + "threshold": 0.2998302532035258 + }, + { + "coverage": 0.467, + "risk": 0.31620271234832265, + "threshold": 0.2989457723166252 + }, + { + "coverage": 0.4673333333333333, + "risk": 0.3159771754636234, + "threshold": 0.2985234097877501 + }, + { + "coverage": 0.4676666666666667, + "risk": 0.315751960085531, + "threshold": 0.29841399540399277 + }, + { + "coverage": 0.468, + "risk": 0.31552706552706555, + "threshold": 0.29839314176830606 + }, + { + "coverage": 0.4683333333333333, + "risk": 0.31530249110320285, + "threshold": 0.2973531976740554 + }, + { + "coverage": 0.4686666666666667, + "risk": 0.3157894736842105, + "threshold": 0.2971679681909759 + }, + { + "coverage": 0.469, + "risk": 0.31556503198294245, + "threshold": 0.2966790572562134 + }, + { + "coverage": 0.4693333333333333, + "risk": 0.3153409090909091, + "threshold": 0.29607107238255753 + }, + { + "coverage": 0.4696666666666667, + "risk": 0.31511710432931156, + "threshold": 0.29606420199459416 + }, + { + "coverage": 0.47, + "risk": 0.3148936170212766, + "threshold": 0.2955528290299982 + }, + { + "coverage": 0.4703333333333333, + "risk": 0.3146704464918498, + "threshold": 0.295236939433682 + }, + { + "coverage": 0.4706666666666667, + "risk": 0.3151558073654391, + "threshold": 0.29523300844368244 + }, + { + "coverage": 0.471, + "risk": 0.31493276716206653, + "threshold": 0.29511073192373144 + }, + { + "coverage": 0.4713333333333333, + "risk": 0.3147100424328147, + "threshold": 0.2950582098237391 + }, + { + "coverage": 0.4716666666666667, + "risk": 0.3151943462897526, + "threshold": 0.2948806278417203 + }, + { + "coverage": 0.472, + "risk": 0.3149717514124294, + "threshold": 0.29480610174433663 + }, + { + "coverage": 0.4723333333333333, + "risk": 0.3147494707127735, + "threshold": 0.2940993660221045 + }, + { + "coverage": 0.4726666666666667, + "risk": 0.3145275035260931, + "threshold": 0.29349298981636407 + }, + { + "coverage": 0.473, + "risk": 0.3143058491895701, + "threshold": 0.2929824202992704 + }, + { + "coverage": 0.47333333333333333, + "risk": 0.31408450704225355, + "threshold": 0.2929701879269462 + }, + { + "coverage": 0.4736666666666667, + "risk": 0.3138634764250528, + "threshold": 0.29235359403651523 + }, + { + "coverage": 0.474, + "risk": 0.3136427566807314, + "threshold": 0.2918864524742877 + }, + { + "coverage": 0.47433333333333333, + "risk": 0.3134223471539002, + "threshold": 0.2917454813109433 + }, + { + "coverage": 0.4746666666666667, + "risk": 0.31390449438202245, + "threshold": 0.29165073328828656 + }, + { + "coverage": 0.475, + "risk": 0.3136842105263158, + "threshold": 0.2913781573706752 + }, + { + "coverage": 0.47533333333333333, + "risk": 0.3134642356241234, + "threshold": 0.29056451981084613 + }, + { + "coverage": 0.4756666666666667, + "risk": 0.3132445690259285, + "threshold": 0.29030439270020614 + }, + { + "coverage": 0.476, + "risk": 0.3130252100840336, + "threshold": 0.2902897664610036 + }, + { + "coverage": 0.47633333333333333, + "risk": 0.31280615815255425, + "threshold": 0.2902418775722106 + }, + { + "coverage": 0.4766666666666667, + "risk": 0.31258741258741257, + "threshold": 0.2898885311449255 + }, + { + "coverage": 0.477, + "risk": 0.31236897274633124, + "threshold": 0.28960646622216407 + }, + { + "coverage": 0.47733333333333333, + "risk": 0.3121508379888268, + "threshold": 0.2895492090538588 + }, + { + "coverage": 0.4776666666666667, + "risk": 0.31193300767620374, + "threshold": 0.2892321139649791 + }, + { + "coverage": 0.478, + "risk": 0.3117154811715481, + "threshold": 0.28847357765214326 + }, + { + "coverage": 0.47833333333333333, + "risk": 0.31149825783972124, + "threshold": 0.28821865582561407 + }, + { + "coverage": 0.4786666666666667, + "risk": 0.3112813370473538, + "threshold": 0.2881383100736937 + }, + { + "coverage": 0.479, + "risk": 0.31106471816283926, + "threshold": 0.2881351569562001 + }, + { + "coverage": 0.47933333333333333, + "risk": 0.31084840055632823, + "threshold": 0.28806007457519806 + }, + { + "coverage": 0.4796666666666667, + "risk": 0.310632383599722, + "threshold": 0.28778232621140215 + }, + { + "coverage": 0.48, + "risk": 0.3104166666666667, + "threshold": 0.2870846313892713 + }, + { + "coverage": 0.48033333333333333, + "risk": 0.3102012491325468, + "threshold": 0.2866814649277119 + }, + { + "coverage": 0.4806666666666667, + "risk": 0.30998613037447986, + "threshold": 0.28661220634647705 + }, + { + "coverage": 0.481, + "risk": 0.3097713097713098, + "threshold": 0.28609609375336403 + }, + { + "coverage": 0.48133333333333334, + "risk": 0.3095567867036011, + "threshold": 0.2854170356212381 + }, + { + "coverage": 0.4816666666666667, + "risk": 0.30934256055363324, + "threshold": 0.28484973753006 + }, + { + "coverage": 0.482, + "risk": 0.3091286307053942, + "threshold": 0.2846199969873313 + }, + { + "coverage": 0.48233333333333334, + "risk": 0.30960608154803043, + "threshold": 0.28460524234923446 + }, + { + "coverage": 0.4826666666666667, + "risk": 0.30939226519337015, + "threshold": 0.2842823819358262 + }, + { + "coverage": 0.483, + "risk": 0.30917874396135264, + "threshold": 0.28415444222165204 + }, + { + "coverage": 0.48333333333333334, + "risk": 0.30896551724137933, + "threshold": 0.2827130767925665 + }, + { + "coverage": 0.4836666666666667, + "risk": 0.3087525844245348, + "threshold": 0.28270878164858276 + }, + { + "coverage": 0.484, + "risk": 0.3085399449035813, + "threshold": 0.2825913720610484 + }, + { + "coverage": 0.48433333333333334, + "risk": 0.3083275980729525, + "threshold": 0.28233011770889843 + }, + { + "coverage": 0.4846666666666667, + "risk": 0.3081155433287483, + "threshold": 0.28225296142176787 + }, + { + "coverage": 0.485, + "risk": 0.30790378006872854, + "threshold": 0.2822272292736077 + }, + { + "coverage": 0.48533333333333334, + "risk": 0.3076923076923077, + "threshold": 0.2820583533377386 + }, + { + "coverage": 0.4856666666666667, + "risk": 0.3074811256005491, + "threshold": 0.2813739274988673 + }, + { + "coverage": 0.486, + "risk": 0.30727023319615915, + "threshold": 0.28111889557376246 + }, + { + "coverage": 0.48633333333333334, + "risk": 0.30705962988348184, + "threshold": 0.2809877806997482 + }, + { + "coverage": 0.4866666666666667, + "risk": 0.30684931506849317, + "threshold": 0.28002080406990976 + }, + { + "coverage": 0.487, + "risk": 0.3066392881587953, + "threshold": 0.279450725798257 + }, + { + "coverage": 0.48733333333333334, + "risk": 0.3064295485636115, + "threshold": 0.2790096361517844 + }, + { + "coverage": 0.4876666666666667, + "risk": 0.3062200956937799, + "threshold": 0.2783385748573059 + }, + { + "coverage": 0.488, + "risk": 0.30601092896174864, + "threshold": 0.27832388498125205 + }, + { + "coverage": 0.48833333333333334, + "risk": 0.30580204778156994, + "threshold": 0.2771208231119552 + }, + { + "coverage": 0.4886666666666667, + "risk": 0.30559345156889495, + "threshold": 0.27690093314896114 + }, + { + "coverage": 0.489, + "risk": 0.305385139740968, + "threshold": 0.2762702907783216 + }, + { + "coverage": 0.48933333333333334, + "risk": 0.30517711171662126, + "threshold": 0.27619744020536247 + }, + { + "coverage": 0.48966666666666664, + "risk": 0.3049693669162696, + "threshold": 0.2759166858230768 + }, + { + "coverage": 0.49, + "risk": 0.3047619047619048, + "threshold": 0.2747818845968582 + }, + { + "coverage": 0.49033333333333334, + "risk": 0.30455472467709044, + "threshold": 0.2746697698064117 + }, + { + "coverage": 0.49066666666666664, + "risk": 0.30434782608695654, + "threshold": 0.27434836022050557 + }, + { + "coverage": 0.491, + "risk": 0.30414120841819414, + "threshold": 0.2735700592545576 + }, + { + "coverage": 0.49133333333333334, + "risk": 0.3039348710990502, + "threshold": 0.27303440048263294 + }, + { + "coverage": 0.49166666666666664, + "risk": 0.303728813559322, + "threshold": 0.272520802285886 + }, + { + "coverage": 0.492, + "risk": 0.3035230352303523, + "threshold": 0.27179550286162774 + }, + { + "coverage": 0.49233333333333335, + "risk": 0.3039945836154367, + "threshold": 0.27129764877479173 + }, + { + "coverage": 0.49266666666666664, + "risk": 0.3037889039242219, + "threshold": 0.27120461198955786 + }, + { + "coverage": 0.493, + "risk": 0.3035835023664638, + "threshold": 0.2709689487965736 + }, + { + "coverage": 0.49333333333333335, + "risk": 0.3033783783783784, + "threshold": 0.2702161368269853 + }, + { + "coverage": 0.49366666666666664, + "risk": 0.30317353139770425, + "threshold": 0.26971765421214866 + }, + { + "coverage": 0.494, + "risk": 0.3029689608636977, + "threshold": 0.26940973156516484 + }, + { + "coverage": 0.49433333333333335, + "risk": 0.30276466621712744, + "threshold": 0.2676933950958837 + }, + { + "coverage": 0.49466666666666664, + "risk": 0.3025606469002695, + "threshold": 0.26729270306035263 + }, + { + "coverage": 0.495, + "risk": 0.30303030303030304, + "threshold": 0.2671285933023483 + }, + { + "coverage": 0.49533333333333335, + "risk": 0.3028263795423957, + "threshold": 0.2664127194869439 + }, + { + "coverage": 0.49566666666666664, + "risk": 0.3026227303295225, + "threshold": 0.2658036799186036 + }, + { + "coverage": 0.496, + "risk": 0.3024193548387097, + "threshold": 0.26537964251812085 + }, + { + "coverage": 0.49633333333333335, + "risk": 0.30288784419073206, + "threshold": 0.26502077140141206 + }, + { + "coverage": 0.49666666666666665, + "risk": 0.30268456375838926, + "threshold": 0.26501251033493783 + }, + { + "coverage": 0.497, + "risk": 0.30315224681421865, + "threshold": 0.2647351130179425 + }, + { + "coverage": 0.49733333333333335, + "risk": 0.30294906166219837, + "threshold": 0.26409560382062713 + }, + { + "coverage": 0.49766666666666665, + "risk": 0.30274614869390487, + "threshold": 0.26403377000112793 + }, + { + "coverage": 0.498, + "risk": 0.30254350736278446, + "threshold": 0.263977295954752 + }, + { + "coverage": 0.49833333333333335, + "risk": 0.30234113712374583, + "threshold": 0.2638987301845584 + }, + { + "coverage": 0.49866666666666665, + "risk": 0.30213903743315507, + "threshold": 0.2638459170634938 + }, + { + "coverage": 0.499, + "risk": 0.301937207748831, + "threshold": 0.26382459267577485 + }, + { + "coverage": 0.49933333333333335, + "risk": 0.30173564753004006, + "threshold": 0.26379560057740037 + }, + { + "coverage": 0.49966666666666665, + "risk": 0.30153435623749164, + "threshold": 0.26346402695545756 + }, + { + "coverage": 0.5, + "risk": 0.30133333333333334, + "threshold": 0.26215010387977944 + }, + { + "coverage": 0.5003333333333333, + "risk": 0.3011325782811459, + "threshold": 0.26202913167208325 + }, + { + "coverage": 0.5006666666666667, + "risk": 0.3015978695073236, + "threshold": 0.2616698184723849 + }, + { + "coverage": 0.501, + "risk": 0.3013972055888224, + "threshold": 0.26115093169510534 + }, + { + "coverage": 0.5013333333333333, + "risk": 0.3011968085106383, + "threshold": 0.2607415302277079 + }, + { + "coverage": 0.5016666666666667, + "risk": 0.3009966777408638, + "threshold": 0.25992261103228137 + }, + { + "coverage": 0.502, + "risk": 0.300796812749004, + "threshold": 0.2595562309277467 + }, + { + "coverage": 0.5023333333333333, + "risk": 0.30059721300597214, + "threshold": 0.2591259223560261 + }, + { + "coverage": 0.5026666666666667, + "risk": 0.3003978779840849, + "threshold": 0.25905447441258306 + }, + { + "coverage": 0.503, + "risk": 0.30019880715705766, + "threshold": 0.25798310185972734 + }, + { + "coverage": 0.5033333333333333, + "risk": 0.3, + "threshold": 0.2577544022854102 + }, + { + "coverage": 0.5036666666666667, + "risk": 0.299801455989411, + "threshold": 0.2576026775052284 + }, + { + "coverage": 0.504, + "risk": 0.2996031746031746, + "threshold": 0.2575463148781393 + }, + { + "coverage": 0.5043333333333333, + "risk": 0.29940515532055517, + "threshold": 0.2573459343699852 + }, + { + "coverage": 0.5046666666666667, + "risk": 0.29920739762219284, + "threshold": 0.2559911492493926 + }, + { + "coverage": 0.505, + "risk": 0.299009900990099, + "threshold": 0.2559703157150911 + }, + { + "coverage": 0.5053333333333333, + "risk": 0.29881266490765174, + "threshold": 0.25594115369953435 + }, + { + "coverage": 0.5056666666666667, + "risk": 0.2992748846407383, + "threshold": 0.25534635117104 + }, + { + "coverage": 0.506, + "risk": 0.29907773386034253, + "threshold": 0.25528834057495803 + }, + { + "coverage": 0.5063333333333333, + "risk": 0.2988808426596445, + "threshold": 0.25472839903103534 + }, + { + "coverage": 0.5066666666666667, + "risk": 0.2986842105263158, + "threshold": 0.25446746487739985 + }, + { + "coverage": 0.507, + "risk": 0.29914529914529914, + "threshold": 0.2542919346094201 + }, + { + "coverage": 0.5073333333333333, + "risk": 0.29894875164257556, + "threshold": 0.254268775546271 + }, + { + "coverage": 0.5076666666666667, + "risk": 0.29875246224556795, + "threshold": 0.2539527936804131 + }, + { + "coverage": 0.508, + "risk": 0.29855643044619423, + "threshold": 0.25378232326422906 + }, + { + "coverage": 0.5083333333333333, + "risk": 0.2983606557377049, + "threshold": 0.25338771050267006 + }, + { + "coverage": 0.5086666666666667, + "risk": 0.2981651376146789, + "threshold": 0.2524619569038181 + }, + { + "coverage": 0.509, + "risk": 0.297969875573019, + "threshold": 0.2510992190200787 + }, + { + "coverage": 0.5093333333333333, + "risk": 0.29777486910994766, + "threshold": 0.25099181184623254 + }, + { + "coverage": 0.5096666666666667, + "risk": 0.2975801177240026, + "threshold": 0.25084570899062725 + }, + { + "coverage": 0.51, + "risk": 0.2973856209150327, + "threshold": 0.2504076742683417 + }, + { + "coverage": 0.5103333333333333, + "risk": 0.29719137818419333, + "threshold": 0.2502554906029289 + }, + { + "coverage": 0.5106666666666667, + "risk": 0.29699738903394257, + "threshold": 0.2497533253975705 + }, + { + "coverage": 0.511, + "risk": 0.2968036529680365, + "threshold": 0.24955385112972167 + }, + { + "coverage": 0.5113333333333333, + "risk": 0.2966101694915254, + "threshold": 0.2489981411783413 + }, + { + "coverage": 0.5116666666666667, + "risk": 0.2964169381107492, + "threshold": 0.2484587854773323 + }, + { + "coverage": 0.512, + "risk": 0.2962239583333333, + "threshold": 0.24841153496133847 + }, + { + "coverage": 0.5123333333333333, + "risk": 0.2960312296681848, + "threshold": 0.24832786457456857 + }, + { + "coverage": 0.5126666666666667, + "risk": 0.2958387516254877, + "threshold": 0.24802187654666832 + }, + { + "coverage": 0.513, + "risk": 0.29564652371669914, + "threshold": 0.24759298750798575 + }, + { + "coverage": 0.5133333333333333, + "risk": 0.29545454545454547, + "threshold": 0.24748210070119186 + }, + { + "coverage": 0.5136666666666667, + "risk": 0.29526281635301754, + "threshold": 0.24740967039514258 + }, + { + "coverage": 0.514, + "risk": 0.29507133592736706, + "threshold": 0.24707973492044275 + }, + { + "coverage": 0.5143333333333333, + "risk": 0.2948801036941024, + "threshold": 0.24702111177261502 + }, + { + "coverage": 0.5146666666666667, + "risk": 0.29468911917098445, + "threshold": 0.24639628768356295 + }, + { + "coverage": 0.515, + "risk": 0.29449838187702265, + "threshold": 0.24622030663347877 + }, + { + "coverage": 0.5153333333333333, + "risk": 0.2943078913324709, + "threshold": 0.24608210131555738 + }, + { + "coverage": 0.5156666666666667, + "risk": 0.29411764705882354, + "threshold": 0.24586514837224818 + }, + { + "coverage": 0.516, + "risk": 0.2939276485788114, + "threshold": 0.24585604928382018 + }, + { + "coverage": 0.5163333333333333, + "risk": 0.29373789541639767, + "threshold": 0.24566708564181852 + }, + { + "coverage": 0.5166666666666667, + "risk": 0.29419354838709677, + "threshold": 0.24559331410160085 + }, + { + "coverage": 0.517, + "risk": 0.2940038684719536, + "threshold": 0.24536092866541376 + }, + { + "coverage": 0.5173333333333333, + "risk": 0.29381443298969073, + "threshold": 0.24527771346917068 + }, + { + "coverage": 0.5176666666666667, + "risk": 0.2936252414681262, + "threshold": 0.24502109169345979 + }, + { + "coverage": 0.518, + "risk": 0.29343629343629346, + "threshold": 0.24488880113383635 + }, + { + "coverage": 0.5183333333333333, + "risk": 0.2932475884244373, + "threshold": 0.24394063052439224 + }, + { + "coverage": 0.5186666666666667, + "risk": 0.29370179948586117, + "threshold": 0.24373372063799112 + }, + { + "coverage": 0.519, + "risk": 0.2935131663455363, + "threshold": 0.24283955941261745 + }, + { + "coverage": 0.5193333333333333, + "risk": 0.2933247753530167, + "threshold": 0.2428105430782153 + }, + { + "coverage": 0.5196666666666667, + "risk": 0.2931366260423348, + "threshold": 0.24268555149185955 + }, + { + "coverage": 0.52, + "risk": 0.29294871794871796, + "threshold": 0.2425348427746383 + }, + { + "coverage": 0.5203333333333333, + "risk": 0.29276105060858426, + "threshold": 0.24215113940785418 + }, + { + "coverage": 0.5206666666666667, + "risk": 0.293213828425096, + "threshold": 0.24209416494839947 + }, + { + "coverage": 0.521, + "risk": 0.2930262316058861, + "threshold": 0.24152226243664954 + }, + { + "coverage": 0.5213333333333333, + "risk": 0.2928388746803069, + "threshold": 0.24104578098637913 + }, + { + "coverage": 0.5216666666666666, + "risk": 0.2926517571884984, + "threshold": 0.24066837350026477 + }, + { + "coverage": 0.522, + "risk": 0.29246487867177523, + "threshold": 0.24003455263952272 + }, + { + "coverage": 0.5223333333333333, + "risk": 0.29227823867262287, + "threshold": 0.2394127654944491 + }, + { + "coverage": 0.5226666666666666, + "risk": 0.29209183673469385, + "threshold": 0.23869466633471975 + }, + { + "coverage": 0.523, + "risk": 0.29190567240280435, + "threshold": 0.2385389137113056 + }, + { + "coverage": 0.5233333333333333, + "risk": 0.2917197452229299, + "threshold": 0.23849826968723514 + }, + { + "coverage": 0.5236666666666666, + "risk": 0.2915340547422024, + "threshold": 0.2384797162065129 + }, + { + "coverage": 0.524, + "risk": 0.2913486005089059, + "threshold": 0.23822545798136438 + }, + { + "coverage": 0.5243333333333333, + "risk": 0.291163382072473, + "threshold": 0.23804796996706898 + }, + { + "coverage": 0.5246666666666666, + "risk": 0.2909783989834816, + "threshold": 0.2375630489246862 + }, + { + "coverage": 0.525, + "risk": 0.29079365079365077, + "threshold": 0.23752987325648325 + }, + { + "coverage": 0.5253333333333333, + "risk": 0.29124365482233505, + "threshold": 0.23611959214296993 + }, + { + "coverage": 0.5256666666666666, + "risk": 0.29105897273303744, + "threshold": 0.2354755623755801 + }, + { + "coverage": 0.526, + "risk": 0.2908745247148289, + "threshold": 0.23538914337206712 + }, + { + "coverage": 0.5263333333333333, + "risk": 0.29069031032298925, + "threshold": 0.23469057089863885 + }, + { + "coverage": 0.5266666666666666, + "risk": 0.29050632911392404, + "threshold": 0.23412918137096486 + }, + { + "coverage": 0.527, + "risk": 0.2903225806451613, + "threshold": 0.2340478291769453 + }, + { + "coverage": 0.5273333333333333, + "risk": 0.29013906447534765, + "threshold": 0.2334242259513932 + }, + { + "coverage": 0.5276666666666666, + "risk": 0.2899557801642451, + "threshold": 0.23306501344325872 + }, + { + "coverage": 0.528, + "risk": 0.2897727272727273, + "threshold": 0.23258770152044667 + }, + { + "coverage": 0.5283333333333333, + "risk": 0.289589905362776, + "threshold": 0.23237061683072507 + }, + { + "coverage": 0.5286666666666666, + "risk": 0.28940731399747793, + "threshold": 0.23158173935017798 + }, + { + "coverage": 0.529, + "risk": 0.2892249527410208, + "threshold": 0.23134039555863087 + }, + { + "coverage": 0.5293333333333333, + "risk": 0.2890428211586902, + "threshold": 0.23090919078407227 + }, + { + "coverage": 0.5296666666666666, + "risk": 0.28886091881686593, + "threshold": 0.23073703112328384 + }, + { + "coverage": 0.53, + "risk": 0.28867924528301886, + "threshold": 0.23063052121938385 + }, + { + "coverage": 0.5303333333333333, + "risk": 0.2891263356379635, + "threshold": 0.22947556108539735 + }, + { + "coverage": 0.5306666666666666, + "risk": 0.28957286432160806, + "threshold": 0.22871784933791497 + }, + { + "coverage": 0.531, + "risk": 0.2893910860012555, + "threshold": 0.2286887121580623 + }, + { + "coverage": 0.5313333333333333, + "risk": 0.28920953575909664, + "threshold": 0.2281899911532815 + }, + { + "coverage": 0.5316666666666666, + "risk": 0.2890282131661442, + "threshold": 0.2276660874760806 + }, + { + "coverage": 0.532, + "risk": 0.28884711779448624, + "threshold": 0.2271291804925865 + }, + { + "coverage": 0.5323333333333333, + "risk": 0.2886662492172824, + "threshold": 0.2268939213793323 + }, + { + "coverage": 0.5326666666666666, + "risk": 0.28848560700876097, + "threshold": 0.22606963129796614 + }, + { + "coverage": 0.533, + "risk": 0.2883051907442151, + "threshold": 0.2260080301676665 + }, + { + "coverage": 0.5333333333333333, + "risk": 0.288125, + "threshold": 0.22575801503651372 + }, + { + "coverage": 0.5336666666666666, + "risk": 0.28794503435352903, + "threshold": 0.2244019086822959 + }, + { + "coverage": 0.534, + "risk": 0.2877652933832709, + "threshold": 0.22409340732408942 + }, + { + "coverage": 0.5343333333333333, + "risk": 0.2875857766687461, + "threshold": 0.22370066076440617 + }, + { + "coverage": 0.5346666666666666, + "risk": 0.2874064837905237, + "threshold": 0.22318625881738505 + }, + { + "coverage": 0.535, + "risk": 0.2872274143302181, + "threshold": 0.22280815247569366 + }, + { + "coverage": 0.5353333333333333, + "risk": 0.28704856787048566, + "threshold": 0.22277931209863452 + }, + { + "coverage": 0.5356666666666666, + "risk": 0.2868699439950218, + "threshold": 0.22256634544072226 + }, + { + "coverage": 0.536, + "risk": 0.2866915422885572, + "threshold": 0.22205341604978457 + }, + { + "coverage": 0.5363333333333333, + "risk": 0.2865133623368552, + "threshold": 0.22145037933611295 + }, + { + "coverage": 0.5366666666666666, + "risk": 0.28633540372670807, + "threshold": 0.22066795443676168 + }, + { + "coverage": 0.537, + "risk": 0.2861576660459342, + "threshold": 0.21963114527110741 + }, + { + "coverage": 0.5373333333333333, + "risk": 0.2859801488833747, + "threshold": 0.2193471940634572 + }, + { + "coverage": 0.5376666666666666, + "risk": 0.2864228146311221, + "threshold": 0.21932207715464722 + }, + { + "coverage": 0.538, + "risk": 0.2862453531598513, + "threshold": 0.21886252249719146 + }, + { + "coverage": 0.5383333333333333, + "risk": 0.28606811145510835, + "threshold": 0.218768749855964 + }, + { + "coverage": 0.5386666666666666, + "risk": 0.28589108910891087, + "threshold": 0.21875462578562258 + }, + { + "coverage": 0.539, + "risk": 0.2857142857142857, + "threshold": 0.21835624015063096 + }, + { + "coverage": 0.5393333333333333, + "risk": 0.28553770086526575, + "threshold": 0.21793073587048725 + }, + { + "coverage": 0.5396666666666666, + "risk": 0.2859789993823348, + "threshold": 0.21762086337096848 + }, + { + "coverage": 0.54, + "risk": 0.2858024691358025, + "threshold": 0.2163701858111045 + }, + { + "coverage": 0.5403333333333333, + "risk": 0.28562615669339914, + "threshold": 0.21616310799992114 + }, + { + "coverage": 0.5406666666666666, + "risk": 0.28545006165228115, + "threshold": 0.21599544594553569 + }, + { + "coverage": 0.541, + "risk": 0.28527418361059764, + "threshold": 0.2159402873578458 + }, + { + "coverage": 0.5413333333333333, + "risk": 0.2850985221674877, + "threshold": 0.21564699130301487 + }, + { + "coverage": 0.5416666666666666, + "risk": 0.28492307692307695, + "threshold": 0.2147475601681376 + }, + { + "coverage": 0.542, + "risk": 0.28474784747847476, + "threshold": 0.2146240669892296 + }, + { + "coverage": 0.5423333333333333, + "risk": 0.28457283343577133, + "threshold": 0.21443921314364894 + }, + { + "coverage": 0.5426666666666666, + "risk": 0.2843980343980344, + "threshold": 0.21436289888254875 + }, + { + "coverage": 0.543, + "risk": 0.2842234499693063, + "threshold": 0.21420712852399362 + }, + { + "coverage": 0.5433333333333333, + "risk": 0.28404907975460125, + "threshold": 0.21418688449012002 + }, + { + "coverage": 0.5436666666666666, + "risk": 0.2838749233599019, + "threshold": 0.21314054822538267 + }, + { + "coverage": 0.544, + "risk": 0.28370098039215685, + "threshold": 0.21304104417209285 + }, + { + "coverage": 0.5443333333333333, + "risk": 0.2835272504592774, + "threshold": 0.21292494967960324 + }, + { + "coverage": 0.5446666666666666, + "risk": 0.2833537331701346, + "threshold": 0.21280862750375845 + }, + { + "coverage": 0.545, + "risk": 0.28318042813455657, + "threshold": 0.21235463118705009 + }, + { + "coverage": 0.5453333333333333, + "risk": 0.2830073349633252, + "threshold": 0.21140469085185068 + }, + { + "coverage": 0.5456666666666666, + "risk": 0.28283445326817347, + "threshold": 0.21108079468372146 + }, + { + "coverage": 0.546, + "risk": 0.28266178266178266, + "threshold": 0.2099993849279692 + }, + { + "coverage": 0.5463333333333333, + "risk": 0.28248932275777916, + "threshold": 0.20972624653565178 + }, + { + "coverage": 0.5466666666666666, + "risk": 0.2823170731707317, + "threshold": 0.20965538976220327 + }, + { + "coverage": 0.547, + "risk": 0.28214503351614867, + "threshold": 0.20961618379414648 + }, + { + "coverage": 0.5473333333333333, + "risk": 0.28197320341047505, + "threshold": 0.2090935510904524 + }, + { + "coverage": 0.5476666666666666, + "risk": 0.2818015824710895, + "threshold": 0.20865071042217115 + }, + { + "coverage": 0.548, + "risk": 0.2816301703163017, + "threshold": 0.2085749002799123 + }, + { + "coverage": 0.5483333333333333, + "risk": 0.2814589665653495, + "threshold": 0.20839887260370826 + }, + { + "coverage": 0.5486666666666666, + "risk": 0.28128797083839613, + "threshold": 0.20833321204271316 + }, + { + "coverage": 0.549, + "risk": 0.28111718275652703, + "threshold": 0.2078574381515635 + }, + { + "coverage": 0.5493333333333333, + "risk": 0.2809466019417476, + "threshold": 0.20682488010631786 + }, + { + "coverage": 0.5496666666666666, + "risk": 0.28077622801698, + "threshold": 0.2067580996417821 + }, + { + "coverage": 0.55, + "risk": 0.2806060606060606, + "threshold": 0.20654857256209386 + }, + { + "coverage": 0.5503333333333333, + "risk": 0.28043609933373714, + "threshold": 0.2057423573707358 + }, + { + "coverage": 0.5506666666666666, + "risk": 0.28026634382566584, + "threshold": 0.20530747233624555 + }, + { + "coverage": 0.551, + "risk": 0.28009679370840895, + "threshold": 0.2052757738988116 + }, + { + "coverage": 0.5513333333333333, + "risk": 0.2799274486094317, + "threshold": 0.20521662958423784 + }, + { + "coverage": 0.5516666666666666, + "risk": 0.2797583081570997, + "threshold": 0.2050559450655948 + }, + { + "coverage": 0.552, + "risk": 0.27958937198067635, + "threshold": 0.2048081589808355 + }, + { + "coverage": 0.5523333333333333, + "risk": 0.27942063971031983, + "threshold": 0.20461196397755507 + }, + { + "coverage": 0.5526666666666666, + "risk": 0.2792521109770808, + "threshold": 0.2042602730513843 + }, + { + "coverage": 0.553, + "risk": 0.27908378541289935, + "threshold": 0.20307178916255772 + }, + { + "coverage": 0.5533333333333333, + "risk": 0.2789156626506024, + "threshold": 0.20304019620559025 + }, + { + "coverage": 0.5536666666666666, + "risk": 0.27874774232390126, + "threshold": 0.20263827562977355 + }, + { + "coverage": 0.554, + "risk": 0.2785800240673887, + "threshold": 0.20245174317346318 + }, + { + "coverage": 0.5543333333333333, + "risk": 0.2784125075165364, + "threshold": 0.20203548847768546 + }, + { + "coverage": 0.5546666666666666, + "risk": 0.2782451923076923, + "threshold": 0.20194743038514354 + }, + { + "coverage": 0.555, + "risk": 0.27807807807807805, + "threshold": 0.2019294387314158 + }, + { + "coverage": 0.5553333333333333, + "risk": 0.27791116446578634, + "threshold": 0.2018164066361734 + }, + { + "coverage": 0.5556666666666666, + "risk": 0.277744451109778, + "threshold": 0.20104140905320272 + }, + { + "coverage": 0.556, + "risk": 0.2775779376498801, + "threshold": 0.20098078488354099 + }, + { + "coverage": 0.5563333333333333, + "risk": 0.2774116237267825, + "threshold": 0.20085886440264997 + }, + { + "coverage": 0.5566666666666666, + "risk": 0.2772455089820359, + "threshold": 0.2005770966202808 + }, + { + "coverage": 0.557, + "risk": 0.2770795930580491, + "threshold": 0.2003114559556877 + }, + { + "coverage": 0.5573333333333333, + "risk": 0.2769138755980861, + "threshold": 0.19988246118907657 + }, + { + "coverage": 0.5576666666666666, + "risk": 0.2767483562462642, + "threshold": 0.19983520301913593 + }, + { + "coverage": 0.558, + "risk": 0.2765830346475508, + "threshold": 0.19982863995491976 + }, + { + "coverage": 0.5583333333333333, + "risk": 0.2764179104477612, + "threshold": 0.1997199304228502 + }, + { + "coverage": 0.5586666666666666, + "risk": 0.2762529832935561, + "threshold": 0.1983670652692227 + }, + { + "coverage": 0.559, + "risk": 0.2766845557543232, + "threshold": 0.19835134757519046 + }, + { + "coverage": 0.5593333333333333, + "risk": 0.2765196662693683, + "threshold": 0.1978825868881046 + }, + { + "coverage": 0.5596666666666666, + "risk": 0.27635497319833235, + "threshold": 0.1978243427598313 + }, + { + "coverage": 0.56, + "risk": 0.2761904761904762, + "threshold": 0.1977372323397185 + }, + { + "coverage": 0.5603333333333333, + "risk": 0.2760261748958953, + "threshold": 0.19764739702552916 + }, + { + "coverage": 0.5606666666666666, + "risk": 0.27586206896551724, + "threshold": 0.19724308325357065 + }, + { + "coverage": 0.561, + "risk": 0.27629233511586454, + "threshold": 0.19721481264330865 + }, + { + "coverage": 0.5613333333333334, + "risk": 0.27612826603325413, + "threshold": 0.19675503528533198 + }, + { + "coverage": 0.5616666666666666, + "risk": 0.27596439169139464, + "threshold": 0.19614938016098254 + }, + { + "coverage": 0.562, + "risk": 0.27580071174377224, + "threshold": 0.1959833189837439 + }, + { + "coverage": 0.5623333333333334, + "risk": 0.27563722584469474, + "threshold": 0.19515764637214084 + }, + { + "coverage": 0.5626666666666666, + "risk": 0.2754739336492891, + "threshold": 0.19472901895061523 + }, + { + "coverage": 0.563, + "risk": 0.275902901124926, + "threshold": 0.19471645911903307 + }, + { + "coverage": 0.5633333333333334, + "risk": 0.2757396449704142, + "threshold": 0.1941691049105874 + }, + { + "coverage": 0.5636666666666666, + "risk": 0.2755765819041987, + "threshold": 0.19341292741839414 + }, + { + "coverage": 0.564, + "risk": 0.27541371158392436, + "threshold": 0.19339999893228554 + }, + { + "coverage": 0.5643333333333334, + "risk": 0.2752510336680449, + "threshold": 0.1932243844480129 + }, + { + "coverage": 0.5646666666666667, + "risk": 0.2756788665879575, + "threshold": 0.19322390317575813 + }, + { + "coverage": 0.565, + "risk": 0.2761061946902655, + "threshold": 0.1929827560003555 + }, + { + "coverage": 0.5653333333333334, + "risk": 0.2759433962264151, + "threshold": 0.19297654148592214 + }, + { + "coverage": 0.5656666666666667, + "risk": 0.27637006482027104, + "threshold": 0.19242915218244927 + }, + { + "coverage": 0.566, + "risk": 0.2767962308598351, + "threshold": 0.19226668000938982 + }, + { + "coverage": 0.5663333333333334, + "risk": 0.2766333137139494, + "threshold": 0.19223686435959084 + }, + { + "coverage": 0.5666666666666667, + "risk": 0.27647058823529413, + "threshold": 0.19209816414447878 + }, + { + "coverage": 0.567, + "risk": 0.27630805408583187, + "threshold": 0.192015194304151 + }, + { + "coverage": 0.5673333333333334, + "risk": 0.27614571092831963, + "threshold": 0.19178397074591064 + }, + { + "coverage": 0.5676666666666667, + "risk": 0.2759835584263065, + "threshold": 0.19167870747594962 + }, + { + "coverage": 0.568, + "risk": 0.27582159624413144, + "threshold": 0.1915495934182556 + }, + { + "coverage": 0.5683333333333334, + "risk": 0.2756598240469208, + "threshold": 0.1914361305947137 + }, + { + "coverage": 0.5686666666666667, + "risk": 0.27549824150058616, + "threshold": 0.19082400173074496 + }, + { + "coverage": 0.569, + "risk": 0.2753368482718219, + "threshold": 0.19080207450496256 + }, + { + "coverage": 0.5693333333333334, + "risk": 0.275175644028103, + "threshold": 0.19039542746359403 + }, + { + "coverage": 0.5696666666666667, + "risk": 0.2750146284376829, + "threshold": 0.19021121756468268 + }, + { + "coverage": 0.57, + "risk": 0.27485380116959063, + "threshold": 0.19014984232861284 + }, + { + "coverage": 0.5703333333333334, + "risk": 0.27469316189362947, + "threshold": 0.18936731865383583 + }, + { + "coverage": 0.5706666666666667, + "risk": 0.27453271028037385, + "threshold": 0.18904620694780366 + }, + { + "coverage": 0.571, + "risk": 0.27495621716287216, + "threshold": 0.18873073999091608 + }, + { + "coverage": 0.5713333333333334, + "risk": 0.2747957992998833, + "threshold": 0.18803243616797194 + }, + { + "coverage": 0.5716666666666667, + "risk": 0.27463556851311954, + "threshold": 0.18734144165587158 + }, + { + "coverage": 0.572, + "risk": 0.2744755244755245, + "threshold": 0.18730411844033368 + }, + { + "coverage": 0.5723333333333334, + "risk": 0.2743156668608037, + "threshold": 0.186534688877912 + }, + { + "coverage": 0.5726666666666667, + "risk": 0.2741559953434226, + "threshold": 0.18632557441801115 + }, + { + "coverage": 0.573, + "risk": 0.2745782431646306, + "threshold": 0.18622274774664566 + }, + { + "coverage": 0.5733333333333334, + "risk": 0.2744186046511628, + "threshold": 0.18614362345947902 + }, + { + "coverage": 0.5736666666666667, + "risk": 0.27425915165601394, + "threshold": 0.18573223704997965 + }, + { + "coverage": 0.574, + "risk": 0.27409988385598144, + "threshold": 0.1854437503778995 + }, + { + "coverage": 0.5743333333333334, + "risk": 0.2739408009286129, + "threshold": 0.18527956582953095 + }, + { + "coverage": 0.5746666666666667, + "risk": 0.2737819025522042, + "threshold": 0.18500720185387748 + }, + { + "coverage": 0.575, + "risk": 0.2736231884057971, + "threshold": 0.18487500524057462 + }, + { + "coverage": 0.5753333333333334, + "risk": 0.2734646581691773, + "threshold": 0.18474769913312325 + }, + { + "coverage": 0.5756666666666667, + "risk": 0.27330631152287205, + "threshold": 0.18459628743271098 + }, + { + "coverage": 0.576, + "risk": 0.27314814814814814, + "threshold": 0.18432929130679193 + }, + { + "coverage": 0.5763333333333334, + "risk": 0.27299016772700985, + "threshold": 0.18381906013033425 + }, + { + "coverage": 0.5766666666666667, + "risk": 0.2728323699421965, + "threshold": 0.1834799490817435 + }, + { + "coverage": 0.577, + "risk": 0.2726747544771808, + "threshold": 0.1829864028631794 + }, + { + "coverage": 0.5773333333333334, + "risk": 0.27251732101616627, + "threshold": 0.1828004091905277 + }, + { + "coverage": 0.5776666666666667, + "risk": 0.2723600692440854, + "threshold": 0.18277944759042142 + }, + { + "coverage": 0.578, + "risk": 0.2722029988465975, + "threshold": 0.18270429429878157 + }, + { + "coverage": 0.5783333333333334, + "risk": 0.27204610951008645, + "threshold": 0.18270226953462768 + }, + { + "coverage": 0.5786666666666667, + "risk": 0.271889400921659, + "threshold": 0.18249509628917845 + }, + { + "coverage": 0.579, + "risk": 0.2717328727691422, + "threshold": 0.18241991698121668 + }, + { + "coverage": 0.5793333333333334, + "risk": 0.2715765247410817, + "threshold": 0.18233644286960002 + }, + { + "coverage": 0.5796666666666667, + "risk": 0.2714203565267395, + "threshold": 0.18213836467848735 + }, + { + "coverage": 0.58, + "risk": 0.271264367816092, + "threshold": 0.18143537596074594 + }, + { + "coverage": 0.5803333333333334, + "risk": 0.27110855829982766, + "threshold": 0.18130913413815847 + }, + { + "coverage": 0.5806666666666667, + "risk": 0.2709529276693456, + "threshold": 0.18125638985953507 + }, + { + "coverage": 0.581, + "risk": 0.2707974756167527, + "threshold": 0.1796386663520457 + }, + { + "coverage": 0.5813333333333334, + "risk": 0.2706422018348624, + "threshold": 0.1792574176823886 + }, + { + "coverage": 0.5816666666666667, + "risk": 0.270487106017192, + "threshold": 0.17922900787291077 + }, + { + "coverage": 0.582, + "risk": 0.27033218785796104, + "threshold": 0.1788171898605288 + }, + { + "coverage": 0.5823333333333334, + "risk": 0.2701774470520893, + "threshold": 0.1786881450531461 + }, + { + "coverage": 0.5826666666666667, + "risk": 0.2700228832951945, + "threshold": 0.17857434277801448 + }, + { + "coverage": 0.583, + "risk": 0.2698684962835906, + "threshold": 0.1783097996339236 + }, + { + "coverage": 0.5833333333333334, + "risk": 0.26971428571428574, + "threshold": 0.17765959731202988 + }, + { + "coverage": 0.5836666666666667, + "risk": 0.26956025128498, + "threshold": 0.17744716766306673 + }, + { + "coverage": 0.584, + "risk": 0.2694063926940639, + "threshold": 0.1772425896100922 + }, + { + "coverage": 0.5843333333333334, + "risk": 0.2692527096406161, + "threshold": 0.17713971960820424 + }, + { + "coverage": 0.5846666666666667, + "risk": 0.2690992018244014, + "threshold": 0.17708769286268597 + }, + { + "coverage": 0.585, + "risk": 0.26894586894586897, + "threshold": 0.1768492964731905 + }, + { + "coverage": 0.5853333333333334, + "risk": 0.26879271070615035, + "threshold": 0.1766832366568963 + }, + { + "coverage": 0.5856666666666667, + "risk": 0.2686397268070575, + "threshold": 0.17667574851654808 + }, + { + "coverage": 0.586, + "risk": 0.2684869169510808, + "threshold": 0.17649956829895394 + }, + { + "coverage": 0.5863333333333334, + "risk": 0.26833428084138716, + "threshold": 0.17610199894387432 + }, + { + "coverage": 0.5866666666666667, + "risk": 0.2681818181818182, + "threshold": 0.1759129541952356 + }, + { + "coverage": 0.587, + "risk": 0.2680295286768881, + "threshold": 0.17564469310141492 + }, + { + "coverage": 0.5873333333333334, + "risk": 0.26787741203178206, + "threshold": 0.17562352242358992 + }, + { + "coverage": 0.5876666666666667, + "risk": 0.26772546795235397, + "threshold": 0.17546730191763943 + }, + { + "coverage": 0.588, + "risk": 0.2675736961451247, + "threshold": 0.1754628581341734 + }, + { + "coverage": 0.5883333333333334, + "risk": 0.2674220963172805, + "threshold": 0.17511371646826382 + }, + { + "coverage": 0.5886666666666667, + "risk": 0.26727066817667045, + "threshold": 0.1750099167295091 + }, + { + "coverage": 0.589, + "risk": 0.26711941143180534, + "threshold": 0.17495590002268255 + }, + { + "coverage": 0.5893333333333334, + "risk": 0.2669683257918552, + "threshold": 0.17460990936153134 + }, + { + "coverage": 0.5896666666666667, + "risk": 0.26681741096664785, + "threshold": 0.1743286768285653 + }, + { + "coverage": 0.59, + "risk": 0.26666666666666666, + "threshold": 0.17429861567588376 + }, + { + "coverage": 0.5903333333333334, + "risk": 0.26651609260304915, + "threshold": 0.17426320559244696 + }, + { + "coverage": 0.5906666666666667, + "risk": 0.26693002257336346, + "threshold": 0.1742102550214454 + }, + { + "coverage": 0.591, + "risk": 0.2667794698251551, + "threshold": 0.17398484194150685 + }, + { + "coverage": 0.5913333333333334, + "risk": 0.2666290868094701, + "threshold": 0.17393154026675509 + }, + { + "coverage": 0.5916666666666667, + "risk": 0.26647887323943664, + "threshold": 0.17385778204633523 + }, + { + "coverage": 0.592, + "risk": 0.26632882882882886, + "threshold": 0.17385561723648482 + }, + { + "coverage": 0.5923333333333334, + "risk": 0.26674169949352844, + "threshold": 0.17309153110814682 + }, + { + "coverage": 0.5926666666666667, + "risk": 0.26659167604049494, + "threshold": 0.17307040788839828 + }, + { + "coverage": 0.593, + "risk": 0.26644182124789206, + "threshold": 0.17263300509148172 + }, + { + "coverage": 0.5933333333333334, + "risk": 0.2662921348314607, + "threshold": 0.17216508173538678 + }, + { + "coverage": 0.5936666666666667, + "risk": 0.26614261650758003, + "threshold": 0.17165708014578068 + }, + { + "coverage": 0.594, + "risk": 0.265993265993266, + "threshold": 0.17152647369596954 + }, + { + "coverage": 0.5943333333333334, + "risk": 0.266404935501963, + "threshold": 0.17139695158623994 + }, + { + "coverage": 0.5946666666666667, + "risk": 0.26625560538116594, + "threshold": 0.17113888878019592 + }, + { + "coverage": 0.595, + "risk": 0.2661064425770308, + "threshold": 0.17095962360650735 + }, + { + "coverage": 0.5953333333333334, + "risk": 0.26595744680851063, + "threshold": 0.17067977246763397 + }, + { + "coverage": 0.5956666666666667, + "risk": 0.26580861779518744, + "threshold": 0.17058427797197073 + }, + { + "coverage": 0.596, + "risk": 0.2656599552572707, + "threshold": 0.17043189532281602 + }, + { + "coverage": 0.5963333333333334, + "risk": 0.2655114589155953, + "threshold": 0.1704292668659618 + }, + { + "coverage": 0.5966666666666667, + "risk": 0.26536312849162014, + "threshold": 0.17030313120241025 + }, + { + "coverage": 0.597, + "risk": 0.26521496370742603, + "threshold": 0.17029874786311647 + }, + { + "coverage": 0.5973333333333334, + "risk": 0.2650669642857143, + "threshold": 0.17016284708682314 + }, + { + "coverage": 0.5976666666666667, + "risk": 0.2649191299498048, + "threshold": 0.17005623811341336 + }, + { + "coverage": 0.598, + "risk": 0.26477146042363436, + "threshold": 0.1693515565971413 + }, + { + "coverage": 0.5983333333333334, + "risk": 0.2646239554317549, + "threshold": 0.1688811638542502 + }, + { + "coverage": 0.5986666666666667, + "risk": 0.26447661469933187, + "threshold": 0.16834847020589172 + }, + { + "coverage": 0.599, + "risk": 0.2643294379521425, + "threshold": 0.16807686590841375 + }, + { + "coverage": 0.5993333333333334, + "risk": 0.26418242491657395, + "threshold": 0.1672200158702993 + }, + { + "coverage": 0.5996666666666667, + "risk": 0.264035575319622, + "threshold": 0.16714753757407888 + }, + { + "coverage": 0.6, + "risk": 0.2638888888888889, + "threshold": 0.16712389103536437 + }, + { + "coverage": 0.6003333333333334, + "risk": 0.2637423653525819, + "threshold": 0.16680691915450793 + }, + { + "coverage": 0.6006666666666667, + "risk": 0.26359600443951164, + "threshold": 0.1667629646560506 + }, + { + "coverage": 0.601, + "risk": 0.2634498058790904, + "threshold": 0.16619287165928626 + }, + { + "coverage": 0.6013333333333334, + "risk": 0.2633037694013304, + "threshold": 0.16588607790309948 + }, + { + "coverage": 0.6016666666666667, + "risk": 0.2631578947368421, + "threshold": 0.16580243265196903 + }, + { + "coverage": 0.602, + "risk": 0.26301218161683276, + "threshold": 0.16557132672764954 + }, + { + "coverage": 0.6023333333333334, + "risk": 0.2628666297731046, + "threshold": 0.1654993283390034 + }, + { + "coverage": 0.6026666666666667, + "risk": 0.2627212389380531, + "threshold": 0.16544242062987388 + }, + { + "coverage": 0.603, + "risk": 0.2625760088446656, + "threshold": 0.1653198461472639 + }, + { + "coverage": 0.6033333333333334, + "risk": 0.26243093922651933, + "threshold": 0.16473780083521108 + }, + { + "coverage": 0.6036666666666667, + "risk": 0.2622860298177802, + "threshold": 0.1646657271961427 + }, + { + "coverage": 0.604, + "risk": 0.2621412803532009, + "threshold": 0.16458395800457912 + }, + { + "coverage": 0.6043333333333333, + "risk": 0.26199669056811914, + "threshold": 0.1642216522817908 + }, + { + "coverage": 0.6046666666666667, + "risk": 0.2618522601984565, + "threshold": 0.16396310458384314 + }, + { + "coverage": 0.605, + "risk": 0.2622589531680441, + "threshold": 0.16250749589119978 + }, + { + "coverage": 0.6053333333333333, + "risk": 0.2621145374449339, + "threshold": 0.16171247789700635 + }, + { + "coverage": 0.6056666666666667, + "risk": 0.2619702806824436, + "threshold": 0.16164254385544669 + }, + { + "coverage": 0.606, + "risk": 0.26182618261826185, + "threshold": 0.161099655010722 + }, + { + "coverage": 0.6063333333333333, + "risk": 0.2616822429906542, + "threshold": 0.16025942984061123 + }, + { + "coverage": 0.6066666666666667, + "risk": 0.26153846153846155, + "threshold": 0.15992894939013666 + }, + { + "coverage": 0.607, + "risk": 0.26194398682042835, + "threshold": 0.15982487331953002 + }, + { + "coverage": 0.6073333333333333, + "risk": 0.26180021953896815, + "threshold": 0.1595003134438034 + }, + { + "coverage": 0.6076666666666667, + "risk": 0.2616566099835436, + "threshold": 0.159436559865025 + }, + { + "coverage": 0.608, + "risk": 0.26151315789473684, + "threshold": 0.1593701876667834 + }, + { + "coverage": 0.6083333333333333, + "risk": 0.26136986301369863, + "threshold": 0.15909654519252658 + }, + { + "coverage": 0.6086666666666667, + "risk": 0.2612267250821468, + "threshold": 0.15908946557015594 + }, + { + "coverage": 0.609, + "risk": 0.26108374384236455, + "threshold": 0.1590504544639492 + }, + { + "coverage": 0.6093333333333333, + "risk": 0.2609409190371991, + "threshold": 0.15900925324813894 + }, + { + "coverage": 0.6096666666666667, + "risk": 0.26079825041006016, + "threshold": 0.1587685370286261 + }, + { + "coverage": 0.61, + "risk": 0.260655737704918, + "threshold": 0.15849209286310822 + }, + { + "coverage": 0.6103333333333333, + "risk": 0.2605133806663026, + "threshold": 0.15848525202615696 + }, + { + "coverage": 0.6106666666666667, + "risk": 0.2603711790393013, + "threshold": 0.1584694187355318 + }, + { + "coverage": 0.611, + "risk": 0.2602291325695581, + "threshold": 0.15828472082049366 + }, + { + "coverage": 0.6113333333333333, + "risk": 0.2600872410032715, + "threshold": 0.15807898750279303 + }, + { + "coverage": 0.6116666666666667, + "risk": 0.25994550408719347, + "threshold": 0.1578564173456681 + }, + { + "coverage": 0.612, + "risk": 0.25980392156862747, + "threshold": 0.15708956603312169 + }, + { + "coverage": 0.6123333333333333, + "risk": 0.25966249319542734, + "threshold": 0.15693571217405664 + }, + { + "coverage": 0.6126666666666667, + "risk": 0.25952121871599565, + "threshold": 0.15683919002881555 + }, + { + "coverage": 0.613, + "risk": 0.25938009787928223, + "threshold": 0.1566700744200666 + }, + { + "coverage": 0.6133333333333333, + "risk": 0.2592391304347826, + "threshold": 0.15564088927383476 + }, + { + "coverage": 0.6136666666666667, + "risk": 0.25909831613253664, + "threshold": 0.15559880912103843 + }, + { + "coverage": 0.614, + "risk": 0.25895765472312704, + "threshold": 0.15551034391301405 + }, + { + "coverage": 0.6143333333333333, + "risk": 0.2588171459576777, + "threshold": 0.15527651954324256 + }, + { + "coverage": 0.6146666666666667, + "risk": 0.2586767895878525, + "threshold": 0.1551596271689842 + }, + { + "coverage": 0.615, + "risk": 0.25853658536585367, + "threshold": 0.15442805970704626 + }, + { + "coverage": 0.6153333333333333, + "risk": 0.2583965330444204, + "threshold": 0.15442307541342304 + }, + { + "coverage": 0.6156666666666667, + "risk": 0.25879805089334057, + "threshold": 0.15414049971607166 + }, + { + "coverage": 0.616, + "risk": 0.25865800865800864, + "threshold": 0.1536302745737584 + }, + { + "coverage": 0.6163333333333333, + "risk": 0.2585181179015684, + "threshold": 0.15361157923814966 + }, + { + "coverage": 0.6166666666666667, + "risk": 0.2583783783783784, + "threshold": 0.15342831949720517 + }, + { + "coverage": 0.617, + "risk": 0.25823878984332793, + "threshold": 0.1532597679577481 + }, + { + "coverage": 0.6173333333333333, + "risk": 0.25863930885529157, + "threshold": 0.15307076448848325 + }, + { + "coverage": 0.6176666666666667, + "risk": 0.25849973016729627, + "threshold": 0.1530184061973117 + }, + { + "coverage": 0.618, + "risk": 0.25836030204962246, + "threshold": 0.1528621242269356 + }, + { + "coverage": 0.6183333333333333, + "risk": 0.2582210242587601, + "threshold": 0.1528301229218827 + }, + { + "coverage": 0.6186666666666667, + "risk": 0.25808189655172414, + "threshold": 0.152197173504099 + }, + { + "coverage": 0.619, + "risk": 0.2579429186860528, + "threshold": 0.15198735102560687 + }, + { + "coverage": 0.6193333333333333, + "risk": 0.25780409041980623, + "threshold": 0.1519258603764105 + }, + { + "coverage": 0.6196666666666667, + "risk": 0.258203335126412, + "threshold": 0.15170341969098397 + }, + { + "coverage": 0.62, + "risk": 0.25806451612903225, + "threshold": 0.1508983310009308 + }, + { + "coverage": 0.6203333333333333, + "risk": 0.25792584631918325, + "threshold": 0.14997498896826336 + }, + { + "coverage": 0.6206666666666667, + "risk": 0.2577873254564984, + "threshold": 0.14900567791491567 + }, + { + "coverage": 0.621, + "risk": 0.2576489533011272, + "threshold": 0.1489879372035335 + }, + { + "coverage": 0.6213333333333333, + "risk": 0.2575107296137339, + "threshold": 0.14892715950065613 + }, + { + "coverage": 0.6216666666666667, + "risk": 0.257372654155496, + "threshold": 0.14882496061598627 + }, + { + "coverage": 0.622, + "risk": 0.2572347266881029, + "threshold": 0.14872361366367 + }, + { + "coverage": 0.6223333333333333, + "risk": 0.2570969469737547, + "threshold": 0.14849935816261134 + }, + { + "coverage": 0.6226666666666667, + "risk": 0.2569593147751606, + "threshold": 0.14767176289346182 + }, + { + "coverage": 0.623, + "risk": 0.2568218298555377, + "threshold": 0.1475527154908326 + }, + { + "coverage": 0.6233333333333333, + "risk": 0.25668449197860965, + "threshold": 0.1474907595704315 + }, + { + "coverage": 0.6236666666666667, + "risk": 0.256547300908605, + "threshold": 0.14726236394097436 + }, + { + "coverage": 0.624, + "risk": 0.2564102564102564, + "threshold": 0.14687969234735568 + }, + { + "coverage": 0.6243333333333333, + "risk": 0.2562733582487987, + "threshold": 0.1465089049095737 + }, + { + "coverage": 0.6246666666666667, + "risk": 0.256136606189968, + "threshold": 0.14630106536720913 + }, + { + "coverage": 0.625, + "risk": 0.256, + "threshold": 0.14600163418707177 + }, + { + "coverage": 0.6253333333333333, + "risk": 0.255863539445629, + "threshold": 0.14538408189391655 + }, + { + "coverage": 0.6256666666666667, + "risk": 0.2557272242940863, + "threshold": 0.14535480968430098 + }, + { + "coverage": 0.626, + "risk": 0.25559105431309903, + "threshold": 0.14510119191675744 + }, + { + "coverage": 0.6263333333333333, + "risk": 0.25545502927088876, + "threshold": 0.14490174833985023 + }, + { + "coverage": 0.6266666666666667, + "risk": 0.2553191489361702, + "threshold": 0.14479455131950275 + }, + { + "coverage": 0.627, + "risk": 0.2551834130781499, + "threshold": 0.1447697905723894 + }, + { + "coverage": 0.6273333333333333, + "risk": 0.255047821466525, + "threshold": 0.14474041011103983 + }, + { + "coverage": 0.6276666666666667, + "risk": 0.2549123738714817, + "threshold": 0.1443601271637328 + }, + { + "coverage": 0.628, + "risk": 0.25477707006369427, + "threshold": 0.1442529171131879 + }, + { + "coverage": 0.6283333333333333, + "risk": 0.2546419098143236, + "threshold": 0.1441548720145627 + }, + { + "coverage": 0.6286666666666667, + "risk": 0.2545068928950159, + "threshold": 0.14402740840993095 + }, + { + "coverage": 0.629, + "risk": 0.2543720190779014, + "threshold": 0.14278020107929917 + }, + { + "coverage": 0.6293333333333333, + "risk": 0.2542372881355932, + "threshold": 0.14269867776073392 + }, + { + "coverage": 0.6296666666666667, + "risk": 0.25463208046585495, + "threshold": 0.1426036016506367 + }, + { + "coverage": 0.63, + "risk": 0.2544973544973545, + "threshold": 0.14243756673409375 + }, + { + "coverage": 0.6303333333333333, + "risk": 0.254362771020624, + "threshold": 0.14234867629857 + }, + { + "coverage": 0.6306666666666667, + "risk": 0.25422832980972515, + "threshold": 0.14231081642103915 + }, + { + "coverage": 0.631, + "risk": 0.25409403063919705, + "threshold": 0.1420799992433982 + }, + { + "coverage": 0.6313333333333333, + "risk": 0.2539598732840549, + "threshold": 0.1419839674874874 + }, + { + "coverage": 0.6316666666666667, + "risk": 0.2538258575197889, + "threshold": 0.141470095863647 + }, + { + "coverage": 0.632, + "risk": 0.25369198312236285, + "threshold": 0.14138661207485212 + }, + { + "coverage": 0.6323333333333333, + "risk": 0.25355824986821296, + "threshold": 0.1404910965829742 + }, + { + "coverage": 0.6326666666666667, + "risk": 0.2534246575342466, + "threshold": 0.14011342804753657 + }, + { + "coverage": 0.633, + "risk": 0.253291205897841, + "threshold": 0.1398230310627345 + }, + { + "coverage": 0.6333333333333333, + "risk": 0.2531578947368421, + "threshold": 0.13977408259137555 + }, + { + "coverage": 0.6336666666666667, + "risk": 0.25302472382956337, + "threshold": 0.13965152775580575 + }, + { + "coverage": 0.634, + "risk": 0.2528916929547844, + "threshold": 0.13960473162671846 + }, + { + "coverage": 0.6343333333333333, + "risk": 0.2527588018917499, + "threshold": 0.1394419343679776 + }, + { + "coverage": 0.6346666666666667, + "risk": 0.25262605042016806, + "threshold": 0.13915962394999404 + }, + { + "coverage": 0.635, + "risk": 0.25249343832021, + "threshold": 0.13895744330036194 + }, + { + "coverage": 0.6353333333333333, + "risk": 0.25236096537250785, + "threshold": 0.13893027515965176 + }, + { + "coverage": 0.6356666666666667, + "risk": 0.25222863135815415, + "threshold": 0.13873111905449975 + }, + { + "coverage": 0.636, + "risk": 0.2520964360587002, + "threshold": 0.13847734315732343 + }, + { + "coverage": 0.6363333333333333, + "risk": 0.25196437925615506, + "threshold": 0.13756570601818127 + }, + { + "coverage": 0.6366666666666667, + "risk": 0.2518324607329843, + "threshold": 0.13745364863446513 + }, + { + "coverage": 0.637, + "risk": 0.25170068027210885, + "threshold": 0.1373333545380024 + }, + { + "coverage": 0.6373333333333333, + "risk": 0.252092050209205, + "threshold": 0.13654687193990037 + }, + { + "coverage": 0.6376666666666667, + "risk": 0.25248301097752224, + "threshold": 0.13617057016717848 + }, + { + "coverage": 0.638, + "risk": 0.25235109717868337, + "threshold": 0.13610519105913227 + }, + { + "coverage": 0.6383333333333333, + "risk": 0.25221932114882506, + "threshold": 0.1359747197763795 + }, + { + "coverage": 0.6386666666666667, + "risk": 0.2520876826722338, + "threshold": 0.13593971350603543 + }, + { + "coverage": 0.639, + "risk": 0.2519561815336463, + "threshold": 0.13577460273214742 + }, + { + "coverage": 0.6393333333333333, + "risk": 0.2518248175182482, + "threshold": 0.13570978054885904 + }, + { + "coverage": 0.6396666666666667, + "risk": 0.2516935904116727, + "threshold": 0.13553295844935934 + }, + { + "coverage": 0.64, + "risk": 0.2515625, + "threshold": 0.13548545739576306 + }, + { + "coverage": 0.6403333333333333, + "risk": 0.2514315460697553, + "threshold": 0.13535567683033234 + }, + { + "coverage": 0.6406666666666667, + "risk": 0.2513007284079084, + "threshold": 0.13479366429773498 + }, + { + "coverage": 0.641, + "risk": 0.25117004680187205, + "threshold": 0.13465986051698997 + }, + { + "coverage": 0.6413333333333333, + "risk": 0.251039501039501, + "threshold": 0.13454777933110199 + }, + { + "coverage": 0.6416666666666667, + "risk": 0.2509090909090909, + "threshold": 0.13426288240597248 + }, + { + "coverage": 0.642, + "risk": 0.2507788161993769, + "threshold": 0.1341035820522742 + }, + { + "coverage": 0.6423333333333333, + "risk": 0.25064867669953295, + "threshold": 0.13381128896221056 + }, + { + "coverage": 0.6426666666666667, + "risk": 0.2505186721991701, + "threshold": 0.13363508106565986 + }, + { + "coverage": 0.643, + "risk": 0.25038880248833595, + "threshold": 0.1329266251263882 + }, + { + "coverage": 0.6433333333333333, + "risk": 0.25025906735751297, + "threshold": 0.1325753575497336 + }, + { + "coverage": 0.6436666666666667, + "risk": 0.2501294665976178, + "threshold": 0.13254396343960764 + }, + { + "coverage": 0.644, + "risk": 0.2505175983436853, + "threshold": 0.1324677826484912 + }, + { + "coverage": 0.6443333333333333, + "risk": 0.25038799793067773, + "threshold": 0.13217350725602048 + }, + { + "coverage": 0.6446666666666667, + "risk": 0.250258531540848, + "threshold": 0.13160221029088368 + }, + { + "coverage": 0.645, + "risk": 0.2501291989664083, + "threshold": 0.13157394246393359 + }, + { + "coverage": 0.6453333333333333, + "risk": 0.25, + "threshold": 0.13142322320855654 + }, + { + "coverage": 0.6456666666666667, + "risk": 0.24987093443469283, + "threshold": 0.1313465744516384 + }, + { + "coverage": 0.646, + "risk": 0.24974200206398348, + "threshold": 0.13133651277403968 + }, + { + "coverage": 0.6463333333333333, + "risk": 0.24961320268179474, + "threshold": 0.13128636218860715 + }, + { + "coverage": 0.6466666666666666, + "risk": 0.24948453608247423, + "threshold": 0.13102391638037697 + }, + { + "coverage": 0.647, + "risk": 0.24935600206079342, + "threshold": 0.13011649137061423 + }, + { + "coverage": 0.6473333333333333, + "risk": 0.24922760041194644, + "threshold": 0.12980123082681658 + }, + { + "coverage": 0.6476666666666666, + "risk": 0.24909933093154915, + "threshold": 0.12977737889391314 + }, + { + "coverage": 0.648, + "risk": 0.24897119341563786, + "threshold": 0.12964412190746763 + }, + { + "coverage": 0.6483333333333333, + "risk": 0.24884318766066837, + "threshold": 0.1294128369641617 + }, + { + "coverage": 0.6486666666666666, + "risk": 0.24871531346351491, + "threshold": 0.12860075789964567 + }, + { + "coverage": 0.649, + "risk": 0.24858757062146894, + "threshold": 0.12834681819429555 + }, + { + "coverage": 0.6493333333333333, + "risk": 0.24845995893223818, + "threshold": 0.12830681456608026 + }, + { + "coverage": 0.6496666666666666, + "risk": 0.2483324781939456, + "threshold": 0.1278459307193644 + }, + { + "coverage": 0.65, + "risk": 0.24820512820512822, + "threshold": 0.12783165695329315 + }, + { + "coverage": 0.6503333333333333, + "risk": 0.24807790876473604, + "threshold": 0.12779934585393565 + }, + { + "coverage": 0.6506666666666666, + "risk": 0.24795081967213115, + "threshold": 0.12766508254188444 + }, + { + "coverage": 0.651, + "risk": 0.24782386072708654, + "threshold": 0.12736263362260075 + }, + { + "coverage": 0.6513333333333333, + "risk": 0.24769703172978505, + "threshold": 0.12719191140724076 + }, + { + "coverage": 0.6516666666666666, + "risk": 0.24757033248081842, + "threshold": 0.12717032988886426 + }, + { + "coverage": 0.652, + "risk": 0.2474437627811861, + "threshold": 0.1269892103143272 + }, + { + "coverage": 0.6523333333333333, + "risk": 0.24731732243229432, + "threshold": 0.12676703666679376 + }, + { + "coverage": 0.6526666666666666, + "risk": 0.24719101123595505, + "threshold": 0.1267562491455534 + }, + { + "coverage": 0.653, + "risk": 0.2470648289943849, + "threshold": 0.12672516341334308 + }, + { + "coverage": 0.6533333333333333, + "risk": 0.24693877551020407, + "threshold": 0.1266904190319236 + }, + { + "coverage": 0.6536666666666666, + "risk": 0.24681285058643548, + "threshold": 0.12622997349774964 + }, + { + "coverage": 0.654, + "risk": 0.24668705402650357, + "threshold": 0.1256401322412789 + }, + { + "coverage": 0.6543333333333333, + "risk": 0.24656138563423333, + "threshold": 0.1250840460185128 + }, + { + "coverage": 0.6546666666666666, + "risk": 0.24643584521384929, + "threshold": 0.12503164967005298 + }, + { + "coverage": 0.655, + "risk": 0.24631043256997456, + "threshold": 0.1248133934059136 + }, + { + "coverage": 0.6553333333333333, + "risk": 0.2461851475076297, + "threshold": 0.12469830653238292 + }, + { + "coverage": 0.6556666666666666, + "risk": 0.24605998983223182, + "threshold": 0.12462099039594711 + }, + { + "coverage": 0.656, + "risk": 0.2459349593495935, + "threshold": 0.12456269349191301 + }, + { + "coverage": 0.6563333333333333, + "risk": 0.24581005586592178, + "threshold": 0.12443736229070589 + }, + { + "coverage": 0.6566666666666666, + "risk": 0.24568527918781727, + "threshold": 0.12425314227603589 + }, + { + "coverage": 0.657, + "risk": 0.24606798579401318, + "threshold": 0.12399341159499384 + }, + { + "coverage": 0.6573333333333333, + "risk": 0.24594320486815416, + "threshold": 0.12396880286745993 + }, + { + "coverage": 0.6576666666666666, + "risk": 0.24581855043081602, + "threshold": 0.12357904699275052 + }, + { + "coverage": 0.658, + "risk": 0.24569402228976697, + "threshold": 0.12321758147142432 + }, + { + "coverage": 0.6583333333333333, + "risk": 0.24556962025316456, + "threshold": 0.12301333562152131 + }, + { + "coverage": 0.6586666666666666, + "risk": 0.24544534412955465, + "threshold": 0.1222244405229186 + }, + { + "coverage": 0.659, + "risk": 0.2453211937278705, + "threshold": 0.12204937418039154 + }, + { + "coverage": 0.6593333333333333, + "risk": 0.24519716885743176, + "threshold": 0.12158034059730373 + }, + { + "coverage": 0.6596666666666666, + "risk": 0.2450732693279434, + "threshold": 0.12127677736021054 + }, + { + "coverage": 0.66, + "risk": 0.24494949494949494, + "threshold": 0.12125339684749369 + }, + { + "coverage": 0.6603333333333333, + "risk": 0.24482584553255932, + "threshold": 0.12099707163541754 + }, + { + "coverage": 0.6606666666666666, + "risk": 0.24470232088799193, + "threshold": 0.12089326451802962 + }, + { + "coverage": 0.661, + "risk": 0.24457892082702976, + "threshold": 0.12065125015093613 + }, + { + "coverage": 0.6613333333333333, + "risk": 0.2444556451612903, + "threshold": 0.12060733047742532 + }, + { + "coverage": 0.6616666666666666, + "risk": 0.24433249370277077, + "threshold": 0.12060645461169259 + }, + { + "coverage": 0.662, + "risk": 0.24420946626384693, + "threshold": 0.12031206781058904 + }, + { + "coverage": 0.6623333333333333, + "risk": 0.24408656265727227, + "threshold": 0.12021548424383101 + }, + { + "coverage": 0.6626666666666666, + "risk": 0.24396378269617705, + "threshold": 0.12007065441558176 + }, + { + "coverage": 0.663, + "risk": 0.24384112619406736, + "threshold": 0.11979724839629995 + }, + { + "coverage": 0.6633333333333333, + "risk": 0.2442211055276382, + "threshold": 0.11973630076620674 + }, + { + "coverage": 0.6636666666666666, + "risk": 0.2440984429934706, + "threshold": 0.11942505656596229 + }, + { + "coverage": 0.664, + "risk": 0.24397590361445784, + "threshold": 0.11933111233943852 + }, + { + "coverage": 0.6643333333333333, + "risk": 0.24385348720521827, + "threshold": 0.11910965241395098 + }, + { + "coverage": 0.6646666666666666, + "risk": 0.24373119358074222, + "threshold": 0.11888936861662583 + }, + { + "coverage": 0.665, + "risk": 0.24360902255639097, + "threshold": 0.11854070554753045 + }, + { + "coverage": 0.6653333333333333, + "risk": 0.24348697394789579, + "threshold": 0.11806178300868225 + }, + { + "coverage": 0.6656666666666666, + "risk": 0.24336504757135705, + "threshold": 0.1180551110976027 + }, + { + "coverage": 0.666, + "risk": 0.24324324324324326, + "threshold": 0.11803855919761179 + }, + { + "coverage": 0.6663333333333333, + "risk": 0.24312156078039018, + "threshold": 0.11754075630186978 + }, + { + "coverage": 0.6666666666666666, + "risk": 0.243, + "threshold": 0.11751872321849892 + }, + { + "coverage": 0.667, + "risk": 0.24287856071964017, + "threshold": 0.11749522055483155 + }, + { + "coverage": 0.6673333333333333, + "risk": 0.24275724275724275, + "threshold": 0.11713014816925639 + }, + { + "coverage": 0.6676666666666666, + "risk": 0.24263604593110336, + "threshold": 0.11691891525933701 + }, + { + "coverage": 0.668, + "risk": 0.24251497005988024, + "threshold": 0.1168943863751581 + }, + { + "coverage": 0.6683333333333333, + "risk": 0.24239401496259352, + "threshold": 0.11666080845354916 + }, + { + "coverage": 0.6686666666666666, + "risk": 0.24227318045862412, + "threshold": 0.11661339915737051 + }, + { + "coverage": 0.669, + "risk": 0.242152466367713, + "threshold": 0.11651592835239374 + }, + { + "coverage": 0.6693333333333333, + "risk": 0.24203187250996017, + "threshold": 0.1159133083903632 + }, + { + "coverage": 0.6696666666666666, + "risk": 0.2419113987058238, + "threshold": 0.11570910171723298 + }, + { + "coverage": 0.67, + "risk": 0.2417910447761194, + "threshold": 0.11548816198273511 + }, + { + "coverage": 0.6703333333333333, + "risk": 0.2416708105420189, + "threshold": 0.1153569123789038 + }, + { + "coverage": 0.6706666666666666, + "risk": 0.2415506958250497, + "threshold": 0.11511965127296088 + }, + { + "coverage": 0.671, + "risk": 0.2414307004470939, + "threshold": 0.11503459162116648 + }, + { + "coverage": 0.6713333333333333, + "risk": 0.2413108242303873, + "threshold": 0.11485302100088957 + }, + { + "coverage": 0.6716666666666666, + "risk": 0.2411910669975186, + "threshold": 0.11465922305529921 + }, + { + "coverage": 0.672, + "risk": 0.24107142857142858, + "threshold": 0.11460063854593357 + }, + { + "coverage": 0.6723333333333333, + "risk": 0.24095190877540903, + "threshold": 0.11449093887742047 + }, + { + "coverage": 0.6726666666666666, + "risk": 0.24083250743310208, + "threshold": 0.11439760260351133 + }, + { + "coverage": 0.673, + "risk": 0.24071322436849926, + "threshold": 0.11438922816812233 + }, + { + "coverage": 0.6733333333333333, + "risk": 0.24059405940594059, + "threshold": 0.11414162038552188 + }, + { + "coverage": 0.6736666666666666, + "risk": 0.24047501237011382, + "threshold": 0.11405258837681237 + }, + { + "coverage": 0.674, + "risk": 0.2403560830860534, + "threshold": 0.1139604776901974 + }, + { + "coverage": 0.6743333333333333, + "risk": 0.2402372713791399, + "threshold": 0.11337528081497836 + }, + { + "coverage": 0.6746666666666666, + "risk": 0.24011857707509882, + "threshold": 0.11300402847917825 + }, + { + "coverage": 0.675, + "risk": 0.24, + "threshold": 0.11298483768258917 + }, + { + "coverage": 0.6753333333333333, + "risk": 0.23988153998025666, + "threshold": 0.11288470711555301 + }, + { + "coverage": 0.6756666666666666, + "risk": 0.23976319684262457, + "threshold": 0.11280802974779508 + }, + { + "coverage": 0.676, + "risk": 0.23964497041420119, + "threshold": 0.11279761243738541 + }, + { + "coverage": 0.6763333333333333, + "risk": 0.23952686052242483, + "threshold": 0.1127024348241956 + }, + { + "coverage": 0.6766666666666666, + "risk": 0.23940886699507388, + "threshold": 0.11268081423130359 + }, + { + "coverage": 0.677, + "risk": 0.23929098966026588, + "threshold": 0.11267692819904757 + }, + { + "coverage": 0.6773333333333333, + "risk": 0.23917322834645668, + "threshold": 0.11260384350023868 + }, + { + "coverage": 0.6776666666666666, + "risk": 0.23905558288243975, + "threshold": 0.11235082800705308 + }, + { + "coverage": 0.678, + "risk": 0.23893805309734514, + "threshold": 0.11196233296985483 + }, + { + "coverage": 0.6783333333333333, + "risk": 0.2388206388206388, + "threshold": 0.11181772485853791 + }, + { + "coverage": 0.6786666666666666, + "risk": 0.2387033398821218, + "threshold": 0.11172132032997933 + }, + { + "coverage": 0.679, + "risk": 0.23858615611192932, + "threshold": 0.1112578779051846 + }, + { + "coverage": 0.6793333333333333, + "risk": 0.23846908734052993, + "threshold": 0.11111921810262282 + }, + { + "coverage": 0.6796666666666666, + "risk": 0.23835213339872485, + "threshold": 0.11108910281579182 + }, + { + "coverage": 0.68, + "risk": 0.23823529411764705, + "threshold": 0.1110626703091163 + }, + { + "coverage": 0.6803333333333333, + "risk": 0.2381185693287604, + "threshold": 0.11084521064709642 + }, + { + "coverage": 0.6806666666666666, + "risk": 0.23800195886385897, + "threshold": 0.11024974517729365 + }, + { + "coverage": 0.681, + "risk": 0.23788546255506607, + "threshold": 0.11010523437918067 + }, + { + "coverage": 0.6813333333333333, + "risk": 0.23776908023483365, + "threshold": 0.11005779926987572 + }, + { + "coverage": 0.6816666666666666, + "risk": 0.23765281173594133, + "threshold": 0.10951676699296416 + }, + { + "coverage": 0.682, + "risk": 0.2375366568914956, + "threshold": 0.10922455526778241 + }, + { + "coverage": 0.6823333333333333, + "risk": 0.23742061553492916, + "threshold": 0.10902100316146214 + }, + { + "coverage": 0.6826666666666666, + "risk": 0.2373046875, + "threshold": 0.10887055724269647 + }, + { + "coverage": 0.683, + "risk": 0.23718887262079064, + "threshold": 0.10875552883888008 + }, + { + "coverage": 0.6833333333333333, + "risk": 0.2370731707317073, + "threshold": 0.10862627371392103 + }, + { + "coverage": 0.6836666666666666, + "risk": 0.23695758166747927, + "threshold": 0.1085883272332208 + }, + { + "coverage": 0.684, + "risk": 0.23684210526315788, + "threshold": 0.10858279165593972 + }, + { + "coverage": 0.6843333333333333, + "risk": 0.23672674135411592, + "threshold": 0.1084504849816798 + }, + { + "coverage": 0.6846666666666666, + "risk": 0.23661148977604674, + "threshold": 0.10843977768528822 + }, + { + "coverage": 0.685, + "risk": 0.2364963503649635, + "threshold": 0.10828308720522152 + }, + { + "coverage": 0.6853333333333333, + "risk": 0.23638132295719844, + "threshold": 0.10826212409397794 + }, + { + "coverage": 0.6856666666666666, + "risk": 0.23626640738940205, + "threshold": 0.10795418329050034 + }, + { + "coverage": 0.686, + "risk": 0.23615160349854228, + "threshold": 0.10787692416497728 + }, + { + "coverage": 0.6863333333333334, + "risk": 0.23603691112190384, + "threshold": 0.10754537301479353 + }, + { + "coverage": 0.6866666666666666, + "risk": 0.23592233009708738, + "threshold": 0.10745012590097083 + }, + { + "coverage": 0.687, + "risk": 0.23580786026200873, + "threshold": 0.10742522820523116 + }, + { + "coverage": 0.6873333333333334, + "risk": 0.23569350145489815, + "threshold": 0.10714850587116212 + }, + { + "coverage": 0.6876666666666666, + "risk": 0.23557925351429956, + "threshold": 0.10698792346526567 + }, + { + "coverage": 0.688, + "risk": 0.23546511627906977, + "threshold": 0.10682746295048891 + }, + { + "coverage": 0.6883333333333334, + "risk": 0.23535108958837772, + "threshold": 0.10676538499934898 + }, + { + "coverage": 0.6886666666666666, + "risk": 0.23523717328170377, + "threshold": 0.10652962041550046 + }, + { + "coverage": 0.689, + "risk": 0.2351233671988389, + "threshold": 0.10611335622942383 + }, + { + "coverage": 0.6893333333333334, + "risk": 0.23500967117988394, + "threshold": 0.10607615508525056 + }, + { + "coverage": 0.6896666666666667, + "risk": 0.23489608506524892, + "threshold": 0.10576380555226716 + }, + { + "coverage": 0.69, + "risk": 0.23478260869565218, + "threshold": 0.10563055925446664 + }, + { + "coverage": 0.6903333333333334, + "risk": 0.23466924191211974, + "threshold": 0.10558343064718226 + }, + { + "coverage": 0.6906666666666667, + "risk": 0.23455598455598456, + "threshold": 0.10504244826183128 + }, + { + "coverage": 0.691, + "risk": 0.23444283646888567, + "threshold": 0.10470143975238723 + }, + { + "coverage": 0.6913333333333334, + "risk": 0.2343297974927676, + "threshold": 0.10436268411792785 + }, + { + "coverage": 0.6916666666666667, + "risk": 0.23421686746987952, + "threshold": 0.10396602600010377 + }, + { + "coverage": 0.692, + "risk": 0.23410404624277456, + "threshold": 0.10395268961367359 + }, + { + "coverage": 0.6923333333333334, + "risk": 0.2339913336543091, + "threshold": 0.10386436698958165 + }, + { + "coverage": 0.6926666666666667, + "risk": 0.23387872954764197, + "threshold": 0.103698452931936 + }, + { + "coverage": 0.693, + "risk": 0.23376623376623376, + "threshold": 0.10346444152849388 + }, + { + "coverage": 0.6933333333333334, + "risk": 0.23365384615384616, + "threshold": 0.10326346491168305 + }, + { + "coverage": 0.6936666666666667, + "risk": 0.23354156655454109, + "threshold": 0.10310215648738758 + }, + { + "coverage": 0.694, + "risk": 0.23390970220941404, + "threshold": 0.10300429530857472 + }, + { + "coverage": 0.6943333333333334, + "risk": 0.23379740758521364, + "threshold": 0.10290834020598122 + }, + { + "coverage": 0.6946666666666667, + "risk": 0.2336852207293666, + "threshold": 0.10290061624276309 + }, + { + "coverage": 0.695, + "risk": 0.23357314148681055, + "threshold": 0.10251019957813937 + }, + { + "coverage": 0.6953333333333334, + "risk": 0.23346116970278044, + "threshold": 0.10237619582502565 + }, + { + "coverage": 0.6956666666666667, + "risk": 0.23334930522280786, + "threshold": 0.1023488507792807 + }, + { + "coverage": 0.696, + "risk": 0.2332375478927203, + "threshold": 0.10216264086437594 + }, + { + "coverage": 0.6963333333333334, + "risk": 0.2331258975586405, + "threshold": 0.10167439276349789 + }, + { + "coverage": 0.6966666666666667, + "risk": 0.23301435406698565, + "threshold": 0.1016292240072735 + }, + { + "coverage": 0.697, + "risk": 0.23290291726446677, + "threshold": 0.10137958848480687 + }, + { + "coverage": 0.6973333333333334, + "risk": 0.23279158699808794, + "threshold": 0.10136042337613338 + }, + { + "coverage": 0.6976666666666667, + "risk": 0.23268036311514573, + "threshold": 0.10132253319455835 + }, + { + "coverage": 0.698, + "risk": 0.23256924546322827, + "threshold": 0.10122967579011513 + }, + { + "coverage": 0.6983333333333334, + "risk": 0.2324582338902148, + "threshold": 0.10119044551888047 + }, + { + "coverage": 0.6986666666666667, + "risk": 0.23234732824427481, + "threshold": 0.10069079128531172 + }, + { + "coverage": 0.699, + "risk": 0.23271340009537433, + "threshold": 0.10012605967031643 + }, + { + "coverage": 0.6993333333333334, + "risk": 0.23260247855100094, + "threshold": 0.09952024328200688 + }, + { + "coverage": 0.6996666666666667, + "risk": 0.23249166269652216, + "threshold": 0.09927911518851537 + }, + { + "coverage": 0.7, + "risk": 0.23238095238095238, + "threshold": 0.09923486547469555 + }, + { + "coverage": 0.7003333333333334, + "risk": 0.23227034745359354, + "threshold": 0.09907746797903191 + }, + { + "coverage": 0.7006666666666667, + "risk": 0.23215984776403426, + "threshold": 0.09834453715740878 + }, + { + "coverage": 0.701, + "risk": 0.2320494531621493, + "threshold": 0.09833179535527752 + }, + { + "coverage": 0.7013333333333334, + "risk": 0.23193916349809887, + "threshold": 0.0983227382646467 + }, + { + "coverage": 0.7016666666666667, + "risk": 0.2318289786223278, + "threshold": 0.0980605306418883 + }, + { + "coverage": 0.702, + "risk": 0.23171889838556506, + "threshold": 0.09776360532192091 + }, + { + "coverage": 0.7023333333333334, + "risk": 0.23160892263882296, + "threshold": 0.09775371958638576 + }, + { + "coverage": 0.7026666666666667, + "risk": 0.2314990512333966, + "threshold": 0.09758912279537825 + }, + { + "coverage": 0.703, + "risk": 0.23138928402086298, + "threshold": 0.09754044809886986 + }, + { + "coverage": 0.7033333333333334, + "risk": 0.23127962085308057, + "threshold": 0.09728509273428596 + }, + { + "coverage": 0.7036666666666667, + "risk": 0.23117006158218853, + "threshold": 0.09719702889502259 + }, + { + "coverage": 0.704, + "risk": 0.23106060606060605, + "threshold": 0.09686784961667726 + }, + { + "coverage": 0.7043333333333334, + "risk": 0.23142451490771415, + "threshold": 0.09676732368255099 + }, + { + "coverage": 0.7046666666666667, + "risk": 0.23131504257332072, + "threshold": 0.0964332793188707 + }, + { + "coverage": 0.705, + "risk": 0.23120567375886525, + "threshold": 0.0962460232076949 + }, + { + "coverage": 0.7053333333333334, + "risk": 0.23109640831758035, + "threshold": 0.09587966595981096 + }, + { + "coverage": 0.7056666666666667, + "risk": 0.2309872461029759, + "threshold": 0.09581959038634964 + }, + { + "coverage": 0.706, + "risk": 0.23087818696883852, + "threshold": 0.09490746235628077 + }, + { + "coverage": 0.7063333333333334, + "risk": 0.23076923076923078, + "threshold": 0.0948765114424623 + }, + { + "coverage": 0.7066666666666667, + "risk": 0.23066037735849057, + "threshold": 0.09481316571025404 + }, + { + "coverage": 0.707, + "risk": 0.23055162659123055, + "threshold": 0.09455063247331842 + }, + { + "coverage": 0.7073333333333334, + "risk": 0.23044297832233743, + "threshold": 0.0945292795128781 + }, + { + "coverage": 0.7076666666666667, + "risk": 0.23033443240697127, + "threshold": 0.09445374204042742 + }, + { + "coverage": 0.708, + "risk": 0.23022598870056496, + "threshold": 0.09427482422765793 + }, + { + "coverage": 0.7083333333333334, + "risk": 0.23011764705882354, + "threshold": 0.09406667233905752 + }, + { + "coverage": 0.7086666666666667, + "risk": 0.23000940733772343, + "threshold": 0.0935457386370184 + }, + { + "coverage": 0.709, + "risk": 0.229901269393512, + "threshold": 0.09344255735082747 + }, + { + "coverage": 0.7093333333333334, + "risk": 0.22979323308270677, + "threshold": 0.09339517986136077 + }, + { + "coverage": 0.7096666666666667, + "risk": 0.22968529826209488, + "threshold": 0.09332560746366476 + }, + { + "coverage": 0.71, + "risk": 0.2295774647887324, + "threshold": 0.09332016887782102 + }, + { + "coverage": 0.7103333333333334, + "risk": 0.22946973251994368, + "threshold": 0.09324250549202538 + }, + { + "coverage": 0.7106666666666667, + "risk": 0.22936210131332083, + "threshold": 0.09286711674678379 + }, + { + "coverage": 0.711, + "risk": 0.22925457102672292, + "threshold": 0.09177049627606196 + }, + { + "coverage": 0.7113333333333334, + "risk": 0.22914714151827553, + "threshold": 0.09140507015449209 + }, + { + "coverage": 0.7116666666666667, + "risk": 0.22903981264637002, + "threshold": 0.09095249594156127 + }, + { + "coverage": 0.712, + "risk": 0.22893258426966293, + "threshold": 0.09081562388270567 + }, + { + "coverage": 0.7123333333333334, + "risk": 0.22882545624707534, + "threshold": 0.0907934908246105 + }, + { + "coverage": 0.7126666666666667, + "risk": 0.22871842843779233, + "threshold": 0.09060816045431355 + }, + { + "coverage": 0.713, + "risk": 0.22861150070126227, + "threshold": 0.09057754255865601 + }, + { + "coverage": 0.7133333333333334, + "risk": 0.22850467289719625, + "threshold": 0.09026398860970052 + }, + { + "coverage": 0.7136666666666667, + "risk": 0.2283979448855675, + "threshold": 0.09019578182051075 + }, + { + "coverage": 0.714, + "risk": 0.22829131652661064, + "threshold": 0.09001533539341752 + }, + { + "coverage": 0.7143333333333334, + "risk": 0.2281847876808213, + "threshold": 0.08987528593212431 + }, + { + "coverage": 0.7146666666666667, + "risk": 0.22807835820895522, + "threshold": 0.08943261053597856 + }, + { + "coverage": 0.715, + "risk": 0.22797202797202798, + "threshold": 0.08941729506220075 + }, + { + "coverage": 0.7153333333333334, + "risk": 0.22786579683131408, + "threshold": 0.08914281795259028 + }, + { + "coverage": 0.7156666666666667, + "risk": 0.22775966464834654, + "threshold": 0.0888877620284582 + }, + { + "coverage": 0.716, + "risk": 0.2276536312849162, + "threshold": 0.08870382248804076 + }, + { + "coverage": 0.7163333333333334, + "risk": 0.2275476966030712, + "threshold": 0.0886071259644621 + }, + { + "coverage": 0.7166666666666667, + "risk": 0.22744186046511627, + "threshold": 0.08854439945474718 + }, + { + "coverage": 0.717, + "risk": 0.22733612273361228, + "threshold": 0.08844717163683798 + }, + { + "coverage": 0.7173333333333334, + "risk": 0.22723048327137546, + "threshold": 0.08839704171367915 + }, + { + "coverage": 0.7176666666666667, + "risk": 0.227124941941477, + "threshold": 0.08791735940952336 + }, + { + "coverage": 0.718, + "risk": 0.22701949860724233, + "threshold": 0.08781589918598454 + }, + { + "coverage": 0.7183333333333334, + "risk": 0.22691415313225058, + "threshold": 0.08768518772471268 + }, + { + "coverage": 0.7186666666666667, + "risk": 0.22680890538033396, + "threshold": 0.08750735084430618 + }, + { + "coverage": 0.719, + "risk": 0.2267037552155772, + "threshold": 0.0872817060505663 + }, + { + "coverage": 0.7193333333333334, + "risk": 0.22659870250231695, + "threshold": 0.0869768232436173 + }, + { + "coverage": 0.7196666666666667, + "risk": 0.22649374710514128, + "threshold": 0.0864031935579622 + }, + { + "coverage": 0.72, + "risk": 0.2263888888888889, + "threshold": 0.08626479306899999 + }, + { + "coverage": 0.7203333333333334, + "risk": 0.22628412771864878, + "threshold": 0.08622401993087106 + }, + { + "coverage": 0.7206666666666667, + "risk": 0.22617946345975948, + "threshold": 0.08607232969052885 + }, + { + "coverage": 0.721, + "risk": 0.2260748959778086, + "threshold": 0.0860439932641645 + }, + { + "coverage": 0.7213333333333334, + "risk": 0.22597042513863216, + "threshold": 0.08569136669150465 + }, + { + "coverage": 0.7216666666666667, + "risk": 0.2258660508083141, + "threshold": 0.08558137718445319 + }, + { + "coverage": 0.722, + "risk": 0.2257617728531856, + "threshold": 0.08557833619020791 + }, + { + "coverage": 0.7223333333333334, + "risk": 0.22565759113982464, + "threshold": 0.08552891222828253 + }, + { + "coverage": 0.7226666666666667, + "risk": 0.22555350553505535, + "threshold": 0.08544654356537176 + }, + { + "coverage": 0.723, + "risk": 0.22544951590594745, + "threshold": 0.08519735104871946 + }, + { + "coverage": 0.7233333333333334, + "risk": 0.22534562211981568, + "threshold": 0.08497015086205452 + }, + { + "coverage": 0.7236666666666667, + "risk": 0.22524182404421925, + "threshold": 0.08469992193464633 + }, + { + "coverage": 0.724, + "risk": 0.22513812154696133, + "threshold": 0.0845960019031058 + }, + { + "coverage": 0.7243333333333334, + "risk": 0.22503451449608836, + "threshold": 0.08436651724617351 + }, + { + "coverage": 0.7246666666666667, + "risk": 0.22493100275988961, + "threshold": 0.08432038336667035 + }, + { + "coverage": 0.725, + "risk": 0.22482758620689655, + "threshold": 0.08375038994843445 + }, + { + "coverage": 0.7253333333333334, + "risk": 0.22472426470588236, + "threshold": 0.08372263368970126 + }, + { + "coverage": 0.7256666666666667, + "risk": 0.2246210381258613, + "threshold": 0.08371979136317385 + }, + { + "coverage": 0.726, + "risk": 0.22451790633608815, + "threshold": 0.0836716205848181 + }, + { + "coverage": 0.7263333333333334, + "risk": 0.22441486920605783, + "threshold": 0.08362982864750394 + }, + { + "coverage": 0.7266666666666667, + "risk": 0.22431192660550459, + "threshold": 0.08342467096324879 + }, + { + "coverage": 0.727, + "risk": 0.22420907840440166, + "threshold": 0.08337052800931721 + }, + { + "coverage": 0.7273333333333334, + "risk": 0.22410632447296058, + "threshold": 0.08287978741226051 + }, + { + "coverage": 0.7276666666666667, + "risk": 0.22400366468163077, + "threshold": 0.08278467541146894 + }, + { + "coverage": 0.728, + "risk": 0.2239010989010989, + "threshold": 0.08266769525090294 + }, + { + "coverage": 0.7283333333333334, + "risk": 0.22379862700228834, + "threshold": 0.08240226215340657 + }, + { + "coverage": 0.7286666666666667, + "risk": 0.22369624885635864, + "threshold": 0.0823535834220237 + }, + { + "coverage": 0.729, + "risk": 0.22359396433470508, + "threshold": 0.08199247196373606 + }, + { + "coverage": 0.7293333333333333, + "risk": 0.22349177330895795, + "threshold": 0.0818531296002707 + }, + { + "coverage": 0.7296666666666667, + "risk": 0.2233896756509822, + "threshold": 0.08180455616100839 + }, + { + "coverage": 0.73, + "risk": 0.2232876712328767, + "threshold": 0.08169778460458214 + }, + { + "coverage": 0.7303333333333333, + "risk": 0.22318575992697398, + "threshold": 0.08147928605726008 + }, + { + "coverage": 0.7306666666666667, + "risk": 0.22308394160583941, + "threshold": 0.08146156611506797 + }, + { + "coverage": 0.731, + "risk": 0.22298221614227087, + "threshold": 0.08136532082152802 + }, + { + "coverage": 0.7313333333333333, + "risk": 0.2228805834092981, + "threshold": 0.08128389675811327 + }, + { + "coverage": 0.7316666666666667, + "risk": 0.22277904328018222, + "threshold": 0.08079661866742022 + }, + { + "coverage": 0.732, + "risk": 0.2226775956284153, + "threshold": 0.08066740434252105 + }, + { + "coverage": 0.7323333333333333, + "risk": 0.22303140646335912, + "threshold": 0.08033698851869091 + }, + { + "coverage": 0.7326666666666667, + "risk": 0.2229299363057325, + "threshold": 0.08031264707103275 + }, + { + "coverage": 0.733, + "risk": 0.22282855843565258, + "threshold": 0.0796845426337104 + }, + { + "coverage": 0.7333333333333333, + "risk": 0.22272727272727272, + "threshold": 0.07964718619632076 + }, + { + "coverage": 0.7336666666666667, + "risk": 0.222626079054975, + "threshold": 0.07899131457183561 + }, + { + "coverage": 0.734, + "risk": 0.22252497729336967, + "threshold": 0.07895460952118111 + }, + { + "coverage": 0.7343333333333333, + "risk": 0.2224239673172946, + "threshold": 0.07863401542408575 + }, + { + "coverage": 0.7346666666666667, + "risk": 0.22232304900181488, + "threshold": 0.07853449278163524 + }, + { + "coverage": 0.735, + "risk": 0.2222222222222222, + "threshold": 0.0785186257057217 + }, + { + "coverage": 0.7353333333333333, + "risk": 0.22212148685403446, + "threshold": 0.07851172041747603 + }, + { + "coverage": 0.7356666666666667, + "risk": 0.22202084277299503, + "threshold": 0.07839134696834943 + }, + { + "coverage": 0.736, + "risk": 0.22192028985507245, + "threshold": 0.07829140719545959 + }, + { + "coverage": 0.7363333333333333, + "risk": 0.22181982797645994, + "threshold": 0.07790027772559203 + }, + { + "coverage": 0.7366666666666667, + "risk": 0.22171945701357465, + "threshold": 0.07781683928482123 + }, + { + "coverage": 0.737, + "risk": 0.22161917684305743, + "threshold": 0.0777640883534564 + }, + { + "coverage": 0.7373333333333333, + "risk": 0.22151898734177214, + "threshold": 0.0775175883280517 + }, + { + "coverage": 0.7376666666666667, + "risk": 0.22141888838680523, + "threshold": 0.07732697173374839 + }, + { + "coverage": 0.738, + "risk": 0.22131887985546522, + "threshold": 0.0771599022806842 + }, + { + "coverage": 0.7383333333333333, + "risk": 0.22121896162528218, + "threshold": 0.0770039340749193 + }, + { + "coverage": 0.7386666666666667, + "risk": 0.22111913357400723, + "threshold": 0.07699851125342906 + }, + { + "coverage": 0.739, + "risk": 0.2210193955796121, + "threshold": 0.07637966234993167 + }, + { + "coverage": 0.7393333333333333, + "risk": 0.22091974752028856, + "threshold": 0.07629835281109083 + }, + { + "coverage": 0.7396666666666667, + "risk": 0.22082018927444794, + "threshold": 0.07613467027698738 + }, + { + "coverage": 0.74, + "risk": 0.22072072072072071, + "threshold": 0.07605531666416819 + }, + { + "coverage": 0.7403333333333333, + "risk": 0.2206213417379559, + "threshold": 0.07565980387009476 + }, + { + "coverage": 0.7406666666666667, + "risk": 0.22052205220522053, + "threshold": 0.07550061847030214 + }, + { + "coverage": 0.741, + "risk": 0.22042285200179937, + "threshold": 0.07548323609473771 + }, + { + "coverage": 0.7413333333333333, + "risk": 0.22032374100719423, + "threshold": 0.07533943374888889 + }, + { + "coverage": 0.7416666666666667, + "risk": 0.2202247191011236, + "threshold": 0.07495881603077206 + }, + { + "coverage": 0.742, + "risk": 0.22012578616352202, + "threshold": 0.07458462169582612 + }, + { + "coverage": 0.7423333333333333, + "risk": 0.22002694207453974, + "threshold": 0.07431948687507689 + }, + { + "coverage": 0.7426666666666667, + "risk": 0.2199281867145422, + "threshold": 0.07414448480860973 + }, + { + "coverage": 0.743, + "risk": 0.21982951996410946, + "threshold": 0.0741101093858588 + }, + { + "coverage": 0.7433333333333333, + "risk": 0.21973094170403587, + "threshold": 0.07402989694618699 + }, + { + "coverage": 0.7436666666666667, + "risk": 0.21963245181532945, + "threshold": 0.07401264954779171 + }, + { + "coverage": 0.744, + "risk": 0.21953405017921146, + "threshold": 0.07374836672535036 + }, + { + "coverage": 0.7443333333333333, + "risk": 0.219435736677116, + "threshold": 0.07372881198393709 + }, + { + "coverage": 0.7446666666666667, + "risk": 0.21933751119068934, + "threshold": 0.07338674193645033 + }, + { + "coverage": 0.745, + "risk": 0.21923937360178972, + "threshold": 0.07292751766035026 + }, + { + "coverage": 0.7453333333333333, + "risk": 0.21914132379248658, + "threshold": 0.072892520447817 + }, + { + "coverage": 0.7456666666666667, + "risk": 0.21904336164506036, + "threshold": 0.07264179325675539 + }, + { + "coverage": 0.746, + "risk": 0.21894548704200179, + "threshold": 0.07201098125697829 + }, + { + "coverage": 0.7463333333333333, + "risk": 0.2188476998660116, + "threshold": 0.07187790490585098 + }, + { + "coverage": 0.7466666666666667, + "risk": 0.21875, + "threshold": 0.0718626607863098 + }, + { + "coverage": 0.747, + "risk": 0.2186523873270861, + "threshold": 0.07157070064653309 + }, + { + "coverage": 0.7473333333333333, + "risk": 0.21855486173059768, + "threshold": 0.07127182282999521 + }, + { + "coverage": 0.7476666666666667, + "risk": 0.21845742309407043, + "threshold": 0.07090226204382749 + }, + { + "coverage": 0.748, + "risk": 0.21836007130124777, + "threshold": 0.07089856948056476 + }, + { + "coverage": 0.7483333333333333, + "risk": 0.2182628062360802, + "threshold": 0.07084637598209738 + }, + { + "coverage": 0.7486666666666667, + "risk": 0.21816562778272486, + "threshold": 0.07083544967311757 + }, + { + "coverage": 0.749, + "risk": 0.21806853582554517, + "threshold": 0.07075481350427879 + }, + { + "coverage": 0.7493333333333333, + "risk": 0.2179715302491103, + "threshold": 0.0705820487453003 + }, + { + "coverage": 0.7496666666666667, + "risk": 0.21787461093819474, + "threshold": 0.07054180731125173 + }, + { + "coverage": 0.75, + "risk": 0.21777777777777776, + "threshold": 0.07045309346336283 + }, + { + "coverage": 0.7503333333333333, + "risk": 0.2176810306530431, + "threshold": 0.07024590952183998 + }, + { + "coverage": 0.7506666666666667, + "risk": 0.21758436944937834, + "threshold": 0.07022810011195804 + }, + { + "coverage": 0.751, + "risk": 0.21748779405237462, + "threshold": 0.07012403356079633 + }, + { + "coverage": 0.7513333333333333, + "risk": 0.21739130434782608, + "threshold": 0.07009313248622415 + }, + { + "coverage": 0.7516666666666667, + "risk": 0.21729490022172948, + "threshold": 0.06981004950048698 + }, + { + "coverage": 0.752, + "risk": 0.21719858156028368, + "threshold": 0.06973954974166142 + }, + { + "coverage": 0.7523333333333333, + "risk": 0.21710234824988922, + "threshold": 0.06959264583735188 + }, + { + "coverage": 0.7526666666666667, + "risk": 0.21700620017714792, + "threshold": 0.06959118125458306 + }, + { + "coverage": 0.753, + "risk": 0.21691013722886232, + "threshold": 0.06946955416828815 + }, + { + "coverage": 0.7533333333333333, + "risk": 0.2168141592920354, + "threshold": 0.0692479220791731 + }, + { + "coverage": 0.7536666666666667, + "risk": 0.21671826625386997, + "threshold": 0.06891742407884531 + }, + { + "coverage": 0.754, + "risk": 0.21662245800176835, + "threshold": 0.06889583578337416 + }, + { + "coverage": 0.7543333333333333, + "risk": 0.21696862571807335, + "threshold": 0.06870111392574901 + }, + { + "coverage": 0.7546666666666667, + "risk": 0.21687279151943462, + "threshold": 0.06863410971646348 + }, + { + "coverage": 0.755, + "risk": 0.21677704194260486, + "threshold": 0.06853845432286759 + }, + { + "coverage": 0.7553333333333333, + "risk": 0.21668137687555164, + "threshold": 0.0684948174729138 + }, + { + "coverage": 0.7556666666666667, + "risk": 0.21658579620644022, + "threshold": 0.0683793358545413 + }, + { + "coverage": 0.756, + "risk": 0.21649029982363316, + "threshold": 0.06820678106295006 + }, + { + "coverage": 0.7563333333333333, + "risk": 0.21639488761568973, + "threshold": 0.06815515368926911 + }, + { + "coverage": 0.7566666666666667, + "risk": 0.21629955947136564, + "threshold": 0.06809517233190578 + }, + { + "coverage": 0.757, + "risk": 0.2162043152796125, + "threshold": 0.06799570774845287 + }, + { + "coverage": 0.7573333333333333, + "risk": 0.21610915492957747, + "threshold": 0.06794070878297398 + }, + { + "coverage": 0.7576666666666667, + "risk": 0.21601407831060274, + "threshold": 0.0679243010983803 + }, + { + "coverage": 0.758, + "risk": 0.21591908531222515, + "threshold": 0.06779378790473922 + }, + { + "coverage": 0.7583333333333333, + "risk": 0.21582417582417582, + "threshold": 0.06777919910072486 + }, + { + "coverage": 0.7586666666666667, + "risk": 0.2157293497363796, + "threshold": 0.06777477643705404 + }, + { + "coverage": 0.759, + "risk": 0.21563460693895475, + "threshold": 0.06744712237323738 + }, + { + "coverage": 0.7593333333333333, + "risk": 0.21553994732221246, + "threshold": 0.06743027247760364 + }, + { + "coverage": 0.7596666666666667, + "risk": 0.21544537077665643, + "threshold": 0.06735982705636986 + }, + { + "coverage": 0.76, + "risk": 0.21535087719298246, + "threshold": 0.0671829781245101 + }, + { + "coverage": 0.7603333333333333, + "risk": 0.21525646646207802, + "threshold": 0.0667882070597335 + }, + { + "coverage": 0.7606666666666667, + "risk": 0.2151621384750219, + "threshold": 0.06678804267866598 + }, + { + "coverage": 0.761, + "risk": 0.21506789312308366, + "threshold": 0.06665237380186759 + }, + { + "coverage": 0.7613333333333333, + "risk": 0.2149737302977233, + "threshold": 0.06657677811372674 + }, + { + "coverage": 0.7616666666666667, + "risk": 0.2148796498905908, + "threshold": 0.06635427778386867 + }, + { + "coverage": 0.762, + "risk": 0.21478565179352582, + "threshold": 0.06598524898875312 + }, + { + "coverage": 0.7623333333333333, + "risk": 0.21469173589855706, + "threshold": 0.06582976149675387 + }, + { + "coverage": 0.7626666666666667, + "risk": 0.2145979020979021, + "threshold": 0.06579033340816304 + }, + { + "coverage": 0.763, + "risk": 0.2145041502839668, + "threshold": 0.06552085831326575 + }, + { + "coverage": 0.7633333333333333, + "risk": 0.21441048034934498, + "threshold": 0.0654230216913685 + }, + { + "coverage": 0.7636666666666667, + "risk": 0.214316892186818, + "threshold": 0.06529458899481841 + }, + { + "coverage": 0.764, + "risk": 0.21422338568935428, + "threshold": 0.06490116000476699 + }, + { + "coverage": 0.7643333333333333, + "risk": 0.21412996075010904, + "threshold": 0.06488377834451453 + }, + { + "coverage": 0.7646666666666667, + "risk": 0.2140366172624237, + "threshold": 0.06471289332136826 + }, + { + "coverage": 0.765, + "risk": 0.2139433551198257, + "threshold": 0.06426066542228731 + }, + { + "coverage": 0.7653333333333333, + "risk": 0.21385017421602787, + "threshold": 0.06405942483746169 + }, + { + "coverage": 0.7656666666666667, + "risk": 0.21375707444492817, + "threshold": 0.06393393885997292 + }, + { + "coverage": 0.766, + "risk": 0.21366405570060923, + "threshold": 0.0638949909961119 + }, + { + "coverage": 0.7663333333333333, + "risk": 0.21357111787733798, + "threshold": 0.06378519554146207 + }, + { + "coverage": 0.7666666666666667, + "risk": 0.21347826086956523, + "threshold": 0.06367643514361208 + }, + { + "coverage": 0.767, + "risk": 0.21338548457192524, + "threshold": 0.06326564706782664 + }, + { + "coverage": 0.7673333333333333, + "risk": 0.21329278887923545, + "threshold": 0.06280808620773419 + }, + { + "coverage": 0.7676666666666667, + "risk": 0.21320017368649588, + "threshold": 0.06266821637037054 + }, + { + "coverage": 0.768, + "risk": 0.2131076388888889, + "threshold": 0.0626625176232214 + }, + { + "coverage": 0.7683333333333333, + "risk": 0.21301518438177874, + "threshold": 0.0624431689566762 + }, + { + "coverage": 0.7686666666666667, + "risk": 0.21292281006071118, + "threshold": 0.0623701388252763 + }, + { + "coverage": 0.769, + "risk": 0.2128305158214131, + "threshold": 0.06221410010993975 + }, + { + "coverage": 0.7693333333333333, + "risk": 0.21317157712305027, + "threshold": 0.06202753721503734 + }, + { + "coverage": 0.7696666666666667, + "risk": 0.21307925508878303, + "threshold": 0.06180221812383352 + }, + { + "coverage": 0.77, + "risk": 0.21298701298701297, + "threshold": 0.06174939656210128 + }, + { + "coverage": 0.7703333333333333, + "risk": 0.21289485071397662, + "threshold": 0.06154062370763047 + }, + { + "coverage": 0.7706666666666667, + "risk": 0.21280276816608998, + "threshold": 0.06150850073499206 + }, + { + "coverage": 0.771, + "risk": 0.2127107652399481, + "threshold": 0.061070574523290976 + }, + { + "coverage": 0.7713333333333333, + "risk": 0.212618841832325, + "threshold": 0.060908792966163534 + }, + { + "coverage": 0.7716666666666666, + "risk": 0.2125269978401728, + "threshold": 0.06054750481470883 + }, + { + "coverage": 0.772, + "risk": 0.21243523316062177, + "threshold": 0.060491852173087064 + }, + { + "coverage": 0.7723333333333333, + "risk": 0.2123435476909797, + "threshold": 0.059946809419571104 + }, + { + "coverage": 0.7726666666666666, + "risk": 0.21225194132873165, + "threshold": 0.05991371692237429 + }, + { + "coverage": 0.773, + "risk": 0.21216041397153945, + "threshold": 0.05984296914703797 + }, + { + "coverage": 0.7733333333333333, + "risk": 0.2120689655172414, + "threshold": 0.05978844283259936 + }, + { + "coverage": 0.7736666666666666, + "risk": 0.2119775958638518, + "threshold": 0.05959963785865953 + }, + { + "coverage": 0.774, + "risk": 0.21188630490956073, + "threshold": 0.059515750862966024 + }, + { + "coverage": 0.7743333333333333, + "risk": 0.21179509255273354, + "threshold": 0.05923585188197162 + }, + { + "coverage": 0.7746666666666666, + "risk": 0.2117039586919105, + "threshold": 0.05917173725991099 + }, + { + "coverage": 0.775, + "risk": 0.21161290322580645, + "threshold": 0.05903704068984388 + }, + { + "coverage": 0.7753333333333333, + "risk": 0.2115219260533104, + "threshold": 0.058748626951637226 + }, + { + "coverage": 0.7756666666666666, + "risk": 0.21143102707348518, + "threshold": 0.05871566031575337 + }, + { + "coverage": 0.776, + "risk": 0.211340206185567, + "threshold": 0.05819006619239882 + }, + { + "coverage": 0.7763333333333333, + "risk": 0.2112494632889652, + "threshold": 0.057997815318321384 + }, + { + "coverage": 0.7766666666666666, + "risk": 0.2111587982832618, + "threshold": 0.057996936260328785 + }, + { + "coverage": 0.777, + "risk": 0.21106821106821108, + "threshold": 0.05740471133796807 + }, + { + "coverage": 0.7773333333333333, + "risk": 0.2109777015437393, + "threshold": 0.05723410008685526 + }, + { + "coverage": 0.7776666666666666, + "risk": 0.21088726960994428, + "threshold": 0.05715115153982211 + }, + { + "coverage": 0.778, + "risk": 0.21079691516709512, + "threshold": 0.057087310674673514 + }, + { + "coverage": 0.7783333333333333, + "risk": 0.2107066381156317, + "threshold": 0.056805118098410066 + }, + { + "coverage": 0.7786666666666666, + "risk": 0.2106164383561644, + "threshold": 0.056440014229914445 + }, + { + "coverage": 0.779, + "risk": 0.21052631578947367, + "threshold": 0.05616988488310841 + }, + { + "coverage": 0.7793333333333333, + "risk": 0.21043627031650983, + "threshold": 0.05610487546602847 + }, + { + "coverage": 0.7796666666666666, + "risk": 0.21034630183839248, + "threshold": 0.055837929594031756 + }, + { + "coverage": 0.78, + "risk": 0.21025641025641026, + "threshold": 0.05579545897903656 + }, + { + "coverage": 0.7803333333333333, + "risk": 0.21016659547202052, + "threshold": 0.05565073766164623 + }, + { + "coverage": 0.7806666666666666, + "risk": 0.21007685738684884, + "threshold": 0.05562420796848043 + }, + { + "coverage": 0.781, + "risk": 0.20998719590268886, + "threshold": 0.05542599073377763 + }, + { + "coverage": 0.7813333333333333, + "risk": 0.2098976109215017, + "threshold": 0.05529337980591934 + }, + { + "coverage": 0.7816666666666666, + "risk": 0.20980810234541578, + "threshold": 0.05505522282800285 + }, + { + "coverage": 0.782, + "risk": 0.20971867007672634, + "threshold": 0.054726798982345976 + }, + { + "coverage": 0.7823333333333333, + "risk": 0.2096293140178952, + "threshold": 0.05452576753607028 + }, + { + "coverage": 0.7826666666666666, + "risk": 0.20954003407155025, + "threshold": 0.0544741896822871 + }, + { + "coverage": 0.783, + "risk": 0.20945083014048532, + "threshold": 0.05412795292140059 + }, + { + "coverage": 0.7833333333333333, + "risk": 0.2093617021276596, + "threshold": 0.054019074597478324 + }, + { + "coverage": 0.7836666666666666, + "risk": 0.20927264993619737, + "threshold": 0.05397632443291061 + }, + { + "coverage": 0.784, + "risk": 0.20918367346938777, + "threshold": 0.05397411430296895 + }, + { + "coverage": 0.7843333333333333, + "risk": 0.20909477263068424, + "threshold": 0.053969193574342905 + }, + { + "coverage": 0.7846666666666666, + "risk": 0.20900594732370434, + "threshold": 0.05380910463293687 + }, + { + "coverage": 0.785, + "risk": 0.2089171974522293, + "threshold": 0.05372423213918554 + }, + { + "coverage": 0.7853333333333333, + "risk": 0.20882852292020374, + "threshold": 0.05370441699951271 + }, + { + "coverage": 0.7856666666666666, + "risk": 0.20873992363173527, + "threshold": 0.05369710006196086 + }, + { + "coverage": 0.786, + "risk": 0.20865139949109415, + "threshold": 0.053413197627667865 + }, + { + "coverage": 0.7863333333333333, + "risk": 0.20856295040271303, + "threshold": 0.05306398708460379 + }, + { + "coverage": 0.7866666666666666, + "risk": 0.20847457627118643, + "threshold": 0.05284442247752218 + }, + { + "coverage": 0.787, + "risk": 0.20838627700127066, + "threshold": 0.05284157645614737 + }, + { + "coverage": 0.7873333333333333, + "risk": 0.20829805249788316, + "threshold": 0.05280154371659959 + }, + { + "coverage": 0.7876666666666666, + "risk": 0.20820990266610243, + "threshold": 0.05253775660158389 + }, + { + "coverage": 0.788, + "risk": 0.20812182741116753, + "threshold": 0.052285656719983864 + }, + { + "coverage": 0.7883333333333333, + "risk": 0.2080338266384778, + "threshold": 0.05226606710849052 + }, + { + "coverage": 0.7886666666666666, + "risk": 0.20794590025359255, + "threshold": 0.051543489849880435 + }, + { + "coverage": 0.789, + "risk": 0.20785804816223066, + "threshold": 0.05139978974745896 + }, + { + "coverage": 0.7893333333333333, + "risk": 0.20777027027027026, + "threshold": 0.051395137492039535 + }, + { + "coverage": 0.7896666666666666, + "risk": 0.20768256648374842, + "threshold": 0.05139012107566072 + }, + { + "coverage": 0.79, + "risk": 0.20759493670886076, + "threshold": 0.05134929567723853 + }, + { + "coverage": 0.7903333333333333, + "risk": 0.2075073808519612, + "threshold": 0.05134667841636336 + }, + { + "coverage": 0.7906666666666666, + "risk": 0.20741989881956155, + "threshold": 0.05128955453154354 + }, + { + "coverage": 0.791, + "risk": 0.20733249051833122, + "threshold": 0.05114000091403513 + }, + { + "coverage": 0.7913333333333333, + "risk": 0.2072451558550969, + "threshold": 0.05098779859137953 + }, + { + "coverage": 0.7916666666666666, + "risk": 0.2071578947368421, + "threshold": 0.05051400284221226 + }, + { + "coverage": 0.792, + "risk": 0.20707070707070707, + "threshold": 0.050269652483256044 + }, + { + "coverage": 0.7923333333333333, + "risk": 0.20698359276398823, + "threshold": 0.050239297468800535 + }, + { + "coverage": 0.7926666666666666, + "risk": 0.20689655172413793, + "threshold": 0.04993719325900616 + }, + { + "coverage": 0.793, + "risk": 0.2068095838587642, + "threshold": 0.049847833069042176 + }, + { + "coverage": 0.7933333333333333, + "risk": 0.20672268907563024, + "threshold": 0.04979955832973766 + }, + { + "coverage": 0.7936666666666666, + "risk": 0.20663586728265435, + "threshold": 0.04968343022847519 + }, + { + "coverage": 0.794, + "risk": 0.20654911838790932, + "threshold": 0.049644649434132083 + }, + { + "coverage": 0.7943333333333333, + "risk": 0.20646244229962232, + "threshold": 0.049637780367878366 + }, + { + "coverage": 0.7946666666666666, + "risk": 0.2063758389261745, + "threshold": 0.049541584190852445 + }, + { + "coverage": 0.795, + "risk": 0.20628930817610064, + "threshold": 0.0493241693840372 + }, + { + "coverage": 0.7953333333333333, + "risk": 0.20620284995808885, + "threshold": 0.04927723178836601 + }, + { + "coverage": 0.7956666666666666, + "risk": 0.2061164641809803, + "threshold": 0.04925618582850732 + }, + { + "coverage": 0.796, + "risk": 0.20603015075376885, + "threshold": 0.048651828362625604 + }, + { + "coverage": 0.7963333333333333, + "risk": 0.20594390958560066, + "threshold": 0.04862122031030273 + }, + { + "coverage": 0.7966666666666666, + "risk": 0.20585774058577405, + "threshold": 0.04837339602277741 + }, + { + "coverage": 0.797, + "risk": 0.205771643663739, + "threshold": 0.04829353863776632 + }, + { + "coverage": 0.7973333333333333, + "risk": 0.205685618729097, + "threshold": 0.04824711853652488 + }, + { + "coverage": 0.7976666666666666, + "risk": 0.2055996656916005, + "threshold": 0.04819961969289596 + }, + { + "coverage": 0.798, + "risk": 0.20551378446115287, + "threshold": 0.047977838068014166 + }, + { + "coverage": 0.7983333333333333, + "risk": 0.20542797494780793, + "threshold": 0.047954358580100276 + }, + { + "coverage": 0.7986666666666666, + "risk": 0.20534223706176963, + "threshold": 0.04782383423663395 + }, + { + "coverage": 0.799, + "risk": 0.20525657071339173, + "threshold": 0.04768564311629331 + }, + { + "coverage": 0.7993333333333333, + "risk": 0.20517097581317764, + "threshold": 0.047660452082643426 + }, + { + "coverage": 0.7996666666666666, + "risk": 0.2050854522717799, + "threshold": 0.047569543740816164 + }, + { + "coverage": 0.8, + "risk": 0.205, + "threshold": 0.0474826303869737 + }, + { + "coverage": 0.8003333333333333, + "risk": 0.204914618908788, + "threshold": 0.047366016326561435 + }, + { + "coverage": 0.8006666666666666, + "risk": 0.2048293089092423, + "threshold": 0.047183098026814564 + }, + { + "coverage": 0.801, + "risk": 0.20474406991260924, + "threshold": 0.047162240704150725 + }, + { + "coverage": 0.8013333333333333, + "risk": 0.20465890183028287, + "threshold": 0.047098852566577164 + }, + { + "coverage": 0.8016666666666666, + "risk": 0.20457380457380459, + "threshold": 0.04703355113792274 + }, + { + "coverage": 0.802, + "risk": 0.20448877805486285, + "threshold": 0.046927616151079175 + }, + { + "coverage": 0.8023333333333333, + "risk": 0.2044038221852929, + "threshold": 0.046845473590663815 + }, + { + "coverage": 0.8026666666666666, + "risk": 0.20431893687707642, + "threshold": 0.04679230128141122 + }, + { + "coverage": 0.803, + "risk": 0.20423412204234123, + "threshold": 0.04635832220978407 + }, + { + "coverage": 0.8033333333333333, + "risk": 0.204149377593361, + "threshold": 0.04628959975589201 + }, + { + "coverage": 0.8036666666666666, + "risk": 0.20406470344255495, + "threshold": 0.04607715079181717 + }, + { + "coverage": 0.804, + "risk": 0.20398009950248755, + "threshold": 0.04585328519686025 + }, + { + "coverage": 0.8043333333333333, + "risk": 0.20389556568586822, + "threshold": 0.04566224782291879 + }, + { + "coverage": 0.8046666666666666, + "risk": 0.20381110190555096, + "threshold": 0.045598473897826446 + }, + { + "coverage": 0.805, + "risk": 0.20372670807453416, + "threshold": 0.04548479561152179 + }, + { + "coverage": 0.8053333333333333, + "risk": 0.20364238410596028, + "threshold": 0.045360764917655935 + }, + { + "coverage": 0.8056666666666666, + "risk": 0.20355812991311542, + "threshold": 0.04534185199491035 + }, + { + "coverage": 0.806, + "risk": 0.20347394540942929, + "threshold": 0.04522675213923307 + }, + { + "coverage": 0.8063333333333333, + "risk": 0.2033898305084746, + "threshold": 0.04489998406551932 + }, + { + "coverage": 0.8066666666666666, + "risk": 0.20330578512396694, + "threshold": 0.044648067980013 + }, + { + "coverage": 0.807, + "risk": 0.20322180916976457, + "threshold": 0.0446115113190639 + }, + { + "coverage": 0.8073333333333333, + "risk": 0.20313790255986788, + "threshold": 0.0445481613527345 + }, + { + "coverage": 0.8076666666666666, + "risk": 0.20305406520841932, + "threshold": 0.04452774457320372 + }, + { + "coverage": 0.808, + "risk": 0.20297029702970298, + "threshold": 0.044523160592929205 + }, + { + "coverage": 0.8083333333333333, + "risk": 0.20288659793814434, + "threshold": 0.04448981765843986 + }, + { + "coverage": 0.8086666666666666, + "risk": 0.20280296784830998, + "threshold": 0.0441487587582431 + }, + { + "coverage": 0.809, + "risk": 0.2027194066749073, + "threshold": 0.044079644625732 + }, + { + "coverage": 0.8093333333333333, + "risk": 0.2030477759472817, + "threshold": 0.04396604857660027 + }, + { + "coverage": 0.8096666666666666, + "risk": 0.20296418279127212, + "threshold": 0.043919910063260896 + }, + { + "coverage": 0.81, + "risk": 0.202880658436214, + "threshold": 0.04353614775107039 + }, + { + "coverage": 0.8103333333333333, + "risk": 0.20279720279720279, + "threshold": 0.043325205875225606 + }, + { + "coverage": 0.8106666666666666, + "risk": 0.20271381578947367, + "threshold": 0.04315193417542717 + }, + { + "coverage": 0.811, + "risk": 0.20263049732840116, + "threshold": 0.04304868957868479 + }, + { + "coverage": 0.8113333333333334, + "risk": 0.20254724732949878, + "threshold": 0.042974892763218565 + }, + { + "coverage": 0.8116666666666666, + "risk": 0.20246406570841888, + "threshold": 0.04285309784081102 + }, + { + "coverage": 0.812, + "risk": 0.20238095238095238, + "threshold": 0.04284559401486834 + }, + { + "coverage": 0.8123333333333334, + "risk": 0.20229790726302832, + "threshold": 0.04259409061337438 + }, + { + "coverage": 0.8126666666666666, + "risk": 0.2022149302707137, + "threshold": 0.042572263928824086 + }, + { + "coverage": 0.813, + "risk": 0.2021320213202132, + "threshold": 0.042476537671194775 + }, + { + "coverage": 0.8133333333333334, + "risk": 0.20204918032786887, + "threshold": 0.04235472613960567 + }, + { + "coverage": 0.8136666666666666, + "risk": 0.20196640721015977, + "threshold": 0.0422616096019987 + }, + { + "coverage": 0.814, + "risk": 0.2018837018837019, + "threshold": 0.04215391510421285 + }, + { + "coverage": 0.8143333333333334, + "risk": 0.20180106426524766, + "threshold": 0.04198266721316387 + }, + { + "coverage": 0.8146666666666667, + "risk": 0.20171849427168576, + "threshold": 0.0419405504466759 + }, + { + "coverage": 0.815, + "risk": 0.2016359918200409, + "threshold": 0.041436969946441624 + }, + { + "coverage": 0.8153333333333334, + "risk": 0.20155355682747342, + "threshold": 0.04136731704521442 + }, + { + "coverage": 0.8156666666666667, + "risk": 0.20147118921127913, + "threshold": 0.04129430858797203 + }, + { + "coverage": 0.816, + "risk": 0.2013888888888889, + "threshold": 0.041239462555195054 + }, + { + "coverage": 0.8163333333333334, + "risk": 0.20130665577786852, + "threshold": 0.04119708013240918 + }, + { + "coverage": 0.8166666666666667, + "risk": 0.20122448979591837, + "threshold": 0.04103653348299373 + }, + { + "coverage": 0.817, + "risk": 0.2011423908608731, + "threshold": 0.04087598819858757 + }, + { + "coverage": 0.8173333333333334, + "risk": 0.20106035889070148, + "threshold": 0.04063771566647936 + }, + { + "coverage": 0.8176666666666667, + "risk": 0.2009783938035059, + "threshold": 0.040633701038157305 + }, + { + "coverage": 0.818, + "risk": 0.20089649551752242, + "threshold": 0.04053288470702292 + }, + { + "coverage": 0.8183333333333334, + "risk": 0.20081466395112016, + "threshold": 0.040444453593804865 + }, + { + "coverage": 0.8186666666666667, + "risk": 0.2007328990228013, + "threshold": 0.040224843571377115 + }, + { + "coverage": 0.819, + "risk": 0.20065120065120065, + "threshold": 0.040223308809295005 + }, + { + "coverage": 0.8193333333333334, + "risk": 0.20056956875508544, + "threshold": 0.03949496313397095 + }, + { + "coverage": 0.8196666666666667, + "risk": 0.20048800325335503, + "threshold": 0.03946855928857184 + }, + { + "coverage": 0.82, + "risk": 0.20040650406504065, + "threshold": 0.03913887218886763 + }, + { + "coverage": 0.8203333333333334, + "risk": 0.20032507110930517, + "threshold": 0.039036435083725035 + }, + { + "coverage": 0.8206666666666667, + "risk": 0.20024370430544272, + "threshold": 0.03903143905221513 + }, + { + "coverage": 0.821, + "risk": 0.2001624035728786, + "threshold": 0.038984416820296325 + }, + { + "coverage": 0.8213333333333334, + "risk": 0.20008116883116883, + "threshold": 0.03893854511024127 + }, + { + "coverage": 0.8216666666666667, + "risk": 0.2, + "threshold": 0.03887441260174038 + }, + { + "coverage": 0.822, + "risk": 0.19991889699918897, + "threshold": 0.03885943549353799 + }, + { + "coverage": 0.8223333333333334, + "risk": 0.19983785974868262, + "threshold": 0.038734646418321486 + }, + { + "coverage": 0.8226666666666667, + "risk": 0.19975688816855752, + "threshold": 0.03867827471044891 + }, + { + "coverage": 0.823, + "risk": 0.19967598217901986, + "threshold": 0.038575592952498135 + }, + { + "coverage": 0.8233333333333334, + "risk": 0.19959514170040485, + "threshold": 0.038519489767059774 + }, + { + "coverage": 0.8236666666666667, + "risk": 0.19951436665317684, + "threshold": 0.038441815942034836 + }, + { + "coverage": 0.824, + "risk": 0.1994336569579288, + "threshold": 0.03836611690408317 + }, + { + "coverage": 0.8243333333333334, + "risk": 0.19935301253538212, + "threshold": 0.0383180362747735 + }, + { + "coverage": 0.8246666666666667, + "risk": 0.19927243330638642, + "threshold": 0.03809116989737152 + }, + { + "coverage": 0.825, + "risk": 0.1991919191919192, + "threshold": 0.03801984553336923 + }, + { + "coverage": 0.8253333333333334, + "risk": 0.1991114701130856, + "threshold": 0.03777925587382282 + }, + { + "coverage": 0.8256666666666667, + "risk": 0.1990310859911183, + "threshold": 0.037667548703966075 + }, + { + "coverage": 0.826, + "risk": 0.1989507667473769, + "threshold": 0.03759402862677727 + }, + { + "coverage": 0.8263333333333334, + "risk": 0.19887051230334812, + "threshold": 0.03755728726737347 + }, + { + "coverage": 0.8266666666666667, + "risk": 0.19879032258064516, + "threshold": 0.03753877957132533 + }, + { + "coverage": 0.827, + "risk": 0.19871019750100766, + "threshold": 0.03738766904526154 + }, + { + "coverage": 0.8273333333333334, + "risk": 0.19863013698630136, + "threshold": 0.0373867322985633 + }, + { + "coverage": 0.8276666666666667, + "risk": 0.19855014095851792, + "threshold": 0.03703013266692184 + }, + { + "coverage": 0.828, + "risk": 0.19847020933977455, + "threshold": 0.03701076865482434 + }, + { + "coverage": 0.8283333333333334, + "risk": 0.19839034205231387, + "threshold": 0.03697197403667917 + }, + { + "coverage": 0.8286666666666667, + "risk": 0.19831053901850362, + "threshold": 0.03681022223404854 + }, + { + "coverage": 0.829, + "risk": 0.19823080016083636, + "threshold": 0.03674300726191598 + }, + { + "coverage": 0.8293333333333334, + "risk": 0.19815112540192925, + "threshold": 0.03657771865789051 + }, + { + "coverage": 0.8296666666666667, + "risk": 0.1980715146645239, + "threshold": 0.036475145161047856 + }, + { + "coverage": 0.83, + "risk": 0.19799196787148593, + "threshold": 0.036333437380894514 + }, + { + "coverage": 0.8303333333333334, + "risk": 0.1979124849458049, + "threshold": 0.036312123503433456 + }, + { + "coverage": 0.8306666666666667, + "risk": 0.1978330658105939, + "threshold": 0.036223078929234265 + }, + { + "coverage": 0.831, + "risk": 0.19775371038908945, + "threshold": 0.03613554008383576 + }, + { + "coverage": 0.8313333333333334, + "risk": 0.19767441860465115, + "threshold": 0.036066879332439426 + }, + { + "coverage": 0.8316666666666667, + "risk": 0.1975951903807615, + "threshold": 0.03589937503160153 + }, + { + "coverage": 0.832, + "risk": 0.19751602564102563, + "threshold": 0.035815446608189416 + }, + { + "coverage": 0.8323333333333334, + "risk": 0.197436924309171, + "threshold": 0.03537212861590756 + }, + { + "coverage": 0.8326666666666667, + "risk": 0.19735788630904724, + "threshold": 0.03490821784960768 + }, + { + "coverage": 0.833, + "risk": 0.19727891156462585, + "threshold": 0.03480564691880903 + }, + { + "coverage": 0.8333333333333334, + "risk": 0.1972, + "threshold": 0.034691781345367094 + }, + { + "coverage": 0.8336666666666667, + "risk": 0.19712115153938425, + "threshold": 0.03468613722228305 + }, + { + "coverage": 0.834, + "risk": 0.19704236610711431, + "threshold": 0.034593044248267424 + }, + { + "coverage": 0.8343333333333334, + "risk": 0.19696364362764682, + "threshold": 0.03455447187375263 + }, + { + "coverage": 0.8346666666666667, + "risk": 0.1968849840255591, + "threshold": 0.03431049506663542 + }, + { + "coverage": 0.835, + "risk": 0.1968063872255489, + "threshold": 0.034308814751680836 + }, + { + "coverage": 0.8353333333333334, + "risk": 0.19672785315243416, + "threshold": 0.03414612291915176 + }, + { + "coverage": 0.8356666666666667, + "risk": 0.19664938173115276, + "threshold": 0.03407959086711247 + }, + { + "coverage": 0.836, + "risk": 0.19657097288676237, + "threshold": 0.03407713811973389 + }, + { + "coverage": 0.8363333333333334, + "risk": 0.19649262654444002, + "threshold": 0.033926202207450074 + }, + { + "coverage": 0.8366666666666667, + "risk": 0.19641434262948207, + "threshold": 0.03374885348367253 + }, + { + "coverage": 0.837, + "risk": 0.19633612106730386, + "threshold": 0.03353553991401304 + }, + { + "coverage": 0.8373333333333334, + "risk": 0.1962579617834395, + "threshold": 0.03348874353473182 + }, + { + "coverage": 0.8376666666666667, + "risk": 0.1961798647035416, + "threshold": 0.03346008328219509 + }, + { + "coverage": 0.838, + "risk": 0.19610182975338106, + "threshold": 0.03330276362910334 + }, + { + "coverage": 0.8383333333333334, + "risk": 0.1960238568588469, + "threshold": 0.03319801765964351 + }, + { + "coverage": 0.8386666666666667, + "risk": 0.19594594594594594, + "threshold": 0.03292222705619326 + }, + { + "coverage": 0.839, + "risk": 0.19586809694080254, + "threshold": 0.03291701864074973 + }, + { + "coverage": 0.8393333333333334, + "risk": 0.19579030976965847, + "threshold": 0.032873832416009344 + }, + { + "coverage": 0.8396666666666667, + "risk": 0.19571258435887257, + "threshold": 0.0328489879544715 + }, + { + "coverage": 0.84, + "risk": 0.19563492063492063, + "threshold": 0.03264790088113709 + }, + { + "coverage": 0.8403333333333334, + "risk": 0.1955573185243951, + "threshold": 0.03262848568609974 + }, + { + "coverage": 0.8406666666666667, + "risk": 0.19547977795400476, + "threshold": 0.032569810716791985 + }, + { + "coverage": 0.841, + "risk": 0.19540229885057472, + "threshold": 0.03256914929878929 + }, + { + "coverage": 0.8413333333333334, + "risk": 0.19532488114104596, + "threshold": 0.03250845631287455 + }, + { + "coverage": 0.8416666666666667, + "risk": 0.19524752475247525, + "threshold": 0.03235117743968437 + }, + { + "coverage": 0.842, + "risk": 0.19517022961203484, + "threshold": 0.03233610581962417 + }, + { + "coverage": 0.8423333333333334, + "risk": 0.19509299564701227, + "threshold": 0.032278035180463016 + }, + { + "coverage": 0.8426666666666667, + "risk": 0.19501582278481014, + "threshold": 0.03207634677101926 + }, + { + "coverage": 0.843, + "risk": 0.19493871095294582, + "threshold": 0.03198684304836223 + }, + { + "coverage": 0.8433333333333334, + "risk": 0.1948616600790514, + "threshold": 0.03189428423858622 + }, + { + "coverage": 0.8436666666666667, + "risk": 0.19478467009087316, + "threshold": 0.031862176090832355 + }, + { + "coverage": 0.844, + "risk": 0.19470774091627172, + "threshold": 0.031857779139605163 + }, + { + "coverage": 0.8443333333333334, + "risk": 0.19463087248322147, + "threshold": 0.031794146217398116 + }, + { + "coverage": 0.8446666666666667, + "risk": 0.19455406471981057, + "threshold": 0.03177938855267624 + }, + { + "coverage": 0.845, + "risk": 0.19447731755424064, + "threshold": 0.03175618130219149 + }, + { + "coverage": 0.8453333333333334, + "risk": 0.1944006309148265, + "threshold": 0.03166047624471945 + }, + { + "coverage": 0.8456666666666667, + "risk": 0.19432400472999606, + "threshold": 0.03162751112135087 + }, + { + "coverage": 0.846, + "risk": 0.19424743892829, + "threshold": 0.03159979010555791 + }, + { + "coverage": 0.8463333333333334, + "risk": 0.19417093343836156, + "threshold": 0.03154831570147383 + }, + { + "coverage": 0.8466666666666667, + "risk": 0.19409448818897637, + "threshold": 0.03148132020359661 + }, + { + "coverage": 0.847, + "risk": 0.1940181031090122, + "threshold": 0.03146494319536536 + }, + { + "coverage": 0.8473333333333334, + "risk": 0.1939417781274587, + "threshold": 0.031447275321999764 + }, + { + "coverage": 0.8476666666666667, + "risk": 0.19386551317341721, + "threshold": 0.031200251290857163 + }, + { + "coverage": 0.848, + "risk": 0.19378930817610063, + "threshold": 0.030942345999743832 + }, + { + "coverage": 0.8483333333333334, + "risk": 0.193713163064833, + "threshold": 0.030909780509644307 + }, + { + "coverage": 0.8486666666666667, + "risk": 0.19363707776904948, + "threshold": 0.030893870375132228 + }, + { + "coverage": 0.849, + "risk": 0.19356105221829603, + "threshold": 0.03065522337835013 + }, + { + "coverage": 0.8493333333333334, + "risk": 0.1934850863422292, + "threshold": 0.03055369928369286 + }, + { + "coverage": 0.8496666666666667, + "risk": 0.19340918007061592, + "threshold": 0.030524396322895497 + }, + { + "coverage": 0.85, + "risk": 0.19333333333333333, + "threshold": 0.030208893032404344 + }, + { + "coverage": 0.8503333333333334, + "risk": 0.1932575460603685, + "threshold": 0.02992638889866764 + }, + { + "coverage": 0.8506666666666667, + "risk": 0.19318181818181818, + "threshold": 0.029917809091254934 + }, + { + "coverage": 0.851, + "risk": 0.19310614962788875, + "threshold": 0.029913367460148433 + }, + { + "coverage": 0.8513333333333334, + "risk": 0.19303054032889586, + "threshold": 0.02982482367843422 + }, + { + "coverage": 0.8516666666666667, + "risk": 0.1929549902152642, + "threshold": 0.029817569943581732 + }, + { + "coverage": 0.852, + "risk": 0.1928794992175274, + "threshold": 0.029797024564905794 + }, + { + "coverage": 0.8523333333333334, + "risk": 0.19280406726632773, + "threshold": 0.029450458040524957 + }, + { + "coverage": 0.8526666666666667, + "risk": 0.19272869429241596, + "threshold": 0.029440963301765346 + }, + { + "coverage": 0.853, + "risk": 0.19265338022665104, + "threshold": 0.029299502638380548 + }, + { + "coverage": 0.8533333333333334, + "risk": 0.192578125, + "threshold": 0.029216206465303013 + }, + { + "coverage": 0.8536666666666667, + "risk": 0.19250292854353768, + "threshold": 0.029182866049331548 + }, + { + "coverage": 0.854, + "risk": 0.19242779078844652, + "threshold": 0.029151589520088506 + }, + { + "coverage": 0.8543333333333333, + "risk": 0.1923527116660164, + "threshold": 0.028908321324910242 + }, + { + "coverage": 0.8546666666666667, + "risk": 0.1922776911076443, + "threshold": 0.028617911923700994 + }, + { + "coverage": 0.855, + "risk": 0.19220272904483432, + "threshold": 0.028588441032807846 + }, + { + "coverage": 0.8553333333333333, + "risk": 0.1921278254091972, + "threshold": 0.028529894816538986 + }, + { + "coverage": 0.8556666666666667, + "risk": 0.19205298013245034, + "threshold": 0.028508007678986393 + }, + { + "coverage": 0.856, + "risk": 0.19197819314641745, + "threshold": 0.02847369869686995 + }, + { + "coverage": 0.8563333333333333, + "risk": 0.1919034643830284, + "threshold": 0.028467217727173963 + }, + { + "coverage": 0.8566666666666667, + "risk": 0.19182879377431908, + "threshold": 0.028405251955430543 + }, + { + "coverage": 0.857, + "risk": 0.19175418125243096, + "threshold": 0.02827994024784048 + }, + { + "coverage": 0.8573333333333333, + "risk": 0.1916796267496112, + "threshold": 0.02822525836456184 + }, + { + "coverage": 0.8576666666666667, + "risk": 0.1916051301982122, + "threshold": 0.02801657487116348 + }, + { + "coverage": 0.858, + "risk": 0.19153069153069152, + "threshold": 0.027818794316333207 + }, + { + "coverage": 0.8583333333333333, + "risk": 0.19145631067961166, + "threshold": 0.02779790682677905 + }, + { + "coverage": 0.8586666666666667, + "risk": 0.19138198757763975, + "threshold": 0.027664494592740275 + }, + { + "coverage": 0.859, + "risk": 0.19130772215754754, + "threshold": 0.02764704550359938 + }, + { + "coverage": 0.8593333333333333, + "risk": 0.191233514352211, + "threshold": 0.027376794658955157 + }, + { + "coverage": 0.8596666666666667, + "risk": 0.19115936409461032, + "threshold": 0.027204381144536428 + }, + { + "coverage": 0.86, + "risk": 0.19108527131782946, + "threshold": 0.026941675446520173 + }, + { + "coverage": 0.8603333333333333, + "risk": 0.19101123595505617, + "threshold": 0.026935758260857798 + }, + { + "coverage": 0.8606666666666667, + "risk": 0.19093725793958172, + "threshold": 0.02684318704175335 + }, + { + "coverage": 0.861, + "risk": 0.19086333720480061, + "threshold": 0.026817081937745346 + }, + { + "coverage": 0.8613333333333333, + "risk": 0.19078947368421054, + "threshold": 0.0266498711147421 + }, + { + "coverage": 0.8616666666666667, + "risk": 0.19071566731141198, + "threshold": 0.026623230340044844 + }, + { + "coverage": 0.862, + "risk": 0.19064191802010827, + "threshold": 0.026591321635967308 + }, + { + "coverage": 0.8623333333333333, + "risk": 0.19056822574410515, + "threshold": 0.02633587559135226 + }, + { + "coverage": 0.8626666666666667, + "risk": 0.19049459041731068, + "threshold": 0.026330842354575383 + }, + { + "coverage": 0.863, + "risk": 0.19042101197373504, + "threshold": 0.02628772604708922 + }, + { + "coverage": 0.8633333333333333, + "risk": 0.19034749034749035, + "threshold": 0.02612947046465373 + }, + { + "coverage": 0.8636666666666667, + "risk": 0.19027402547279043, + "threshold": 0.026093821043107375 + }, + { + "coverage": 0.864, + "risk": 0.19020061728395063, + "threshold": 0.026053230623653147 + }, + { + "coverage": 0.8643333333333333, + "risk": 0.19012726571538757, + "threshold": 0.025868779844610373 + }, + { + "coverage": 0.8646666666666667, + "risk": 0.1900539707016191, + "threshold": 0.025855283838558504 + }, + { + "coverage": 0.865, + "risk": 0.18998073217726397, + "threshold": 0.025754268719490443 + }, + { + "coverage": 0.8653333333333333, + "risk": 0.1899075500770416, + "threshold": 0.025611981891344195 + }, + { + "coverage": 0.8656666666666667, + "risk": 0.18983442433577205, + "threshold": 0.025592128386625175 + }, + { + "coverage": 0.866, + "risk": 0.18976135488837567, + "threshold": 0.02536130917676633 + }, + { + "coverage": 0.8663333333333333, + "risk": 0.18968834166987303, + "threshold": 0.02528276947008569 + }, + { + "coverage": 0.8666666666666667, + "risk": 0.1896153846153846, + "threshold": 0.02527914763494649 + }, + { + "coverage": 0.867, + "risk": 0.1895424836601307, + "threshold": 0.025141948173506866 + }, + { + "coverage": 0.8673333333333333, + "risk": 0.1894696387394312, + "threshold": 0.02502647130975468 + }, + { + "coverage": 0.8676666666666667, + "risk": 0.18939684978870533, + "threshold": 0.024878890532984112 + }, + { + "coverage": 0.868, + "risk": 0.18932411674347158, + "threshold": 0.024749689231743033 + }, + { + "coverage": 0.8683333333333333, + "risk": 0.18925143953934742, + "threshold": 0.02461673798826794 + }, + { + "coverage": 0.8686666666666667, + "risk": 0.18917881811204912, + "threshold": 0.024483933162050633 + }, + { + "coverage": 0.869, + "risk": 0.18910625239739165, + "threshold": 0.024393531832410754 + }, + { + "coverage": 0.8693333333333333, + "risk": 0.18903374233128833, + "threshold": 0.024301152633056735 + }, + { + "coverage": 0.8696666666666667, + "risk": 0.18896128784975086, + "threshold": 0.02417605917358333 + }, + { + "coverage": 0.87, + "risk": 0.18888888888888888, + "threshold": 0.024170266598844175 + }, + { + "coverage": 0.8703333333333333, + "risk": 0.18881654538491, + "threshold": 0.024085461958591 + }, + { + "coverage": 0.8706666666666667, + "risk": 0.18874425727411945, + "threshold": 0.02406792017328658 + }, + { + "coverage": 0.871, + "risk": 0.18867202449292, + "threshold": 0.024067901807146196 + }, + { + "coverage": 0.8713333333333333, + "risk": 0.1885998469778118, + "threshold": 0.02402112316944882 + }, + { + "coverage": 0.8716666666666667, + "risk": 0.18852772466539197, + "threshold": 0.023998515068654854 + }, + { + "coverage": 0.872, + "risk": 0.18845565749235474, + "threshold": 0.02395579599904831 + }, + { + "coverage": 0.8723333333333333, + "risk": 0.18838364539549102, + "threshold": 0.023840953112668244 + }, + { + "coverage": 0.8726666666666667, + "risk": 0.18831168831168832, + "threshold": 0.023777651879492052 + }, + { + "coverage": 0.873, + "risk": 0.1882397861779305, + "threshold": 0.023749316401790187 + }, + { + "coverage": 0.8733333333333333, + "risk": 0.1881679389312977, + "threshold": 0.02349334970365614 + }, + { + "coverage": 0.8736666666666667, + "risk": 0.18809614650896606, + "threshold": 0.023485009668840127 + }, + { + "coverage": 0.874, + "risk": 0.18802440884820748, + "threshold": 0.02346409599385897 + }, + { + "coverage": 0.8743333333333333, + "risk": 0.18795272588638964, + "threshold": 0.02344069727773237 + }, + { + "coverage": 0.8746666666666667, + "risk": 0.1878810975609756, + "threshold": 0.023351449344807814 + }, + { + "coverage": 0.875, + "risk": 0.18780952380952382, + "threshold": 0.023206578257544144 + }, + { + "coverage": 0.8753333333333333, + "risk": 0.18773800456968773, + "threshold": 0.023087028739790766 + }, + { + "coverage": 0.8756666666666667, + "risk": 0.18766653977921582, + "threshold": 0.023013552180908094 + }, + { + "coverage": 0.876, + "risk": 0.1875951293759513, + "threshold": 0.0229300346363425 + }, + { + "coverage": 0.8763333333333333, + "risk": 0.18752377329783187, + "threshold": 0.022879943105534598 + }, + { + "coverage": 0.8766666666666667, + "risk": 0.18745247148288974, + "threshold": 0.022665767516587575 + }, + { + "coverage": 0.877, + "risk": 0.18738122386925124, + "threshold": 0.022477378315015707 + }, + { + "coverage": 0.8773333333333333, + "risk": 0.18731003039513677, + "threshold": 0.02239910106912936 + }, + { + "coverage": 0.8776666666666667, + "risk": 0.18723889099886062, + "threshold": 0.02237470670010106 + }, + { + "coverage": 0.878, + "risk": 0.18716780561883067, + "threshold": 0.022300644875476925 + }, + { + "coverage": 0.8783333333333333, + "risk": 0.1870967741935484, + "threshold": 0.022231405149210768 + }, + { + "coverage": 0.8786666666666667, + "risk": 0.18702579666160848, + "threshold": 0.022104586251519717 + }, + { + "coverage": 0.879, + "risk": 0.1869548729616989, + "threshold": 0.022083017564794544 + }, + { + "coverage": 0.8793333333333333, + "risk": 0.18688400303260044, + "threshold": 0.021864935609178147 + }, + { + "coverage": 0.8796666666666667, + "risk": 0.18681318681318682, + "threshold": 0.021861251171931478 + }, + { + "coverage": 0.88, + "risk": 0.18674242424242424, + "threshold": 0.021549591577153956 + }, + { + "coverage": 0.8803333333333333, + "risk": 0.18667171525937146, + "threshold": 0.021502828756351525 + }, + { + "coverage": 0.8806666666666667, + "risk": 0.1866010598031794, + "threshold": 0.02146218222502007 + }, + { + "coverage": 0.881, + "risk": 0.1865304578130912, + "threshold": 0.021441388827513293 + }, + { + "coverage": 0.8813333333333333, + "risk": 0.18645990922844174, + "threshold": 0.021388981543043663 + }, + { + "coverage": 0.8816666666666667, + "risk": 0.18638941398865785, + "threshold": 0.02117065393226372 + }, + { + "coverage": 0.882, + "risk": 0.18631897203325776, + "threshold": 0.021136395576084488 + }, + { + "coverage": 0.8823333333333333, + "risk": 0.18624858330185115, + "threshold": 0.021082015686807863 + }, + { + "coverage": 0.8826666666666667, + "risk": 0.18617824773413896, + "threshold": 0.021019663334884798 + }, + { + "coverage": 0.883, + "risk": 0.18610796526991316, + "threshold": 0.02101375793851029 + }, + { + "coverage": 0.8833333333333333, + "risk": 0.1860377358490566, + "threshold": 0.02097285766241925 + }, + { + "coverage": 0.8836666666666667, + "risk": 0.1859675594115428, + "threshold": 0.020904816386120597 + }, + { + "coverage": 0.884, + "risk": 0.1858974358974359, + "threshold": 0.020857438137292576 + }, + { + "coverage": 0.8843333333333333, + "risk": 0.1858273652468903, + "threshold": 0.020801500420109302 + }, + { + "coverage": 0.8846666666666667, + "risk": 0.1857573474001507, + "threshold": 0.020700840381741362 + }, + { + "coverage": 0.885, + "risk": 0.1856873822975518, + "threshold": 0.020627555373780755 + }, + { + "coverage": 0.8853333333333333, + "risk": 0.18561746987951808, + "threshold": 0.0204551369998336 + }, + { + "coverage": 0.8856666666666667, + "risk": 0.18554761008656379, + "threshold": 0.020272588603261334 + }, + { + "coverage": 0.886, + "risk": 0.1854778028592927, + "threshold": 0.0202706349625523 + }, + { + "coverage": 0.8863333333333333, + "risk": 0.1854080481383979, + "threshold": 0.020262255094319784 + }, + { + "coverage": 0.8866666666666667, + "risk": 0.18533834586466166, + "threshold": 0.020220417133150623 + }, + { + "coverage": 0.887, + "risk": 0.1852686959789553, + "threshold": 0.020212804093654505 + }, + { + "coverage": 0.8873333333333333, + "risk": 0.18519909842223892, + "threshold": 0.020089647657541 + }, + { + "coverage": 0.8876666666666667, + "risk": 0.1851295531355614, + "threshold": 0.02006494215389249 + }, + { + "coverage": 0.888, + "risk": 0.18506006006006007, + "threshold": 0.020030084007052907 + }, + { + "coverage": 0.8883333333333333, + "risk": 0.1849906191369606, + "threshold": 0.02002070102989747 + }, + { + "coverage": 0.8886666666666667, + "risk": 0.1849212303075769, + "threshold": 0.01999099822248061 + }, + { + "coverage": 0.889, + "risk": 0.18485189351331083, + "threshold": 0.01978588213967454 + }, + { + "coverage": 0.8893333333333333, + "risk": 0.18478260869565216, + "threshold": 0.019746615468766597 + }, + { + "coverage": 0.8896666666666667, + "risk": 0.18471337579617833, + "threshold": 0.019666061890960554 + }, + { + "coverage": 0.89, + "risk": 0.1846441947565543, + "threshold": 0.01963241177734491 + }, + { + "coverage": 0.8903333333333333, + "risk": 0.1845750655185324, + "threshold": 0.019542481002164214 + }, + { + "coverage": 0.8906666666666667, + "risk": 0.1845059880239521, + "threshold": 0.019435611497566726 + }, + { + "coverage": 0.891, + "risk": 0.18443696221473999, + "threshold": 0.019367373341206258 + }, + { + "coverage": 0.8913333333333333, + "risk": 0.1843679880329095, + "threshold": 0.01923238078502829 + }, + { + "coverage": 0.8916666666666667, + "risk": 0.18429906542056074, + "threshold": 0.01916711737328591 + }, + { + "coverage": 0.892, + "risk": 0.1842301943198804, + "threshold": 0.01915914238236343 + }, + { + "coverage": 0.8923333333333333, + "risk": 0.1841613746731416, + "threshold": 0.019087427751523337 + }, + { + "coverage": 0.8926666666666667, + "risk": 0.1840926064227035, + "threshold": 0.019056815776671263 + }, + { + "coverage": 0.893, + "risk": 0.18402388951101156, + "threshold": 0.01896017048007987 + }, + { + "coverage": 0.8933333333333333, + "risk": 0.183955223880597, + "threshold": 0.01883044607546156 + }, + { + "coverage": 0.8936666666666667, + "risk": 0.18388660947407684, + "threshold": 0.018805853809112212 + }, + { + "coverage": 0.894, + "risk": 0.18381804623415363, + "threshold": 0.01879907241486046 + }, + { + "coverage": 0.8943333333333333, + "risk": 0.18374953410361536, + "threshold": 0.018692869956970577 + }, + { + "coverage": 0.8946666666666667, + "risk": 0.18368107302533532, + "threshold": 0.018685263936526114 + }, + { + "coverage": 0.895, + "risk": 0.18361266294227188, + "threshold": 0.01866906525215442 + }, + { + "coverage": 0.8953333333333333, + "risk": 0.18354430379746836, + "threshold": 0.01866108385233521 + }, + { + "coverage": 0.8956666666666667, + "risk": 0.18347599553405286, + "threshold": 0.018577664680371097 + }, + { + "coverage": 0.896, + "risk": 0.18340773809523808, + "threshold": 0.018525369727203762 + }, + { + "coverage": 0.8963333333333333, + "risk": 0.1833395314243213, + "threshold": 0.01852532132336899 + }, + { + "coverage": 0.8966666666666666, + "risk": 0.18327137546468403, + "threshold": 0.018501181701660242 + }, + { + "coverage": 0.897, + "risk": 0.1832032701597919, + "threshold": 0.01840119558595175 + }, + { + "coverage": 0.8973333333333333, + "risk": 0.18313521545319464, + "threshold": 0.0183814057172939 + }, + { + "coverage": 0.8976666666666666, + "risk": 0.1830672112885258, + "threshold": 0.018195791267363724 + }, + { + "coverage": 0.898, + "risk": 0.18299925760950259, + "threshold": 0.018173205017206852 + }, + { + "coverage": 0.8983333333333333, + "risk": 0.1829313543599258, + "threshold": 0.018066256280513487 + }, + { + "coverage": 0.8986666666666666, + "risk": 0.18286350148367952, + "threshold": 0.01804593837212949 + }, + { + "coverage": 0.899, + "risk": 0.1827956989247312, + "threshold": 0.018041320723575498 + }, + { + "coverage": 0.8993333333333333, + "risk": 0.1827279466271312, + "threshold": 0.017884588960071515 + }, + { + "coverage": 0.8996666666666666, + "risk": 0.18266024453501298, + "threshold": 0.01784595927690371 + }, + { + "coverage": 0.9, + "risk": 0.18259259259259258, + "threshold": 0.0177954433126585 + }, + { + "coverage": 0.9003333333333333, + "risk": 0.18252499074416884, + "threshold": 0.017766221501053777 + }, + { + "coverage": 0.9006666666666666, + "risk": 0.18245743893412286, + "threshold": 0.017676605964452235 + }, + { + "coverage": 0.901, + "risk": 0.18238993710691823, + "threshold": 0.01765266999787001 + }, + { + "coverage": 0.9013333333333333, + "risk": 0.1823224852071006, + "threshold": 0.017578090996918278 + }, + { + "coverage": 0.9016666666666666, + "risk": 0.1822550831792976, + "threshold": 0.017572073226426354 + }, + { + "coverage": 0.902, + "risk": 0.18255728011825573, + "threshold": 0.01714877142526065 + }, + { + "coverage": 0.9023333333333333, + "risk": 0.18248984115256742, + "threshold": 0.017140296184977814 + }, + { + "coverage": 0.9026666666666666, + "risk": 0.18242245199409157, + "threshold": 0.01700873382301706 + }, + { + "coverage": 0.903, + "risk": 0.18235511258767073, + "threshold": 0.016972436903930112 + }, + { + "coverage": 0.9033333333333333, + "risk": 0.18228782287822878, + "threshold": 0.016887539206008263 + }, + { + "coverage": 0.9036666666666666, + "risk": 0.18222058281077094, + "threshold": 0.016610526477068777 + }, + { + "coverage": 0.904, + "risk": 0.18215339233038347, + "threshold": 0.016541058717339907 + }, + { + "coverage": 0.9043333333333333, + "risk": 0.1820862513822337, + "threshold": 0.01647038058411047 + }, + { + "coverage": 0.9046666666666666, + "risk": 0.18201915991156964, + "threshold": 0.01640426306181593 + }, + { + "coverage": 0.905, + "risk": 0.18195211786372006, + "threshold": 0.016361460367104876 + }, + { + "coverage": 0.9053333333333333, + "risk": 0.18188512518409425, + "threshold": 0.016159895837551508 + }, + { + "coverage": 0.9056666666666666, + "risk": 0.18181818181818182, + "threshold": 0.016151811055692574 + }, + { + "coverage": 0.906, + "risk": 0.1817512877115526, + "threshold": 0.015984650180168373 + }, + { + "coverage": 0.9063333333333333, + "risk": 0.18168444280985657, + "threshold": 0.01595491035992694 + }, + { + "coverage": 0.9066666666666666, + "risk": 0.18161764705882352, + "threshold": 0.01589643295937549 + }, + { + "coverage": 0.907, + "risk": 0.18155090040426314, + "threshold": 0.01582487537427875 + }, + { + "coverage": 0.9073333333333333, + "risk": 0.18148420279206465, + "threshold": 0.015810756116381698 + }, + { + "coverage": 0.9076666666666666, + "risk": 0.18141755416819685, + "threshold": 0.015711624713035274 + }, + { + "coverage": 0.908, + "risk": 0.18135095447870778, + "threshold": 0.015686342762079496 + }, + { + "coverage": 0.9083333333333333, + "risk": 0.18128440366972476, + "threshold": 0.015651824554526555 + }, + { + "coverage": 0.9086666666666666, + "risk": 0.18121790168745414, + "threshold": 0.015625989460128574 + }, + { + "coverage": 0.909, + "risk": 0.18115144847818115, + "threshold": 0.015625499916621776 + }, + { + "coverage": 0.9093333333333333, + "risk": 0.1810850439882698, + "threshold": 0.015623700972306448 + }, + { + "coverage": 0.9096666666666666, + "risk": 0.1810186881641627, + "threshold": 0.015546134624124502 + }, + { + "coverage": 0.91, + "risk": 0.18095238095238095, + "threshold": 0.015485898297467288 + }, + { + "coverage": 0.9103333333333333, + "risk": 0.18088612229952397, + "threshold": 0.015451364809050232 + }, + { + "coverage": 0.9106666666666666, + "risk": 0.1808199121522694, + "threshold": 0.015448250100588775 + }, + { + "coverage": 0.911, + "risk": 0.18075375045737285, + "threshold": 0.015405664848457441 + }, + { + "coverage": 0.9113333333333333, + "risk": 0.18068763716166789, + "threshold": 0.015301319441837078 + }, + { + "coverage": 0.9116666666666666, + "risk": 0.18062157221206582, + "threshold": 0.01525811530737671 + }, + { + "coverage": 0.912, + "risk": 0.18055555555555555, + "threshold": 0.01515609715616834 + }, + { + "coverage": 0.9123333333333333, + "risk": 0.18048958713920352, + "threshold": 0.015100901165802063 + }, + { + "coverage": 0.9126666666666666, + "risk": 0.1804236669101534, + "threshold": 0.015014909009108129 + }, + { + "coverage": 0.913, + "risk": 0.18035779481562614, + "threshold": 0.015001512711249233 + }, + { + "coverage": 0.9133333333333333, + "risk": 0.1802919708029197, + "threshold": 0.014933207291330143 + }, + { + "coverage": 0.9136666666666666, + "risk": 0.18022619481940896, + "threshold": 0.014845820489723325 + }, + { + "coverage": 0.914, + "risk": 0.18016046681254558, + "threshold": 0.014823088940783276 + }, + { + "coverage": 0.9143333333333333, + "risk": 0.18009478672985782, + "threshold": 0.014795414025483157 + }, + { + "coverage": 0.9146666666666666, + "risk": 0.18002915451895043, + "threshold": 0.014747341885217279 + }, + { + "coverage": 0.915, + "risk": 0.17996357012750455, + "threshold": 0.014554031824128826 + }, + { + "coverage": 0.9153333333333333, + "risk": 0.1798980335032775, + "threshold": 0.014516738279466044 + }, + { + "coverage": 0.9156666666666666, + "risk": 0.17983254459410267, + "threshold": 0.014499562124163893 + }, + { + "coverage": 0.916, + "risk": 0.17976710334788937, + "threshold": 0.014413435761233107 + }, + { + "coverage": 0.9163333333333333, + "risk": 0.17970170971262278, + "threshold": 0.01432519156260573 + }, + { + "coverage": 0.9166666666666666, + "risk": 0.17963636363636365, + "threshold": 0.014323762642089476 + }, + { + "coverage": 0.917, + "risk": 0.17957106506724826, + "threshold": 0.014307098673560475 + }, + { + "coverage": 0.9173333333333333, + "risk": 0.17950581395348839, + "threshold": 0.014226308160989883 + }, + { + "coverage": 0.9176666666666666, + "risk": 0.17944061024337088, + "threshold": 0.014181892918674293 + }, + { + "coverage": 0.918, + "risk": 0.1793754538852578, + "threshold": 0.01416928303844646 + }, + { + "coverage": 0.9183333333333333, + "risk": 0.1793103448275862, + "threshold": 0.01405800167780165 + }, + { + "coverage": 0.9186666666666666, + "risk": 0.1792452830188679, + "threshold": 0.014022851865838295 + }, + { + "coverage": 0.919, + "risk": 0.17918026840768952, + "threshold": 0.013952334738139898 + }, + { + "coverage": 0.9193333333333333, + "risk": 0.1791153009427121, + "threshold": 0.013929904890323497 + }, + { + "coverage": 0.9196666666666666, + "risk": 0.17905038057267125, + "threshold": 0.013874582777735431 + }, + { + "coverage": 0.92, + "risk": 0.17898550724637682, + "threshold": 0.013838224011859338 + }, + { + "coverage": 0.9203333333333333, + "risk": 0.1789206809127128, + "threshold": 0.013825278484435334 + }, + { + "coverage": 0.9206666666666666, + "risk": 0.1788559015206372, + "threshold": 0.013662783940333469 + }, + { + "coverage": 0.921, + "risk": 0.17879116901918204, + "threshold": 0.013654239269823381 + }, + { + "coverage": 0.9213333333333333, + "risk": 0.17872648335745298, + "threshold": 0.013595241628332028 + }, + { + "coverage": 0.9216666666666666, + "risk": 0.1786618444846293, + "threshold": 0.013495174469438266 + }, + { + "coverage": 0.922, + "risk": 0.17859725234996385, + "threshold": 0.013425742163397852 + }, + { + "coverage": 0.9223333333333333, + "risk": 0.1785327069027828, + "threshold": 0.013382538193689538 + }, + { + "coverage": 0.9226666666666666, + "risk": 0.17846820809248554, + "threshold": 0.01321341224200467 + }, + { + "coverage": 0.923, + "risk": 0.1784037558685446, + "threshold": 0.01319163083035171 + }, + { + "coverage": 0.9233333333333333, + "risk": 0.17833935018050542, + "threshold": 0.013107643456578506 + }, + { + "coverage": 0.9236666666666666, + "risk": 0.1782749909779863, + "threshold": 0.013077649855340838 + }, + { + "coverage": 0.924, + "risk": 0.1782106782106782, + "threshold": 0.013010523513414538 + }, + { + "coverage": 0.9243333333333333, + "risk": 0.17814641182834476, + "threshold": 0.013010489555826439 + }, + { + "coverage": 0.9246666666666666, + "risk": 0.1780821917808219, + "threshold": 0.013008913626777802 + }, + { + "coverage": 0.925, + "risk": 0.17801801801801803, + "threshold": 0.012993841949861763 + }, + { + "coverage": 0.9253333333333333, + "risk": 0.17795389048991356, + "threshold": 0.012958991542873445 + }, + { + "coverage": 0.9256666666666666, + "risk": 0.17788980914656105, + "threshold": 0.012956258313302441 + }, + { + "coverage": 0.926, + "risk": 0.17782577393808496, + "threshold": 0.012888224525032847 + }, + { + "coverage": 0.9263333333333333, + "risk": 0.17776178481468155, + "threshold": 0.012846144056609892 + }, + { + "coverage": 0.9266666666666666, + "risk": 0.1776978417266187, + "threshold": 0.012752216049963612 + }, + { + "coverage": 0.927, + "risk": 0.17763394462423587, + "threshold": 0.01273764781847543 + }, + { + "coverage": 0.9273333333333333, + "risk": 0.17757009345794392, + "threshold": 0.012621676201878923 + }, + { + "coverage": 0.9276666666666666, + "risk": 0.17750628817822495, + "threshold": 0.012593211715752194 + }, + { + "coverage": 0.928, + "risk": 0.17744252873563218, + "threshold": 0.012560960377086783 + }, + { + "coverage": 0.9283333333333333, + "risk": 0.17737881508078995, + "threshold": 0.012437520465783577 + }, + { + "coverage": 0.9286666666666666, + "risk": 0.1773151471643934, + "threshold": 0.012395004725690092 + }, + { + "coverage": 0.929, + "risk": 0.17725152493720847, + "threshold": 0.012323965450204146 + }, + { + "coverage": 0.9293333333333333, + "risk": 0.17718794835007173, + "threshold": 0.012283114729549984 + }, + { + "coverage": 0.9296666666666666, + "risk": 0.17712441735389028, + "threshold": 0.012247264680051578 + }, + { + "coverage": 0.93, + "risk": 0.1770609318996416, + "threshold": 0.012221536293334227 + }, + { + "coverage": 0.9303333333333333, + "risk": 0.17699749193837333, + "threshold": 0.012219300026717343 + }, + { + "coverage": 0.9306666666666666, + "risk": 0.17693409742120345, + "threshold": 0.012211770285814596 + }, + { + "coverage": 0.931, + "risk": 0.17687074829931973, + "threshold": 0.012102718275198713 + }, + { + "coverage": 0.9313333333333333, + "risk": 0.17680744452397995, + "threshold": 0.01207966275962612 + }, + { + "coverage": 0.9316666666666666, + "risk": 0.17674418604651163, + "threshold": 0.011980196825777271 + }, + { + "coverage": 0.932, + "risk": 0.17668097281831188, + "threshold": 0.01192379841942079 + }, + { + "coverage": 0.9323333333333333, + "risk": 0.17661780479084735, + "threshold": 0.011897745152481765 + }, + { + "coverage": 0.9326666666666666, + "risk": 0.17655468191565404, + "threshold": 0.01184658695258093 + }, + { + "coverage": 0.933, + "risk": 0.17649160414433726, + "threshold": 0.011767655267132639 + }, + { + "coverage": 0.9333333333333333, + "risk": 0.17642857142857143, + "threshold": 0.011619689586971288 + }, + { + "coverage": 0.9336666666666666, + "risk": 0.17636558372009997, + "threshold": 0.011611595621991032 + }, + { + "coverage": 0.934, + "risk": 0.1763026409707352, + "threshold": 0.011506084516832151 + }, + { + "coverage": 0.9343333333333333, + "risk": 0.17623974313235818, + "threshold": 0.011486224137896189 + }, + { + "coverage": 0.9346666666666666, + "risk": 0.17617689015691868, + "threshold": 0.011465436096954666 + }, + { + "coverage": 0.935, + "risk": 0.17611408199643494, + "threshold": 0.011328901473951951 + }, + { + "coverage": 0.9353333333333333, + "risk": 0.17605131860299358, + "threshold": 0.0112991766754235 + }, + { + "coverage": 0.9356666666666666, + "risk": 0.17598859992874955, + "threshold": 0.011286446893596647 + }, + { + "coverage": 0.936, + "risk": 0.17592592592592593, + "threshold": 0.011186049529693437 + }, + { + "coverage": 0.9363333333333334, + "risk": 0.17586329654681382, + "threshold": 0.011180140433174083 + }, + { + "coverage": 0.9366666666666666, + "risk": 0.17580071174377224, + "threshold": 0.011166770435427537 + }, + { + "coverage": 0.937, + "risk": 0.17573817146922804, + "threshold": 0.011138567629441206 + }, + { + "coverage": 0.9373333333333334, + "risk": 0.17567567567567569, + "threshold": 0.011131749379214089 + }, + { + "coverage": 0.9376666666666666, + "risk": 0.17561322431567722, + "threshold": 0.011091389898213023 + }, + { + "coverage": 0.938, + "risk": 0.17555081734186212, + "threshold": 0.011086328426838602 + }, + { + "coverage": 0.9383333333333334, + "risk": 0.17548845470692717, + "threshold": 0.011057074490267969 + }, + { + "coverage": 0.9386666666666666, + "risk": 0.17542613636363635, + "threshold": 0.01094751909016712 + }, + { + "coverage": 0.939, + "risk": 0.17536386226482073, + "threshold": 0.010936998678699851 + }, + { + "coverage": 0.9393333333333334, + "risk": 0.17530163236337828, + "threshold": 0.010912816196151155 + }, + { + "coverage": 0.9396666666666667, + "risk": 0.17523944661227386, + "threshold": 0.010697437642545488 + }, + { + "coverage": 0.94, + "risk": 0.175177304964539, + "threshold": 0.010574528621004128 + }, + { + "coverage": 0.9403333333333334, + "risk": 0.17511520737327188, + "threshold": 0.010561136576116468 + }, + { + "coverage": 0.9406666666666667, + "risk": 0.17505315379163713, + "threshold": 0.01052412342178001 + }, + { + "coverage": 0.941, + "risk": 0.17499114417286574, + "threshold": 0.010461083452791228 + }, + { + "coverage": 0.9413333333333334, + "risk": 0.17492917847025496, + "threshold": 0.010281560647686844 + }, + { + "coverage": 0.9416666666666667, + "risk": 0.17486725663716815, + "threshold": 0.010257908566101138 + }, + { + "coverage": 0.942, + "risk": 0.17480537862703469, + "threshold": 0.010231873465292588 + }, + { + "coverage": 0.9423333333333334, + "risk": 0.17474354439334985, + "threshold": 0.010016909201113327 + }, + { + "coverage": 0.9426666666666667, + "risk": 0.1746817538896747, + "threshold": 0.009949666837318398 + }, + { + "coverage": 0.943, + "risk": 0.1746200070696359, + "threshold": 0.00981546800872775 + }, + { + "coverage": 0.9433333333333334, + "risk": 0.1745583038869258, + "threshold": 0.009757040197911671 + }, + { + "coverage": 0.9436666666666667, + "risk": 0.174496644295302, + "threshold": 0.009746012995759619 + }, + { + "coverage": 0.944, + "risk": 0.17443502824858756, + "threshold": 0.009677507494122476 + }, + { + "coverage": 0.9443333333333334, + "risk": 0.17437345570067067, + "threshold": 0.009584095391978324 + }, + { + "coverage": 0.9446666666666667, + "risk": 0.1743119266055046, + "threshold": 0.009558893467718581 + }, + { + "coverage": 0.945, + "risk": 0.1742504409171076, + "threshold": 0.009484106939624615 + }, + { + "coverage": 0.9453333333333334, + "risk": 0.17418899858956277, + "threshold": 0.009462394389304728 + }, + { + "coverage": 0.9456666666666667, + "risk": 0.17412759957701798, + "threshold": 0.009455580302907237 + }, + { + "coverage": 0.946, + "risk": 0.17406624383368569, + "threshold": 0.009336922378967263 + }, + { + "coverage": 0.9463333333333334, + "risk": 0.1740049313138429, + "threshold": 0.009263745024693182 + }, + { + "coverage": 0.9466666666666667, + "risk": 0.17394366197183098, + "threshold": 0.00919906611396123 + }, + { + "coverage": 0.947, + "risk": 0.17388243576205561, + "threshold": 0.008917308792583489 + }, + { + "coverage": 0.9473333333333334, + "risk": 0.17382125263898662, + "threshold": 0.008887056785693785 + }, + { + "coverage": 0.9476666666666667, + "risk": 0.17376011255715793, + "threshold": 0.008850972101486693 + }, + { + "coverage": 0.948, + "risk": 0.17369901547116737, + "threshold": 0.00883760648965616 + }, + { + "coverage": 0.9483333333333334, + "risk": 0.17363796133567663, + "threshold": 0.008724006997037704 + }, + { + "coverage": 0.9486666666666667, + "risk": 0.1735769501054111, + "threshold": 0.008714169511748722 + }, + { + "coverage": 0.949, + "risk": 0.1735159817351598, + "threshold": 0.008707379376620122 + }, + { + "coverage": 0.9493333333333334, + "risk": 0.17345505617977527, + "threshold": 0.008680586844742905 + }, + { + "coverage": 0.9496666666666667, + "risk": 0.1733941733941734, + "threshold": 0.008672701949111404 + }, + { + "coverage": 0.95, + "risk": 0.17333333333333334, + "threshold": 0.008657145744312264 + }, + { + "coverage": 0.9503333333333334, + "risk": 0.17327253595229744, + "threshold": 0.008538673407023986 + }, + { + "coverage": 0.9506666666666667, + "risk": 0.17321178120617112, + "threshold": 0.008508408752252068 + }, + { + "coverage": 0.951, + "risk": 0.17315106905012267, + "threshold": 0.008500932884138012 + }, + { + "coverage": 0.9513333333333334, + "risk": 0.1730903994393833, + "threshold": 0.008376436342198791 + }, + { + "coverage": 0.9516666666666667, + "risk": 0.17302977232924693, + "threshold": 0.008206540759297805 + }, + { + "coverage": 0.952, + "risk": 0.17296918767507002, + "threshold": 0.008089723387616104 + }, + { + "coverage": 0.9523333333333334, + "risk": 0.1729086454322716, + "threshold": 0.008021928417560225 + }, + { + "coverage": 0.9526666666666667, + "risk": 0.1728481455563331, + "threshold": 0.008017217600860072 + }, + { + "coverage": 0.953, + "risk": 0.17278768800279817, + "threshold": 0.007981238380864767 + }, + { + "coverage": 0.9533333333333334, + "risk": 0.17272727272727273, + "threshold": 0.007921893216914028 + }, + { + "coverage": 0.9536666666666667, + "risk": 0.17266689968542467, + "threshold": 0.007893582920189018 + }, + { + "coverage": 0.954, + "risk": 0.17260656883298392, + "threshold": 0.007798493785057089 + }, + { + "coverage": 0.9543333333333334, + "risk": 0.17254628012574222, + "threshold": 0.0077845647320728874 + }, + { + "coverage": 0.9546666666666667, + "risk": 0.17248603351955308, + "threshold": 0.007765793480225813 + }, + { + "coverage": 0.955, + "risk": 0.1724258289703316, + "threshold": 0.007765347325285562 + }, + { + "coverage": 0.9553333333333334, + "risk": 0.17236566643405443, + "threshold": 0.0077340757621599835 + }, + { + "coverage": 0.9556666666666667, + "risk": 0.17230554586675967, + "threshold": 0.007688508758277502 + }, + { + "coverage": 0.956, + "risk": 0.17224546722454673, + "threshold": 0.00755114908081811 + }, + { + "coverage": 0.9563333333333334, + "risk": 0.17218543046357615, + "threshold": 0.00753810684314696 + }, + { + "coverage": 0.9566666666666667, + "risk": 0.17212543554006968, + "threshold": 0.007457038471145356 + }, + { + "coverage": 0.957, + "risk": 0.17206548241031, + "threshold": 0.007420295541922741 + }, + { + "coverage": 0.9573333333333334, + "risk": 0.17200557103064068, + "threshold": 0.007386524972565427 + }, + { + "coverage": 0.9576666666666667, + "risk": 0.17194570135746606, + "threshold": 0.007373681165362861 + }, + { + "coverage": 0.958, + "risk": 0.17188587334725122, + "threshold": 0.007360848970612749 + }, + { + "coverage": 0.9583333333333334, + "risk": 0.17182608695652174, + "threshold": 0.007343803245294024 + }, + { + "coverage": 0.9586666666666667, + "risk": 0.1717663421418637, + "threshold": 0.007321125032574761 + }, + { + "coverage": 0.959, + "risk": 0.17170663885992354, + "threshold": 0.007263182178275967 + }, + { + "coverage": 0.9593333333333334, + "risk": 0.17164697706740792, + "threshold": 0.007083492563430478 + }, + { + "coverage": 0.9596666666666667, + "risk": 0.17158735672108372, + "threshold": 0.0070274345476333926 + }, + { + "coverage": 0.96, + "risk": 0.17152777777777778, + "threshold": 0.006981166427575704 + }, + { + "coverage": 0.9603333333333334, + "risk": 0.17146824019437695, + "threshold": 0.006916412278117509 + }, + { + "coverage": 0.9606666666666667, + "risk": 0.1714087439278279, + "threshold": 0.0068844165645753335 + }, + { + "coverage": 0.961, + "risk": 0.171349288935137, + "threshold": 0.006865057970361638 + }, + { + "coverage": 0.9613333333333334, + "risk": 0.17128987517337033, + "threshold": 0.006863762324825616 + }, + { + "coverage": 0.9616666666666667, + "risk": 0.1712305025996534, + "threshold": 0.006845701305641802 + }, + { + "coverage": 0.962, + "risk": 0.17117117117117117, + "threshold": 0.0068451461531128166 + }, + { + "coverage": 0.9623333333333334, + "risk": 0.17111188084516799, + "threshold": 0.006818277479151654 + }, + { + "coverage": 0.9626666666666667, + "risk": 0.17105263157894737, + "threshold": 0.006766957361306546 + }, + { + "coverage": 0.963, + "risk": 0.17099342332987194, + "threshold": 0.006620448602443383 + }, + { + "coverage": 0.9633333333333334, + "risk": 0.17093425605536333, + "threshold": 0.006614600091875795 + }, + { + "coverage": 0.9636666666666667, + "risk": 0.1708751297129021, + "threshold": 0.006580678862271318 + }, + { + "coverage": 0.964, + "risk": 0.17081604426002767, + "threshold": 0.0065456616440852725 + }, + { + "coverage": 0.9643333333333334, + "risk": 0.17075699965433805, + "threshold": 0.006485000580747179 + }, + { + "coverage": 0.9646666666666667, + "risk": 0.17069799585349, + "threshold": 0.0064126918809576 + }, + { + "coverage": 0.965, + "risk": 0.17063903281519863, + "threshold": 0.0061992812955319766 + }, + { + "coverage": 0.9653333333333334, + "risk": 0.17058011049723756, + "threshold": 0.006139444103613369 + }, + { + "coverage": 0.9656666666666667, + "risk": 0.17052122885743873, + "threshold": 0.0060524255888211665 + }, + { + "coverage": 0.966, + "risk": 0.1704623878536922, + "threshold": 0.00605142799217619 + }, + { + "coverage": 0.9663333333333334, + "risk": 0.17040358744394618, + "threshold": 0.006042555589193283 + }, + { + "coverage": 0.9666666666666667, + "risk": 0.1703448275862069, + "threshold": 0.00597137760876558 + }, + { + "coverage": 0.967, + "risk": 0.17028610823853843, + "threshold": 0.005968406979301946 + }, + { + "coverage": 0.9673333333333334, + "risk": 0.1702274293590627, + "threshold": 0.00594309539320495 + }, + { + "coverage": 0.9676666666666667, + "risk": 0.17016879090595935, + "threshold": 0.005849704448174114 + }, + { + "coverage": 0.968, + "risk": 0.17011019283746556, + "threshold": 0.005799263560322745 + }, + { + "coverage": 0.9683333333333334, + "risk": 0.17005163511187607, + "threshold": 0.005716464553177768 + }, + { + "coverage": 0.9686666666666667, + "risk": 0.16999311768754302, + "threshold": 0.0056797873723510904 + }, + { + "coverage": 0.969, + "risk": 0.16993464052287582, + "threshold": 0.005622520232921386 + }, + { + "coverage": 0.9693333333333334, + "risk": 0.16987620357634112, + "threshold": 0.005580832090355399 + }, + { + "coverage": 0.9696666666666667, + "risk": 0.1698178068064627, + "threshold": 0.005560826492499625 + }, + { + "coverage": 0.97, + "risk": 0.1697594501718213, + "threshold": 0.005459628744201745 + }, + { + "coverage": 0.9703333333333334, + "risk": 0.16970113363105463, + "threshold": 0.005438437109403812 + }, + { + "coverage": 0.9706666666666667, + "risk": 0.16964285714285715, + "threshold": 0.005438140115907972 + }, + { + "coverage": 0.971, + "risk": 0.1695846206659801, + "threshold": 0.00542002985579263 + }, + { + "coverage": 0.9713333333333334, + "risk": 0.1695264241592313, + "threshold": 0.005376750962287704 + }, + { + "coverage": 0.9716666666666667, + "risk": 0.16946826758147512, + "threshold": 0.005352635281067446 + }, + { + "coverage": 0.972, + "risk": 0.16941015089163236, + "threshold": 0.005339821719777975 + }, + { + "coverage": 0.9723333333333334, + "risk": 0.16935207404868016, + "threshold": 0.005307191120900886 + }, + { + "coverage": 0.9726666666666667, + "risk": 0.16929403701165183, + "threshold": 0.005260934267427338 + }, + { + "coverage": 0.973, + "risk": 0.16923603973963686, + "threshold": 0.005239408060514732 + }, + { + "coverage": 0.9733333333333334, + "risk": 0.16917808219178082, + "threshold": 0.005223580532763886 + }, + { + "coverage": 0.9736666666666667, + "risk": 0.1691201643272852, + "threshold": 0.005209566195239626 + }, + { + "coverage": 0.974, + "risk": 0.16906228610540724, + "threshold": 0.005106114023632553 + }, + { + "coverage": 0.9743333333333334, + "risk": 0.16900444748546015, + "threshold": 0.005104027072263737 + }, + { + "coverage": 0.9746666666666667, + "risk": 0.1689466484268126, + "threshold": 0.005068611151762725 + }, + { + "coverage": 0.975, + "risk": 0.1688888888888889, + "threshold": 0.0050592321852995946 + }, + { + "coverage": 0.9753333333333334, + "risk": 0.16883116883116883, + "threshold": 0.005055218598923757 + }, + { + "coverage": 0.9756666666666667, + "risk": 0.16877348821318755, + "threshold": 0.005023850993589009 + }, + { + "coverage": 0.976, + "risk": 0.1687158469945355, + "threshold": 0.004971541095674734 + }, + { + "coverage": 0.9763333333333334, + "risk": 0.1686582451348583, + "threshold": 0.0049156116022687915 + }, + { + "coverage": 0.9766666666666667, + "risk": 0.16860068259385666, + "threshold": 0.004822653726711263 + }, + { + "coverage": 0.977, + "risk": 0.16854315933128625, + "threshold": 0.004787514220557723 + }, + { + "coverage": 0.9773333333333334, + "risk": 0.1684856753069577, + "threshold": 0.004743830337404391 + }, + { + "coverage": 0.9776666666666667, + "risk": 0.16842823048073643, + "threshold": 0.004720453690163472 + }, + { + "coverage": 0.978, + "risk": 0.1683708248125426, + "threshold": 0.00459126247412541 + }, + { + "coverage": 0.9783333333333334, + "risk": 0.16831345826235095, + "threshold": 0.004556439423182028 + }, + { + "coverage": 0.9786666666666667, + "risk": 0.16825613079019072, + "threshold": 0.004498531732066736 + }, + { + "coverage": 0.979, + "risk": 0.16819884235614574, + "threshold": 0.004375601674275334 + }, + { + "coverage": 0.9793333333333333, + "risk": 0.168141592920354, + "threshold": 0.004271048398391377 + }, + { + "coverage": 0.9796666666666667, + "risk": 0.16808438244300783, + "threshold": 0.004246873371867178 + }, + { + "coverage": 0.98, + "risk": 0.16802721088435374, + "threshold": 0.00414859745486226 + }, + { + "coverage": 0.9803333333333333, + "risk": 0.16797007820469229, + "threshold": 0.004130900635414592 + }, + { + "coverage": 0.9806666666666667, + "risk": 0.167912984364378, + "threshold": 0.004096464991567567 + }, + { + "coverage": 0.981, + "risk": 0.16785592932381924, + "threshold": 0.004095260789170414 + }, + { + "coverage": 0.9813333333333333, + "risk": 0.16779891304347827, + "threshold": 0.004044666254702659 + }, + { + "coverage": 0.9816666666666667, + "risk": 0.16774193548387098, + "threshold": 0.003983602126516787 + }, + { + "coverage": 0.982, + "risk": 0.16768499660556688, + "threshold": 0.0039046125690171875 + }, + { + "coverage": 0.9823333333333333, + "risk": 0.16762809636918902, + "threshold": 0.003883423342250592 + }, + { + "coverage": 0.9826666666666667, + "risk": 0.16757123473541383, + "threshold": 0.003737748147559855 + }, + { + "coverage": 0.983, + "risk": 0.16751441166497116, + "threshold": 0.003671810347370307 + }, + { + "coverage": 0.9833333333333333, + "risk": 0.16745762711864406, + "threshold": 0.0036463227464102 + }, + { + "coverage": 0.9836666666666667, + "risk": 0.16740088105726872, + "threshold": 0.0034664719531777187 + }, + { + "coverage": 0.984, + "risk": 0.1673441734417344, + "threshold": 0.003461722378083765 + }, + { + "coverage": 0.9843333333333333, + "risk": 0.1672875042329834, + "threshold": 0.003445110868104408 + }, + { + "coverage": 0.9846666666666667, + "risk": 0.16723087339201084, + "threshold": 0.0034437049466615457 + }, + { + "coverage": 0.985, + "risk": 0.16717428087986463, + "threshold": 0.003395002640040515 + }, + { + "coverage": 0.9853333333333333, + "risk": 0.16711772665764546, + "threshold": 0.003384277003457615 + }, + { + "coverage": 0.9856666666666667, + "risk": 0.1670612106865066, + "threshold": 0.0033489195022319185 + }, + { + "coverage": 0.986, + "risk": 0.16700473292765383, + "threshold": 0.0033438756920916688 + }, + { + "coverage": 0.9863333333333333, + "risk": 0.1669482933423454, + "threshold": 0.0033233852513713507 + }, + { + "coverage": 0.9866666666666667, + "risk": 0.16689189189189188, + "threshold": 0.0031960539273737307 + }, + { + "coverage": 0.987, + "risk": 0.1668355285376562, + "threshold": 0.0031692243260875337 + }, + { + "coverage": 0.9873333333333333, + "risk": 0.16677920324105333, + "threshold": 0.0031500530350011113 + }, + { + "coverage": 0.9876666666666667, + "risk": 0.16672291596355046, + "threshold": 0.003129416226493738 + }, + { + "coverage": 0.988, + "risk": 0.16666666666666666, + "threshold": 0.002939023018927831 + }, + { + "coverage": 0.9883333333333333, + "risk": 0.16661045531197302, + "threshold": 0.002907507618371468 + }, + { + "coverage": 0.9886666666666667, + "risk": 0.1665542818610924, + "threshold": 0.0028952474003424563 + }, + { + "coverage": 0.989, + "risk": 0.16649814627569937, + "threshold": 0.0028827455087978206 + }, + { + "coverage": 0.9893333333333333, + "risk": 0.1664420485175202, + "threshold": 0.0028509187581370107 + }, + { + "coverage": 0.9896666666666667, + "risk": 0.16638598854833278, + "threshold": 0.002825191559396151 + }, + { + "coverage": 0.99, + "risk": 0.16632996632996633, + "threshold": 0.0027842397039670196 + }, + { + "coverage": 0.9903333333333333, + "risk": 0.1662739818243016, + "threshold": 0.0026999943968549402 + }, + { + "coverage": 0.9906666666666667, + "risk": 0.16621803499327054, + "threshold": 0.002688452699994671 + }, + { + "coverage": 0.991, + "risk": 0.16616212579885636, + "threshold": 0.0025003073912069373 + }, + { + "coverage": 0.9913333333333333, + "risk": 0.16610625420309347, + "threshold": 0.002488942709947486 + }, + { + "coverage": 0.9916666666666667, + "risk": 0.16605042016806723, + "threshold": 0.002450612434259328 + }, + { + "coverage": 0.992, + "risk": 0.16599462365591397, + "threshold": 0.0024184903605484484 + }, + { + "coverage": 0.9923333333333333, + "risk": 0.16593886462882096, + "threshold": 0.0023871864782089875 + }, + { + "coverage": 0.9926666666666667, + "risk": 0.16588314304902618, + "threshold": 0.0023566734184880854 + }, + { + "coverage": 0.993, + "risk": 0.1658274588788184, + "threshold": 0.002276869416118303 + }, + { + "coverage": 0.9933333333333333, + "risk": 0.1657718120805369, + "threshold": 0.002246091790704538 + }, + { + "coverage": 0.9936666666666667, + "risk": 0.16571620261657163, + "threshold": 0.002202814395978525 + }, + { + "coverage": 0.994, + "risk": 0.16566063044936286, + "threshold": 0.002115599385667492 + }, + { + "coverage": 0.9943333333333333, + "risk": 0.16560509554140126, + "threshold": 0.002101571851994059 + }, + { + "coverage": 0.9946666666666667, + "risk": 0.16554959785522788, + "threshold": 0.0020144183502358564 + }, + { + "coverage": 0.995, + "risk": 0.16549413735343382, + "threshold": 0.001900569951178951 + }, + { + "coverage": 0.9953333333333333, + "risk": 0.16543871399866042, + "threshold": 0.0017751305710343088 + }, + { + "coverage": 0.9956666666666667, + "risk": 0.16538332775359893, + "threshold": 0.001667723751630765 + }, + { + "coverage": 0.996, + "risk": 0.16532797858099063, + "threshold": 0.0016476212723657716 + }, + { + "coverage": 0.9963333333333333, + "risk": 0.16527266644362662, + "threshold": 0.0016175643081979818 + }, + { + "coverage": 0.9966666666666667, + "risk": 0.16521739130434782, + "threshold": 0.0015903695774096601 + }, + { + "coverage": 0.997, + "risk": 0.1651621531260448, + "threshold": 0.0015525906683869798 + }, + { + "coverage": 0.9973333333333333, + "risk": 0.16510695187165775, + "threshold": 0.0014484024229360858 + }, + { + "coverage": 0.9976666666666667, + "risk": 0.1650517875041764, + "threshold": 0.001444086205640527 + }, + { + "coverage": 0.998, + "risk": 0.16499665998663995, + "threshold": 0.0014294509754502979 + }, + { + "coverage": 0.9983333333333333, + "risk": 0.1649415692821369, + "threshold": 0.0012677550650445105 + }, + { + "coverage": 0.9986666666666667, + "risk": 0.16488651535380508, + "threshold": 0.0011716294049438645 + }, + { + "coverage": 0.999, + "risk": 0.1648314981648315, + "threshold": 0.000919168485786853 + }, + { + "coverage": 0.9993333333333333, + "risk": 0.1647765176784523, + "threshold": 0.0009005505649267603 + }, + { + "coverage": 0.9996666666666667, + "risk": 0.16472157385795266, + "threshold": 0.0008815808043069625 + }, + { + "coverage": 1.0, + "risk": 0.16466666666666666, + "threshold": 0.0008208035312067058 + } + ], + "style_accuracy": { + "lower": 0.7003333333333334, + "mean": 0.7176666666666667, + "speaker_count": 1500.0, + "upper": 0.734 + }, + "style_brier": 0.18738474400010494, + "style_ece": 0.04268690432235598, + "support": { + "accuracy": 0.7913333333333333, + "brier": 0.1419104675412005, + "count": 3000, + "ece": 0.08411749658000188, + "f1": 0.7205357142857143, + "precision": 0.6508064516129032, + "recall": 0.807 + } + }, + "task_scope": "style+support+conflict+citation", + "threshold": 0.966101122793104 +} diff --git a/supplement/language_provenance/artifacts/snli_semantic_mlp_seed2.json b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed2.json new file mode 100644 index 0000000..733db34 --- /dev/null +++ b/supplement/language_provenance/artifacts/snli_semantic_mlp_seed2.json @@ -0,0 +1,137695 @@ +{ + "accept_temperature": 1.0, + "acceptance_mode": "selective_head", + "calibration_file": "checkpoints/snli_semantic_mlp_seed2.calibration.json", + "checkpoint_data_fingerprint": "sha256:0f4c8bb05648174215d0c23ab9ea29d88c97116c40b33b95e05b70f566c51a6c", + "checkpoint_epoch": 2, + "citation_threshold": 0.6989091038703918, + "citation_top_k": 2, + "evaluation_data_fingerprint": "sha256:0b6094089901215bf51c37e8c38ec81617fe96bbf5c512363f94b1d84827effe", + "evaluation_splits": [ + "test" + ], + "gate_threshold": 0.5, + "rows": [ + { + "accept_correct": 0, + "accept_probability": 0.7914836784973289, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16591216623783112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25683504343032837, + 0.6728581786155701, + 0.9544432163238525, + 0.40878012776374817, + 0.05665045976638794 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 1, + "style_probability": 0.6240877509117126, + "style_target": 1, + "support_probability": 0.8678779602050781, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8860337575535198, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11201808601617813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20622098445892334, + 0.71327805519104, + 0.9811655282974243, + 0.3467210531234741, + 0.03925073891878128 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6", + "split": "test", + "style_correct": 0, + "style_probability": 0.7804449796676636, + "style_target": 0, + "support_probability": 0.9079492688179016, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6636317614133986, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2255995273590088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26778942346572876, + 0.8964880108833313, + 0.12183824181556702, + 0.3305830955505371, + 0.19235657155513763, + 0.0488675981760025 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 1, + "style_probability": 0.5118183493614197, + "style_target": 1, + "support_probability": 0.8412665128707886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5600588444033576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12448848783969879, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:13:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24046207964420319, + 0.9029892683029175, + 0.13377003371715546, + 0.3203881084918976, + 0.14970824122428894, + 0.18252675235271454 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:13", + "split": "test", + "style_correct": 1, + "style_probability": 0.37934330105781555, + "style_target": 0, + "support_probability": 0.838085412979126, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.3055277191536021, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3178795874118805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31163471937179565, + 0.37893059849739075, + 0.8358156681060791, + 0.15163485705852509, + 0.0876290425658226, + 0.10401733964681625, + 0.11080900579690933, + 0.22045765817165375, + 0.3234783709049225 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 1, + "style_probability": 0.6247338652610779, + "style_target": 1, + "support_probability": 0.38400930166244507, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16927481427038593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08006796985864639, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:26:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3773058354854584, + 0.35867759585380554, + 0.9331041574478149, + 0.1345454752445221, + 0.07192093878984451, + 0.12883658707141876, + 0.1729191094636917, + 0.11615218222141266, + 0.22181299328804016 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:26", + "split": "test", + "style_correct": 1, + "style_probability": 0.1848902404308319, + "style_target": 0, + "support_probability": 0.7180892825126648, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.279347949471731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.712958574295044, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4317830204963684, + 0.3834473788738251, + 0.7933929562568665, + 0.35280004143714905, + 0.14241942763328552, + 0.04084182158112526, + 0.08324800431728363, + 0.07393297553062439, + 0.08833359181880951, + 0.12712813913822174, + 0.05788937211036682 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 1, + "style_probability": 0.6073436141014099, + "style_target": 1, + "support_probability": 0.2778885066509247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5461655773047269, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6589988470077515, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:30:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4576396644115448, + 0.38817399740219116, + 0.8089510202407837, + 0.3373493254184723, + 0.19157356023788452, + 0.1198282316327095, + 0.06380888819694519, + 0.0664527639746666, + 0.055779289454221725, + 0.04967372491955757, + 0.05647306144237518 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:30", + "split": "test", + "style_correct": 0, + "style_probability": 0.8249453902244568, + "style_target": 0, + "support_probability": 0.3757806122303009, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9189374614883867, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17626222968101501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9568224549293518, + 0.43769922852516174, + 0.48539838194847107, + 0.6590520143508911, + 0.04909928888082504 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.8465243577957153, + "style_target": 1, + "support_probability": 0.889820396900177, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5269434285567306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22781097888946533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:32:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9674469828605652, + 0.5348324775695801, + 0.542864978313446, + 0.646239697933197, + 0.11762379109859467 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:32", + "split": "test", + "style_correct": 1, + "style_probability": 0.29016852378845215, + "style_target": 0, + "support_probability": 0.8932913541793823, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18442559900387678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3071248233318329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15072192251682281, + 0.5139399766921997, + 0.7863191366195679, + 0.15723766386508942, + 0.05112200975418091, + 0.0693541094660759 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.830111563205719, + "style_target": 1, + "support_probability": 0.21366345882415771, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06940297832247344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3194633424282074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:34:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18193712830543518, + 0.5390480756759644, + 0.7412630915641785, + 0.1552007794380188, + 0.06444045901298523, + 0.059336237609386444 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:34", + "split": "test", + "style_correct": 1, + "style_probability": 0.4320546090602875, + "style_target": 0, + "support_probability": 0.24741314351558685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8465944578385347, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05250706151127815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9918489456176758, + 0.32382267713546753, + 0.348234087228775, + 0.3873286843299866, + 0.08226742595434189, + 0.06638775020837784, + 0.05886600911617279, + 0.07617571949958801, + 0.08645787090063095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.8531724214553833, + "style_target": 1, + "support_probability": 0.6104093790054321, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5816272191728302, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.026347314938902855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:36:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.987165093421936, + 0.3286057114601135, + 0.34776023030281067, + 0.37190741300582886, + 0.14167940616607666, + 0.0672248974442482, + 0.09151012450456619, + 0.04458794742822647, + 0.10226712375879288 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:36", + "split": "test", + "style_correct": 1, + "style_probability": 0.3871350884437561, + "style_target": 0, + "support_probability": 0.7269140481948853, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42921978434393365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16540497541427612, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22029326856136322, + 0.2471882402896881, + 0.9037477374076843, + 0.3390028178691864, + 0.26408466696739197, + 0.168316051363945, + 0.12496457248926163, + 0.10967197269201279, + 0.11542216688394547, + 0.08253490924835205, + 0.11009727418422699 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.7656455039978027, + "style_target": 1, + "support_probability": 0.31436148285865784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021018988815784016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37118566036224365, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:38:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27414649724960327, + 0.30153515934944153, + 0.9219298958778381, + 0.38136181235313416, + 0.09049615263938904, + 0.10316278785467148, + 0.1310323178768158, + 0.1383029818534851, + 0.10432359576225281, + 0.09357281029224396 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:38", + "split": "test", + "style_correct": 1, + "style_probability": 0.16815327107906342, + "style_target": 0, + "support_probability": 0.378824919462204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11475186986412654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18401548266410828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7814727425575256, + 0.2975006103515625, + 0.36781907081604004, + 0.49180173873901367, + 0.07069339603185654 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 0, + "style_probability": 0.3037388026714325, + "style_target": 1, + "support_probability": 0.40481919050216675, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00756866532613171, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32013601064682007, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:40:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.884523332118988, + 0.37579819560050964, + 0.40985390543937683, + 0.5995201468467712, + 0.12761403620243073 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:40", + "split": "test", + "style_correct": 1, + "style_probability": 0.08706041425466537, + "style_target": 0, + "support_probability": 0.3084973096847534, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8691336458469934, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.043315522372722626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:48:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9294155240058899, + 0.2969183325767517, + 0.609768807888031, + 0.23697149753570557, + 0.10442466288805008, + 0.054739806801080704 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 1, + "style_probability": 0.7655298113822937, + "style_target": 1, + "support_probability": 0.9535330533981323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9497876229343589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02894076146185398, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:48:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9691850543022156, + 0.3303215801715851, + 0.5724886655807495, + 0.23605772852897644, + 0.1321094036102295, + 0.09726549685001373 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:48", + "split": "test", + "style_correct": 0, + "style_probability": 0.8806892037391663, + "style_target": 0, + "support_probability": 0.9685046076774597, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008004182943998297, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8339669108390808, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1808883398771286, + 0.13430126011371613, + 0.5062332153320312, + 0.5101908445358276, + 0.096654012799263, + 0.2727988660335541, + 0.09809844940900803, + 0.06742947548627853, + 0.041847847402095795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 1, + "style_probability": 0.5600188970565796, + "style_target": 1, + "support_probability": 0.04731987044215202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010905443365515355, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7650552988052368, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:55:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15013247728347778, + 0.12612608075141907, + 0.42282092571258545, + 0.5665140748023987, + 0.10726016014814377, + 0.07193543761968613, + 0.0704604834318161, + 0.05378708615899086, + 0.06039614602923393 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:55", + "split": "test", + "style_correct": 1, + "style_probability": 0.233059823513031, + "style_target": 0, + "support_probability": 0.06802856922149658, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8111033345742982, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20253056287765503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9466270804405212, + 0.5763862729072571, + 0.3207382261753082, + 0.5717838406562805, + 0.08988824486732483, + 0.0526307076215744, + 0.1031879335641861, + 0.38140612840652466, + 0.06331980228424072, + 0.0787743628025055, + 0.05564415454864502 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.5486513376235962, + "style_target": 1, + "support_probability": 0.8770780563354492, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13913919307642963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17794325947761536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:56:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9859958291053772, + 0.485831081867218, + 0.36969640851020813, + 0.6870033740997314, + 0.1152302697300911, + 0.09115220606327057, + 0.08586820214986801, + 0.123334139585495, + 0.06997925788164139, + 0.1344362199306488 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:56", + "split": "test", + "style_correct": 1, + "style_probability": 0.11305859684944153, + "style_target": 0, + "support_probability": 0.8447650074958801, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5181695023681243, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.23104512691497803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5284233093261719, + 0.2687038481235504, + 0.8567118644714355, + 0.3443226218223572, + 0.07414307445287704 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 1, + "style_probability": 0.615399956703186, + "style_target": 1, + "support_probability": 0.5298029780387878, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6275352607659279, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2911084294319153, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:62:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5296366810798645, + 0.24422486126422882, + 0.9201080203056335, + 0.39166077971458435, + 0.15395380556583405 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:62", + "split": "test", + "style_correct": 0, + "style_probability": 0.8551974892616272, + "style_target": 0, + "support_probability": 0.40821313858032227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15930826764843078, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7244560718536377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5794180035591125, + 0.30321675539016724, + 0.33217471837997437, + 0.358439177274704, + 0.16315889358520508, + 0.14394758641719818 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.6690016388893127, + "style_target": 1, + "support_probability": 0.21170185506343842, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01313617649212517, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8398933410644531, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:68:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5135879516601562, + 0.3244970142841339, + 0.2784601151943207, + 0.36822643876075745, + 0.16755859553813934, + 0.13587290048599243 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:68", + "split": "test", + "style_correct": 1, + "style_probability": 0.26407691836357117, + "style_target": 0, + "support_probability": 0.07838398963212967, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3682361404772223, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4412233531475067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9408993721008301, + 0.4417921006679535, + 0.40869879722595215, + 0.3359057903289795, + 0.03375353291630745, + 0.08409825712442398, + 0.09406914561986923, + 0.04686130955815315, + 0.0377497561275959 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 0, + "style_probability": 0.3260771632194519, + "style_target": 1, + "support_probability": 0.7314808964729309, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.019194651926390586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4958657920360565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:76:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9789499044418335, + 0.5740635395050049, + 0.48468831181526184, + 0.4023875296115875, + 0.06874828040599823, + 0.11817692220211029, + 0.07567856460809708, + 0.05421334505081177, + 0.0939793586730957 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:76", + "split": "test", + "style_correct": 1, + "style_probability": 0.0638001561164856, + "style_target": 0, + "support_probability": 0.6263620257377625, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6754644662106912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16199210286140442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48993733525276184, + 0.6040446162223816, + 0.9797471761703491, + 0.5854758620262146, + 0.11187603324651718, + 0.041387081146240234, + 0.06543820351362228, + 0.07237936556339264, + 0.0704069435596466, + 0.04751107841730118, + 0.02922586165368557 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 1, + "style_probability": 0.6061055064201355, + "style_target": 1, + "support_probability": 0.7211357951164246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8186525327675084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11436954140663147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:86:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5344283580780029, + 0.6551992893218994, + 0.9927674531936646, + 0.5835517644882202, + 0.030267419293522835, + 0.031119823455810547, + 0.030364245176315308, + 0.12936808168888092, + 0.06590186059474945, + 0.10085713118314743 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:86", + "split": "test", + "style_correct": 0, + "style_probability": 0.7064490914344788, + "style_target": 0, + "support_probability": 0.8135635256767273, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8110221946830087, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18151406943798065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2761918604373932, + 0.8907482028007507, + 0.4430331587791443, + 0.36638838052749634, + 0.10321765393018723 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 1, + "style_probability": 0.6362621784210205, + "style_target": 1, + "support_probability": 0.9054403901100159, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7221066847872493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06996206939220428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:89:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2802998125553131, + 0.9330427646636963, + 0.48816847801208496, + 0.3297703266143799, + 0.05391600728034973 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:89", + "split": "test", + "style_correct": 1, + "style_probability": 0.3546559810638428, + "style_target": 0, + "support_probability": 0.9139235615730286, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007773378972013509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8874192237854004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24298669397830963, + 0.30764535069465637, + 0.30211636424064636, + 0.6530604958534241, + 0.09660963714122772, + 0.09920617192983627 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 0, + "style_probability": 0.48091399669647217, + "style_target": 1, + "support_probability": 0.0035145196598023176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00010266806832005815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9354883432388306, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:95:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28330403566360474, + 0.30888834595680237, + 0.34806913137435913, + 0.7451897859573364, + 0.08081439137458801, + 0.09912724047899246 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:95", + "split": "test", + "style_correct": 1, + "style_probability": 0.19587059319019318, + "style_target": 0, + "support_probability": 0.008167662657797337, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26333799447011863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16578951478004456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2536001205444336, + 0.14488601684570312, + 0.935275137424469, + 0.2536001205444336, + 0.16779330372810364, + 0.0535479374229908, + 0.08897870033979416, + 0.0641433596611023, + 0.07160171866416931 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 1, + "style_probability": 0.6560332179069519, + "style_target": 1, + "support_probability": 0.2945493459701538, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46733033377940814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1466582715511322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:114:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2107124626636505, + 0.13419383764266968, + 0.9538901448249817, + 0.2107124626636505, + 0.08469182997941971, + 0.043391622602939606, + 0.08745895326137543, + 0.08308806270360947, + 0.10075510293245316 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:114", + "split": "test", + "style_correct": 0, + "style_probability": 0.8410081267356873, + "style_target": 0, + "support_probability": 0.3959241509437561, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8008949006843373, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10475227981805801, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38757801055908203, + 0.9328122735023499, + 0.27127665281295776, + 0.38757801055908203, + 0.07539410144090652, + 0.09891203045845032, + 0.13450860977172852, + 0.0797126367688179, + 0.09178251773118973, + 0.06392098218202591, + 0.062015313655138016 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 1, + "style_probability": 0.5535451769828796, + "style_target": 1, + "support_probability": 0.8683573603630066, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5929083431403299, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10335838794708252, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:121:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39672529697418213, + 0.9341975450515747, + 0.2684032917022705, + 0.39672529697418213, + 0.09503822773694992, + 0.1348004937171936, + 0.07687808573246002, + 0.082557313144207, + 0.06324786692857742, + 0.08046035468578339, + 0.0408954955637455 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:121", + "split": "test", + "style_correct": 1, + "style_probability": 0.3272053301334381, + "style_target": 0, + "support_probability": 0.8441875576972961, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04442577344260012, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5885614156723022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7327976822853088, + 0.6696425080299377, + 0.19139720499515533, + 0.23752425611019135, + 0.05145815387368202 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 0, + "style_probability": 0.48699796199798584, + "style_target": 1, + "support_probability": 0.15885210037231445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004173635602805148, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5999155044555664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7993984818458557, + 0.758848249912262, + 0.24789312481880188, + 0.2294999361038208, + 0.11340469866991043 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:157", + "split": "test", + "style_correct": 1, + "style_probability": 0.15531711280345917, + "style_target": 0, + "support_probability": 0.15802563726902008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01783775098720764, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7021269798278809, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5238266587257385, + 0.31916436553001404, + 0.5010780096054077, + 0.36542025208473206, + 0.10493070632219315, + 0.08096647262573242 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 1, + "style_probability": 0.6918110251426697, + "style_target": 1, + "support_probability": 0.04855883866548538, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09092304201988634, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8707731366157532, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5927316546440125, + 0.27301955223083496, + 0.5520632266998291, + 0.3391000032424927, + 0.08161604404449463, + 0.3517618179321289 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:160", + "split": "test", + "style_correct": 0, + "style_probability": 0.8551594614982605, + "style_target": 0, + "support_probability": 0.08401906490325928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019049627787101734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.655258297920227, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47257110476493835, + 0.5622876882553101, + 0.3198230564594269, + 0.5943471789360046, + 0.15109579265117645, + 0.03890605643391609, + 0.10906942188739777, + 0.12230505049228668, + 0.11892299354076385 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 1, + "style_probability": 0.6320281624794006, + "style_target": 1, + "support_probability": 0.08600088953971863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010260896778084174, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7339867353439331, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:167:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41800639033317566, + 0.5479718446731567, + 0.30838632583618164, + 0.5651470422744751, + 0.1528470814228058, + 0.14637638628482819, + 0.13670113682746887, + 0.07894857972860336, + 0.13667802512645721 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:167", + "split": "test", + "style_correct": 1, + "style_probability": 0.4255116283893585, + "style_target": 0, + "support_probability": 0.07294269651174545, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006784320299322745, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7792167663574219, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2544735372066498, + 0.1762198656797409, + 0.5136089324951172, + 0.21258467435836792, + 0.11383134871721268, + 0.04869648069143295, + 0.17460155487060547, + 0.07338491827249527, + 0.08367235958576202, + 0.10701114684343338, + 0.13229306042194366 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 1, + "style_probability": 0.6153141856193542, + "style_target": 1, + "support_probability": 0.05640886351466179, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002995755496865745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7433149218559265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:170:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31794318556785583, + 0.23607237637043, + 0.8258412480354309, + 0.2462620884180069, + 0.09117753803730011, + 0.05180007219314575, + 0.05143943428993225, + 0.08286076784133911, + 0.09065137058496475, + 0.14219720661640167 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:170", + "split": "test", + "style_correct": 1, + "style_probability": 0.22774732112884521, + "style_target": 0, + "support_probability": 0.036776017397642136, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9135067200522825, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11659285426139832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6920743584632874, + 0.9808407425880432, + 0.18874289095401764, + 0.0908656120300293, + 0.08863081783056259 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.8067299723625183, + "style_target": 1, + "support_probability": 0.8866810202598572, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6703390550084798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08725973218679428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6100612878799438, + 0.9618316292762756, + 0.21017779409885406, + 0.09363479912281036, + 0.08843015879392624 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:173", + "split": "test", + "style_correct": 1, + "style_probability": 0.23018783330917358, + "style_target": 0, + "support_probability": 0.8901034593582153, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4900779364247824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08474808931350708, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9986175298690796, + 0.29306700825691223, + 0.36032992601394653, + 0.26340746879577637, + 0.06854327768087387, + 0.05968897417187691 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.6684994101524353, + "style_target": 1, + "support_probability": 0.4084520637989044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2625584216697803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09428577870130539, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:180:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9995083808898926, + 0.3436255156993866, + 0.35343676805496216, + 0.22867730259895325, + 0.07644190639257431, + 0.1942405104637146 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:180", + "split": "test", + "style_correct": 1, + "style_probability": 0.3598655164241791, + "style_target": 0, + "support_probability": 0.5605834722518921, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7449596703326123, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2329898327589035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3082016408443451, + 0.37291619181632996, + 0.9708021283149719, + 0.4843800365924835, + 0.06975995004177094, + 0.10515549778938293, + 0.06367998570203781, + 0.060795072466135025, + 0.07729270309209824 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 1, + "style_probability": 0.8420481085777283, + "style_target": 1, + "support_probability": 0.526331901550293, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6508883693312484, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24692276120185852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:182:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28788453340530396, + 0.3657936453819275, + 0.9524494409561157, + 0.4731554090976715, + 0.19174310564994812, + 0.22303782403469086, + 0.21247299015522003, + 0.13251468539237976, + 0.17142003774642944 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:182", + "split": "test", + "style_correct": 0, + "style_probability": 0.6899073719978333, + "style_target": 0, + "support_probability": 0.5480471253395081, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8447323685370448, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1289215236902237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2807355225086212, + 0.18597033619880676, + 0.9842387437820435, + 0.2674713730812073, + 0.13659152388572693, + 0.14836899936199188, + 0.08322802931070328, + 0.0968158170580864, + 0.07477226853370667, + 0.06909205764532089, + 0.12219695746898651 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.8599686622619629, + "style_target": 1, + "support_probability": 0.5321835279464722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41680017099273664, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11170267313718796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:186:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30976980924606323, + 0.21780695021152496, + 0.9708057045936584, + 0.2789202034473419, + 0.09262804687023163, + 0.12602435052394867, + 0.12749749422073364, + 0.11297457665205002, + 0.1381804198026657, + 0.13325291872024536, + 0.19635559618473053 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:186", + "split": "test", + "style_correct": 1, + "style_probability": 0.36079832911491394, + "style_target": 0, + "support_probability": 0.5220283269882202, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007560948490532457, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9474366903305054, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.405825674533844, + 0.272107869386673, + 0.23234479129314423, + 0.5552859902381897, + 0.18226958811283112 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.7979640364646912, + "style_target": 1, + "support_probability": 0.023786095902323723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003236985129671242, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9634683728218079, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5358978509902954, + 0.3648763597011566, + 0.3743197023868561, + 0.5652956962585449, + 0.14280734956264496 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:188", + "split": "test", + "style_correct": 1, + "style_probability": 0.15400481224060059, + "style_target": 0, + "support_probability": 0.014654157683253288, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20339939136917448, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6583158373832703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17113858461380005, + 0.6217718124389648, + 0.39845365285873413, + 0.17113858461380005, + 0.10069543123245239, + 0.16746467351913452 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 1, + "style_probability": 0.8965246081352234, + "style_target": 1, + "support_probability": 0.168627068400383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08646171216935185, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4481186270713806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:193:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1712159812450409, + 0.624305784702301, + 0.4001370668411255, + 0.1712159812450409, + 0.05945447459816933, + 0.08471423387527466 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:193", + "split": "test", + "style_correct": 0, + "style_probability": 0.5432695746421814, + "style_target": 0, + "support_probability": 0.2529658377170563, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8418988471242911, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20093044638633728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47671636939048767, + 0.37651488184928894, + 0.9923840165138245, + 0.3163319528102875, + 0.05277782678604126, + 0.04339971765875816, + 0.22243699431419373, + 0.08040393888950348, + 0.07236044108867645 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.6880379915237427, + "style_target": 1, + "support_probability": 0.5620347261428833, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6416054712587993, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15394681692123413, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47428083419799805, + 0.3498687744140625, + 0.9869396686553955, + 0.3456161320209503, + 0.04051430895924568, + 0.07170495390892029, + 0.049566857516765594, + 0.17146749794483185, + 0.08697973936796188 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:198", + "split": "test", + "style_correct": 1, + "style_probability": 0.24942940473556519, + "style_target": 0, + "support_probability": 0.67950439453125, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031767311336521604, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9104115962982178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5693020820617676, + 0.7474257349967957, + 0.3756411671638489, + 0.5693020820617676, + 0.21532896161079407, + 0.1452139914035797, + 0.10368812084197998, + 0.08296987414360046, + 0.21660387516021729, + 0.18226025998592377, + 0.26540547609329224 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.7314848899841309, + "style_target": 1, + "support_probability": 0.003541261423379183, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005087515993116298, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8821420669555664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:205:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5634781718254089, + 0.7930831909179688, + 0.3536974787712097, + 0.5634781718254089, + 0.08081267029047012, + 0.0663609579205513, + 0.17576806247234344, + 0.11560308933258057, + 0.06933295726776123, + 0.1268875002861023 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:205", + "split": "test", + "style_correct": 1, + "style_probability": 0.27197444438934326, + "style_target": 0, + "support_probability": 0.004614138510078192, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28220187183766576, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23839610815048218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6994122266769409, + 0.7184084057807922, + 0.24506287276744843, + 0.2817387878894806, + 0.06998811662197113 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 0, + "style_probability": 0.4065605700016022, + "style_target": 1, + "support_probability": 0.7121390700340271, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015326382596704491, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31965693831443787, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:212:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.783144474029541, + 0.8109531402587891, + 0.2218341827392578, + 0.3071454167366028, + 0.14473369717597961 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:212", + "split": "test", + "style_correct": 1, + "style_probability": 0.1342698186635971, + "style_target": 0, + "support_probability": 0.6644033789634705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.846324825297586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14881648123264313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35375070571899414, + 0.20824821293354034, + 0.31373661756515503, + 0.9906193017959595, + 0.0854782685637474, + 0.08879681676626205 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.4968571364879608, + "style_target": 1, + "support_probability": 0.8636647462844849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9010265786922212, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22041188180446625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:215:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.326124906539917, + 0.18575112521648407, + 0.2888033390045166, + 0.9971854090690613, + 0.15313579142093658, + 0.04873427003622055 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:215", + "split": "test", + "style_correct": 0, + "style_probability": 0.8190563321113586, + "style_target": 0, + "support_probability": 0.7742433547973633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05229958358832977, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6463993191719055, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17549942433834076, + 0.8022117018699646, + 0.25382331013679504, + 0.2900235652923584, + 0.14267203211784363, + 0.16018818318843842, + 0.08119320869445801, + 0.08915868401527405, + 0.21871960163116455 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 1, + "style_probability": 0.5574911236763, + "style_target": 1, + "support_probability": 0.10352818667888641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.089893357280495, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7226451635360718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:225:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15210065245628357, + 0.8557760715484619, + 0.20852212607860565, + 0.3040599822998047, + 0.08015929907560349, + 0.09469137340784073, + 0.04558892175555229, + 0.06286230683326721, + 0.05786670744419098 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:225", + "split": "test", + "style_correct": 0, + "style_probability": 0.5197438597679138, + "style_target": 0, + "support_probability": 0.1673383265733719, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2477380987148017, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7729928493499756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5735008120536804, + 0.7645663022994995, + 0.7230889797210693, + 0.5735008120536804, + 0.12216848134994507, + 0.08046836405992508, + 0.08911029994487762, + 0.11367787420749664, + 0.25565415620803833, + 0.24176636338233948, + 0.13442464172840118 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.5195596814155579, + "style_target": 1, + "support_probability": 0.3919012248516083, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0054126621312264875, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7753174901008606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6513400673866272, + 0.7522287964820862, + 0.7635159492492676, + 0.6513400673866272, + 0.1607544869184494, + 0.08011290431022644, + 0.20475882291793823, + 0.11788718402385712, + 0.08923507481813431, + 0.1279219388961792 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:238", + "split": "test", + "style_correct": 1, + "style_probability": 0.053023044019937515, + "style_target": 0, + "support_probability": 0.5597859621047974, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8583723942086685, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05753900483250618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9809468388557434, + 0.3861314356327057, + 0.7643466591835022, + 0.2235250473022461, + 0.37212708592414856 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.8184100985527039, + "style_target": 1, + "support_probability": 0.8096540570259094, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.43653461933090115, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07841678708791733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9694154858589172, + 0.3749185800552368, + 0.7249740362167358, + 0.23456232249736786, + 0.24711553752422333 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:240", + "split": "test", + "style_correct": 1, + "style_probability": 0.28585392236709595, + "style_target": 0, + "support_probability": 0.7724139094352722, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12402811630536095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2646558880805969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21883277595043182, + 0.2997671067714691, + 0.9760740995407104, + 0.21883277595043182, + 0.22004473209381104, + 0.33561083674430847 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.7249661684036255, + "style_target": 1, + "support_probability": 0.11776652932167053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008225892844714752, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3934372663497925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:242:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24239236116409302, + 0.34764283895492554, + 0.9882960915565491, + 0.24239236116409302, + 0.18288567662239075, + 0.2479090690612793 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:242", + "split": "test", + "style_correct": 1, + "style_probability": 0.15687942504882812, + "style_target": 0, + "support_probability": 0.0919414535164833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016321081068566553, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8017416596412659, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6012436747550964, + 0.3324594795703888, + 0.22174091637134552, + 0.47010740637779236, + 0.1453969031572342, + 0.14757512509822845, + 0.12757135927677155, + 0.09344607591629028, + 0.10622819513082504 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 1, + "style_probability": 0.8997344970703125, + "style_target": 1, + "support_probability": 0.04086177423596382, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027724300165962164, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7985610961914062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:248:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49717599153518677, + 0.28415659070014954, + 0.19503290951251984, + 0.45544829964637756, + 0.2801962196826935, + 0.11511099338531494, + 0.08455536514520645, + 0.060323771089315414, + 0.06734912842512131 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:248", + "split": "test", + "style_correct": 0, + "style_probability": 0.5986983180046082, + "style_target": 0, + "support_probability": 0.06206175312399864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012543925542418672, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8881300091743469, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31063157320022583, + 0.2322462648153305, + 0.3539084494113922, + 0.473072350025177, + 0.05503471568226814, + 0.11621557921171188, + 0.08169083297252655, + 0.15751495957374573, + 0.09213707596063614, + 0.1086735725402832 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.8267803192138672, + "style_target": 1, + "support_probability": 0.030530788004398346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007097307671827645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7568868398666382, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:253:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30571505427360535, + 0.2601901590824127, + 0.3302139937877655, + 0.4289076626300812, + 0.06877212226390839, + 0.10255362838506699, + 0.15117554366588593, + 0.10830218344926834, + 0.05556243658065796, + 0.13722237944602966 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:253", + "split": "test", + "style_correct": 1, + "style_probability": 0.2374117523431778, + "style_target": 0, + "support_probability": 0.04421161115169525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058675243579648914, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3202899694442749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4072891175746918, + 0.44177383184432983, + 0.499849796295166, + 0.8086422681808472, + 0.08657326549291611 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.6161584258079529, + "style_target": 1, + "support_probability": 0.45264580845832825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0022739730345351507, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4475712776184082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:263:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4114651083946228, + 0.453870564699173, + 0.5253332257270813, + 0.8292209506034851, + 0.10917005687952042 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:263", + "split": "test", + "style_correct": 1, + "style_probability": 0.10735782235860825, + "style_target": 0, + "support_probability": 0.4339643120765686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24328968571387222, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42337918281555176, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5786743760108948, + 0.764886736869812, + 0.6631576418876648, + 0.465067058801651, + 0.054494213312864304, + 0.13904008269309998 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 1, + "style_probability": 0.8154454231262207, + "style_target": 1, + "support_probability": 0.3468957841396332, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19544569706545548, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.38103634119033813, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:270:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5311907529830933, + 0.7080601453781128, + 0.6512038111686707, + 0.45806562900543213, + 0.03955854848027229, + 0.10697879642248154 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:270", + "split": "test", + "style_correct": 0, + "style_probability": 0.6994731426239014, + "style_target": 0, + "support_probability": 0.3441448509693146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17879845829720864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08815637975931168, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17231754958629608, + 0.7675521373748779, + 0.21406036615371704, + 0.1376277357339859, + 0.06946178525686264, + 0.05411005765199661, + 0.08099326491355896, + 0.08338312059640884, + 0.05215318128466606 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.8307576179504395, + "style_target": 1, + "support_probability": 0.10650093853473663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11039104702310798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0650465339422226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:281:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1915559321641922, + 0.7877820134162903, + 0.22786834836006165, + 0.15153490006923676, + 0.07444315403699875, + 0.13885383307933807, + 0.10159014910459518, + 0.07702488452196121, + 0.05310629680752754 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:281", + "split": "test", + "style_correct": 1, + "style_probability": 0.37130603194236755, + "style_target": 0, + "support_probability": 0.21420520544052124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5134085050381145, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19171860814094543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6077156662940979, + 0.775936484336853, + 0.7156161665916443, + 0.3863948881626129, + 0.03083375282585621, + 0.12186020612716675, + 0.037074632942676544, + 0.030183542519807816, + 0.08346480876207352, + 0.06444840878248215, + 0.07587949186563492 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.744277834892273, + "style_target": 1, + "support_probability": 0.622822105884552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04082528826298315, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16159382462501526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:282:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7262448668479919, + 0.7882765531539917, + 0.7255969047546387, + 0.46796751022338867, + 0.06560623645782471, + 0.06115390360355377, + 0.09495580941438675, + 0.05647987127304077, + 0.04387214034795761, + 0.11686623841524124 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:282", + "split": "test", + "style_correct": 1, + "style_probability": 0.11236222833395004, + "style_target": 0, + "support_probability": 0.6321896314620972, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8646075799386495, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10680253803730011, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.990318775177002, + 0.45208051800727844, + 0.37421396374702454, + 0.2459368258714676, + 0.12130600214004517 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 1, + "style_probability": 0.6488625407218933, + "style_target": 1, + "support_probability": 0.9363791942596436, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8019302795006027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1555671989917755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:284:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9730318784713745, + 0.44533097743988037, + 0.3411569893360138, + 0.25814568996429443, + 0.02972204051911831 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:284", + "split": "test", + "style_correct": 0, + "style_probability": 0.5544814467430115, + "style_target": 0, + "support_probability": 0.9056709408760071, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6154205011236868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31669968366622925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4354855418205261, + 0.8019775152206421, + 0.18270842730998993, + 0.4209778308868408, + 0.04275299981236458, + 0.06419716775417328 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 1, + "style_probability": 0.8034296631813049, + "style_target": 1, + "support_probability": 0.6709991097450256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.579497423324483, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2641741633415222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:289:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46049267053604126, + 0.8008651733398438, + 0.19870202243328094, + 0.4256301820278168, + 0.06449021399021149, + 0.07830099761486053 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:289", + "split": "test", + "style_correct": 0, + "style_probability": 0.5411251783370972, + "style_target": 0, + "support_probability": 0.8085072040557861, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03291168691621316, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8646618723869324, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42884010076522827, + 0.3582272231578827, + 0.3957095742225647, + 0.6042777299880981, + 0.04184043034911156, + 0.06971222162246704, + 0.053217727690935135, + 0.06902320683002472, + 0.06656918674707413 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.6573731303215027, + "style_target": 1, + "support_probability": 0.13815416395664215, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014965002686931491, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8937860131263733, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:295:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4534640610218048, + 0.39071306586265564, + 0.4294184148311615, + 0.5421603322029114, + 0.11565340310335159, + 0.14374643564224243, + 0.08727782964706421, + 0.08428287506103516, + 0.18099340796470642 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:295", + "split": "test", + "style_correct": 1, + "style_probability": 0.13549590110778809, + "style_target": 0, + "support_probability": 0.1093902736902237, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7728935110677959, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30193009972572327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20930194854736328, + 0.45943427085876465, + 0.23547104001045227, + 0.9393059611320496, + 0.0923994779586792, + 0.03129299730062485, + 0.062002282589673996, + 0.08449142426252365, + 0.05523652955889702, + 0.04845081642270088, + 0.11973708868026733 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 1, + "style_probability": 0.8661103844642639, + "style_target": 1, + "support_probability": 0.6188424229621887, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5356762580898433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31949418783187866, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:299:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17270386219024658, + 0.42647039890289307, + 0.25011971592903137, + 0.9061359763145447, + 0.05131484568119049, + 0.15153443813323975, + 0.1628611981868744, + 0.1426420509815216, + 0.09071668237447739, + 0.08387413620948792, + 0.0796324610710144 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:299", + "split": "test", + "style_correct": 0, + "style_probability": 0.622266948223114, + "style_target": 0, + "support_probability": 0.5713239312171936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45144682430578703, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18758819997310638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9964244961738586, + 0.4629995822906494, + 0.4301893413066864, + 0.35764068365097046, + 0.18571072816848755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 1, + "style_probability": 0.6183223128318787, + "style_target": 1, + "support_probability": 0.5419732332229614, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5742351247146971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21127921342849731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:300:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9995887875556946, + 0.42252930998802185, + 0.46119025349617004, + 0.38309797644615173, + 0.045447301119565964 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:300", + "split": "test", + "style_correct": 0, + "style_probability": 0.7284342646598816, + "style_target": 0, + "support_probability": 0.6230836510658264, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25901143692905393, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40610912442207336, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2719643712043762, + 0.9920267462730408, + 0.39982813596725464, + 0.3140347898006439, + 0.21896275877952576, + 0.15756739675998688 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 1, + "style_probability": 0.6613895893096924, + "style_target": 1, + "support_probability": 0.29521018266677856, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17494229337156852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.441980242729187, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:301:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26439008116722107, + 0.9959200620651245, + 0.32946714758872986, + 0.24347138404846191, + 0.14990636706352234, + 0.11719894409179688 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:301", + "split": "test", + "style_correct": 1, + "style_probability": 0.4652181565761566, + "style_target": 0, + "support_probability": 0.3424680829048157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3661382808753686, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30497726798057556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09011565148830414, + 0.8375070691108704, + 0.09011565148830414, + 0.20217269659042358, + 0.1420726627111435, + 0.11485154181718826, + 0.15295250713825226, + 0.13984094560146332, + 0.13377152383327484 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 1, + "style_probability": 0.7950140237808228, + "style_target": 1, + "support_probability": 0.4028870463371277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12078764294076816, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22653335332870483, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:309:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07239682972431183, + 0.9188627600669861, + 0.07239682972431183, + 0.18416732549667358, + 0.06463989615440369, + 0.1066797524690628, + 0.09955943375825882, + 0.06823776662349701, + 0.11582525819540024 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:309", + "split": "test", + "style_correct": 0, + "style_probability": 0.5967491865158081, + "style_target": 0, + "support_probability": 0.4899948537349701, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2801643411000754, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40376850962638855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12106330692768097, + 0.1927444189786911, + 0.8561491966247559, + 0.12106330692768097, + 0.03951772302389145, + 0.23427772521972656, + 0.1890232264995575, + 0.13018935918807983, + 0.17071647942066193, + 0.16310934722423553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 1, + "style_probability": 0.6367019414901733, + "style_target": 1, + "support_probability": 0.3167366087436676, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30673821401965284, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40888741612434387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.08240832388401031, + 0.17442171275615692, + 0.9176414012908936, + 0.08240832388401031, + 0.06320700794458389, + 0.08925788104534149, + 0.04164250195026398, + 0.08092406392097473, + 0.12154995650053024, + 0.0689164325594902 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:310", + "split": "test", + "style_correct": 0, + "style_probability": 0.70162433385849, + "style_target": 0, + "support_probability": 0.4049719572067261, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01462184307226377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8899748921394348, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23213857412338257, + 0.2522384822368622, + 0.10622186213731766, + 0.8210535645484924, + 0.24480858445167542 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.7023299336433411, + "style_target": 1, + "support_probability": 0.09830953925848007, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005230193209157576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8453655242919922, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30180051922798157, + 0.2677309513092041, + 0.12045868486166, + 0.8716191053390503, + 0.10924369096755981 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:311", + "split": "test", + "style_correct": 1, + "style_probability": 0.27668747305870056, + "style_target": 0, + "support_probability": 0.15088209509849548, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6084340066991238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.500745415687561, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.601923406124115, + 0.3592677116394043, + 0.7732208371162415, + 0.20333077013492584, + 0.031203819438815117, + 0.14200237393379211 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 1, + "style_probability": 0.9106380343437195, + "style_target": 1, + "support_probability": 0.5598164200782776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18405666770040785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6149473190307617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5078175663948059, + 0.31509628891944885, + 0.7740241885185242, + 0.2029782235622406, + 0.153366357088089, + 0.15603284537792206 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:314", + "split": "test", + "style_correct": 0, + "style_probability": 0.6217907667160034, + "style_target": 0, + "support_probability": 0.3802891671657562, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8784528207842471, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15827062726020813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2173500806093216, + 0.3134379982948303, + 0.24163343012332916, + 0.9410805702209473, + 0.06749439239501953, + 0.09511057287454605, + 0.15509864687919617, + 0.06508021056652069, + 0.21131734549999237 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 1, + "style_probability": 0.886560320854187, + "style_target": 1, + "support_probability": 0.7202762365341187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.715777329530578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08147528022527695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2116745412349701, + 0.29221364855766296, + 0.21307653188705444, + 0.9225262999534607, + 0.1647510677576065, + 0.06610286235809326, + 0.06970459967851639, + 0.13668188452720642, + 0.08760971575975418 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:319", + "split": "test", + "style_correct": 0, + "style_probability": 0.5006933212280273, + "style_target": 0, + "support_probability": 0.7503275275230408, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8612836590516934, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04612717777490616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6251676082611084, + 0.9966946840286255, + 0.7766818404197693, + 0.3475344181060791, + 0.048652201890945435, + 0.057241443544626236, + 0.08877653628587723, + 0.04269598424434662, + 0.033682841807603836, + 0.10233490914106369 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 1, + "style_probability": 0.6459799408912659, + "style_target": 1, + "support_probability": 0.9563514590263367, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33150189437033184, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05447005480527878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:321:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6666818857192993, + 0.9967735409736633, + 0.8325201869010925, + 0.37695902585983276, + 0.16272473335266113, + 0.05772615596652031, + 0.09589166939258575, + 0.13209715485572815, + 0.08613617718219757, + 0.09232259541749954 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:321", + "split": "test", + "style_correct": 1, + "style_probability": 0.15253305435180664, + "style_target": 0, + "support_probability": 0.9537693858146667, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14644229571013057, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8272390961647034, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2979418933391571, + 0.45188257098197937, + 0.7705069780349731, + 0.28739476203918457, + 0.1342191994190216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.8017222285270691, + "style_target": 1, + "support_probability": 0.16160932183265686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17812372054009987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5521787405014038, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:326:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3448117673397064, + 0.42277607321739197, + 0.7693367004394531, + 0.29909035563468933, + 0.09127955138683319 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:326", + "split": "test", + "style_correct": 1, + "style_probability": 0.4734915792942047, + "style_target": 0, + "support_probability": 0.33994850516319275, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4593340114414133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1678018718957901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7797496318817139, + 0.5254285931587219, + 0.3918038010597229, + 0.4763471186161041, + 0.10199645906686783, + 0.16913029551506042 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.8514875769615173, + "style_target": 1, + "support_probability": 0.38567298650741577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20058073140636526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12770572304725647, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:332:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7815274000167847, + 0.4997429847717285, + 0.46334370970726013, + 0.5461912155151367, + 0.07941727340221405, + 0.051033563911914825 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:332", + "split": "test", + "style_correct": 1, + "style_probability": 0.4121125042438507, + "style_target": 0, + "support_probability": 0.45377233624458313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17409490800287752, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4565054476261139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31491580605506897, + 0.4008619487285614, + 0.3286707103252411, + 0.3098481297492981, + 0.1415283977985382, + 0.0636105090379715, + 0.08555136620998383, + 0.2197289913892746, + 0.05182671919465065 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.8912969827651978, + "style_target": 1, + "support_probability": 0.26731398701667786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007519258953887498, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5980732440948486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:333:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3285529911518097, + 0.5373710989952087, + 0.39626678824424744, + 0.23095691204071045, + 0.07505568861961365, + 0.09588512033224106, + 0.09164192527532578, + 0.07139939069747925, + 0.10146110504865646 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:333", + "split": "test", + "style_correct": 1, + "style_probability": 0.24906840920448303, + "style_target": 0, + "support_probability": 0.18839263916015625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019265944253644136, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8779272437095642, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3008960783481598, + 0.37824752926826477, + 0.43049949407577515, + 0.3008960783481598, + 0.05713121220469475, + 0.12947562336921692, + 0.04986070096492767, + 0.08572189509868622, + 0.17967307567596436, + 0.128718763589859 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 1, + "style_probability": 0.8017802238464355, + "style_target": 1, + "support_probability": 0.007417172659188509, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001748586850101902, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8909374475479126, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2385273426771164, + 0.3589027225971222, + 0.4179082214832306, + 0.2385273426771164, + 0.08946196734905243, + 0.09349548071622849, + 0.1408585011959076, + 0.11335432529449463, + 0.054807472974061966, + 0.058469802141189575, + 0.08593525737524033 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:334", + "split": "test", + "style_correct": 0, + "style_probability": 0.6325859427452087, + "style_target": 0, + "support_probability": 0.006308326497673988, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09072722358463992, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3444530963897705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2347005307674408, + 0.3208279013633728, + 0.8137288093566895, + 0.338427871465683, + 0.1282198131084442 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 1, + "style_probability": 0.5420689582824707, + "style_target": 1, + "support_probability": 0.20953528583049774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12231815378307155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.49170076847076416, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2209535390138626, + 0.381168007850647, + 0.8112029433250427, + 0.41329291462898254, + 0.07041939347982407 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:342", + "split": "test", + "style_correct": 0, + "style_probability": 0.869485080242157, + "style_target": 0, + "support_probability": 0.09407759457826614, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02243892545898429, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4570169746875763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6330884099006653, + 0.3487360179424286, + 0.14520612359046936, + 0.26846879720687866, + 0.07161874324083328, + 0.09061753749847412 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 0, + "style_probability": 0.4358949363231659, + "style_target": 1, + "support_probability": 0.18840785324573517, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03342224299903274, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.233493372797966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6567928194999695, + 0.3810942471027374, + 0.14605890214443207, + 0.2887893319129944, + 0.12081966549158096, + 0.1252184361219406 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:348", + "split": "test", + "style_correct": 1, + "style_probability": 0.23787595331668854, + "style_target": 0, + "support_probability": 0.347419798374176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3424365026230092, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12970280647277832, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5978047251701355, + 0.2996581792831421, + 0.8494248986244202, + 0.5731540322303772, + 0.06253508478403091, + 0.21314236521720886, + 0.7329516410827637, + 0.10006393492221832, + 0.07850992679595947 + ], + "selected_evidence_indices": [ + 2, + 6 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 1, + "style_probability": 0.5977564454078674, + "style_target": 1, + "support_probability": 0.493873655796051, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019915857030720963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09666406363248825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:354:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5673478841781616, + 0.30844539403915405, + 0.940646767616272, + 0.5845940113067627, + 0.12925098836421967, + 0.11581138521432877, + 0.29611289501190186, + 0.19558455049991608, + 0.17096343636512756 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:354", + "split": "test", + "style_correct": 1, + "style_probability": 0.10369882732629776, + "style_target": 0, + "support_probability": 0.5158126950263977, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3389221805184697, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.419019877910614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14504170417785645, + 0.13532881438732147, + 0.564929723739624, + 0.8964095711708069, + 0.08689692616462708, + 0.12503591179847717, + 0.08700498938560486, + 0.09475419670343399, + 0.42713963985443115, + 0.11795040220022202, + 0.15384188294410706 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 1, + "style_probability": 0.5946635007858276, + "style_target": 1, + "support_probability": 0.5188596844673157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7052248753424948, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2722957730293274, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:367:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1119108498096466, + 0.1313372552394867, + 0.56449294090271, + 0.94395911693573, + 0.05728159844875336, + 0.12575440108776093, + 0.08771955966949463, + 0.1118289977312088, + 0.07162679731845856, + 0.11394713073968887, + 0.07819025218486786 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:367", + "split": "test", + "style_correct": 0, + "style_probability": 0.8130881190299988, + "style_target": 0, + "support_probability": 0.7418190836906433, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3300886278629953, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30286872386932373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35142189264297485, + 0.24855907261371613, + 0.8341768980026245, + 0.33629170060157776, + 0.04632012918591499 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 0, + "style_probability": 0.403067022562027, + "style_target": 1, + "support_probability": 0.7485214471817017, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25133786092777655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2416573166847229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:378:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3687223792076111, + 0.2533702850341797, + 0.868819534778595, + 0.34414955973625183, + 0.13837558031082153 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:378", + "split": "test", + "style_correct": 1, + "style_probability": 0.2176838219165802, + "style_target": 0, + "support_probability": 0.813209593296051, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034926415396142594, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8548681139945984, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2665962278842926, + 0.36670705676078796, + 0.19166773557662964, + 0.4292767643928528, + 0.061211034655570984, + 0.09647735208272934 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 0, + "style_probability": 0.41835498809814453, + "style_target": 1, + "support_probability": 0.022165793925523758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003210505130673335, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7535733580589294, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3603059947490692, + 0.4756530821323395, + 0.23882432281970978, + 0.5754309296607971, + 0.09320665895938873, + 0.11580844968557358 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:383", + "split": "test", + "style_correct": 1, + "style_probability": 0.29795292019844055, + "style_target": 0, + "support_probability": 0.07837658375501633, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8652499891841288, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10969790071249008, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:386:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2756543457508087, + 0.4981721341609955, + 0.9722532629966736, + 0.49629732966423035, + 0.09330933541059494, + 0.09074117243289948, + 0.04879254102706909, + 0.04625917598605156, + 0.08706339448690414 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 1, + "style_probability": 0.7544039487838745, + "style_target": 1, + "support_probability": 0.8139786720275879, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9471066365663248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15382738411426544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:386:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2505881190299988, + 0.48194828629493713, + 0.9956427812576294, + 0.47743725776672363, + 0.09543799608945847, + 0.05212092027068138, + 0.07601113617420197, + 0.11572328954935074, + 0.056554075330495834 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:386", + "split": "test", + "style_correct": 0, + "style_probability": 0.8947204351425171, + "style_target": 0, + "support_probability": 0.8528879284858704, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09381507509919196, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6194167733192444, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3105921745300293, + 0.2992372214794159, + 0.39193475246429443, + 0.6587381958961487, + 0.08813061565160751, + 0.09332332760095596, + 0.0963289886713028, + 0.08561201393604279, + 0.03671150654554367, + 0.06450137495994568, + 0.16175976395606995 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 1, + "style_probability": 0.5463451743125916, + "style_target": 1, + "support_probability": 0.12936925888061523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08772491180275656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3742671310901642, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:391:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31793853640556335, + 0.3455834686756134, + 0.40160566568374634, + 0.6985598206520081, + 0.06898413598537445, + 0.1786905825138092, + 0.05234147235751152, + 0.09688317775726318, + 0.06213811784982681, + 0.09514850378036499, + 0.0961955115199089 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:391", + "split": "test", + "style_correct": 1, + "style_probability": 0.3404594659805298, + "style_target": 0, + "support_probability": 0.2931840419769287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13292869213439576, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.41680553555488586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19258227944374084, + 0.9724195003509521, + 0.3960798382759094, + 0.19258227944374084, + 0.054391320794820786 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 1, + "style_probability": 0.6077530384063721, + "style_target": 1, + "support_probability": 0.3006751239299774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3416490042222975, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27463844418525696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:393:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17846611142158508, + 0.9485098719596863, + 0.3595002591609955, + 0.17846611142158508, + 0.07295534014701843 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:393", + "split": "test", + "style_correct": 0, + "style_probability": 0.6110436320304871, + "style_target": 0, + "support_probability": 0.4024345576763153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0389432944602599, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.887497067451477, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18185271322727203, + 0.42057815194129944, + 0.24215099215507507, + 0.21895286440849304, + 0.16251784563064575, + 0.12268802523612976 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.8980224132537842, + "style_target": 1, + "support_probability": 0.07561541348695755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005009186339179793, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9287997484207153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:397:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16919279098510742, + 0.38523635268211365, + 0.2051471620798111, + 0.19752241671085358, + 0.1602684110403061, + 0.09405171871185303 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:397", + "split": "test", + "style_correct": 1, + "style_probability": 0.34750983119010925, + "style_target": 0, + "support_probability": 0.06195388734340668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18944094358913066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.177251398563385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8527815937995911, + 0.22265930473804474, + 0.2880813777446747, + 0.28118056058883667, + 0.0812515839934349, + 0.052452098578214645, + 0.06190834194421768, + 0.19475975632667542, + 0.06829240173101425 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.8206705451011658, + "style_target": 1, + "support_probability": 0.12443535029888153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006559679822415039, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3639994263648987, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8861166834831238, + 0.23650017380714417, + 0.39262351393699646, + 0.23541952669620514, + 0.11575815081596375, + 0.07005400955677032, + 0.08751649409532547, + 0.1049288660287857, + 0.1326833963394165 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:412", + "split": "test", + "style_correct": 1, + "style_probability": 0.1226143091917038, + "style_target": 0, + "support_probability": 0.10416414588689804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4939015857956974, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5519484281539917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8226568698883057, + 0.6867976188659668, + 0.4558276832103729, + 0.5821315050125122, + 0.10804510116577148, + 0.05635632202029228, + 0.0685521811246872, + 0.038594819605350494, + 0.12352548539638519, + 0.04588714614510536 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 1, + "style_probability": 0.7594149708747864, + "style_target": 1, + "support_probability": 0.6023390293121338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3044335204157118, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5811426043510437, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:417:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.753182590007782, + 0.6894806027412415, + 0.3640305697917938, + 0.5051714181900024, + 0.07262104749679565, + 0.0585511140525341, + 0.05198788642883301, + 0.06471259891986847, + 0.11877897381782532, + 0.07648947089910507, + 0.0795038640499115 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:417", + "split": "test", + "style_correct": 0, + "style_probability": 0.5687307715415955, + "style_target": 0, + "support_probability": 0.5697035789489746, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20164012176625692, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6182687282562256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8292375206947327, + 0.298850953578949, + 0.5127434730529785, + 0.4207245111465454, + 0.06992943584918976 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.8364050984382629, + "style_target": 1, + "support_probability": 0.22420699894428253, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06234659045802893, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5633997917175293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:424:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8013001680374146, + 0.27948272228240967, + 0.5187134742736816, + 0.4045885503292084, + 0.15931908786296844 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:424", + "split": "test", + "style_correct": 1, + "style_probability": 0.4770388603210449, + "style_target": 0, + "support_probability": 0.18847140669822693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5978732916833671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12451335042715073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9634796977043152, + 0.08370169997215271, + 0.26596900820732117, + 0.3671754002571106, + 0.05323869362473488, + 0.05802031606435776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.7910957336425781, + "style_target": 1, + "support_probability": 0.5585100054740906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04190302827269531, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17026787996292114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:432:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9751079678535461, + 0.1051502674818039, + 0.32058608531951904, + 0.35020777583122253, + 0.0796666368842125, + 0.09583307057619095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:432", + "split": "test", + "style_correct": 1, + "style_probability": 0.20630215108394623, + "style_target": 0, + "support_probability": 0.5119762420654297, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7778528849300592, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10048016905784607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11057756841182709, + 0.9474095702171326, + 0.869631290435791, + 0.19256600737571716, + 0.17545409500598907, + 0.050677910447120667, + 0.05005454644560814, + 0.028471702709794044, + 0.04988880455493927 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 1, + "style_probability": 0.7511029243469238, + "style_target": 1, + "support_probability": 0.8657630085945129, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6298173974908181, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.154727965593338, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:433:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15894745290279388, + 0.9072490334510803, + 0.7414692640304565, + 0.1910950094461441, + 0.04850942641496658, + 0.07819917052984238, + 0.25104445219039917, + 0.04831667244434357, + 0.07506512105464935 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:433", + "split": "test", + "style_correct": 0, + "style_probability": 0.5124943256378174, + "style_target": 0, + "support_probability": 0.81789231300354, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00152170510216193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9837506413459778, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.591124415397644, + 0.2836378216743469, + 0.7480915784835815, + 0.12798342108726501, + 0.1033984124660492, + 0.13923463225364685, + 0.2594626247882843, + 0.09756258130073547, + 0.09092721343040466, + 0.06944742053747177, + 0.10015042871236801 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.6297901272773743, + "style_target": 1, + "support_probability": 0.006189935840666294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012159328702110448, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9869686961174011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:434:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5739014148712158, + 0.35140836238861084, + 0.7218065857887268, + 0.144942507147789, + 0.07989982515573502, + 0.13430000841617584, + 0.07885497808456421, + 0.12990018725395203, + 0.11843635141849518, + 0.08172047138214111, + 0.13613468408584595 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:434", + "split": "test", + "style_correct": 1, + "style_probability": 0.29272139072418213, + "style_target": 0, + "support_probability": 0.004085430409759283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004349364070954432, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9324093461036682, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.600544273853302, + 0.13707630336284637, + 0.2542794346809387, + 0.399154394865036, + 0.10395944118499756 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.7492164969444275, + "style_target": 1, + "support_probability": 0.015412391163408756, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001156992305590646, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9375746846199036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:452:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6335028409957886, + 0.13154283165931702, + 0.2723838984966278, + 0.4526422917842865, + 0.1677660346031189 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:452", + "split": "test", + "style_correct": 1, + "style_probability": 0.28911638259887695, + "style_target": 0, + "support_probability": 0.02426975592970848, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029386395092405596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9043393135070801, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.264838844537735, + 0.5289891958236694, + 0.30689969658851624, + 0.264838844537735, + 0.11192774027585983, + 0.1219245195388794 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 1, + "style_probability": 0.8631017208099365, + "style_target": 1, + "support_probability": 0.06019948050379753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03196375750122838, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8430801630020142, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:469:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27690809965133667, + 0.5290247797966003, + 0.297254353761673, + 0.27690809965133667, + 0.08257676661014557, + 0.22354920208454132 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:469", + "split": "test", + "style_correct": 0, + "style_probability": 0.7096642851829529, + "style_target": 0, + "support_probability": 0.09248463064432144, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8356215087541939, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07350035756826401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2514346241950989, + 0.25259068608283997, + 0.20691779255867004, + 0.9697657823562622, + 0.1040843054652214, + 0.09485416859388351, + 0.05729261040687561, + 0.06912913918495178, + 0.08293736726045609 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.8253703117370605, + "style_target": 1, + "support_probability": 0.6659898161888123, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5302193366787549, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044339366257190704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:475:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2721986472606659, + 0.24511471390724182, + 0.205471470952034, + 0.953877329826355, + 0.07930771261453629, + 0.09835587441921234, + 0.08169718086719513, + 0.1453755497932434, + 0.15811844170093536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:475", + "split": "test", + "style_correct": 1, + "style_probability": 0.34055352210998535, + "style_target": 0, + "support_probability": 0.7009453177452087, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.901523773144028, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17040623724460602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6518530249595642, + 0.21372999250888824, + 0.9696076512336731, + 0.2853432297706604, + 0.12334594875574112, + 0.0544145293533802, + 0.04150504991412163, + 0.09489530324935913, + 0.13546662032604218, + 0.09182507544755936 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.8345212936401367, + "style_target": 1, + "support_probability": 0.7628498077392578, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18052693146206256, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24966523051261902, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:502:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6297662854194641, + 0.25732940435409546, + 0.9844797253608704, + 0.3032699525356293, + 0.0863121896982193, + 0.09225709736347198, + 0.09570538997650146, + 0.15206091105937958, + 0.08183684200048447, + 0.0894017368555069 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:502", + "split": "test", + "style_correct": 1, + "style_probability": 0.15819646418094635, + "style_target": 0, + "support_probability": 0.7536524534225464, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04798723491245837, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8848847150802612, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5847582817077637, + 0.39412084221839905, + 0.19365741312503815, + 0.34867537021636963, + 0.055013470351696014 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 1, + "style_probability": 0.8444707989692688, + "style_target": 1, + "support_probability": 0.1496458500623703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008607319614148509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9204481244087219, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:521:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5271204710006714, + 0.3863856792449951, + 0.21777993440628052, + 0.3829163610935211, + 0.08097298443317413 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:521", + "split": "test", + "style_correct": 0, + "style_probability": 0.5687362551689148, + "style_target": 0, + "support_probability": 0.07250900566577911, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014978615051248204, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8613061904907227, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.516145646572113, + 0.35534149408340454, + 0.44421032071113586, + 0.2782767117023468, + 0.05666555464267731, + 0.05042605474591255 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.7118702530860901, + "style_target": 1, + "support_probability": 0.06666044145822525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004364871878268227, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8706881999969482, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:538:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.49226564168930054, + 0.33561187982559204, + 0.4120092988014221, + 0.316459059715271, + 0.129167839884758, + 0.10032150149345398 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:538", + "split": "test", + "style_correct": 1, + "style_probability": 0.326616108417511, + "style_target": 0, + "support_probability": 0.03381102904677391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3448796842897828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4836267828941345, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3894515931606293, + 0.5289446115493774, + 0.12645027041435242, + 0.7557505369186401, + 0.11042875051498413, + 0.06131473183631897, + 0.0887562707066536, + 0.04165605455636978, + 0.05985588580369949 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.9059911966323853, + "style_target": 1, + "support_probability": 0.23054231703281403, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.080103913887001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23022891581058502, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40040525794029236, + 0.604594886302948, + 0.10781669616699219, + 0.8435640335083008, + 0.0874425545334816, + 0.15463887155056, + 0.05807320773601532, + 0.08739010244607925, + 0.11734280735254288 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:543", + "split": "test", + "style_correct": 1, + "style_probability": 0.3439024090766907, + "style_target": 0, + "support_probability": 0.538818359375, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5911055395788409, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28747376799583435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4690217971801758, + 0.4803912341594696, + 0.6296454071998596, + 0.5129750370979309, + 0.04438600316643715, + 0.03188934922218323, + 0.09743338823318481, + 0.08138217031955719, + 0.026344353333115578, + 0.030024684965610504, + 0.033275045454502106 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.8590257167816162, + "style_target": 1, + "support_probability": 0.82191401720047, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05953515352604022, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7889571785926819, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:546:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31306004524230957, + 0.41013506054878235, + 0.465177446603775, + 0.3942159116268158, + 0.05704502761363983, + 0.08708507567644119, + 0.04478122666478157, + 0.040777817368507385, + 0.06624839454889297, + 0.07159584015607834, + 0.09521833062171936 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:546", + "split": "test", + "style_correct": 1, + "style_probability": 0.4343664348125458, + "style_target": 0, + "support_probability": 0.22540321946144104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10576413729450172, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.651613712310791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48311421275138855, + 0.4163915812969208, + 0.4599071145057678, + 0.4163915812969208, + 0.357889324426651 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.2741664946079254, + "style_target": 1, + "support_probability": 0.3765864670276642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4649349507271978, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3055552542209625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:548:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5633445382118225, + 0.49104154109954834, + 0.466431587934494, + 0.49104154109954834, + 0.1547900140285492 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:548", + "split": "test", + "style_correct": 0, + "style_probability": 0.861351728439331, + "style_target": 0, + "support_probability": 0.6401551365852356, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1079945517264953, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6296635270118713, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5344060659408569, + 0.21438011527061462, + 0.268160879611969, + 0.31687480211257935, + 0.09811511635780334, + 0.21184208989143372 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 0, + "style_probability": 0.4771648645401001, + "style_target": 1, + "support_probability": 0.28064391016960144, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009154940368967194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9323580265045166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:563:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.49837788939476013, + 0.276319295167923, + 0.31460991501808167, + 0.3840336501598358, + 0.08997202664613724, + 0.19367727637290955 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:563", + "split": "test", + "style_correct": 1, + "style_probability": 0.3670574724674225, + "style_target": 0, + "support_probability": 0.16996781527996063, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6515509840832117, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07773777097463608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9539477229118347, + 0.36089640855789185, + 0.4026933014392853, + 0.19492323696613312, + 0.07911191880702972, + 0.07839425653219223, + 0.08089800179004669, + 0.13730013370513916, + 0.24910423159599304 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.39818650484085083, + "style_target": 1, + "support_probability": 0.8802120089530945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.848594044314723, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14347240328788757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:564:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9525518417358398, + 0.3824973404407501, + 0.38148945569992065, + 0.20378006994724274, + 0.10278632491827011, + 0.14126747846603394, + 0.17232567071914673, + 0.07741675525903702, + 0.045135192573070526 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:564", + "split": "test", + "style_correct": 0, + "style_probability": 0.731724739074707, + "style_target": 0, + "support_probability": 0.8669551610946655, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24041081854261398, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2290266901254654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14875011146068573, + 0.14875011146068573, + 0.23747503757476807, + 0.910069465637207, + 0.0509926937520504, + 0.14016297459602356, + 0.04787377640604973, + 0.07207295298576355, + 0.07378903031349182, + 0.13130810856819153, + 0.3747771978378296 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 1, + "style_probability": 0.5151233673095703, + "style_target": 1, + "support_probability": 0.22462980449199677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.59692559623306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16374392807483673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:567:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15403525531291962, + 0.15403525531291962, + 0.1964854747056961, + 0.9419383406639099, + 0.2201617956161499, + 0.07557094842195511, + 0.04729728028178215, + 0.06492380052804947, + 0.0855143666267395, + 0.22340159118175507, + 0.19071540236473083 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:567", + "split": "test", + "style_correct": 0, + "style_probability": 0.8462871313095093, + "style_target": 0, + "support_probability": 0.3874829113483429, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7468658863826754, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06912526488304138, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17732004821300507, + 0.9255046248435974, + 0.18452765047550201, + 0.21253669261932373, + 0.11271237581968307 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 1, + "style_probability": 0.5607587695121765, + "style_target": 1, + "support_probability": 0.8745855093002319, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7945020756923142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.051720183342695236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:569:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2701297998428345, + 0.9601966142654419, + 0.1588195115327835, + 0.19614765048027039, + 0.04478068649768829 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:569", + "split": "test", + "style_correct": 0, + "style_probability": 0.5083383321762085, + "style_target": 0, + "support_probability": 0.9015044569969177, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.599200108515987, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22154909372329712, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3068089187145233, + 0.13423874974250793, + 0.3068089187145233, + 0.8841556310653687, + 0.05062797665596008, + 0.08662732690572739 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 0, + "style_probability": 0.32565218210220337, + "style_target": 1, + "support_probability": 0.8336437940597534, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40379549163940703, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17439712584018707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:575:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41887637972831726, + 0.13854585587978363, + 0.41887637972831726, + 0.9671375751495361, + 0.03671109303832054, + 0.07131063938140869 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:575", + "split": "test", + "style_correct": 1, + "style_probability": 0.25110262632369995, + "style_target": 0, + "support_probability": 0.8548107147216797, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14889702651981196, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5474209189414978, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4317235052585602, + 0.6581678986549377, + 0.3844529092311859, + 0.45589518547058105, + 0.19984076917171478, + 0.09632028639316559, + 0.08015794306993484, + 0.0631776824593544, + 0.06570214033126831 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6302769184112549, + "style_target": 1, + "support_probability": 0.14751207828521729, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15084337759017075, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5827054381370544, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:577:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48610636591911316, + 0.6804940700531006, + 0.4188341796398163, + 0.5030415058135986, + 0.11409641057252884, + 0.08906226605176926, + 0.06988249719142914, + 0.1495961993932724, + 0.15184739232063293 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:577", + "split": "test", + "style_correct": 0, + "style_probability": 0.8112754225730896, + "style_target": 0, + "support_probability": 0.09683071821928024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12905989789634464, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49400854110717773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5890425443649292, + 0.8345556855201721, + 0.17322948575019836, + 0.2064889818429947, + 0.07863110303878784, + 0.28157615661621094, + 0.611788272857666, + 0.08320443332195282, + 0.07454665005207062, + 0.05121475085616112, + 0.10610509663820267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 0, + "style_probability": 0.4310503900051117, + "style_target": 1, + "support_probability": 0.3264102339744568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047511300775175076, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5971407294273376, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:581:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5747966766357422, + 0.7903507947921753, + 0.20548395812511444, + 0.18520499765872955, + 0.13697662949562073, + 0.18706060945987701, + 0.2434287965297699, + 0.08195705711841583, + 0.12520292401313782, + 0.05567863956093788, + 0.08828733116388321 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:581", + "split": "test", + "style_correct": 1, + "style_probability": 0.3158642649650574, + "style_target": 0, + "support_probability": 0.1448024958372116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1183055026879583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23417969048023224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7675007581710815, + 0.16893990337848663, + 0.36211442947387695, + 0.2902790904045105, + 0.03548521548509598 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 1, + "style_probability": 0.6164531111717224, + "style_target": 1, + "support_probability": 0.2942895293235779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11311181483499409, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22955188155174255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:588:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7275396585464478, + 0.16005484759807587, + 0.3711100220680237, + 0.26361677050590515, + 0.11635789275169373 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:588", + "split": "test", + "style_correct": 0, + "style_probability": 0.5673573017120361, + "style_target": 0, + "support_probability": 0.204600989818573, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22466840620372608, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4737992286682129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35260510444641113, + 0.21786178648471832, + 0.35260510444641113, + 0.7770541906356812, + 0.0932684987783432, + 0.06086336821317673 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.6958490610122681, + "style_target": 1, + "support_probability": 0.27380168437957764, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16669203754835138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30024194717407227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3729354739189148, + 0.2194901555776596, + 0.3729354739189148, + 0.7971770763397217, + 0.09349921345710754, + 0.11054513603448868 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:591", + "split": "test", + "style_correct": 1, + "style_probability": 0.261558473110199, + "style_target": 0, + "support_probability": 0.4086228609085083, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4988709729875644, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3661035895347595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30079418420791626, + 0.561532199382782, + 0.5050817131996155, + 0.1843249499797821, + 0.0539681538939476, + 0.026119370013475418, + 0.026447519659996033, + 0.047335781157016754, + 0.032105494290590286 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.7367592453956604, + "style_target": 1, + "support_probability": 0.8897907137870789, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06041380330746194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39961862564086914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:598:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42864900827407837, + 0.6906746625900269, + 0.636725902557373, + 0.24779769778251648, + 0.08954416960477829, + 0.054733023047447205, + 0.045808225870132446, + 0.04493989795446396, + 0.11120118200778961 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:598", + "split": "test", + "style_correct": 1, + "style_probability": 0.11024856567382812, + "style_target": 0, + "support_probability": 0.8783625364303589, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010158597480858783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32529759407043457, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4408054053783417, + 0.7267275452613831, + 0.255429208278656, + 0.18174313008785248, + 0.030236389487981796, + 0.06251706928014755, + 0.03404991328716278, + 0.07049158215522766, + 0.06360775232315063, + 0.042070142924785614 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 1, + "style_probability": 0.6630632281303406, + "style_target": 1, + "support_probability": 0.031998105347156525, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033595895199626025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3294979929924011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:605:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4798136055469513, + 0.7475270628929138, + 0.3047965466976166, + 0.18599021434783936, + 0.1775001436471939, + 0.04985133558511734, + 0.05690719559788704, + 0.044989779591560364, + 0.039379093796014786, + 0.1223432645201683 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:605", + "split": "test", + "style_correct": 0, + "style_probability": 0.6711830496788025, + "style_target": 0, + "support_probability": 0.056549303233623505, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7298559346320133, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14492672681808472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3458675444126129, + 0.07382240891456604, + 0.8719569444656372, + 0.15497367084026337, + 0.1656189113855362 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.8516098260879517, + "style_target": 1, + "support_probability": 0.6318694353103638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21526772089074694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09223655611276627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39385733008384705, + 0.08978259563446045, + 0.846825897693634, + 0.16096340119838715, + 0.23352332413196564 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:606", + "split": "test", + "style_correct": 1, + "style_probability": 0.18965817987918854, + "style_target": 0, + "support_probability": 0.5173110365867615, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4158713524731467, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16474078595638275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.856450080871582, + 0.5876908302307129, + 0.6433653235435486, + 0.5607108473777771, + 0.029706981033086777, + 0.1091177761554718 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 0, + "style_probability": 0.4914621412754059, + "style_target": 1, + "support_probability": 0.6352725028991699, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10241124537432268, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24050305783748627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.893865704536438, + 0.6348086595535278, + 0.6409847140312195, + 0.6457273960113525, + 0.04956211894750595, + 0.10230718553066254 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:616", + "split": "test", + "style_correct": 1, + "style_probability": 0.21009795367717743, + "style_target": 0, + "support_probability": 0.6180028915405273, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08973592053223012, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7187471389770508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8191603422164917, + 0.17596793174743652, + 0.3018934428691864, + 0.21425338089466095, + 0.05892768129706383, + 0.12501154839992523, + 0.1194499284029007, + 0.09390980750322342, + 0.1846182644367218 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 1, + "style_probability": 0.7529296875, + "style_target": 1, + "support_probability": 0.10852983593940735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.061976353667255055, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7781293392181396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:624:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7598600387573242, + 0.16391374170780182, + 0.3044262230396271, + 0.22221577167510986, + 0.19310779869556427, + 0.16706141829490662, + 0.09849414974451065, + 0.07227252423763275, + 0.08235777914524078 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:624", + "split": "test", + "style_correct": 0, + "style_probability": 0.7499974370002747, + "style_target": 0, + "support_probability": 0.04570016264915466, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018145511433470537, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.846994161605835, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3259590268135071, + 0.1532243937253952, + 0.3101434111595154, + 0.24922724068164825, + 0.049396175891160965, + 0.04994934797286987, + 0.05648528039455414, + 0.11418132483959198, + 0.051271744072437286, + 0.0887666642665863 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 1, + "style_probability": 0.7787735462188721, + "style_target": 1, + "support_probability": 0.0688139870762825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016107534935065785, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7670647501945496, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:626:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3091561496257782, + 0.1940719336271286, + 0.26839572191238403, + 0.24716560542583466, + 0.039239466190338135, + 0.05843926593661308, + 0.058844853192567825, + 0.05716920271515846, + 0.06359457969665527, + 0.08565695583820343, + 0.14459370076656342 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:626", + "split": "test", + "style_correct": 1, + "style_probability": 0.3736927807331085, + "style_target": 0, + "support_probability": 0.062491536140441895, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012430997568179114, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8852838277816772, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29620569944381714, + 0.3764576017856598, + 0.6418889760971069, + 0.29620569944381714, + 0.07730023562908173 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 1, + "style_probability": 0.8254755735397339, + "style_target": 1, + "support_probability": 0.017072318121790886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0067924840285743535, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8572813868522644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:641:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3485541343688965, + 0.4694689214229584, + 0.6745246052742004, + 0.3485541343688965, + 0.08779849857091904 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:641", + "split": "test", + "style_correct": 0, + "style_probability": 0.5713562369346619, + "style_target": 0, + "support_probability": 0.03489113971590996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16947670761011047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5367206931114197, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34812262654304504, + 0.36519378423690796, + 0.5099080801010132, + 0.963567852973938, + 0.062228817492723465, + 0.05270160362124443 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 1, + "style_probability": 0.7224512100219727, + "style_target": 1, + "support_probability": 0.291517972946167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.099167731094127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5948087573051453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:647:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34589701890945435, + 0.351117879152298, + 0.5079156756401062, + 0.9565143585205078, + 0.08180488646030426, + 0.1411954164505005 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:647", + "split": "test", + "style_correct": 0, + "style_probability": 0.5316978693008423, + "style_target": 0, + "support_probability": 0.22764483094215393, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9091279625810459, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.146223783493042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9413818120956421, + 0.22271773219108582, + 0.22327226400375366, + 0.22271773219108582, + 0.08395807445049286, + 0.1158294528722763, + 0.05698991194367409, + 0.043792709708213806, + 0.10570303350687027 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.8866132497787476, + "style_target": 1, + "support_probability": 0.9092767238616943, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.642151691825049, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.260387122631073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:660:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.895374059677124, + 0.17948880791664124, + 0.31596842408180237, + 0.17948880791664124, + 0.06140543892979622, + 0.14919908344745636, + 0.06581643223762512, + 0.06942857801914215, + 0.14399023354053497 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:660", + "split": "test", + "style_correct": 1, + "style_probability": 0.3963221311569214, + "style_target": 0, + "support_probability": 0.8256836533546448, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08103069225762737, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7672004699707031, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.691925048828125, + 0.26732978224754333, + 0.691925048828125, + 0.49269071221351624, + 0.16536368429660797, + 0.050950005650520325, + 0.08263295888900757, + 0.05560000613331795, + 0.06826724112033844, + 0.06904776394367218, + 0.050733260810375214 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.7804156541824341, + "style_target": 1, + "support_probability": 0.09846311807632446, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002511180319826334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8770232200622559, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:667:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6184111833572388, + 0.31244316697120667, + 0.6184111833572388, + 0.592017650604248, + 0.11561509966850281, + 0.06204920634627342, + 0.10496935993432999, + 0.1011645644903183, + 0.061807479709386826, + 0.1325577348470688 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:667", + "split": "test", + "style_correct": 1, + "style_probability": 0.10769796371459961, + "style_target": 0, + "support_probability": 0.09174982458353043, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03740964225301724, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8012058734893799, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4880845844745636, + 0.2754586637020111, + 0.354176789522171, + 0.5587661862373352, + 0.1653810292482376 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.6746324300765991, + "style_target": 1, + "support_probability": 0.043005093932151794, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002770898248189823, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9600047469139099, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:671:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4318487346172333, + 0.28282999992370605, + 0.3474140465259552, + 0.5751993060112, + 0.10680967569351196 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:671", + "split": "test", + "style_correct": 1, + "style_probability": 0.29996779561042786, + "style_target": 0, + "support_probability": 0.042241327464580536, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.44635833624602134, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2679279148578644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5052719712257385, + 0.5684768557548523, + 0.2382075935602188, + 0.08703053742647171, + 0.033043116331100464, + 0.08467140048742294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.4071434736251831, + "style_target": 1, + "support_probability": 0.87749844789505, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6646238370101436, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23510804772377014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:673:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5161789655685425, + 0.6991159319877625, + 0.26457884907722473, + 0.0625276044011116, + 0.06391429156064987, + 0.035617586225271225 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:673", + "split": "test", + "style_correct": 0, + "style_probability": 0.8777909874916077, + "style_target": 0, + "support_probability": 0.8199000358581543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33254471254424445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28880804777145386, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42347192764282227, + 0.9775422811508179, + 0.31388694047927856, + 0.30204617977142334, + 0.1181250512599945, + 0.11864073574542999, + 0.09076188504695892, + 0.11001153290271759, + 0.10685358196496964 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 0, + "style_probability": 0.475772887468338, + "style_target": 1, + "support_probability": 0.3341124355792999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20442349879039234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21781891584396362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43446677923202515, + 0.9833763241767883, + 0.21134407818317413, + 0.2716793119907379, + 0.11305703967809677, + 0.09780304878950119, + 0.06430942565202713, + 0.1660405397415161, + 0.13101305067539215 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:681", + "split": "test", + "style_correct": 1, + "style_probability": 0.24372754991054535, + "style_target": 0, + "support_probability": 0.36014652252197266, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6777640099041803, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2616512179374695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5462778806686401, + 0.8906418085098267, + 0.3027561604976654, + 0.26627469062805176, + 0.07080100476741791, + 0.07226083427667618, + 0.16966572403907776, + 0.12077561020851135, + 0.20503351092338562, + 0.08499807864427567, + 0.11988803744316101 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 1, + "style_probability": 0.5658700466156006, + "style_target": 1, + "support_probability": 0.7285159826278687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08249546180355694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28081199526786804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:685:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6728442311286926, + 0.9799312949180603, + 0.5223050117492676, + 0.2772209644317627, + 0.1333458572626114, + 0.1805916726589203, + 0.06171872094273567, + 0.10435915738344193, + 0.054180167615413666, + 0.11477367579936981 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:685", + "split": "test", + "style_correct": 1, + "style_probability": 0.0785994604229927, + "style_target": 0, + "support_probability": 0.7771376967430115, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7968125740090812, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14093855023384094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3318886458873749, + 0.2365405559539795, + 0.22590923309326172, + 0.9446894526481628, + 0.06515809893608093 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 1, + "style_probability": 0.7282790541648865, + "style_target": 1, + "support_probability": 0.7184187173843384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4487969544755987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20276246964931488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:687:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32519394159317017, + 0.26436126232147217, + 0.28391504287719727, + 0.9153732657432556, + 0.11253032088279724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:687", + "split": "test", + "style_correct": 1, + "style_probability": 0.33775943517684937, + "style_target": 0, + "support_probability": 0.6252134442329407, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.630471326556048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06422553211450577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23549127578735352, + 0.3379007875919342, + 0.9832733869552612, + 0.3365252912044525, + 0.16908596456050873, + 0.09741911292076111 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.7908082008361816, + "style_target": 1, + "support_probability": 0.5511002540588379, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09137168756126357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09416837245225906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:690:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25020208954811096, + 0.3410934507846832, + 0.9918498992919922, + 0.3207153379917145, + 0.07059679180383682, + 0.1126248762011528 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:690", + "split": "test", + "style_correct": 1, + "style_probability": 0.3051745295524597, + "style_target": 0, + "support_probability": 0.5215328335762024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06701129137846885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5976391434669495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27237316966056824, + 0.7722486257553101, + 0.2321333885192871, + 0.5194697976112366, + 0.10488756746053696, + 0.12751439213752747, + 0.0895577222108841, + 0.21691325306892395, + 0.1216454803943634 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 1, + "style_probability": 0.8186342120170593, + "style_target": 1, + "support_probability": 0.08365461230278015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044827365713253194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5597268342971802, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:693:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3047649562358856, + 0.6365118026733398, + 0.2671456038951874, + 0.49011510610580444, + 0.1744980812072754, + 0.17346231639385223, + 0.11935584992170334, + 0.1641872674226761, + 0.23089730739593506 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:693", + "split": "test", + "style_correct": 0, + "style_probability": 0.6787424683570862, + "style_target": 0, + "support_probability": 0.08501113951206207, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006166174837767712, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6534044742584229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4838034510612488, + 0.19599662721157074, + 0.5502640604972839, + 0.7787687182426453, + 0.04671584814786911, + 0.07001952081918716, + 0.15846149623394012, + 0.06572376191616058, + 0.09229308366775513, + 0.14546677470207214, + 0.08649949729442596 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.8969004154205322, + "style_target": 1, + "support_probability": 0.01275205984711647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028281084114760427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5207725167274475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:695:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4249417185783386, + 0.18696333467960358, + 0.5108696818351746, + 0.7431374192237854, + 0.07422096282243729, + 0.06434054672718048, + 0.19335122406482697, + 0.11764124035835266, + 0.06525144726037979, + 0.06450161337852478, + 0.06909336149692535 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:695", + "split": "test", + "style_correct": 1, + "style_probability": 0.38572874665260315, + "style_target": 0, + "support_probability": 0.0382423922419548, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06549142165817522, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3496505618095398, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35011208057403564, + 0.3132016360759735, + 0.8083081841468811, + 0.35011208057403564, + 0.05319331958889961 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 1, + "style_probability": 0.7687065005302429, + "style_target": 1, + "support_probability": 0.11109938472509384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011747931329047947, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.480537474155426, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:698:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3003057837486267, + 0.2763982117176056, + 0.7522894144058228, + 0.3003057837486267, + 0.11340399831533432 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:698", + "split": "test", + "style_correct": 0, + "style_probability": 0.574245035648346, + "style_target": 0, + "support_probability": 0.059857580810785294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014752432406610165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9272613525390625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5670019388198853, + 0.215861514210701, + 0.3106231987476349, + 0.37815535068511963, + 0.07436458766460419, + 0.04031483083963394 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 0, + "style_probability": 0.3519545793533325, + "style_target": 1, + "support_probability": 0.12734417617321014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00035352078244350887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9495264887809753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:700:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6126192212104797, + 0.2811303734779358, + 0.33631521463394165, + 0.368873655796051, + 0.09287724643945694, + 0.1154075562953949 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:700", + "split": "test", + "style_correct": 1, + "style_probability": 0.0579925999045372, + "style_target": 0, + "support_probability": 0.05514467507600784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46853423094345453, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16918222606182098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4513442814350128, + 0.5954946279525757, + 0.09403277933597565, + 0.3118244707584381, + 0.041948311030864716, + 0.15555338561534882, + 0.06483244150876999, + 0.06409897655248642, + 0.03654732182621956 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 1, + "style_probability": 0.7124083042144775, + "style_target": 1, + "support_probability": 0.8240283727645874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5116513127939442, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25933459401130676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:705:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4574093818664551, + 0.5800783038139343, + 0.1211925521492958, + 0.2888931930065155, + 0.12647809088230133, + 0.028232920914888382, + 0.14726944267749786, + 0.0986265242099762, + 0.06936771422624588 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:705", + "split": "test", + "style_correct": 0, + "style_probability": 0.9343531727790833, + "style_target": 0, + "support_probability": 0.5885272026062012, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10924453893444142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15563204884529114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9203187823295593, + 0.2539319396018982, + 0.2367260754108429, + 0.32020583748817444, + 0.0754009485244751, + 0.06550274044275284, + 0.05442366749048233, + 0.19969108700752258, + 0.05851555988192558, + 0.058222297579050064, + 0.03939283266663551 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 0, + "style_probability": 0.44614148139953613, + "style_target": 1, + "support_probability": 0.15482863783836365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21987715958495446, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06264325231313705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:708:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9388708472251892, + 0.23188982903957367, + 0.20004291832447052, + 0.3196142911911011, + 0.13988229632377625, + 0.047039322555065155, + 0.042590513825416565, + 0.10509362816810608, + 0.0674438551068306, + 0.067848801612854, + 0.10103031992912292 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:708", + "split": "test", + "style_correct": 1, + "style_probability": 0.296181857585907, + "style_target": 0, + "support_probability": 0.31727561354637146, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6090510253368961, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18526217341423035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9289599657058716, + 0.20140905678272247, + 0.5335589647293091, + 0.34435200691223145, + 0.060835499316453934 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 0, + "style_probability": 0.2999769151210785, + "style_target": 1, + "support_probability": 0.8316708207130432, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3631262316886392, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36451566219329834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8863492012023926, + 0.21921877562999725, + 0.4523542821407318, + 0.28691601753234863, + 0.08045666664838791 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:712", + "split": "test", + "style_correct": 1, + "style_probability": 0.3224172294139862, + "style_target": 0, + "support_probability": 0.6681546568870544, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.092334978492004, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6873462200164795, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2932380139827728, + 0.22640283405780792, + 0.4077572822570801, + 0.8970572352409363, + 0.055915702134370804, + 0.11138976365327835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 0, + "style_probability": 0.3574448227882385, + "style_target": 1, + "support_probability": 0.16644306480884552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2345542422791258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7770442962646484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:723:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32359883189201355, + 0.21425330638885498, + 0.398924320936203, + 0.9249302744865417, + 0.14136016368865967, + 0.07817497849464417 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:723", + "split": "test", + "style_correct": 0, + "style_probability": 0.840252697467804, + "style_target": 0, + "support_probability": 0.1972692608833313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10772493569676114, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5597657561302185, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6373611688613892, + 0.16157078742980957, + 0.6408194899559021, + 0.6373611688613892, + 0.10178199410438538, + 0.07219579815864563, + 0.09826972335577011, + 0.024077272042632103, + 0.06600502133369446 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 0, + "style_probability": 0.45945459604263306, + "style_target": 1, + "support_probability": 0.31328317523002625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009753155579809044, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7771400213241577, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:734:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7081178426742554, + 0.17565612494945526, + 0.6854681372642517, + 0.7081178426742554, + 0.0669715404510498, + 0.08067136257886887, + 0.10687726736068726, + 0.058499615639448166, + 0.08214801549911499 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:734", + "split": "test", + "style_correct": 1, + "style_probability": 0.1658458411693573, + "style_target": 0, + "support_probability": 0.2797614634037018, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0515528012466282, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6690863966941833, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31185707449913025, + 0.3338340222835541, + 0.3721802532672882, + 0.22312724590301514, + 0.06955743581056595, + 0.21752572059631348, + 0.14223147928714752, + 0.06884980946779251, + 0.10203524678945541, + 0.044865842908620834, + 0.11553937941789627 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 0, + "style_probability": 0.4437675476074219, + "style_target": 1, + "support_probability": 0.16743475198745728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05952861752989724, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7530532479286194, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2790205776691437, + 0.3199191987514496, + 0.31401047110557556, + 0.2394007444381714, + 0.07341817766427994, + 0.1732909083366394, + 0.16432614624500275, + 0.11024349182844162, + 0.10202877968549728, + 0.07464049011468887, + 0.04046255722641945 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:738", + "split": "test", + "style_correct": 0, + "style_probability": 0.637652575969696, + "style_target": 0, + "support_probability": 0.1856933981180191, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22938096623036583, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33616942167282104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41207727789878845, + 0.9807491302490234, + 0.2626934051513672, + 0.15465286374092102, + 0.06942517310380936 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 1, + "style_probability": 0.7910792827606201, + "style_target": 1, + "support_probability": 0.34202083945274353, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16949372912414162, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1320626437664032, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:749:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.361325740814209, + 0.9704842567443848, + 0.29081153869628906, + 0.15638679265975952, + 0.12974469363689423 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:749", + "split": "test", + "style_correct": 1, + "style_probability": 0.35095810890197754, + "style_target": 0, + "support_probability": 0.46924838423728943, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.660788503446895, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25794264674186707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2861698269844055, + 0.7785959243774414, + 0.45620816946029663, + 0.26578786969184875, + 0.08725287765264511, + 0.03505156189203262 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.8624789118766785, + "style_target": 1, + "support_probability": 0.6533187627792358, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03723865310853047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28938835859298706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4175378680229187, + 0.886962354183197, + 0.5272371172904968, + 0.29196396470069885, + 0.05942931026220322, + 0.09348264336585999 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:753", + "split": "test", + "style_correct": 1, + "style_probability": 0.06461090594530106, + "style_target": 0, + "support_probability": 0.8099398016929626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010463794769870688, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7756635546684265, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32893696427345276, + 0.25196683406829834, + 0.42495816946029663, + 0.506671130657196, + 0.10963790863752365, + 0.06396964937448502, + 0.046346668154001236, + 0.14971907436847687, + 0.08073293417692184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 1, + "style_probability": 0.903679370880127, + "style_target": 1, + "support_probability": 0.015814613550901413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00891692088455276, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7288935780525208, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:754:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31728577613830566, + 0.28445473313331604, + 0.41565147042274475, + 0.4866384267807007, + 0.08283928036689758, + 0.1081390455365181, + 0.12488521635532379, + 0.11032689362764359, + 0.07525243610143661 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:754", + "split": "test", + "style_correct": 0, + "style_probability": 0.6114524006843567, + "style_target": 0, + "support_probability": 0.018660619854927063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0466192800025822, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2557421624660492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38624730706214905, + 0.21901670098304749, + 0.9213033318519592, + 0.4098462164402008, + 0.07036220282316208, + 0.08674842119216919, + 0.15751025080680847, + 0.07292693108320236, + 0.0599031001329422, + 0.09160143882036209, + 0.05691938102245331 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.7239949107170105, + "style_target": 1, + "support_probability": 0.03554129973053932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008455320618028398, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6269211769104004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:778:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38475340604782104, + 0.2311355024576187, + 0.8721404671669006, + 0.37416496872901917, + 0.18119703233242035, + 0.2060495764017105, + 0.11214042454957962, + 0.11588741838932037, + 0.06386052072048187, + 0.16228871047496796 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:778", + "split": "test", + "style_correct": 1, + "style_probability": 0.34713754057884216, + "style_target": 0, + "support_probability": 0.014583291485905647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23587777623326645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2285950928926468, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47083207964897156, + 0.6982405185699463, + 0.5659415125846863, + 0.47083207964897156, + 0.0674009844660759 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.8529026508331299, + "style_target": 1, + "support_probability": 0.20079270005226135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02226426097153555, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2499285191297531, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.481884241104126, + 0.7194896936416626, + 0.5586568713188171, + 0.481884241104126, + 0.09314684569835663 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:781", + "split": "test", + "style_correct": 1, + "style_probability": 0.2331182211637497, + "style_target": 0, + "support_probability": 0.26492682099342346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027951990961782025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3898089826107025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32289841771125793, + 0.24646547436714172, + 0.29674988985061646, + 0.943266749382019, + 0.04864170774817467, + 0.10619184374809265 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 0, + "style_probability": 0.34531623125076294, + "style_target": 1, + "support_probability": 0.05478525534272194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0982458078840791, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2473725527524948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:783:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32165059447288513, + 0.24529854953289032, + 0.27490633726119995, + 0.9174896478652954, + 0.15592460334300995, + 0.1106267049908638 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:783", + "split": "test", + "style_correct": 0, + "style_probability": 0.5187241435050964, + "style_target": 0, + "support_probability": 0.09712940454483032, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8387955276728577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15894852578639984, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16773410141468048, + 0.3138674199581146, + 0.9658594131469727, + 0.33552294969558716, + 0.029564451426267624, + 0.2127755880355835, + 0.1329309046268463, + 0.0674978718161583, + 0.04699641838669777 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.8839481472969055, + "style_target": 1, + "support_probability": 0.7737467885017395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2980915703813025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15385498106479645, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:798:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1566302627325058, + 0.32614603638648987, + 0.9546038508415222, + 0.31423044204711914, + 0.10579866915941238, + 0.06906305253505707, + 0.31964072585105896, + 0.33402344584465027, + 0.14566092193126678 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:798", + "split": "test", + "style_correct": 1, + "style_probability": 0.35029008984565735, + "style_target": 0, + "support_probability": 0.6043696403503418, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2691435517897389, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4997219145298004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22599999606609344, + 0.8231394290924072, + 0.2895144522190094, + 0.23826757073402405, + 0.1424982100725174, + 0.398449569940567, + 0.06950246542692184, + 0.1632997989654541, + 0.13324102759361267, + 0.1413414478302002, + 0.09726062417030334 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.7981132864952087, + "style_target": 1, + "support_probability": 0.2645227313041687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11377981055032382, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3687472343444824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:801:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2893138527870178, + 0.873109757900238, + 0.34465423226356506, + 0.28643444180488586, + 0.10528694093227386, + 0.1168987825512886, + 0.1115407794713974, + 0.1207113191485405, + 0.08893566578626633, + 0.15270936489105225 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:801", + "split": "test", + "style_correct": 1, + "style_probability": 0.2621961236000061, + "style_target": 0, + "support_probability": 0.7085837125778198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016054080660377337, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9036827683448792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5222514867782593, + 0.3918323218822479, + 0.2703748047351837, + 0.36633631587028503, + 0.05130361393094063 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 0, + "style_probability": 0.354268342256546, + "style_target": 1, + "support_probability": 0.03125612437725067, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009633859649256338, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8461651802062988, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:808:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6062083840370178, + 0.49646592140197754, + 0.3212149441242218, + 0.44160228967666626, + 0.2583094537258148 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:808", + "split": "test", + "style_correct": 0, + "style_probability": 0.554949939250946, + "style_target": 0, + "support_probability": 0.08130586892366409, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5934025730124297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08508285880088806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9050845503807068, + 0.3260686695575714, + 0.36279234290122986, + 0.3260686695575714, + 0.03650924190878868, + 0.07457590103149414 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.41124480962753296, + "style_target": 1, + "support_probability": 0.9261638522148132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6616828703939781, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14866206049919128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:820:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.931113064289093, + 0.30264538526535034, + 0.3507944643497467, + 0.30264538526535034, + 0.051496610045433044, + 0.14236511290073395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:820", + "split": "test", + "style_correct": 0, + "style_probability": 0.6349577903747559, + "style_target": 0, + "support_probability": 0.8938209414482117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.226609098896823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11974958330392838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11858568340539932, + 0.17838504910469055, + 0.40751567482948303, + 0.9410557746887207, + 0.13352422416210175, + 0.1465623676776886, + 0.04435810074210167, + 0.04362223669886589, + 0.10230083763599396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 0, + "style_probability": 0.420097678899765, + "style_target": 1, + "support_probability": 0.22606220841407776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20281525257764257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14018160104751587, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:831:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1318245232105255, + 0.18921969830989838, + 0.42102888226509094, + 0.9700941443443298, + 0.0670892745256424, + 0.07301288843154907, + 0.09244683384895325, + 0.08037496358156204, + 0.100326769053936 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:831", + "split": "test", + "style_correct": 1, + "style_probability": 0.4278978109359741, + "style_target": 0, + "support_probability": 0.2400699257850647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1602417606515978, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2489696741104126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20483778417110443, + 0.7621306777000427, + 0.3242264688014984, + 0.5026285648345947, + 0.05766313150525093, + 0.10522611439228058, + 0.05440443009138107, + 0.11243651807308197, + 0.07298041880130768, + 0.03658898547291756, + 0.07306376844644547 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 0, + "style_probability": 0.21654196083545685, + "style_target": 1, + "support_probability": 0.516510009765625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33115951850331976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19060134887695312, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:841:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.252665251493454, + 0.7982639670372009, + 0.3376975357532501, + 0.4887550175189972, + 0.03913164138793945, + 0.03647609055042267, + 0.07106330990791321, + 0.03411662206053734, + 0.05421312898397446, + 0.039732109755277634, + 0.25940024852752686 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:841", + "split": "test", + "style_correct": 1, + "style_probability": 0.23156128823757172, + "style_target": 0, + "support_probability": 0.6982489824295044, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7372166404856629, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3147211968898773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:856:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9414488077163696, + 0.2110469937324524, + 0.3407636284828186, + 0.3204578459262848, + 0.21582365036010742 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 1, + "style_probability": 0.839455246925354, + "style_target": 1, + "support_probability": 0.5090802907943726, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8669672328895928, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2553526759147644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:856:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9674921631813049, + 0.23235037922859192, + 0.34167391061782837, + 0.28808102011680603, + 0.05926768481731415 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:856", + "split": "test", + "style_correct": 0, + "style_probability": 0.9489860534667969, + "style_target": 0, + "support_probability": 0.6423689126968384, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.564669477492469, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.041249632835388184, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.938283383846283, + 0.3452645242214203, + 0.2621055543422699, + 0.2469877153635025, + 0.14175447821617126, + 0.13844841718673706 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 1, + "style_probability": 0.7296138405799866, + "style_target": 1, + "support_probability": 0.4583003520965576, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20828891529826518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061231646686792374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9201757907867432, + 0.3134307265281677, + 0.22276267409324646, + 0.2612089514732361, + 0.12943102419376373, + 0.1303502768278122 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:860", + "split": "test", + "style_correct": 1, + "style_probability": 0.43986329436302185, + "style_target": 0, + "support_probability": 0.35760998725891113, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04238497096195016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6618843674659729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3427783250808716, + 0.8719955682754517, + 0.21665725111961365, + 0.2692279517650604, + 0.04645451158285141, + 0.07167410850524902, + 0.09036827087402344, + 0.13720126450061798, + 0.09320927411317825 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 0, + "style_probability": 0.4897380769252777, + "style_target": 1, + "support_probability": 0.1353500485420227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006046685384695438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4856211543083191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:865:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5216520428657532, + 0.953845739364624, + 0.2890349328517914, + 0.30360937118530273, + 0.0794353187084198, + 0.12729373574256897, + 0.09916292876005173, + 0.07720594108104706, + 0.16446805000305176 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:865", + "split": "test", + "style_correct": 1, + "style_probability": 0.12260850518941879, + "style_target": 0, + "support_probability": 0.4150685966014862, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.349985547317587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39111536741256714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7531221508979797, + 0.2779303193092346, + 0.41892626881599426, + 0.2363336980342865, + 0.06788468360900879, + 0.15070047974586487, + 0.12191969901323318, + 0.09532884508371353, + 0.08865470439195633, + 0.11539352685213089, + 0.10656658560037613 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.4336787462234497, + "style_target": 1, + "support_probability": 0.4852750301361084, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6486930177261634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23537954688072205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:876:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7675839066505432, + 0.3205423355102539, + 0.4980323612689972, + 0.27063772082328796, + 0.09305047988891602, + 0.05255903676152229, + 0.06884962320327759, + 0.025194955989718437, + 0.06886321306228638, + 0.11835237592458725, + 0.05400211364030838 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:876", + "split": "test", + "style_correct": 0, + "style_probability": 0.7914659380912781, + "style_target": 0, + "support_probability": 0.5819745659828186, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21566063215509776, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3618457019329071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8187125325202942, + 0.15886934101581573, + 0.18257290124893188, + 0.31744900345802307, + 0.12129731476306915 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.8276135325431824, + "style_target": 1, + "support_probability": 0.3532573878765106, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017314044902797818, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4591282904148102, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:892:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8691943883895874, + 0.19790583848953247, + 0.1357799619436264, + 0.33034276962280273, + 0.1752997487783432 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:892", + "split": "test", + "style_correct": 1, + "style_probability": 0.08124542981386185, + "style_target": 0, + "support_probability": 0.28107357025146484, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6603479232351788, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1658547967672348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1593301147222519, + 0.24558040499687195, + 0.20408423244953156, + 0.871711254119873, + 0.05742555111646652, + 0.12769244611263275 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 1, + "style_probability": 0.6729353666305542, + "style_target": 1, + "support_probability": 0.7788420915603638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5698069510968296, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2851182222366333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:895:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2092307060956955, + 0.24241648614406586, + 0.18702177703380585, + 0.809815526008606, + 0.4378637671470642, + 0.0970190241932869 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:895", + "split": "test", + "style_correct": 1, + "style_probability": 0.4978741705417633, + "style_target": 0, + "support_probability": 0.7577094435691833, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6598909948402714, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3131274878978729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20031216740608215, + 0.4077744781970978, + 0.8712522387504578, + 0.5381144285202026, + 0.0256937425583601, + 0.07743503898382187, + 0.04539027065038681, + 0.03332994133234024, + 0.1441720575094223 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.8035011291503906, + "style_target": 1, + "support_probability": 0.6873019933700562, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5111179528287693, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23582565784454346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26334911584854126, + 0.42196908593177795, + 0.8626978397369385, + 0.5248700380325317, + 0.05487434193491936, + 0.18247376382350922, + 0.038300883024930954, + 0.07275435328483582, + 0.09631317853927612 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:902", + "split": "test", + "style_correct": 1, + "style_probability": 0.42122748494148254, + "style_target": 0, + "support_probability": 0.733390212059021, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5912068419611778, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2533986568450928, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8942436575889587, + 0.6937267780303955, + 0.40221625566482544, + 0.32000380754470825, + 0.04415109381079674, + 0.07188363373279572, + 0.05464478209614754, + 0.04852644354104996, + 0.024846458807587624, + 0.04628829285502434, + 0.11317259073257446 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.7585867047309875, + "style_target": 1, + "support_probability": 0.6377868056297302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041568040783521484, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19151324033737183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:908:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9194660782814026, + 0.7559809684753418, + 0.38952797651290894, + 0.35129815340042114, + 0.06015251576900482, + 0.08768229931592941, + 0.07958154380321503, + 0.08998075127601624, + 0.0511527918279171, + 0.08794482052326202 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:908", + "split": "test", + "style_correct": 1, + "style_probability": 0.18223604559898376, + "style_target": 0, + "support_probability": 0.6599288582801819, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6361383380450611, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0794692188501358, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9901004433631897, + 0.6231594085693359, + 0.38067081570625305, + 0.5597625970840454, + 0.05041433498263359 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 1, + "style_probability": 0.5139264464378357, + "style_target": 1, + "support_probability": 0.644670307636261, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5804637839587201, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10779993236064911, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:928:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9691527485847473, + 0.6128158569335938, + 0.4226336181163788, + 0.5750355124473572, + 0.42803332209587097 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:928", + "split": "test", + "style_correct": 0, + "style_probability": 0.5560725927352905, + "style_target": 0, + "support_probability": 0.5566026568412781, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03264696863905026, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8061274290084839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27370479702949524, + 0.6320965886116028, + 0.3764301538467407, + 0.3865901231765747, + 0.10276728123426437, + 0.0774901807308197 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.7760652899742126, + "style_target": 1, + "support_probability": 0.03828262910246849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007797312528943448, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5256904363632202, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:937:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28506234288215637, + 0.6146248579025269, + 0.39508047699928284, + 0.3221299350261688, + 0.20488089323043823, + 0.12226888537406921 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:937", + "split": "test", + "style_correct": 1, + "style_probability": 0.23025061190128326, + "style_target": 0, + "support_probability": 0.059600625187158585, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7985962585732622, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11779990047216415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6224672198295593, + 0.8584169745445251, + 0.4803529977798462, + 0.4418657422065735, + 0.06401176005601883, + 0.020387234166264534, + 0.027283858507871628, + 0.07084494829177856, + 0.04996008798480034 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.7789392471313477, + "style_target": 1, + "support_probability": 0.8902606964111328, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2814653211011411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16323785483837128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:941:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6540930271148682, + 0.9146908521652222, + 0.4609132409095764, + 0.47380608320236206, + 0.052693285048007965, + 0.06395244598388672, + 0.06061355769634247, + 0.03779890760779381, + 0.04720288887619972 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:941", + "split": "test", + "style_correct": 1, + "style_probability": 0.168363556265831, + "style_target": 0, + "support_probability": 0.8794443607330322, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9590390185915253, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028203533962368965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41809317469596863, + 0.30783581733703613, + 0.9802529811859131, + 0.5559169054031372, + 0.03934311494231224, + 0.17721496522426605, + 0.04777444526553154, + 0.08035533875226974, + 0.026366086676716805, + 0.04955264925956726, + 0.08975852280855179 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 1, + "style_probability": 0.8314934372901917, + "style_target": 1, + "support_probability": 0.9760326147079468, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8706192833670023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05939033254981041, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:942:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.34565284848213196, + 0.2648043930530548, + 0.9516224265098572, + 0.5848878026008606, + 0.08440990746021271, + 0.06378090381622314, + 0.06972284615039825, + 0.10194399207830429, + 0.09587862342596054, + 0.1619645357131958 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:942", + "split": "test", + "style_correct": 0, + "style_probability": 0.6072598099708557, + "style_target": 0, + "support_probability": 0.9577138423919678, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.347219877409401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25065284967422485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7004008293151855, + 0.6250178813934326, + 0.5072140097618103, + 0.4395596385002136, + 0.06972631067037582 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 1, + "style_probability": 0.5109675526618958, + "style_target": 1, + "support_probability": 0.5818250179290771, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.712133942530798, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14706449210643768, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:944:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7390559911727905, + 0.6650945544242859, + 0.5826929807662964, + 0.369457483291626, + 0.06971334666013718 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:944", + "split": "test", + "style_correct": 0, + "style_probability": 0.9396278262138367, + "style_target": 0, + "support_probability": 0.8687930703163147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0977290635886612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27684861421585083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16428042948246002, + 0.2916242480278015, + 0.1356632113456726, + 0.8817089200019836, + 0.14295655488967896, + 0.08398210257291794 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 1, + "style_probability": 0.5146759748458862, + "style_target": 1, + "support_probability": 0.2742804288864136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03519004270504201, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29625311493873596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:983:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15853601694107056, + 0.27338308095932007, + 0.17384973168373108, + 0.9550517201423645, + 0.08178680390119553, + 0.1728300303220749 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:983", + "split": "test", + "style_correct": 1, + "style_probability": 0.36578279733657837, + "style_target": 0, + "support_probability": 0.5020330548286438, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.42099956517209586, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31121763586997986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4226720333099365, + 0.35266542434692383, + 0.8969157338142395, + 0.45716479420661926, + 0.05662841349840164, + 0.04528342932462692, + 0.1304999589920044, + 0.10532894730567932, + 0.24230246245861053 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 1, + "style_probability": 0.5293238162994385, + "style_target": 1, + "support_probability": 0.5116267800331116, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6684243597595838, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33075302839279175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:994:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39330747723579407, + 0.3458707332611084, + 0.9064557552337646, + 0.4520070552825928, + 0.043349768966436386, + 0.05395219847559929, + 0.21085308492183685, + 0.13792216777801514, + 0.04617536440491676 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:994", + "split": "test", + "style_correct": 0, + "style_probability": 0.6216091513633728, + "style_target": 0, + "support_probability": 0.641054093837738, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8754713522269598, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031443580985069275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.302141934633255, + 0.9789661169052124, + 0.23586371541023254, + 0.19080813229084015, + 0.16372238099575043, + 0.4718649387359619, + 0.09016647189855576, + 0.1333848536014557, + 0.07957221567630768, + 0.08726274967193604, + 0.05906816944479942 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.33329862356185913, + "style_target": 1, + "support_probability": 0.9564563632011414, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.963871882971123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07697943598031998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1009:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2874540686607361, + 0.9788413643836975, + 0.21470972895622253, + 0.22831061482429504, + 0.07811500132083893, + 0.05682932212948799, + 0.09141059219837189, + 0.052481576800346375, + 0.04995431378483772, + 0.04469127207994461, + 0.18128162622451782 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1009", + "split": "test", + "style_correct": 0, + "style_probability": 0.9435138702392578, + "style_target": 0, + "support_probability": 0.9288682341575623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26474370278526044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3084864914417267, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20896029472351074, + 0.31515008211135864, + 0.8404597043991089, + 0.21320658922195435, + 0.19016499817371368 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 0, + "style_probability": 0.40619605779647827, + "style_target": 1, + "support_probability": 0.6173319816589355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.168907006905739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29642871022224426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19572198390960693, + 0.28143900632858276, + 0.8426281213760376, + 0.16617773473262787, + 0.10463377833366394 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1010", + "split": "test", + "style_correct": 1, + "style_probability": 0.28172415494918823, + "style_target": 0, + "support_probability": 0.5370475053787231, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1629567354766266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.31459593772888184, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18110719323158264, + 0.23648801445960999, + 0.18110719323158264, + 0.8974497318267822, + 0.07955093681812286, + 0.45726925134658813 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 1, + "style_probability": 0.5407284498214722, + "style_target": 1, + "support_probability": 0.245993509888649, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0057935790037021455, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.40977784991264343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3018280565738678, + 0.29561856389045715, + 0.3018280565738678, + 0.9505134224891663, + 0.05408359691500664, + 0.10672677308320999 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1019", + "split": "test", + "style_correct": 1, + "style_probability": 0.14650793373584747, + "style_target": 0, + "support_probability": 0.3036690652370453, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7098775704152587, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09422580897808075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5789139866828918, + 0.17115427553653717, + 0.67718905210495, + 0.908158004283905, + 0.030544277280569077, + 0.12238501757383347, + 0.07523926347494125, + 0.1007397249341011, + 0.18823771178722382 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 0, + "style_probability": 0.4316619634628296, + "style_target": 1, + "support_probability": 0.8384974598884583, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.816880958818551, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08068989962339401, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5791540741920471, + 0.1830097734928131, + 0.7175682187080383, + 0.937324583530426, + 0.04825930297374725, + 0.03001200221478939, + 0.08924739807844162, + 0.04264365881681442, + 0.035353075712919235 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1035", + "split": "test", + "style_correct": 0, + "style_probability": 0.6535548567771912, + "style_target": 0, + "support_probability": 0.8191226720809937, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.326024140712164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11666534841060638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36240720748901367, + 0.2921190559864044, + 0.8516140580177307, + 0.21679911017417908, + 0.0691467747092247, + 0.07074135541915894, + 0.046565406024456024, + 0.052682578563690186, + 0.05707632377743721, + 0.16699166595935822, + 0.1560414731502533 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 1, + "style_probability": 0.5790577530860901, + "style_target": 1, + "support_probability": 0.39324381947517395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2418531376242683, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08582428842782974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1042:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3288675844669342, + 0.26629725098609924, + 0.8728711605072021, + 0.24916011095046997, + 0.06513216346502304, + 0.0713215321302414, + 0.13136647641658783, + 0.08039578050374985, + 0.16252848505973816, + 0.14911815524101257, + 0.14741753041744232 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1042", + "split": "test", + "style_correct": 1, + "style_probability": 0.367769330739975, + "style_target": 0, + "support_probability": 0.38733428716659546, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8946220672040894, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07122328877449036, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9789365530014038, + 0.34176647663116455, + 0.43626126646995544, + 0.603253185749054, + 0.07483173161745071 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 1, + "style_probability": 0.6839542984962463, + "style_target": 1, + "support_probability": 0.8274657726287842, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8657525724168557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06975001841783524, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9668263792991638, + 0.36479413509368896, + 0.4552120864391327, + 0.573631763458252, + 0.10454553365707397 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1044", + "split": "test", + "style_correct": 0, + "style_probability": 0.5515032410621643, + "style_target": 0, + "support_probability": 0.8411162495613098, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10511101259061695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5586605072021484, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.540494441986084, + 0.8756701350212097, + 0.25502651929855347, + 0.3736434578895569, + 0.13516080379486084, + 0.07269074767827988 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.8930521607398987, + "style_target": 1, + "support_probability": 0.13093872368335724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009773450487064796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6854156851768494, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1045:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5271353125572205, + 0.8536099195480347, + 0.25260084867477417, + 0.33399519324302673, + 0.06927010416984558, + 0.13111193478107452 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1045", + "split": "test", + "style_correct": 1, + "style_probability": 0.4653364419937134, + "style_target": 0, + "support_probability": 0.04703598842024803, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8839480649583108, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0929655209183693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1050:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3483448624610901, + 0.36538243293762207, + 0.9935504198074341, + 0.34260979294776917, + 0.10265811532735825, + 0.08493600785732269, + 0.07211196422576904, + 0.09081634879112244, + 0.11739343404769897 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.7569869756698608, + "style_target": 1, + "support_probability": 0.8551570773124695, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3250763561465817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10943250358104706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1050:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42581430077552795, + 0.47660282254219055, + 0.9984583854675293, + 0.38749343156814575, + 0.1099434494972229, + 0.03667326644062996, + 0.09721503406763077, + 0.06968370079994202, + 0.08191150426864624 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1050", + "split": "test", + "style_correct": 1, + "style_probability": 0.18852894008159637, + "style_target": 0, + "support_probability": 0.8498560786247253, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5960100375453976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3794160783290863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24295751750469208, + 0.6211749911308289, + 0.8585681915283203, + 0.26116275787353516, + 0.059552669525146484, + 0.06309380382299423, + 0.04931716248393059, + 0.07909097522497177, + 0.1146639883518219, + 0.15531133115291595 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 1, + "style_probability": 0.8670632243156433, + "style_target": 1, + "support_probability": 0.4265044927597046, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27590622094006, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31867316365242004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1054:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21108919382095337, + 0.5483604073524475, + 0.8242761492729187, + 0.26836633682250977, + 0.19246642291545868, + 0.07078851759433746, + 0.08132472634315491, + 0.07207001000642776, + 0.09473840892314911, + 0.08776523917913437 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1054", + "split": "test", + "style_correct": 1, + "style_probability": 0.36199864745140076, + "style_target": 0, + "support_probability": 0.44576436281204224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07404842788984213, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7563377618789673, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29412853717803955, + 0.6540637016296387, + 0.2388259917497635, + 0.41141822934150696, + 0.11954864859580994 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 1, + "style_probability": 0.5091065764427185, + "style_target": 1, + "support_probability": 0.18997210264205933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.285417065623232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5007910132408142, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1061:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34376123547554016, + 0.7279971837997437, + 0.26047998666763306, + 0.4785468280315399, + 0.06506219506263733 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1061", + "split": "test", + "style_correct": 0, + "style_probability": 0.7688833475112915, + "style_target": 0, + "support_probability": 0.2717108428478241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1544927151281481, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2632160186767578, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4663050174713135, + 0.1800471991300583, + 0.33225730061531067, + 0.884264349937439, + 0.07102253288030624, + 0.19322805106639862 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.528698205947876, + "style_target": 1, + "support_probability": 0.22905799746513367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0789719659667014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2839871346950531, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1067:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42425861954689026, + 0.17361076176166534, + 0.34524834156036377, + 0.8891561031341553, + 0.1136569082736969, + 0.11525338888168335 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1067", + "split": "test", + "style_correct": 1, + "style_probability": 0.2740359306335449, + "style_target": 0, + "support_probability": 0.19941119849681854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4996529059150312, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14910532534122467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8105875849723816, + 0.3425564467906952, + 0.14209046959877014, + 0.3834521174430847, + 0.033269040286540985, + 0.03528870642185211, + 0.05906319245696068, + 0.06990846991539001, + 0.15059348940849304 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 1, + "style_probability": 0.55191969871521, + "style_target": 1, + "support_probability": 0.837914228439331, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09163243449048064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10259800404310226, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8508886098861694, + 0.3866380751132965, + 0.17271701991558075, + 0.3743261992931366, + 0.07016018033027649, + 0.07019949704408646, + 0.07232552021741867, + 0.08204568177461624, + 0.06856384128332138 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1068", + "split": "test", + "style_correct": 1, + "style_probability": 0.24984095990657806, + "style_target": 0, + "support_probability": 0.8400877714157104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32802325587362624, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4013434946537018, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5585275292396545, + 0.4025518000125885, + 0.7182806134223938, + 0.7053372263908386, + 0.05896713584661484, + 0.06335975229740143, + 0.05053816735744476, + 0.03455254063010216, + 0.06957443803548813, + 0.09284991770982742, + 0.1751626580953598 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 1, + "style_probability": 0.6002073287963867, + "style_target": 1, + "support_probability": 0.6894082427024841, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39294166019618615, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4132402241230011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6642774343490601, + 0.48809507489204407, + 0.7735944986343384, + 0.7374043464660645, + 0.1268671154975891, + 0.07232758402824402, + 0.04516535624861717, + 0.0677235797047615, + 0.07645174115896225, + 0.12432220578193665 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1070", + "split": "test", + "style_correct": 0, + "style_probability": 0.8900967240333557, + "style_target": 0, + "support_probability": 0.5051155090332031, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.35800623488410105, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5768133997917175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3221210241317749, + 0.6676022410392761, + 0.2738019526004791, + 0.33953413367271423, + 0.08349982649087906 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 1, + "style_probability": 0.5866062641143799, + "style_target": 1, + "support_probability": 0.6996042728424072, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26561386395946585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4806036949157715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1093:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3577860891819, + 0.7002891302108765, + 0.2697664797306061, + 0.2694631814956665, + 0.1251140534877777 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1093", + "split": "test", + "style_correct": 1, + "style_probability": 0.44890159368515015, + "style_target": 0, + "support_probability": 0.7258382439613342, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5024208518131701, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.062231145799160004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12960149347782135, + 0.26261046528816223, + 0.9844957590103149, + 0.31057241559028625, + 0.18601544201374054, + 0.07815179228782654 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.6396523118019104, + "style_target": 1, + "support_probability": 0.3883818984031677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12446744083506364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10892200469970703, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15230922400951385, + 0.33719420433044434, + 0.9977648258209229, + 0.3372623920440674, + 0.10359258949756622, + 0.12386224418878555 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1110", + "split": "test", + "style_correct": 1, + "style_probability": 0.24726930260658264, + "style_target": 0, + "support_probability": 0.3590737581253052, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6244878260371962, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07326781004667282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23287074267864227, + 0.1394592672586441, + 0.49362146854400635, + 0.9773779511451721, + 0.13577628135681152, + 0.15258759260177612, + 0.10087825357913971, + 0.24864570796489716, + 0.0976712629199028 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 1, + "style_probability": 0.5471715331077576, + "style_target": 1, + "support_probability": 0.6351010799407959, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.404606634887017, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20758405327796936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1111:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24362166225910187, + 0.14275892078876495, + 0.4837077558040619, + 0.9883284568786621, + 0.07256235182285309, + 0.08547567576169968, + 0.08236376196146011, + 0.09366302192211151, + 0.16734372079372406 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1111", + "split": "test", + "style_correct": 0, + "style_probability": 0.5254158973693848, + "style_target": 0, + "support_probability": 0.4328572750091553, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028428435691216784, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7396723031997681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6838616728782654, + 0.1713111847639084, + 0.09148533642292023, + 0.3247208297252655, + 0.04195008426904678, + 0.04224654659628868, + 0.09210985153913498, + 0.0751425251364708, + 0.060040589421987534, + 0.14524051547050476, + 0.09205174446105957 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 0, + "style_probability": 0.4901975691318512, + "style_target": 1, + "support_probability": 0.05609442666172981, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025160163565125515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6906542181968689, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1116:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7883710265159607, + 0.19000931084156036, + 0.0871758908033371, + 0.32253822684288025, + 0.1336047500371933, + 0.1353832334280014, + 0.3866136968135834, + 0.06611660867929459, + 0.16084370017051697, + 0.0670509785413742, + 0.14712148904800415 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1116", + "split": "test", + "style_correct": 1, + "style_probability": 0.27551379799842834, + "style_target": 0, + "support_probability": 0.08151218295097351, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.336890283383422, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5825188755989075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2724732458591461, + 0.41244855523109436, + 0.31128090620040894, + 0.6543392539024353, + 0.16380997002124786 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 1, + "style_probability": 0.9328746795654297, + "style_target": 1, + "support_probability": 0.26692894101142883, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19346252080690718, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5694786310195923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1127:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2554194927215576, + 0.4045082628726959, + 0.3151436746120453, + 0.5594272017478943, + 0.07947064936161041 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1127", + "split": "test", + "style_correct": 0, + "style_probability": 0.6728602647781372, + "style_target": 0, + "support_probability": 0.33376309275627136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1986761916621731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.321081280708313, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7491381764411926, + 0.15895968675613403, + 0.15546779334545135, + 0.3484036922454834, + 0.13839787244796753, + 0.08668332546949387 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.680346667766571, + "style_target": 1, + "support_probability": 0.25694406032562256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16012918262527262, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19741110503673553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1128:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7410026788711548, + 0.18451164662837982, + 0.23803876340389252, + 0.31703364849090576, + 0.08420465141534805, + 0.12461505830287933 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1128", + "split": "test", + "style_correct": 1, + "style_probability": 0.31484243273735046, + "style_target": 0, + "support_probability": 0.27494192123413086, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7210283552410601, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12777958810329437, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4159829020500183, + 0.9007148742675781, + 0.19325943291187286, + 0.09053389728069305, + 0.049518462270498276, + 0.04797811433672905, + 0.08237415552139282, + 0.14573347568511963, + 0.10562236607074738 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.6233119368553162, + "style_target": 1, + "support_probability": 0.6151047348976135, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22705196509755685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12088385969400406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1129:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41288915276527405, + 0.9217403531074524, + 0.2526850998401642, + 0.1044749841094017, + 0.08386988192796707, + 0.12674270570278168, + 0.10817600786685944, + 0.0830315351486206, + 0.10138527303934097 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1129", + "split": "test", + "style_correct": 1, + "style_probability": 0.12827417254447937, + "style_target": 0, + "support_probability": 0.691495954990387, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7247065431404209, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15646810829639435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7541647553443909, + 0.1673218011856079, + 0.40311938524246216, + 0.3829881548881531, + 0.07050745189189911, + 0.022602107375860214, + 0.03751716762781143, + 0.0424673929810524, + 0.044112350791692734, + 0.07448027282953262, + 0.08973405510187149 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.8747864365577698, + "style_target": 1, + "support_probability": 0.8575953245162964, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7051379485543698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09737074375152588, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1132:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7840413451194763, + 0.20890580117702484, + 0.33012035489082336, + 0.3334902226924896, + 0.0795305147767067, + 0.1081334799528122, + 0.0702253207564354, + 0.05099889636039734, + 0.06720291823148727, + 0.11902135610580444, + 0.05549126863479614 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1132", + "split": "test", + "style_correct": 1, + "style_probability": 0.4698924422264099, + "style_target": 0, + "support_probability": 0.8846874237060547, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04897652269932306, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8121283650398254, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19501793384552002, + 0.5815054774284363, + 0.1811484843492508, + 0.34049180150032043, + 0.15659116208553314 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.4749940037727356, + "style_target": 1, + "support_probability": 0.15477590262889862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01977241021373111, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9582921862602234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20053403079509735, + 0.5522245764732361, + 0.1805438995361328, + 0.417538046836853, + 0.09818539023399353 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1133", + "split": "test", + "style_correct": 0, + "style_probability": 0.8402950167655945, + "style_target": 0, + "support_probability": 0.035951633006334305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6246469188861958, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15031889081001282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27784302830696106, + 0.21654020249843597, + 0.904478132724762, + 0.29807353019714355, + 0.16853296756744385, + 0.15440291166305542 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.5468920469284058, + "style_target": 1, + "support_probability": 0.7678520679473877, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5504694627305575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08477124571800232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25792407989501953, + 0.2095501571893692, + 0.9163511395454407, + 0.3049027621746063, + 0.27985817193984985, + 0.1211390271782875 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1142", + "split": "test", + "style_correct": 1, + "style_probability": 0.34606418013572693, + "style_target": 0, + "support_probability": 0.8338956832885742, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8310615648110442, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09151443094015121, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19655385613441467, + 0.9282478094100952, + 0.20835787057876587, + 0.19655385613441467, + 0.07239191979169846, + 0.15946270525455475, + 0.058613888919353485, + 0.20516932010650635, + 0.10609142482280731 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.6551283001899719, + "style_target": 1, + "support_probability": 0.8809215426445007, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2870837535062165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07583164423704147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1145:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18691127002239227, + 0.9782029986381531, + 0.1921883523464203, + 0.18691127002239227, + 0.05000218749046326, + 0.057725802063941956, + 0.09340973198413849, + 0.12288088351488113, + 0.13080382347106934 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1145", + "split": "test", + "style_correct": 1, + "style_probability": 0.20588292181491852, + "style_target": 0, + "support_probability": 0.8964329361915588, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08223508119195701, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7862740159034729, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2338341623544693, + 0.5377681255340576, + 0.48096275329589844, + 0.38707199692726135, + 0.040530186146497726, + 0.07561496645212173, + 0.055287424474954605, + 0.10293767601251602, + 0.07863865047693253, + 0.17343981564044952, + 0.09893473982810974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 1, + "style_probability": 0.5434006452560425, + "style_target": 1, + "support_probability": 0.3149859309196472, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058071422190055795, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8528355360031128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1149:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27601081132888794, + 0.5941230654716492, + 0.5790759921073914, + 0.42894211411476135, + 0.10061003267765045, + 0.08408105373382568, + 0.06989476829767227, + 0.07147688418626785, + 0.08063532412052155, + 0.12977758049964905, + 0.05942999944090843 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1149", + "split": "test", + "style_correct": 0, + "style_probability": 0.8207356929779053, + "style_target": 0, + "support_probability": 0.09615667909383774, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9025385749002269, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07032570242881775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9804835915565491, + 0.43858498334884644, + 0.45877745747566223, + 0.1100587546825409, + 0.06113383546471596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 0, + "style_probability": 0.47296085953712463, + "style_target": 1, + "support_probability": 0.9063946604728699, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7707288245991961, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07770967483520508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1156:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9766403436660767, + 0.5209789276123047, + 0.5373778343200684, + 0.14278030395507812, + 0.20144519209861755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1156", + "split": "test", + "style_correct": 1, + "style_probability": 0.3625852167606354, + "style_target": 0, + "support_probability": 0.8405125737190247, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022812724874349044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45570963621139526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7526037693023682, + 0.1352720856666565, + 0.32819902896881104, + 0.19962307810783386, + 0.058030012995004654, + 0.05607769265770912 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 0, + "style_probability": 0.4010259509086609, + "style_target": 1, + "support_probability": 0.13711166381835938, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007010312290272934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4463057816028595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8319464921951294, + 0.1592230349779129, + 0.3364465534687042, + 0.26991817355155945, + 0.06663040071725845, + 0.06406456977128983 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1160", + "split": "test", + "style_correct": 1, + "style_probability": 0.1783924400806427, + "style_target": 0, + "support_probability": 0.1719455122947693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0351942284074196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6523643136024475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2318764626979828, + 0.76105135679245, + 0.3530136048793793, + 0.38851258158683777, + 0.06403177231550217, + 0.11020224541425705, + 0.33158156275749207, + 0.11116297543048859, + 0.09511588513851166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 1, + "style_probability": 0.5504432916641235, + "style_target": 1, + "support_probability": 0.11510337889194489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02531356973972665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7840047478675842, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1161:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1998155415058136, + 0.7627913355827332, + 0.2923417389392853, + 0.3069174885749817, + 0.055120956152677536, + 0.11599238216876984, + 0.09803805500268936, + 0.060489553958177567, + 0.09499865770339966 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1161", + "split": "test", + "style_correct": 0, + "style_probability": 0.8718962669372559, + "style_target": 0, + "support_probability": 0.07649584114551544, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005237405846919176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9339284896850586, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.894760012626648, + 0.15021388232707977, + 0.42335933446884155, + 0.15021388232707977, + 0.08999902755022049, + 0.06829457730054855, + 0.042398493736982346, + 0.04810617119073868, + 0.07741796225309372, + 0.07083107531070709, + 0.06289065629243851 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 1, + "style_probability": 0.6558762788772583, + "style_target": 1, + "support_probability": 0.010185019113123417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013619989539524811, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8708572387695312, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1172:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.890571117401123, + 0.14476802945137024, + 0.31771135330200195, + 0.14476802945137024, + 0.07071869820356369, + 0.058867767453193665, + 0.07209612429141998, + 0.21312806010246277, + 0.3150573670864105, + 0.09861145168542862, + 0.28588399291038513 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1172", + "split": "test", + "style_correct": 1, + "style_probability": 0.37545427680015564, + "style_target": 0, + "support_probability": 0.02844918891787529, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26565710852107394, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5732406973838806, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1663213074207306, + 0.9192727208137512, + 0.34583982825279236, + 0.17457729578018188, + 0.17246806621551514 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 1, + "style_probability": 0.827523410320282, + "style_target": 1, + "support_probability": 0.16788916289806366, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11232600877832866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5099405646324158, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1173:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17394466698169708, + 0.920260488986969, + 0.3425882160663605, + 0.19785483181476593, + 0.11018282175064087 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1173", + "split": "test", + "style_correct": 0, + "style_probability": 0.5021658539772034, + "style_target": 0, + "support_probability": 0.14299438893795013, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8633638177498391, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.052101701498031616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16895343363285065, + 0.09772717207670212, + 0.5426869988441467, + 0.9974562525749207, + 0.05168894678354263, + 0.11836405098438263 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 1, + "style_probability": 0.5200378894805908, + "style_target": 1, + "support_probability": 0.9155864715576172, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.595019673826851, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06494128704071045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1175:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.205649271607399, + 0.1329135298728943, + 0.5557424426078796, + 0.9912449717521667, + 0.058639850467443466, + 0.08216013014316559 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1175", + "split": "test", + "style_correct": 1, + "style_probability": 0.20710650086402893, + "style_target": 0, + "support_probability": 0.8903741836547852, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6064247234983213, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47698694467544556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4003875255584717, + 0.8202521204948425, + 0.07874967157840729, + 0.332868367433548, + 0.10400283336639404, + 0.04633855074644089, + 0.11357997357845306, + 0.0385301299393177, + 0.0812535434961319 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.8429015874862671, + "style_target": 1, + "support_probability": 0.37078821659088135, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015242412054280556, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5403044819831848, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1177:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4144159257411957, + 0.8641045689582825, + 0.12261224538087845, + 0.3553420603275299, + 0.15061433613300323, + 0.05542847141623497, + 0.10628603398799896, + 0.06448682397603989, + 0.1326548010110855 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1177", + "split": "test", + "style_correct": 1, + "style_probability": 0.09076611697673798, + "style_target": 0, + "support_probability": 0.2418212741613388, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28218518520549907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3659760355949402, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2807168960571289, + 0.4307275116443634, + 0.28757044672966003, + 0.8791862726211548, + 0.048397161066532135, + 0.04000268131494522, + 0.09855704754590988, + 0.08143819123506546, + 0.06686453521251678, + 0.045937273651361465, + 0.07045123726129532 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 1, + "style_probability": 0.8585046529769897, + "style_target": 1, + "support_probability": 0.4219949543476105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08524226345829788, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44337090849876404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1179:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2636859118938446, + 0.4176192581653595, + 0.31131646037101746, + 0.8349912166595459, + 0.08709720522165298, + 0.05313190072774887, + 0.04529838636517525, + 0.15712393820285797, + 0.10727190226316452, + 0.05135376378893852 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1179", + "split": "test", + "style_correct": 0, + "style_probability": 0.5055339336395264, + "style_target": 0, + "support_probability": 0.3627548813819885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009244136926963572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5387404561042786, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7130062580108643, + 0.2780078649520874, + 0.2436017394065857, + 0.1612551063299179, + 0.08437575399875641 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.37158340215682983, + "style_target": 1, + "support_probability": 0.04433554783463478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03706948882589934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4294750988483429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1180:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7457605004310608, + 0.3237980008125305, + 0.24348121881484985, + 0.1502072662115097, + 0.035190399736166 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1180", + "split": "test", + "style_correct": 0, + "style_probability": 0.816927433013916, + "style_target": 0, + "support_probability": 0.11595407128334045, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.22631380633951798, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2003391683101654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18320110440254211, + 0.9222658276557922, + 0.19185598194599152, + 0.23640713095664978, + 0.1325072944164276, + 0.17572392523288727 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 0, + "style_probability": 0.29314202070236206, + "style_target": 1, + "support_probability": 0.4992085099220276, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08321597098424773, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29322606325149536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22468309104442596, + 0.978667676448822, + 0.2872510254383087, + 0.30565527081489563, + 0.110182024538517, + 0.1658090204000473 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1189", + "split": "test", + "style_correct": 1, + "style_probability": 0.12167598307132721, + "style_target": 0, + "support_probability": 0.7245311141014099, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007116970612042089, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9841471910476685, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44766202569007874, + 0.1976862996816635, + 0.731292188167572, + 0.234143927693367, + 0.11830870807170868, + 0.10906900465488434, + 0.11682736873626709, + 0.08995234966278076, + 0.10874297469854355 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.3721884489059448, + "style_target": 1, + "support_probability": 0.003365297568961978, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00539990693256012, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9713597297668457, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1194:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4414237141609192, + 0.2151203602552414, + 0.7835058569908142, + 0.2220582664012909, + 0.10012228786945343, + 0.1504836082458496, + 0.3674463629722595, + 0.1384151428937912, + 0.09782259166240692 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1194", + "split": "test", + "style_correct": 0, + "style_probability": 0.6127872467041016, + "style_target": 0, + "support_probability": 0.009694522246718407, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.45409033699304047, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03469960018992424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17792974412441254, + 0.4893239736557007, + 0.39967116713523865, + 0.9729363918304443, + 0.08508701622486115, + 0.1065126433968544, + 0.06767355650663376, + 0.0887189507484436, + 0.10096931457519531, + 0.06450986117124557, + 0.06862086057662964 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.2834916114807129, + "style_target": 1, + "support_probability": 0.8929410576820374, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8982117128173321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03554246947169304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1195:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17343451082706451, + 0.5186387896537781, + 0.4205971360206604, + 0.9812986254692078, + 0.15925675630569458, + 0.07111848145723343, + 0.03968372568488121, + 0.05311432108283043, + 0.09363538026809692, + 0.06552869081497192, + 0.053875453770160675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1195", + "split": "test", + "style_correct": 0, + "style_probability": 0.8516190648078918, + "style_target": 0, + "support_probability": 0.9104365706443787, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1563613257601446, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5887646675109863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16066588461399078, + 0.298473596572876, + 0.23973402380943298, + 0.5303228497505188, + 0.04404284432530403 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.8161810636520386, + "style_target": 1, + "support_probability": 0.24544398486614227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018230690496481565, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7306508421897888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1203:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19562053680419922, + 0.33811289072036743, + 0.23252946138381958, + 0.6535788178443909, + 0.07989898324012756 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1203", + "split": "test", + "style_correct": 1, + "style_probability": 0.11083023995161057, + "style_target": 0, + "support_probability": 0.21844464540481567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.52127918167578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44521015882492065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34127557277679443, + 0.8414697051048279, + 0.15820877254009247, + 0.3820507228374481, + 0.0562661774456501, + 0.058012302964925766 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 1, + "style_probability": 0.6976719498634338, + "style_target": 1, + "support_probability": 0.7038028836250305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48966499178224865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38378527760505676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1205:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3778442442417145, + 0.7614465951919556, + 0.2031157910823822, + 0.38047507405281067, + 0.06486951559782028, + 0.04461868107318878 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1205", + "split": "test", + "style_correct": 0, + "style_probability": 0.5149194002151489, + "style_target": 0, + "support_probability": 0.7787126302719116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10749184580662607, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29491469264030457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5806479454040527, + 0.17178478837013245, + 0.5806479454040527, + 0.8559974431991577, + 0.04882409796118736, + 0.05529261380434036, + 0.07435369491577148, + 0.043526940047740936, + 0.09557485580444336 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.6338887214660645, + "style_target": 1, + "support_probability": 0.5445535778999329, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006498218505775143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8006689548492432, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1219:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5159711837768555, + 0.18130344152450562, + 0.5159711837768555, + 0.7837916612625122, + 0.10166650265455246, + 0.15102244913578033, + 0.11569298803806305, + 0.10239072144031525, + 0.11755183339118958 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1219", + "split": "test", + "style_correct": 1, + "style_probability": 0.2503334581851959, + "style_target": 0, + "support_probability": 0.06700785458087921, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017347478094724145, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9522519111633301, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6009660959243774, + 0.46827179193496704, + 0.2903553545475006, + 0.46827179193496704, + 0.03245796263217926, + 0.14321567118167877, + 0.050830718129873276, + 0.07079388201236725, + 0.11219876259565353, + 0.052334342151880264, + 0.06007365137338638 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.7513622641563416, + "style_target": 1, + "support_probability": 0.006345472764223814, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006133882546325712, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9496209621429443, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1224:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6969373226165771, + 0.563011884689331, + 0.24751122295856476, + 0.563011884689331, + 0.08023390173912048, + 0.06980997323989868, + 0.12475050240755081, + 0.06697548925876617, + 0.07353357970714569, + 0.10306790471076965 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1224", + "split": "test", + "style_correct": 1, + "style_probability": 0.1911323219537735, + "style_target": 0, + "support_probability": 0.019378094002604485, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4141399660873901, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2102222591638565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9173731803894043, + 0.34871017932891846, + 0.31830546259880066, + 0.34871017932891846, + 0.11928222328424454 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.5894991159439087, + "style_target": 1, + "support_probability": 0.5015206336975098, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03617714485663282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17562392354011536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1228:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9662082195281982, + 0.3091057240962982, + 0.29631340503692627, + 0.3091057240962982, + 0.09113039076328278 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1228", + "split": "test", + "style_correct": 1, + "style_probability": 0.16402079164981842, + "style_target": 0, + "support_probability": 0.5077950954437256, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00638280110601075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9656449556350708, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2786542773246765, + 0.7000184059143066, + 0.16336098313331604, + 0.5326025485992432, + 0.08429437130689621, + 0.08080960810184479 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 1, + "style_probability": 0.5845347046852112, + "style_target": 1, + "support_probability": 0.015079750679433346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008094224960243644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.982909083366394, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1241:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2683577835559845, + 0.6716415286064148, + 0.1615591049194336, + 0.5055270791053772, + 0.05821086838841438, + 0.12227920442819595 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1241", + "split": "test", + "style_correct": 0, + "style_probability": 0.8622806072235107, + "style_target": 0, + "support_probability": 0.008674653246998787, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6804055172473689, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.125249445438385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26632779836654663, + 0.771177351474762, + 0.4369044005870819, + 0.4872438907623291, + 0.058362413197755814, + 0.08546342700719833, + 0.18535088002681732, + 0.056477952748537064, + 0.07523394376039505 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 0, + "style_probability": 0.4160633385181427, + "style_target": 1, + "support_probability": 0.9153041839599609, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7086675984385771, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057745311409235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1245:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26196056604385376, + 0.8271178603172302, + 0.4840657711029053, + 0.5378337502479553, + 0.1333080530166626, + 0.08898353576660156, + 0.07286351174116135, + 0.03234117105603218, + 0.11163564771413803 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1245", + "split": "test", + "style_correct": 1, + "style_probability": 0.289406418800354, + "style_target": 0, + "support_probability": 0.9416030645370483, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0064333114376601275, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9827631711959839, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6604438424110413, + 0.22453144192695618, + 0.20745670795440674, + 0.5119874477386475, + 0.07703043520450592, + 0.05590559542179108, + 0.07075778394937515, + 0.15167191624641418, + 0.1719014048576355, + 0.0732702687382698, + 0.08412409573793411 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 1, + "style_probability": 0.5942692756652832, + "style_target": 1, + "support_probability": 0.03136372193694115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00529028961219244, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9420942664146423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1248:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.717846691608429, + 0.24525132775306702, + 0.19556476175785065, + 0.5117106437683105, + 0.2427028864622116, + 0.12807869911193848, + 0.45079249143600464, + 0.07553994655609131, + 0.06667332351207733, + 0.1698460876941681 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1248", + "split": "test", + "style_correct": 1, + "style_probability": 0.35719218850135803, + "style_target": 0, + "support_probability": 0.11320332437753677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008367580865815684, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8724787831306458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.46259260177612305, + 0.19975368678569794, + 0.18710669875144958, + 0.19975368678569794, + 0.1191960945725441 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 0, + "style_probability": 0.41005218029022217, + "style_target": 1, + "support_probability": 0.06377902626991272, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032896475475817435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7590848803520203, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5169146656990051, + 0.24615783989429474, + 0.1744936853647232, + 0.24615783989429474, + 0.05552823096513748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1256", + "split": "test", + "style_correct": 0, + "style_probability": 0.7422382235527039, + "style_target": 0, + "support_probability": 0.0468435063958168, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020532090904963333, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5856278538703918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4676796495914459, + 0.18887798488140106, + 0.5236321687698364, + 0.27900436520576477, + 0.11930340528488159, + 0.1378350406885147 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 1, + "style_probability": 0.5535988211631775, + "style_target": 1, + "support_probability": 0.06892437487840652, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020292880412992268, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24047023057937622, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5580016374588013, + 0.18016666173934937, + 0.5495519638061523, + 0.24423861503601074, + 0.13841122388839722, + 0.2994919419288635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1260", + "split": "test", + "style_correct": 1, + "style_probability": 0.30291473865509033, + "style_target": 0, + "support_probability": 0.12268852442502975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4783115359960668, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3901618719100952, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26834210753440857, + 0.7671775221824646, + 0.2070005089044571, + 0.25620517134666443, + 0.043383389711380005, + 0.0705576092004776, + 0.05885903164744377, + 0.06297903507947922, + 0.13988646864891052 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.5969972014427185, + "style_target": 1, + "support_probability": 0.7483987212181091, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011415338780463021, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5311724543571472, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29246798157691956, + 0.7619789242744446, + 0.2137463241815567, + 0.326495498418808, + 0.050358183681964874, + 0.0418974906206131, + 0.054382968693971634, + 0.06991146504878998, + 0.11748429387807846 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1261", + "split": "test", + "style_correct": 1, + "style_probability": 0.159291073679924, + "style_target": 0, + "support_probability": 0.4782698452472687, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8716880125288284, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.014274712651968002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.329881876707077, + 0.329881876707077, + 0.3951130211353302, + 0.9973984956741333, + 0.027498578652739525, + 0.04484543949365616, + 0.05278843268752098, + 0.057775817811489105, + 0.04605812579393387, + 0.04066745191812515, + 0.06017843261361122 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 1, + "style_probability": 0.6314899921417236, + "style_target": 1, + "support_probability": 0.8881692886352539, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7667214014963406, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009146274998784065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1263:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4249754846096039, + 0.4249754846096039, + 0.42024096846580505, + 0.999663233757019, + 0.029714424163103104, + 0.04877233877778053, + 0.08250582963228226, + 0.028157750144600868, + 0.034972548484802246, + 0.09377089887857437, + 0.040622685104608536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1263", + "split": "test", + "style_correct": 0, + "style_probability": 0.6304400563240051, + "style_target": 0, + "support_probability": 0.8375376462936401, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8135544879269041, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2408582717180252, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44272685050964355, + 0.873434841632843, + 0.5317786931991577, + 0.8376867771148682, + 0.1190585196018219 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.9269196391105652, + "style_target": 1, + "support_probability": 0.7599288821220398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08002412608765574, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2682676911354065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1265:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.544241189956665, + 0.9209306240081787, + 0.6278629302978516, + 0.8717458248138428, + 0.1020786315202713 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1265", + "split": "test", + "style_correct": 1, + "style_probability": 0.1844659298658371, + "style_target": 0, + "support_probability": 0.6964293122291565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6879554600150186, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20623831450939178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9142093658447266, + 0.18084773421287537, + 0.39476218819618225, + 0.5719133615493774, + 0.06935983151197433, + 0.154371440410614 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 1, + "style_probability": 0.9179585576057434, + "style_target": 1, + "support_probability": 0.4336494505405426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5805758684102962, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14589184522628784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1280:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9204627871513367, + 0.16792930662631989, + 0.39013904333114624, + 0.4963776171207428, + 0.05754420533776283, + 0.05619414523243904 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1280", + "split": "test", + "style_correct": 0, + "style_probability": 0.5910999178886414, + "style_target": 0, + "support_probability": 0.629725456237793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0041451821840232115, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8771980404853821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4777783155441284, + 0.3184891939163208, + 0.20125409960746765, + 0.6491230726242065, + 0.13317526876926422, + 0.09406500309705734, + 0.051105257123708725, + 0.0706634372472763, + 0.0908598005771637 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 1, + "style_probability": 0.8386343717575073, + "style_target": 1, + "support_probability": 0.006338576320558786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0022301442275268757, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9030843377113342, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1283:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44737496972084045, + 0.265032559633255, + 0.22527992725372314, + 0.6098846197128296, + 0.1658766269683838, + 0.079163558781147, + 0.08055650442838669, + 0.11686141788959503, + 0.062269821763038635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1283", + "split": "test", + "style_correct": 1, + "style_probability": 0.4504164457321167, + "style_target": 0, + "support_probability": 0.004987939260900021, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08937593005871049, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3183291554450989, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2869429886341095, + 0.27699965238571167, + 0.4111030101776123, + 0.8955642580986023, + 0.06827425956726074, + 0.12518295645713806, + 0.08270766586065292, + 0.042776238173246384, + 0.0597170926630497, + 0.08302590250968933, + 0.11958582699298859 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.7369064092636108, + "style_target": 1, + "support_probability": 0.17285463213920593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0042645302923147175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3653205633163452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1291:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26085346937179565, + 0.29727745056152344, + 0.4093310534954071, + 0.9165984392166138, + 0.10367635637521744, + 0.08741868287324905, + 0.19860337674617767, + 0.09260118007659912, + 0.08743245154619217, + 0.08041393756866455 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1291", + "split": "test", + "style_correct": 1, + "style_probability": 0.13991807401180267, + "style_target": 0, + "support_probability": 0.1434607356786728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10170235609484737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42637503147125244, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7313671112060547, + 0.4262668192386627, + 0.44900357723236084, + 0.36115217208862305, + 0.11101453006267548 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 0, + "style_probability": 0.3043064475059509, + "style_target": 1, + "support_probability": 0.4776451289653778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2184836492900644, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5349003076553345, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1292:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7306209802627563, + 0.4309821128845215, + 0.40952441096305847, + 0.34409934282302856, + 0.17271576821804047 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1292", + "split": "test", + "style_correct": 0, + "style_probability": 0.5161505341529846, + "style_target": 0, + "support_probability": 0.3309684693813324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42509377324330444, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09538640826940536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.186492919921875, + 0.22435450553894043, + 0.681990921497345, + 0.201904296875, + 0.06919077038764954, + 0.15819332003593445 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.4121360778808594, + "style_target": 1, + "support_probability": 0.7498686909675598, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6898141699372146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08335854113101959, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1294:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.169088214635849, + 0.24872709810733795, + 0.740524172782898, + 0.19579902291297913, + 0.038581833243370056, + 0.08093587309122086 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1294", + "split": "test", + "style_correct": 0, + "style_probability": 0.8952880501747131, + "style_target": 0, + "support_probability": 0.7715806365013123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1911970846940019, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20291191339492798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.284091979265213, + 0.8881129026412964, + 0.5104457139968872, + 0.284091979265213, + 0.08864787966012955, + 0.11019301414489746, + 0.09801594167947769, + 0.03831298649311066, + 0.06892538070678711 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 1, + "style_probability": 0.587640643119812, + "style_target": 1, + "support_probability": 0.407856822013855, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08705446691476125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2259579449892044, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1301:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28391703963279724, + 0.9149993658065796, + 0.5659378170967102, + 0.28391703963279724, + 0.05454697459936142, + 0.05841652303934097, + 0.06680707633495331, + 0.054338239133358, + 0.0714501440525055 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1301", + "split": "test", + "style_correct": 1, + "style_probability": 0.4559483528137207, + "style_target": 0, + "support_probability": 0.4962296783924103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003226562236918973, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.95037841796875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24190320074558258, + 0.2772892117500305, + 0.2554260194301605, + 0.5303134918212891, + 0.14608560502529144, + 0.12125198543071747, + 0.11598337441682816, + 0.1412084549665451, + 0.21676576137542725, + 0.14786435663700104, + 0.18264572322368622 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 0, + "style_probability": 0.36972519755363464, + "style_target": 1, + "support_probability": 0.019629409536719322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02632340140471312, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9380608201026917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1307:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23859894275665283, + 0.31783002614974976, + 0.27970799803733826, + 0.5956551432609558, + 0.14247629046440125, + 0.090399831533432, + 0.14687687158584595, + 0.16640175879001617, + 0.1587168276309967, + 0.2950170636177063 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1307", + "split": "test", + "style_correct": 0, + "style_probability": 0.5607930421829224, + "style_target": 0, + "support_probability": 0.061042651534080505, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8626501230303419, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6204817295074463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28169703483581543, + 0.3363434672355652, + 0.19568446278572083, + 0.9389358758926392, + 0.09226691722869873 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.9171348810195923, + "style_target": 1, + "support_probability": 0.623392641544342, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5087419856261874, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.39965933561325073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1311:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24940015375614166, + 0.28900668025016785, + 0.21049416065216064, + 0.9322084188461304, + 0.15164610743522644 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1311", + "split": "test", + "style_correct": 1, + "style_probability": 0.24080970883369446, + "style_target": 0, + "support_probability": 0.7370871901512146, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6651936531644084, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14267614483833313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2107003927230835, + 0.2291916310787201, + 0.9671686887741089, + 0.12827028334140778, + 0.1247435137629509, + 0.15761031210422516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.8572349548339844, + "style_target": 1, + "support_probability": 0.6884396076202393, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.023422119192898043, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17285682260990143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1314:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22365985810756683, + 0.2152472287416458, + 0.9855962991714478, + 0.17272064089775085, + 0.06579761207103729, + 0.1673167198896408 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1314", + "split": "test", + "style_correct": 1, + "style_probability": 0.1377609223127365, + "style_target": 0, + "support_probability": 0.6273701786994934, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6295762653642213, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1676749736070633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6498749852180481, + 0.3066938817501068, + 0.7282366752624512, + 0.06280064582824707, + 0.05159257352352142, + 0.09124526381492615, + 0.035945527255535126, + 0.022147471085190773, + 0.11634793877601624 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 1, + "style_probability": 0.8055949807167053, + "style_target": 1, + "support_probability": 0.8324588537216187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5441838296326829, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14336280524730682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6926640868186951, + 0.29829543828964233, + 0.6439552307128906, + 0.07963715493679047, + 0.058880515396595, + 0.032857853919267654, + 0.09993113577365875, + 0.053124427795410156, + 0.02725217677652836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1322", + "split": "test", + "style_correct": 1, + "style_probability": 0.49538546800613403, + "style_target": 0, + "support_probability": 0.878859281539917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3440504231888166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2390257865190506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19094456732273102, + 0.08266422897577286, + 0.5800020098686218, + 0.8370329737663269, + 0.0432559959590435, + 0.03769372031092644, + 0.1300285905599594, + 0.041802022606134415, + 0.02831876464188099, + 0.03429006412625313 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.6783855557441711, + "style_target": 1, + "support_probability": 0.5823352932929993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2871778015646438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21680551767349243, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1339:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18013641238212585, + 0.10457785427570343, + 0.5407576560974121, + 0.8095688223838806, + 0.07047978788614273, + 0.05988088622689247, + 0.03576837480068207, + 0.21340131759643555, + 0.10689693689346313, + 0.08731123805046082, + 0.061987802386283875 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1339", + "split": "test", + "style_correct": 1, + "style_probability": 0.434701144695282, + "style_target": 0, + "support_probability": 0.5488477349281311, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3039388385025106, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4965727925300598, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3574282228946686, + 0.44411978125572205, + 0.485171914100647, + 0.4187910854816437, + 0.05406782776117325 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.8276780247688293, + "style_target": 1, + "support_probability": 0.47410064935684204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013552164969922842, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6338658928871155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1342:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35756105184555054, + 0.5013937950134277, + 0.5770597457885742, + 0.44628289341926575, + 0.07610050588846207 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1342", + "split": "test", + "style_correct": 1, + "style_probability": 0.18808609247207642, + "style_target": 0, + "support_probability": 0.2866389751434326, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.696456600801188, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04737357422709465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7164817452430725, + 0.21208669245243073, + 0.924424946308136, + 0.39309588074684143, + 0.18724235892295837, + 0.0958968997001648 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 1, + "style_probability": 0.8006453514099121, + "style_target": 1, + "support_probability": 0.7509983777999878, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45423578350430205, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06548608839511871, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1362:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7149715423583984, + 0.21143056452274323, + 0.9124743938446045, + 0.4110857844352722, + 0.16303129494190216, + 0.08514535427093506 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1362", + "split": "test", + "style_correct": 0, + "style_probability": 0.6248512864112854, + "style_target": 0, + "support_probability": 0.7038828134536743, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.141864325979963, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37217363715171814, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31491369009017944, + 0.4269973933696747, + 0.577218770980835, + 0.24412381649017334, + 0.09386961162090302, + 0.057252056896686554, + 0.03737536817789078, + 0.039972685277462006, + 0.03276979923248291 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.7043144702911377, + "style_target": 1, + "support_probability": 0.2666701376438141, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03148874785522615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4452124834060669, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1372:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3439396917819977, + 0.420540452003479, + 0.5147929191589355, + 0.25564971566200256, + 0.04057345911860466, + 0.20171746611595154, + 0.06412450224161148, + 0.10380394756793976, + 0.21339528262615204 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1372", + "split": "test", + "style_correct": 1, + "style_probability": 0.3812635540962219, + "style_target": 0, + "support_probability": 0.13767029345035553, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.005581799254741837, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9019584059715271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.256288081407547, + 0.6810529828071594, + 0.33457428216934204, + 0.4992368519306183, + 0.04566744342446327, + 0.029780711978673935, + 0.07679370045661926, + 0.06136161461472511, + 0.049633391201496124, + 0.049000926315784454, + 0.033064283430576324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.7273557186126709, + "style_target": 1, + "support_probability": 0.02177475020289421, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015301878253602963, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.920271635055542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.29046303033828735, + 0.6728705167770386, + 0.36328211426734924, + 0.5353466272354126, + 0.06778782606124878, + 0.03621365875005722, + 0.1027107834815979, + 0.05461938679218292, + 0.04355580359697342, + 0.08671443164348602 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1373", + "split": "test", + "style_correct": 1, + "style_probability": 0.3477526903152466, + "style_target": 0, + "support_probability": 0.021848713979125023, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.052418732988297016, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4684925377368927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19692355394363403, + 0.6774551272392273, + 0.14726771414279938, + 0.3460768461227417, + 0.10857290774583817 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 0, + "style_probability": 0.3159567415714264, + "style_target": 1, + "support_probability": 0.20430141687393188, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.021478787897626472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4022175073623657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24655964970588684, + 0.8838934302330017, + 0.23999981582164764, + 0.4167484641075134, + 0.2407664656639099 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1417", + "split": "test", + "style_correct": 1, + "style_probability": 0.31442463397979736, + "style_target": 0, + "support_probability": 0.18661703169345856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012081013353605346, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7284489274024963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4973357617855072, + 0.36091986298561096, + 0.3847605288028717, + 0.4973357617855072, + 0.095170758664608, + 0.09998493641614914 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.49296674132347107, + "style_target": 1, + "support_probability": 0.07626441866159439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06755596772745422, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6005270481109619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43457165360450745, + 0.4136432409286499, + 0.4867812693119049, + 0.43457165360450745, + 0.11399590224027634, + 0.10524695366621017 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1418", + "split": "test", + "style_correct": 0, + "style_probability": 0.8572752475738525, + "style_target": 0, + "support_probability": 0.1413096934556961, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.372942168455932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3857024610042572, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8736174702644348, + 0.1833290308713913, + 0.1745971143245697, + 0.5637674331665039, + 0.06825617700815201, + 0.05888624116778374, + 0.07396654784679413, + 0.07814346998929977, + 0.10479729622602463 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.567160964012146, + "style_target": 1, + "support_probability": 0.6248205304145813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17696215351686106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4774470925331116, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1420:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.888343870639801, + 0.17016001045703888, + 0.2194937765598297, + 0.5341389775276184, + 0.10301896929740906, + 0.0984063521027565, + 0.15526488423347473, + 0.12181837111711502, + 0.12734319269657135 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1420", + "split": "test", + "style_correct": 1, + "style_probability": 0.3248000144958496, + "style_target": 0, + "support_probability": 0.5980923175811768, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009407934668298536, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8830438852310181, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22661641240119934, + 0.6802243590354919, + 0.3604884743690491, + 0.22661641240119934, + 0.06952270865440369, + 0.3441086709499359, + 0.06637490540742874, + 0.08745205402374268, + 0.09626856446266174, + 0.09674359858036041, + 0.08666188269853592 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 0, + "style_probability": 0.4932200014591217, + "style_target": 1, + "support_probability": 0.03901474177837372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005860816379378021, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9822307229042053, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17457397282123566, + 0.7631335258483887, + 0.4334832727909088, + 0.17457397282123566, + 0.19766715168952942, + 0.07832352072000504, + 0.15103043615818024, + 0.17758287489414215, + 0.06990309804677963, + 0.13970619440078735 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1441", + "split": "test", + "style_correct": 1, + "style_probability": 0.24826599657535553, + "style_target": 0, + "support_probability": 0.01492308545857668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16676515076486, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7032597661018372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2535441815853119, + 0.4197448790073395, + 0.5187292098999023, + 0.45378777384757996, + 0.06084795296192169 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.8949488401412964, + "style_target": 1, + "support_probability": 0.22099584341049194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005860648978869315, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8682172894477844, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1454:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2334822118282318, + 0.4374246895313263, + 0.4760902225971222, + 0.4175969064235687, + 0.089945487678051 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1454", + "split": "test", + "style_correct": 1, + "style_probability": 0.34964752197265625, + "style_target": 0, + "support_probability": 0.03928098455071449, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5453856766890297, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.28171104192733765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6187273263931274, + 0.4310714900493622, + 0.24923132359981537, + 0.69981849193573, + 0.07321958243846893, + 0.06710214912891388 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.848013162612915, + "style_target": 1, + "support_probability": 0.5953930020332336, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08977784842804226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.35068899393081665, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1455:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6871698498725891, + 0.4565151333808899, + 0.2745891213417053, + 0.8121891021728516, + 0.12550054490566254, + 0.1725957691669464 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1455", + "split": "test", + "style_correct": 1, + "style_probability": 0.30110570788383484, + "style_target": 0, + "support_probability": 0.5913161039352417, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1783133092100524, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.265799880027771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.474740207195282, + 0.962664008140564, + 0.35139375925064087, + 0.24770821630954742, + 0.10095987468957901, + 0.1861652135848999, + 0.1770411729812622, + 0.07385089248418808, + 0.1397853046655655 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.5854045152664185, + "style_target": 1, + "support_probability": 0.2603814899921417, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07630537372497415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3468483090400696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1461:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4905485510826111, + 0.9368014335632324, + 0.3608734905719757, + 0.26393383741378784, + 0.07364650070667267, + 0.0987354964017868, + 0.07220403850078583, + 0.19364789128303528, + 0.07994641363620758 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1461", + "split": "test", + "style_correct": 1, + "style_probability": 0.29459425806999207, + "style_target": 0, + "support_probability": 0.20575487613677979, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8168436370075048, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045417677611112595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3800777792930603, + 0.42702433466911316, + 0.6608556509017944, + 0.9715160727500916, + 0.08764012157917023, + 0.1350259631872177, + 0.08870121836662292, + 0.147704616189003, + 0.0719965398311615, + 0.10816926509141922, + 0.053378570824861526 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.6000008583068848, + "style_target": 1, + "support_probability": 0.8489464521408081, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.569834754341457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0979531928896904, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1467:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.4239923059940338, + 0.4230674207210541, + 0.6478644013404846, + 0.9594423770904541, + 0.049030985683202744, + 0.13478481769561768, + 0.08508206158876419, + 0.21486134827136993, + 0.06686456501483917, + 0.1675010472536087, + 0.09212317317724228, + 0.10651440918445587 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1467", + "split": "test", + "style_correct": 1, + "style_probability": 0.43852630257606506, + "style_target": 0, + "support_probability": 0.6893591284751892, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45839710311108206, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24337780475616455, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1583499014377594, + 0.26741328835487366, + 0.33754631876945496, + 0.7977004051208496, + 0.10518639534711838 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 1, + "style_probability": 0.7913382649421692, + "style_target": 1, + "support_probability": 0.2521812617778778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26730130314109424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20023582875728607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1471:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13422304391860962, + 0.36599984765052795, + 0.47101491689682007, + 0.9050610661506653, + 0.09592685848474503 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1471", + "split": "test", + "style_correct": 0, + "style_probability": 0.556312620639801, + "style_target": 0, + "support_probability": 0.32102537155151367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24945833497189088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.830970048904419, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3851660192012787, + 0.36709582805633545, + 0.7426490187644958, + 0.5357186794281006, + 0.058827146887779236, + 0.07053860276937485 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.8984431028366089, + "style_target": 1, + "support_probability": 0.4737189710140228, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017796018646475398, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9120115041732788, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1498:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30732613801956177, + 0.3569972813129425, + 0.6413552165031433, + 0.41881054639816284, + 0.0541999414563179, + 0.16573329269886017 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1498", + "split": "test", + "style_correct": 1, + "style_probability": 0.38255712389945984, + "style_target": 0, + "support_probability": 0.3728199899196625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7411248099369877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2181483805179596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9231247305870056, + 0.47715097665786743, + 0.1747010350227356, + 0.18488989770412445, + 0.1349317729473114, + 0.03506062924861908, + 0.06815087795257568, + 0.08259191364049911, + 0.14577285945415497 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.9314883947372437, + "style_target": 1, + "support_probability": 0.6022892594337463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2264096065630898, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1616566926240921, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1500:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9075437188148499, + 0.45890864729881287, + 0.16748426854610443, + 0.1994599848985672, + 0.06516039371490479, + 0.0689823254942894, + 0.17571653425693512, + 0.0632157251238823, + 0.1006145253777504 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1500", + "split": "test", + "style_correct": 1, + "style_probability": 0.44671931862831116, + "style_target": 0, + "support_probability": 0.5898484587669373, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11059266413785983, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6986973881721497, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13857191801071167, + 0.2579508423805237, + 0.7470645308494568, + 0.761105477809906, + 0.07154077291488647, + 0.25609028339385986, + 0.18358519673347473, + 0.17968077957630157, + 0.1173359602689743, + 0.07035773992538452, + 0.12495426833629608 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.8453571200370789, + "style_target": 1, + "support_probability": 0.22804902493953705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004674470298784981, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7065490484237671, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1503:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16927552223205566, + 0.2926781475543976, + 0.8081934452056885, + 0.6790674924850464, + 0.07774240523576736, + 0.07647565007209778, + 0.10828887671232224, + 0.1412210464477539, + 0.08999300003051758, + 0.16049711406230927 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1503", + "split": "test", + "style_correct": 1, + "style_probability": 0.19170542061328888, + "style_target": 0, + "support_probability": 0.298241525888443, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.845176991306719, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.054551202803850174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2250228226184845, + 0.9539657235145569, + 0.35017770528793335, + 0.30606842041015625, + 0.038417983800172806 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 1, + "style_probability": 0.808878481388092, + "style_target": 1, + "support_probability": 0.8674589991569519, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7024772745960632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08162517845630646, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1513:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1967744529247284, + 0.9200208187103271, + 0.2517324984073639, + 0.23264223337173462, + 0.08050026744604111 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1513", + "split": "test", + "style_correct": 0, + "style_probability": 0.669579267501831, + "style_target": 0, + "support_probability": 0.8473854064941406, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15946080426368248, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.30558520555496216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26433059573173523, + 0.23734688758850098, + 0.9369584918022156, + 0.4532449543476105, + 0.08479827642440796, + 0.10235970467329025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.7563257217407227, + "style_target": 1, + "support_probability": 0.1311456263065338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08081661799034424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21752314269542694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1526:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27274569869041443, + 0.23395781219005585, + 0.9319655299186707, + 0.4261225759983063, + 0.07419846206903458, + 0.10689248144626617 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1526", + "split": "test", + "style_correct": 1, + "style_probability": 0.26868751645088196, + "style_target": 0, + "support_probability": 0.20790264010429382, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5784925331465728, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05255711451172829, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40180855989456177, + 0.2755884528160095, + 0.9084710478782654, + 0.37425124645233154, + 0.13737832009792328, + 0.13306131958961487, + 0.0675555020570755, + 0.03623577207326889, + 0.04827987402677536 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.8722969889640808, + "style_target": 1, + "support_probability": 0.43720898032188416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2019002807873278, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04254427179694176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1534:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40118223428726196, + 0.28954648971557617, + 0.9489455223083496, + 0.39182302355766296, + 0.06321953982114792, + 0.12546125054359436, + 0.12754827737808228, + 0.13245560228824615, + 0.21974654495716095 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1534", + "split": "test", + "style_correct": 1, + "style_probability": 0.3882859945297241, + "style_target": 0, + "support_probability": 0.6517617702484131, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2093295098120862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1608457714319229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16319386661052704, + 0.8698632717132568, + 0.3275187015533447, + 0.49852532148361206, + 0.06677865982055664, + 0.19219006597995758, + 0.07920893281698227, + 0.1585845798254013, + 0.06820106506347656, + 0.07494229078292847, + 0.03542467579245567 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 1, + "style_probability": 0.7097506523132324, + "style_target": 1, + "support_probability": 0.21838431060314178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05147250592918783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2828441262245178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1557:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16156214475631714, + 0.8141587376594543, + 0.24110133945941925, + 0.46394863724708557, + 0.054741743952035904, + 0.0825824961066246, + 0.059005219489336014, + 0.20797692239284515, + 0.09415699541568756, + 0.1262383759021759, + 0.11614968627691269 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1557", + "split": "test", + "style_correct": 1, + "style_probability": 0.4745537042617798, + "style_target": 0, + "support_probability": 0.11238079518079758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3746224565445466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.429776132106781, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2914944291114807, + 0.2912834584712982, + 0.8590615391731262, + 0.6288409233093262, + 0.09497439116239548 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.816035807132721, + "style_target": 1, + "support_probability": 0.4591764509677887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1674713817011835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3250389099121094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1562:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2801758050918579, + 0.2705781161785126, + 0.8446796536445618, + 0.5996432900428772, + 0.04923669621348381 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1562", + "split": "test", + "style_correct": 1, + "style_probability": 0.4235670268535614, + "style_target": 0, + "support_probability": 0.4926762878894806, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2689457928121648, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1948339343070984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41654837131500244, + 0.2836834490299225, + 0.9321435689926147, + 0.41654837131500244, + 0.05269680917263031, + 0.06289573758840561 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 1, + "style_probability": 0.5730054378509521, + "style_target": 1, + "support_probability": 0.3768038749694824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07110196529661163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1848514825105667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1570:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.447700172662735, + 0.3145529329776764, + 0.9432294964790344, + 0.447700172662735, + 0.04378607124090195, + 0.09427829831838608 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1570", + "split": "test", + "style_correct": 1, + "style_probability": 0.2448931783437729, + "style_target": 0, + "support_probability": 0.43084436655044556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10562606361269242, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9134788513183594, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8880853056907654, + 0.15480437874794006, + 0.15480437874794006, + 0.6216617822647095, + 0.12704423069953918, + 0.0764557346701622, + 0.1337375044822693, + 0.04384069889783859, + 0.08542046695947647 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.8299728035926819, + "style_target": 1, + "support_probability": 0.12330611795186996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0763979958601806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8179656267166138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.884559690952301, + 0.13017041981220245, + 0.13017041981220245, + 0.6304240822792053, + 0.10809224098920822, + 0.10603415220975876, + 0.105792336165905, + 0.07287687808275223, + 0.06054788455367088 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1572", + "split": "test", + "style_correct": 1, + "style_probability": 0.4700393080711365, + "style_target": 0, + "support_probability": 0.322287917137146, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7347421102724141, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14577986299991608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6314330101013184, + 0.9336556792259216, + 0.06757533550262451, + 0.6314330101013184, + 0.033096689730882645, + 0.05945265665650368, + 0.13084399700164795, + 0.06166831776499748, + 0.058539602905511856, + 0.03851662948727608, + 0.08536605536937714 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.8315122127532959, + "style_target": 1, + "support_probability": 0.7494089603424072, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.357731238155619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15731197595596313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1573:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5984667539596558, + 0.9138952493667603, + 0.08025282621383667, + 0.5984667539596558, + 0.05424760282039642, + 0.11190017312765121, + 0.06633159518241882, + 0.09596695005893707, + 0.06155797094106674, + 0.06902924180030823, + 0.024909144267439842 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1573", + "split": "test", + "style_correct": 1, + "style_probability": 0.3085331618785858, + "style_target": 0, + "support_probability": 0.7125086784362793, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006503883274314381, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9814088940620422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26563823223114014, + 0.6695059537887573, + 0.22488664090633392, + 0.2896437644958496, + 0.11101291328668594 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.6496070027351379, + "style_target": 1, + "support_probability": 0.0025633603800088167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005383120304340574, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9850260615348816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1581:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30311495065689087, + 0.7013201713562012, + 0.26971542835235596, + 0.31074953079223633, + 0.15692327916622162 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1581", + "split": "test", + "style_correct": 1, + "style_probability": 0.309734046459198, + "style_target": 0, + "support_probability": 0.003465446876361966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3343687576639862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10631991177797318, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.221077099442482, + 0.19738802313804626, + 0.8335224986076355, + 0.221077099442482, + 0.2503064274787903, + 0.07416801899671555 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 1, + "style_probability": 0.6002935767173767, + "style_target": 1, + "support_probability": 0.5797138810157776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28281885875744095, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15575358271598816, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1590:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19038307666778564, + 0.22362883388996124, + 0.7970501184463501, + 0.19038307666778564, + 0.07493843883275986, + 0.12855952978134155 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1590", + "split": "test", + "style_correct": 1, + "style_probability": 0.49616000056266785, + "style_target": 0, + "support_probability": 0.5537609457969666, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3494700922678699, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37199369072914124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2707796096801758, + 0.2707796096801758, + 0.30884671211242676, + 0.8394050002098083, + 0.05536055564880371, + 0.13905830681324005, + 0.1451091170310974, + 0.05658993124961853, + 0.06991983950138092 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.8818060755729675, + "style_target": 1, + "support_probability": 0.29988935589790344, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09413471183123744, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29118281602859497, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1599:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2548789381980896, + 0.2548789381980896, + 0.29731684923171997, + 0.8247042298316956, + 0.08203130215406418, + 0.08839991688728333, + 0.08249545842409134, + 0.09770815074443817, + 0.07353700697422028 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1599", + "split": "test", + "style_correct": 1, + "style_probability": 0.40283840894699097, + "style_target": 0, + "support_probability": 0.3560958504676819, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5078090483052442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1561698168516159, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9438371062278748, + 0.5946178436279297, + 0.506016194820404, + 0.21286699175834656, + 0.055666081607341766, + 0.05654977262020111, + 0.029869141057133675, + 0.061754174530506134, + 0.09244685620069504, + 0.03866253420710564, + 0.029249023646116257 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.8641122579574585, + "style_target": 1, + "support_probability": 0.5138989090919495, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04124937897765379, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21824201941490173, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1600:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9552446603775024, + 0.6560244560241699, + 0.5536285042762756, + 0.22813564538955688, + 0.056732457131147385, + 0.037153054028749466, + 0.15169790387153625, + 0.03946743160486221, + 0.038232460618019104, + 0.06036285683512688 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1600", + "split": "test", + "style_correct": 1, + "style_probability": 0.30697500705718994, + "style_target": 0, + "support_probability": 0.5799517631530762, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.91253541001862, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06298889964818954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1626:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1482418179512024, + 0.23292139172554016, + 0.9699697494506836, + 0.12566789984703064, + 0.14153216779232025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 1, + "style_probability": 0.8155368566513062, + "style_target": 1, + "support_probability": 0.8208057880401611, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6626485648584435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07514458149671555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1626:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20548076927661896, + 0.3192376494407654, + 0.9300348162651062, + 0.09672180563211441, + 0.10091967135667801 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1626", + "split": "test", + "style_correct": 1, + "style_probability": 0.4442826807498932, + "style_target": 0, + "support_probability": 0.7445565462112427, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7025394124451374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0737076923251152, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3905613422393799, + 0.882376492023468, + 0.12231200188398361, + 0.3001798987388611, + 0.06489869207143784, + 0.26845598220825195 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.6810249090194702, + "style_target": 1, + "support_probability": 0.8466636538505554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.592257571933196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06304506957530975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1633:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41299140453338623, + 0.8602973222732544, + 0.1225995421409607, + 0.30586323142051697, + 0.11907506734132767, + 0.06528906524181366 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1633", + "split": "test", + "style_correct": 1, + "style_probability": 0.4671776294708252, + "style_target": 0, + "support_probability": 0.8665364384651184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027736151256672503, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5263813138008118, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.382323682308197, + 0.38895758986473083, + 0.6316426992416382, + 0.3483349680900574, + 0.2921217978000641, + 0.19382300972938538, + 0.17427754402160645, + 0.12566769123077393, + 0.1571015864610672 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.6044638752937317, + "style_target": 1, + "support_probability": 0.14010003209114075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03196344389880679, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40481698513031006, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1634:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3966943323612213, + 0.41841819882392883, + 0.6755335330963135, + 0.4141162931919098, + 0.06819663941860199, + 0.055143583565950394, + 0.0563323013484478, + 0.07088322192430496, + 0.1748797446489334 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1634", + "split": "test", + "style_correct": 1, + "style_probability": 0.40905672311782837, + "style_target": 0, + "support_probability": 0.3018161356449127, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8545136396216968, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18876655399799347, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1636:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.990044891834259, + 0.3690177798271179, + 0.18436944484710693, + 0.4558143615722656, + 0.12020223587751389, + 0.06154477596282959, + 0.21046455204486847, + 0.12211695313453674, + 0.07790519297122955, + 0.0628940537571907, + 0.13502857089042664 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 1, + "style_probability": 0.7020023465156555, + "style_target": 1, + "support_probability": 0.8475725650787354, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8241612138685498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19697634875774384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1636:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9773681163787842, + 0.4285096228122711, + 0.18530233204364777, + 0.5361842513084412, + 0.057774659246206284, + 0.05430792644619942, + 0.04876168817281723, + 0.0419045090675354, + 0.05539628490805626, + 0.19546957314014435, + 0.10957441478967667 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1636", + "split": "test", + "style_correct": 0, + "style_probability": 0.530434250831604, + "style_target": 0, + "support_probability": 0.8774899244308472, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3988423568714303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.44948887825012207, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4818575382232666, + 0.3798539340496063, + 0.9420193433761597, + 0.29300999641418457, + 0.09000074863433838 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 1, + "style_probability": 0.5482738018035889, + "style_target": 1, + "support_probability": 0.7196084260940552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5064572183856342, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5090124011039734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1638:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4272105097770691, + 0.42147597670555115, + 0.9505743980407715, + 0.28655266761779785, + 0.13950417935848236 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1638", + "split": "test", + "style_correct": 0, + "style_probability": 0.8310548067092896, + "style_target": 0, + "support_probability": 0.5937548279762268, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3531622407418915, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.48550209403038025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3495311439037323, + 0.2614617943763733, + 0.4050370454788208, + 0.6895832419395447, + 0.06503654271364212, + 0.09509828686714172 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.6230508089065552, + "style_target": 1, + "support_probability": 0.6569287180900574, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1620047063276564, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5179618000984192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1639:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32821694016456604, + 0.30923715233802795, + 0.46704280376434326, + 0.6929680705070496, + 0.11141137778759003, + 0.12856262922286987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1639", + "split": "test", + "style_correct": 1, + "style_probability": 0.4428724944591522, + "style_target": 0, + "support_probability": 0.47519728541374207, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5774188865215208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22403573989868164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8892210125923157, + 0.4820796847343445, + 0.35679173469543457, + 0.38072651624679565, + 0.10437367856502533, + 0.24220171570777893, + 0.07676184177398682, + 0.026738161221146584, + 0.039480578154325485 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.6870465278625488, + "style_target": 1, + "support_probability": 0.6314305067062378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15747616482743257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18870916962623596, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1640:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9000961184501648, + 0.46172547340393066, + 0.4104628562927246, + 0.41238176822662354, + 0.05701584368944168, + 0.03207232430577278, + 0.07181287556886673, + 0.04934382066130638, + 0.10960956662893295 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1640", + "split": "test", + "style_correct": 1, + "style_probability": 0.2983214855194092, + "style_target": 0, + "support_probability": 0.6989774703979492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005506472316445243, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9345727562904358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3413338363170624, + 0.228939026594162, + 0.3907588720321655, + 0.40290504693984985, + 0.0816325768828392, + 0.1485682874917984, + 0.10751223564147949, + 0.15744104981422424, + 0.058163635432720184, + 0.07908231019973755, + 0.06780347228050232 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 1, + "style_probability": 0.5422452092170715, + "style_target": 1, + "support_probability": 0.03323432058095932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018885430732895392, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8969860076904297, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2824007570743561, + 0.20461033284664154, + 0.3806439936161041, + 0.44695618748664856, + 0.11485999822616577, + 0.06843052804470062, + 0.08944687247276306, + 0.07400232553482056, + 0.044404249638319016, + 0.061805360019207 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1643", + "split": "test", + "style_correct": 0, + "style_probability": 0.8500533103942871, + "style_target": 0, + "support_probability": 0.04867328330874443, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7212356608434796, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5807046890258789, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6256047487258911, + 0.15330037474632263, + 0.23350532352924347, + 0.945629894733429, + 0.057215988636016846 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.8095142245292664, + "style_target": 1, + "support_probability": 0.5957182049751282, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02466567324473745, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6382557153701782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7719377875328064, + 0.18411032855510712, + 0.30016282200813293, + 0.978636622428894, + 0.1901417076587677 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:1647", + "split": "test", + "style_correct": 1, + "style_probability": 0.12362445890903473, + "style_target": 0, + "support_probability": 0.4620705544948578, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3994710682287277, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26314401626586914, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6825501918792725, + 0.7788456082344055, + 0.21382303535938263, + 0.9462878704071045, + 0.08813711255788803, + 0.17807981371879578 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 1, + "style_probability": 0.749161958694458, + "style_target": 1, + "support_probability": 0.579624593257904, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6658203879244411, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14830230176448822, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5795106291770935, + 0.7470431923866272, + 0.26095473766326904, + 0.9334192276000977, + 0.06998979300260544, + 0.0784657672047615 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1659", + "split": "test", + "style_correct": 0, + "style_probability": 0.6609693765640259, + "style_target": 0, + "support_probability": 0.7202414870262146, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4638391479834248, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3921513259410858, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.47660282254219055, + 0.41715332865715027, + 0.47660282254219055, + 0.6855621933937073, + 0.04925577715039253, + 0.028893548995256424, + 0.09779347479343414, + 0.08924762904644012, + 0.07248555123806 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.8676478862762451, + "style_target": 1, + "support_probability": 0.4250969886779785, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09074649752428511, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40025436878204346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37655115127563477, + 0.3643293082714081, + 0.37655115127563477, + 0.6375342011451721, + 0.07896099239587784, + 0.09724776446819305, + 0.0430181547999382, + 0.1389700472354889, + 0.10563676804304123 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1663", + "split": "test", + "style_correct": 1, + "style_probability": 0.2879449129104614, + "style_target": 0, + "support_probability": 0.30305618047714233, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6143154115974561, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25026583671569824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33113256096839905, + 0.2732001543045044, + 0.6287220120429993, + 0.8556439280509949, + 0.05736445635557175, + 0.056868989020586014, + 0.04196827858686447, + 0.03182407468557358, + 0.0480622798204422, + 0.10711795091629028, + 0.0676582008600235 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.8408342599868774, + "style_target": 1, + "support_probability": 0.6716948747634888, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0636577029442549, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.388202041387558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.39835917949676514, + 0.28879407048225403, + 0.6263812780380249, + 0.8987409472465515, + 0.08079754561185837, + 0.0680147036910057, + 0.1206337958574295, + 0.15891991555690765, + 0.11094248294830322, + 0.10218414664268494 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1687", + "split": "test", + "style_correct": 1, + "style_probability": 0.1909978836774826, + "style_target": 0, + "support_probability": 0.599376916885376, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5469126532027566, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1894676387310028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25290513038635254, + 0.5899523496627808, + 0.3098761737346649, + 0.5973017811775208, + 0.06533421576023102 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 1, + "style_probability": 0.9010770916938782, + "style_target": 1, + "support_probability": 0.7816381454467773, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4480480031759981, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1547417938709259, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1689:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23310385644435883, + 0.5542738437652588, + 0.26844891905784607, + 0.5430794954299927, + 0.042123161256313324 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1689", + "split": "test", + "style_correct": 0, + "style_probability": 0.6107984185218811, + "style_target": 0, + "support_probability": 0.8265264630317688, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5177336720313974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22731494903564453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8826231956481934, + 0.4164295792579651, + 0.5260794162750244, + 0.440876841545105, + 0.05271724611520767, + 0.14777813851833344 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 1, + "style_probability": 0.6594406962394714, + "style_target": 1, + "support_probability": 0.6839255690574646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18843023017909655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25419625639915466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1692:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8502817749977112, + 0.40506643056869507, + 0.48228418827056885, + 0.4428779184818268, + 0.08741837739944458, + 0.05423807352781296 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1692", + "split": "test", + "style_correct": 1, + "style_probability": 0.31265366077423096, + "style_target": 0, + "support_probability": 0.57747882604599, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46214476709580404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.220375195145607, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33850806951522827, + 0.379046767950058, + 0.9419019818305969, + 0.4054620563983917, + 0.055614493787288666, + 0.07969672232866287, + 0.06735946983098984, + 0.049692995846271515, + 0.13787004351615906 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 0, + "style_probability": 0.47061726450920105, + "style_target": 1, + "support_probability": 0.7463198900222778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09609505999039093, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33645522594451904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1694:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.311006635427475, + 0.4103919267654419, + 0.9407233595848083, + 0.4312644302845001, + 0.07816905528306961, + 0.12257137149572372, + 0.07257719337940216, + 0.06901772320270538, + 0.11152662336826324 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1694", + "split": "test", + "style_correct": 1, + "style_probability": 0.1622518002986908, + "style_target": 0, + "support_probability": 0.7339363694190979, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07475771805281407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19031013548374176, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2870232164859772, + 0.7456054091453552, + 0.20926210284233093, + 0.23362912237644196, + 0.09356249868869781, + 0.05838318541646004, + 0.07364913076162338, + 0.07353401929140091, + 0.05502304807305336, + 0.0683637261390686, + 0.1498619168996811 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 1, + "style_probability": 0.735154390335083, + "style_target": 1, + "support_probability": 0.07217895239591599, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03672583844030064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33830979466438293, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1697:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34381216764450073, + 0.7011651396751404, + 0.23069453239440918, + 0.25404226779937744, + 0.07824338972568512, + 0.0580466091632843, + 0.07827404886484146, + 0.05956627428531647, + 0.06261193007230759, + 0.07994411140680313, + 0.054894812405109406 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1697", + "split": "test", + "style_correct": 0, + "style_probability": 0.649478554725647, + "style_target": 0, + "support_probability": 0.05136726424098015, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6159180433681684, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1740501970052719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.900802493095398, + 0.32884228229522705, + 0.392031729221344, + 0.28272485733032227, + 0.05940941721200943 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 1, + "style_probability": 0.5091531276702881, + "style_target": 1, + "support_probability": 0.812417209148407, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7451914126828285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14843951165676117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1700:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9450457692146301, + 0.34216055274009705, + 0.5378323197364807, + 0.31413087248802185, + 0.06992585211992264 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1700", + "split": "test", + "style_correct": 0, + "style_probability": 0.7432177662849426, + "style_target": 0, + "support_probability": 0.8236404061317444, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5539043059899313, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12366864085197449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10186094045639038, + 0.9080419540405273, + 0.09017570316791534, + 0.249643474817276, + 0.09219177067279816, + 0.15319693088531494 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 1, + "style_probability": 0.6770495176315308, + "style_target": 1, + "support_probability": 0.5723992586135864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07589250003788983, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34229806065559387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1276659220457077, + 0.9093143939971924, + 0.11900836229324341, + 0.24528607726097107, + 0.20609448850154877, + 0.2594992220401764 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1705", + "split": "test", + "style_correct": 1, + "style_probability": 0.4357452094554901, + "style_target": 0, + "support_probability": 0.2353651076555252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4246624239778285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12707912921905518, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1974658966064453, + 0.1999710351228714, + 0.9207859635353088, + 0.18357044458389282, + 0.07146845012903214, + 0.18091312050819397, + 0.10002648085355759, + 0.15841734409332275, + 0.09221801161766052 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 1, + "style_probability": 0.5014009475708008, + "style_target": 1, + "support_probability": 0.5638918876647949, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05340693028941245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10307299345731735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1710:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.210361510515213, + 0.24830132722854614, + 0.974661111831665, + 0.27507245540618896, + 0.1660965532064438, + 0.09481160342693329, + 0.13475444912910461, + 0.09782471507787704, + 0.12522538006305695 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1710", + "split": "test", + "style_correct": 1, + "style_probability": 0.12343736737966537, + "style_target": 0, + "support_probability": 0.724314272403717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03720706136903119, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7278309464454651, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3794752359390259, + 0.22748404741287231, + 0.6195354461669922, + 0.23939166963100433, + 0.05530951917171478, + 0.04862700030207634, + 0.04933302104473114, + 0.09100213646888733, + 0.06470588594675064, + 0.06555817276239395, + 0.034607890993356705 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 1, + "style_probability": 0.6413711309432983, + "style_target": 1, + "support_probability": 0.04635164141654968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011984915849746928, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8357694745063782, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1716:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3485722243785858, + 0.19631652534008026, + 0.575107753276825, + 0.23706623911857605, + 0.054255541414022446, + 0.06568444520235062, + 0.04629398509860039, + 0.048656538128852844, + 0.10611925274133682, + 0.06830659508705139, + 0.03992806375026703 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1716", + "split": "test", + "style_correct": 0, + "style_probability": 0.812441349029541, + "style_target": 0, + "support_probability": 0.023984270170331, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8639770078620933, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.046246662735939026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9783914685249329, + 0.3182031512260437, + 0.4951190650463104, + 0.5741956233978271, + 0.11836733669042587 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 1, + "style_probability": 0.7043960690498352, + "style_target": 1, + "support_probability": 0.9354594349861145, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8577412430773278, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023711228743195534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1720:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9874206185340881, + 0.26633286476135254, + 0.5149810314178467, + 0.6451801657676697, + 0.25049781799316406 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1720", + "split": "test", + "style_correct": 0, + "style_probability": 0.5446410179138184, + "style_target": 0, + "support_probability": 0.9628723859786987, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02112268326756528, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.519503653049469, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45817089080810547, + 0.5120168924331665, + 0.3443061411380768, + 0.45817089080810547, + 0.04921863228082657, + 0.13247813284397125 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 1, + "style_probability": 0.5975058078765869, + "style_target": 1, + "support_probability": 0.04442228376865387, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009895822891002881, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5534195303916931, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1721:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5024155974388123, + 0.6340458393096924, + 0.3537304401397705, + 0.5024155974388123, + 0.08788798004388809, + 0.07610645145177841 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1721", + "split": "test", + "style_correct": 0, + "style_probability": 0.5046316981315613, + "style_target": 0, + "support_probability": 0.038449157029390335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1075294025200435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7403522729873657, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6450709104537964, + 0.4648493528366089, + 0.27567407488822937, + 0.281766414642334, + 0.12127801030874252, + 0.19571445882320404, + 0.13576671481132507, + 0.08807523548603058, + 0.23090185225009918 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 1, + "style_probability": 0.5504828095436096, + "style_target": 1, + "support_probability": 0.18765097856521606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.059527872943430093, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9505285024642944, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1736:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.514322817325592, + 0.3526155948638916, + 0.3068438768386841, + 0.26173868775367737, + 0.06952086091041565, + 0.15769341588020325, + 0.1786765158176422, + 0.14265058934688568, + 0.0916484147310257 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1736", + "split": "test", + "style_correct": 0, + "style_probability": 0.9053013920783997, + "style_target": 0, + "support_probability": 0.04038843885064125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004811543713825887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9666632413864136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6929388642311096, + 0.4642069637775421, + 0.27212345600128174, + 0.4642069637775421, + 0.15727148950099945, + 0.10993090271949768, + 0.08610834926366806, + 0.06907109916210175, + 0.05113523080945015, + 0.12677901983261108 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 1, + "style_probability": 0.5742387175559998, + "style_target": 1, + "support_probability": 0.014028673060238361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0069613983738964205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9493327736854553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1740:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7809397578239441, + 0.4876411557197571, + 0.21365247666835785, + 0.4876411557197571, + 0.1144830584526062, + 0.0961538553237915, + 0.1842568963766098, + 0.10265680402517319, + 0.15640388429164886, + 0.16305260360240936 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1740", + "split": "test", + "style_correct": 1, + "style_probability": 0.4041234254837036, + "style_target": 0, + "support_probability": 0.023976387456059456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21319586568511542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.703125536441803, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.539413332939148, + 0.4493233561515808, + 0.4711395800113678, + 0.31339436769485474, + 0.07175058871507645 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 1, + "style_probability": 0.8523488640785217, + "style_target": 1, + "support_probability": 0.2367028295993805, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1141541870400591, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7244563698768616, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1748:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5382322669029236, + 0.4955523908138275, + 0.5309920907020569, + 0.37160855531692505, + 0.13424253463745117 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1748", + "split": "test", + "style_correct": 0, + "style_probability": 0.559050440788269, + "style_target": 0, + "support_probability": 0.268595814704895, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.493726144827356, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5620570182800293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13174617290496826, + 0.25707414746284485, + 0.335696280002594, + 0.7591437697410583, + 0.03353891149163246, + 0.06270447373390198 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.7923927307128906, + "style_target": 1, + "support_probability": 0.541523814201355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11586256439668573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7622600197792053, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15610818564891815, + 0.2539080083370209, + 0.3756219744682312, + 0.7213445901870728, + 0.17563065886497498, + 0.09279037266969681 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1763", + "split": "test", + "style_correct": 1, + "style_probability": 0.30773359537124634, + "style_target": 0, + "support_probability": 0.28839075565338135, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8986279106916442, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06450658291578293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9599440097808838, + 0.17682015895843506, + 0.2885254919528961, + 0.4541385769844055, + 0.09301982074975967, + 0.12202085554599762, + 0.07583196461200714, + 0.08599625527858734, + 0.04141039773821831 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.8862529993057251, + "style_target": 1, + "support_probability": 0.8596502542495728, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2650381592166031, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.048175372183322906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1764:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.990145742893219, + 0.17856784164905548, + 0.3380960524082184, + 0.5586342215538025, + 0.14379912614822388, + 0.05927300825715065, + 0.12607379257678986, + 0.1088801920413971, + 0.1286245435476303 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1764", + "split": "test", + "style_correct": 1, + "style_probability": 0.14719463884830475, + "style_target": 0, + "support_probability": 0.9423342943191528, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002959310816155052, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8743364810943604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6145145893096924, + 0.6145145893096924, + 0.6976017355918884, + 0.32778987288475037, + 0.04590432345867157, + 0.2275618314743042, + 0.07515184581279755, + 0.09524200856685638, + 0.10797817260026932, + 0.04793115705251694, + 0.10598324984312057 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 1, + "style_probability": 0.8313581943511963, + "style_target": 1, + "support_probability": 0.008664981462061405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003926935811200518, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8709651231765747, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1770:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6188063621520996, + 0.6188063621520996, + 0.6496740579605103, + 0.3424353301525116, + 0.051048241555690765, + 0.06983065605163574, + 0.06833430379629135, + 0.09988124668598175, + 0.06956050544977188, + 0.046546269208192825, + 0.049955956637859344 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1770", + "split": "test", + "style_correct": 0, + "style_probability": 0.6062395572662354, + "style_target": 0, + "support_probability": 0.00911806058138609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003462433485887596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6606146693229675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.697104811668396, + 0.3496166467666626, + 0.5761285424232483, + 0.26316186785697937, + 0.08030971139669418 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 0, + "style_probability": 0.44174498319625854, + "style_target": 1, + "support_probability": 0.029837923124432564, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003332441411437627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7208254337310791, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7241393327713013, + 0.31039780378341675, + 0.6234177947044373, + 0.23596756160259247, + 0.0970790684223175 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1772", + "split": "test", + "style_correct": 0, + "style_probability": 0.73685622215271, + "style_target": 0, + "support_probability": 0.02029281295835972, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6782045882017016, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14930975437164307, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7976948022842407, + 0.2591876685619354, + 0.1757047027349472, + 0.3354370892047882, + 0.05383794754743576, + 0.04047112166881561 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 1, + "style_probability": 0.7774710059165955, + "style_target": 1, + "support_probability": 0.7797377109527588, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.652825137263946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10763494670391083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1784:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8081838488578796, + 0.2826511263847351, + 0.18924909830093384, + 0.34490901231765747, + 0.08789046853780746, + 0.05081169307231903 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1784", + "split": "test", + "style_correct": 0, + "style_probability": 0.6127915978431702, + "style_target": 0, + "support_probability": 0.8224198222160339, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.775656027885752, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0700528472661972, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2359539121389389, + 0.24170012772083282, + 0.8812907934188843, + 0.393085777759552, + 0.36902087926864624, + 0.08527933061122894, + 0.06390050053596497, + 0.04420493543148041, + 0.06540603190660477 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 1, + "style_probability": 0.5983024835586548, + "style_target": 1, + "support_probability": 0.8278749585151672, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19561479882290553, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0828988254070282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1786:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19672659039497375, + 0.2844519019126892, + 0.938606321811676, + 0.4423716068267822, + 0.11017092317342758, + 0.06433174759149551, + 0.07297369092702866, + 0.04380660131573677, + 0.15230564773082733 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1786", + "split": "test", + "style_correct": 1, + "style_probability": 0.19929343461990356, + "style_target": 0, + "support_probability": 0.7395244836807251, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.774389212897777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17097775638103485, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30349481105804443, + 0.29257094860076904, + 0.30349481105804443, + 0.9041498303413391, + 0.06183741241693497, + 0.3420061469078064, + 0.09891021996736526, + 0.06864695250988007, + 0.04224327951669693, + 0.060553960502147675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 1, + "style_probability": 0.6098321080207825, + "style_target": 1, + "support_probability": 0.6783730387687683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6338160342470616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33269232511520386, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1787:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24267517030239105, + 0.27828261256217957, + 0.24267517030239105, + 0.9140841960906982, + 0.02662644162774086, + 0.05548150837421417, + 0.06379588693380356, + 0.09489546716213226, + 0.029904842376708984, + 0.03643146529793739, + 0.04843323305249214 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1787", + "split": "test", + "style_correct": 0, + "style_probability": 0.7636560201644897, + "style_target": 0, + "support_probability": 0.4635690748691559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8950230099574803, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04809160903096199, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9742472767829895, + 0.2408865988254547, + 0.457560658454895, + 0.17868191003799438, + 0.08684156835079193 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 1, + "style_probability": 0.6665828227996826, + "style_target": 1, + "support_probability": 0.8812991380691528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8562037793925086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02336721681058407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9825698137283325, + 0.20120862126350403, + 0.45073285698890686, + 0.17980848252773285, + 0.04523516073822975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1788", + "split": "test", + "style_correct": 1, + "style_probability": 0.40094345808029175, + "style_target": 0, + "support_probability": 0.921710729598999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11892179514629221, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3212226331233978, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13037937879562378, + 0.6743395924568176, + 0.26892420649528503, + 0.3681255578994751, + 0.03989367559552193, + 0.05990277975797653 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.6120221018791199, + "style_target": 1, + "support_probability": 0.3861376941204071, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016642478983682985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25379815697669983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12414389848709106, + 0.7958248853683472, + 0.29493269324302673, + 0.3845631778240204, + 0.07212457060813904, + 0.19589115679264069 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1789", + "split": "test", + "style_correct": 1, + "style_probability": 0.16117213666439056, + "style_target": 0, + "support_probability": 0.6310691833496094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012744821935715356, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8452538847923279, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5554558634757996, + 0.16822922229766846, + 0.3249660134315491, + 0.3696732521057129, + 0.11499239504337311, + 0.0714312195777893, + 0.08769677579402924, + 0.05420142039656639, + 0.1239725723862648 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 1, + "style_probability": 0.5070404410362244, + "style_target": 1, + "support_probability": 0.07708282023668289, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.037669459458082415, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7948212623596191, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1792:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5256878137588501, + 0.1592075526714325, + 0.3876355290412903, + 0.3182852268218994, + 0.04363125190138817, + 0.06872396916151047, + 0.03640811890363693, + 0.06795808672904968, + 0.1352107971906662 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1792", + "split": "test", + "style_correct": 0, + "style_probability": 0.8443279266357422, + "style_target": 0, + "support_probability": 0.1100086122751236, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.03743563649254344, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5662263035774231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3970882296562195, + 0.18456830084323883, + 0.28101998567581177, + 0.5848499536514282, + 0.2737208902835846, + 0.06256735324859619, + 0.0712270438671112, + 0.06998299807310104, + 0.07278124988079071, + 0.05793342739343643, + 0.08592842519283295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 1, + "style_probability": 0.5221191644668579, + "style_target": 1, + "support_probability": 0.16507795453071594, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03982977912474306, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3649562895298004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1795:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.40480026602745056, + 0.17804457247257233, + 0.30844658613204956, + 0.6380744576454163, + 0.09435632824897766, + 0.1684342473745346, + 0.09978866577148438, + 0.11012817919254303, + 0.10899064689874649, + 0.2303520292043686, + 0.12163766473531723 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1795", + "split": "test", + "style_correct": 1, + "style_probability": 0.37681466341018677, + "style_target": 0, + "support_probability": 0.13132546842098236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7155387358510444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08507714420557022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38632383942604065, + 0.8797867894172668, + 0.6315141320228577, + 0.3026553988456726, + 0.041787080466747284 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 1, + "style_probability": 0.8220140337944031, + "style_target": 1, + "support_probability": 0.7453761696815491, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3079764562924652, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15574000775814056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1797:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3125860393047333, + 0.8003597259521484, + 0.5189443230628967, + 0.26169392466545105, + 0.04903543367981911 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1797", + "split": "test", + "style_correct": 1, + "style_probability": 0.4662601351737976, + "style_target": 0, + "support_probability": 0.6285946369171143, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6323955296158452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2656598687171936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3489518165588379, + 0.48869216442108154, + 0.8832220435142517, + 0.16783367097377777, + 0.054002419114112854, + 0.03507654368877411 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.7204747200012207, + "style_target": 1, + "support_probability": 0.8413466215133667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37253351287781733, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19255948066711426, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1802:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34494638442993164, + 0.47394442558288574, + 0.8647119402885437, + 0.1891918033361435, + 0.03685279190540314, + 0.12836748361587524 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1802", + "split": "test", + "style_correct": 1, + "style_probability": 0.3178076446056366, + "style_target": 0, + "support_probability": 0.8795172572135925, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9299527883945657, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010334671474993229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5586947202682495, + 0.5288786888122559, + 0.9987911581993103, + 0.41389158368110657, + 0.034585706889629364, + 0.05282481759786606, + 0.029087180271744728, + 0.04180315136909485, + 0.03394480422139168 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 1, + "style_probability": 0.8569092154502869, + "style_target": 1, + "support_probability": 0.8509839177131653, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7656808953989724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010121870785951614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1806:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5839222073554993, + 0.4922599792480469, + 0.999320387840271, + 0.4244309663772583, + 0.04865703359246254, + 0.02944605052471161, + 0.05294335260987282, + 0.047136563807725906, + 0.08601214736700058 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1806", + "split": "test", + "style_correct": 0, + "style_probability": 0.6205723881721497, + "style_target": 0, + "support_probability": 0.8666010499000549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26007378264061387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4499395489692688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18056292831897736, + 0.7945014238357544, + 0.1889309585094452, + 0.16905072331428528, + 0.06671985238790512, + 0.06716141104698181, + 0.07882491499185562, + 0.11493677645921707, + 0.056415993720293045, + 0.13466587662696838, + 0.06659073382616043 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 1, + "style_probability": 0.6781638860702515, + "style_target": 1, + "support_probability": 0.6713687777519226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1844398038364012, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5179741382598877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20464737713336945, + 0.6419109106063843, + 0.22021634876728058, + 0.21367350220680237, + 0.07068819552659988, + 0.09798144549131393, + 0.13016757369041443, + 0.07967033982276917, + 0.07180572301149368, + 0.057152654975652695, + 0.09022940695285797 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1817", + "split": "test", + "style_correct": 0, + "style_probability": 0.6094681024551392, + "style_target": 0, + "support_probability": 0.5680516362190247, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7155869920255284, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0549350343644619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.968616783618927, + 0.13233742117881775, + 0.5934720039367676, + 0.47434043884277344, + 0.08099587261676788 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.34344443678855896, + "style_target": 1, + "support_probability": 0.8610385656356812, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.845804448822065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07004272192716599, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.981359601020813, + 0.1353297233581543, + 0.6485757231712341, + 0.5177700519561768, + 0.04199615493416786 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1820", + "split": "test", + "style_correct": 0, + "style_probability": 0.7011309266090393, + "style_target": 0, + "support_probability": 0.7783599495887756, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.740199248462929, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023661717772483826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8303645849227905, + 0.25473594665527344, + 0.4238472282886505, + 0.4168252944946289, + 0.09347739070653915, + 0.10346018522977829 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 0, + "style_probability": 0.4319649934768677, + "style_target": 1, + "support_probability": 0.9538022875785828, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16108619069424535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057525407522916794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1832:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9005737900733948, + 0.31221693754196167, + 0.5104395151138306, + 0.44352930784225464, + 0.08054213225841522, + 0.14552854001522064 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1832", + "split": "test", + "style_correct": 1, + "style_probability": 0.1638440638780594, + "style_target": 0, + "support_probability": 0.8661314845085144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12880995772498238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3235194683074951, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3451595902442932, + 0.9826998710632324, + 0.43675076961517334, + 0.3451595902442932, + 0.136200949549675, + 0.07528456300497055, + 0.14051231741905212, + 0.09001479297876358, + 0.0819375067949295 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 0, + "style_probability": 0.4600115716457367, + "style_target": 1, + "support_probability": 0.3209630846977234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30981791839539063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.254677951335907, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3456195592880249, + 0.9801376461982727, + 0.4673386812210083, + 0.3456195592880249, + 0.17715589702129364, + 0.37394025921821594, + 0.13890500366687775, + 0.19714213907718658, + 0.15528780221939087 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1841", + "split": "test", + "style_correct": 0, + "style_probability": 0.6194179058074951, + "style_target": 0, + "support_probability": 0.3385324478149414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07462168623717513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28704988956451416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1976458728313446, + 0.6801411509513855, + 0.6497884392738342, + 0.6166375875473022, + 0.06441934406757355, + 0.0458010770380497, + 0.07195700705051422, + 0.05043047294020653, + 0.12168236821889877, + 0.03390898182988167, + 0.057871390134096146 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.1832716315984726, + "style_target": 1, + "support_probability": 0.2529885172843933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22634573525727045, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3095795214176178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1849:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16081933677196503, + 0.6522722840309143, + 0.6500200033187866, + 0.5716223120689392, + 0.061848413199186325, + 0.03753845393657684, + 0.023325227200984955, + 0.02291686460375786, + 0.026576021686196327, + 0.07387759536504745, + 0.03537367656826973 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1849", + "split": "test", + "style_correct": 0, + "style_probability": 0.7019146084785461, + "style_target": 0, + "support_probability": 0.31392189860343933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05592258307573595, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5174556374549866, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3984473943710327, + 0.14248505234718323, + 0.6318259835243225, + 0.6043512225151062, + 0.12634523212909698 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 0, + "style_probability": 0.4526248276233673, + "style_target": 1, + "support_probability": 0.175139918923378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012335625923124203, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5753588676452637, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1867:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3358651399612427, + 0.15236277878284454, + 0.6987948417663574, + 0.6873264312744141, + 0.04495886713266373 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1867", + "split": "test", + "style_correct": 1, + "style_probability": 0.25000783801078796, + "style_target": 0, + "support_probability": 0.3009791672229767, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0440652763983378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5229550004005432, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33033400774002075, + 0.9129242300987244, + 0.35965391993522644, + 0.29723313450813293, + 0.18274153769016266, + 0.11813061684370041 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 0, + "style_probability": 0.36566993594169617, + "style_target": 1, + "support_probability": 0.1913328617811203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09840558765078634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49203580617904663, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3756687343120575, + 0.8844420909881592, + 0.43053245544433594, + 0.27567362785339355, + 0.07823842018842697, + 0.1097579374909401 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1905", + "split": "test", + "style_correct": 0, + "style_probability": 0.6178027987480164, + "style_target": 0, + "support_probability": 0.16739004850387573, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3553049175715977, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08739882707595825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.711000382900238, + 0.5026170015335083, + 0.9290534853935242, + 0.28836777806282043, + 0.09203925728797913, + 0.08308916538953781, + 0.14379242062568665, + 0.28795766830444336, + 0.10060233622789383 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.2612679600715637, + "style_target": 1, + "support_probability": 0.7063006162643433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.575818268512478, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.156232550740242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7448285222053528, + 0.5074756741523743, + 0.945274829864502, + 0.28061944246292114, + 0.09179293364286423, + 0.07401508837938309, + 0.07146656513214111, + 0.06257306784391403, + 0.15420575439929962 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1910", + "split": "test", + "style_correct": 0, + "style_probability": 0.6599383354187012, + "style_target": 0, + "support_probability": 0.6106863617897034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014820333650600979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.749213695526123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19169940054416656, + 0.18084697425365448, + 0.24782364070415497, + 0.6997672319412231, + 0.14809788763523102, + 0.1516479104757309, + 0.10179528594017029, + 0.13190846145153046, + 0.08358693867921829, + 0.14260096848011017, + 0.10955958068370819 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 0, + "style_probability": 0.4020540416240692, + "style_target": 1, + "support_probability": 0.03960490971803665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005938700558848878, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8538317084312439, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1911:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20697681605815887, + 0.18998901546001434, + 0.268502414226532, + 0.7369194626808167, + 0.09485723823308945, + 0.03657311201095581, + 0.06215604022145271, + 0.14899024367332458, + 0.11082904785871506, + 0.15915066003799438 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1911", + "split": "test", + "style_correct": 1, + "style_probability": 0.3464817702770233, + "style_target": 0, + "support_probability": 0.10337474197149277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6786197531392197, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09178551286458969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9411913752555847, + 0.470098078250885, + 0.13234743475914001, + 0.4922177493572235, + 0.04229611158370972 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 1, + "style_probability": 0.8467125296592712, + "style_target": 1, + "support_probability": 0.7080895304679871, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49000334533809564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13306279480457306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9203472137451172, + 0.40270981192588806, + 0.1245436817407608, + 0.4364081621170044, + 0.121392160654068 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1912", + "split": "test", + "style_correct": 0, + "style_probability": 0.6156877875328064, + "style_target": 0, + "support_probability": 0.6535183787345886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9741064625707168, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06350771337747574, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:1930:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5094866752624512, + 0.343811571598053, + 0.9886960983276367, + 0.31776711344718933, + 0.04057285562157631, + 0.029322482645511627 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 1, + "style_probability": 0.9197498559951782, + "style_target": 1, + "support_probability": 0.9540539383888245, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7339105322443659, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.042418062686920166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1930:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4957803785800934, + 0.3836239278316498, + 0.9799768924713135, + 0.2633970081806183, + 0.02106647379696369, + 0.03402841091156006 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:1930", + "split": "test", + "style_correct": 1, + "style_probability": 0.2585010230541229, + "style_target": 0, + "support_probability": 0.9454385638237, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4665691133086405, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21072588860988617, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3665615916252136, + 0.7083616256713867, + 0.3665615916252136, + 0.9428312182426453, + 0.026426248252391815, + 0.037640586495399475, + 0.06915300339460373, + 0.036420274525880814, + 0.05653751641511917 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.720190167427063, + "style_target": 1, + "support_probability": 0.5034376382827759, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05468564860475867, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2299794703722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1931:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5019499659538269, + 0.7899479866027832, + 0.5019499659538269, + 0.9700070023536682, + 0.04639480635523796, + 0.09425220638513565, + 0.04291143640875816, + 0.06057821959257126, + 0.07211808860301971 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:1931", + "split": "test", + "style_correct": 1, + "style_probability": 0.1300022006034851, + "style_target": 0, + "support_probability": 0.6242737174034119, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8349624109614524, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12095102667808533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5795613527297974, + 0.9271292686462402, + 0.4478636384010315, + 0.375796377658844, + 0.16063626110553741, + 0.06667369604110718, + 0.027665961533784866, + 0.02627032995223999, + 0.09121678024530411, + 0.04345334693789482, + 0.031954459846019745 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 1, + "style_probability": 0.8563776612281799, + "style_target": 1, + "support_probability": 0.9163334965705872, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7195245619865817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24964044988155365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1945:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6379600167274475, + 0.8561424016952515, + 0.4601428806781769, + 0.4276215732097626, + 0.028545096516609192, + 0.07094471901655197, + 0.056692447513341904, + 0.04893742501735687, + 0.06627795845270157, + 0.06391580402851105, + 0.05422737076878548 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1945", + "split": "test", + "style_correct": 0, + "style_probability": 0.6506785154342651, + "style_target": 0, + "support_probability": 0.920741617679596, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007959073128094128, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9042391777038574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1161850094795227, + 0.2405610978603363, + 0.2799989879131317, + 0.6911994814872742, + 0.30770522356033325 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.2954738140106201, + "style_target": 1, + "support_probability": 0.07154787331819534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011369550056515191, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9629685878753662, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1947:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1322476714849472, + 0.1981147676706314, + 0.2674219310283661, + 0.7136027216911316, + 0.12242202460765839 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:1947", + "split": "test", + "style_correct": 0, + "style_probability": 0.8569916486740112, + "style_target": 0, + "support_probability": 0.04922090470790863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3512076672945277, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12621340155601501, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7496698498725891, + 0.2177603542804718, + 0.7999652028083801, + 0.47366535663604736, + 0.12896475195884705, + 0.06759961694478989 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 0, + "style_probability": 0.21458861231803894, + "style_target": 1, + "support_probability": 0.8517586588859558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05111029011612445, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2616187036037445, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:1954:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7928777933120728, + 0.3121068477630615, + 0.8889429569244385, + 0.4499477446079254, + 0.045788586139678955, + 0.08416883647441864 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1954", + "split": "test", + "style_correct": 1, + "style_probability": 0.11815650761127472, + "style_target": 0, + "support_probability": 0.6420650482177734, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.36585299277429506, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4788137376308441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8399621844291687, + 0.4249498248100281, + 0.4249498248100281, + 0.18224148452281952, + 0.10606426000595093, + 0.1676131635904312, + 0.07427165657281876, + 0.17061631381511688, + 0.10583792626857758 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 0, + "style_probability": 0.4215015769004822, + "style_target": 1, + "support_probability": 0.6579142212867737, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4636841248829526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5727736353874207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1956:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.803857684135437, + 0.45248711109161377, + 0.45248711109161377, + 0.21732330322265625, + 0.049079399555921555, + 0.19449424743652344, + 0.06484824419021606, + 0.07260725647211075, + 0.05046341195702553 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1956", + "split": "test", + "style_correct": 0, + "style_probability": 0.5334354639053345, + "style_target": 0, + "support_probability": 0.6678382158279419, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001041243955055941, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8907549381256104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18840697407722473, + 0.40430209040641785, + 0.39143526554107666, + 0.2369687855243683, + 0.06209524720907211, + 0.0964914858341217, + 0.11398342996835709, + 0.15090446174144745, + 0.07837096601724625, + 0.104991115629673, + 0.10579515993595123 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.30780860781669617, + "style_target": 1, + "support_probability": 0.004710169043391943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014205941832536738, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8775098323822021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1958:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1951681673526764, + 0.39903125166893005, + 0.4207623600959778, + 0.22195395827293396, + 0.06448081880807877, + 0.046196356415748596, + 0.03438347205519676, + 0.07473032176494598, + 0.03696706146001816, + 0.05816890299320221 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:1958", + "split": "test", + "style_correct": 0, + "style_probability": 0.8387449979782104, + "style_target": 0, + "support_probability": 0.004991163499653339, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33889365968028073, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24235393106937408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9764142036437988, + 0.1810428649187088, + 0.39247649908065796, + 0.11091847717761993, + 0.1593230813741684 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.7769593596458435, + "style_target": 1, + "support_probability": 0.22620359063148499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030007279026342704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2635051906108856, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9883679747581482, + 0.2529495358467102, + 0.40538322925567627, + 0.13674482703208923, + 0.07451283931732178 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1972", + "split": "test", + "style_correct": 1, + "style_probability": 0.17197513580322266, + "style_target": 0, + "support_probability": 0.28276556730270386, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7323756079343298, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12501142919063568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.120252326130867, + 0.8227609395980835, + 0.2533053457736969, + 0.24081504344940186, + 0.04406402260065079, + 0.04656761884689331 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 1, + "style_probability": 0.759381115436554, + "style_target": 1, + "support_probability": 0.7927533984184265, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4284365494198461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4178721606731415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1973:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1238449364900589, + 0.7046234011650085, + 0.26422110199928284, + 0.21775777637958527, + 0.11648103594779968, + 0.1092861145734787 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:1973", + "split": "test", + "style_correct": 0, + "style_probability": 0.5513817071914673, + "style_target": 0, + "support_probability": 0.6294962763786316, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06035303166738081, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1879175901412964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.941669225692749, + 0.69974285364151, + 0.4697069823741913, + 0.22955338656902313, + 0.09139411896467209, + 0.13669206202030182, + 0.06014663726091385, + 0.04134640097618103, + 0.047561898827552795 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 1, + "style_probability": 0.7716172933578491, + "style_target": 1, + "support_probability": 0.048079006373882294, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03915703787874581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11763032525777817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9294105768203735, + 0.6565424799919128, + 0.4413364827632904, + 0.2529263496398926, + 0.3187209665775299, + 0.07332003861665726, + 0.12590326368808746, + 0.11099598556756973, + 0.07340030372142792 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:1976", + "split": "test", + "style_correct": 1, + "style_probability": 0.43784552812576294, + "style_target": 0, + "support_probability": 0.09041757136583328, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8766987348783069, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1324918419122696, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9412669539451599, + 0.6078892350196838, + 0.7759365439414978, + 0.4770105183124542, + 0.07237110286951065, + 0.07150967419147491, + 0.06203799694776535, + 0.07800076901912689, + 0.05774865671992302, + 0.06446249783039093, + 0.06899797171354294 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.8148136138916016, + "style_target": 1, + "support_probability": 0.868894636631012, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17239549213423902, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14481844007968903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:1996:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9486061930656433, + 0.6522011756896973, + 0.8195464611053467, + 0.5095310807228088, + 0.11385973542928696, + 0.060680244117975235, + 0.07730939239263535, + 0.1274046003818512, + 0.12256025522947311, + 0.10205873101949692 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:1996", + "split": "test", + "style_correct": 1, + "style_probability": 0.1354316771030426, + "style_target": 0, + "support_probability": 0.9052912592887878, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5848630078098331, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2516480088233948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24011759459972382, + 0.9074848294258118, + 0.11832497268915176, + 0.47154369950294495, + 0.081744484603405 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 1, + "style_probability": 0.5458787083625793, + "style_target": 1, + "support_probability": 0.8484578728675842, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.781459405147924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2888677418231964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2013:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25074121356010437, + 0.8992919325828552, + 0.13440246880054474, + 0.4466277062892914, + 0.0977981686592102 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2013", + "split": "test", + "style_correct": 0, + "style_probability": 0.7090448141098022, + "style_target": 0, + "support_probability": 0.9118406772613525, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5448618577153184, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17925092577934265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2017:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32658642530441284, + 0.9656744003295898, + 0.289711058139801, + 0.437913179397583, + 0.06259854137897491, + 0.07137911021709442 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 0, + "style_probability": 0.476828008890152, + "style_target": 1, + "support_probability": 0.6334739923477173, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8313071965320447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20760884881019592, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2017:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33855003118515015, + 0.9744848012924194, + 0.33536556363105774, + 0.3967039883136749, + 0.07132518291473389, + 0.10232269018888474 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2017", + "split": "test", + "style_correct": 0, + "style_probability": 0.8460065722465515, + "style_target": 0, + "support_probability": 0.5847190022468567, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009638181650797887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.850169837474823, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36024272441864014, + 0.3814239501953125, + 0.4468187689781189, + 0.3152214586734772, + 0.10573825985193253, + 0.11704787611961365, + 0.10923104733228683, + 0.0764654129743576, + 0.13842901587486267 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 0, + "style_probability": 0.1787564754486084, + "style_target": 1, + "support_probability": 0.061894118785858154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005376940152898359, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8143323063850403, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2018:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42072945833206177, + 0.46491292119026184, + 0.6276519894599915, + 0.34745150804519653, + 0.05961998924612999, + 0.1144585907459259, + 0.13506530225276947, + 0.08716904371976852, + 0.17495857179164886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2018", + "split": "test", + "style_correct": 1, + "style_probability": 0.12569820880889893, + "style_target": 0, + "support_probability": 0.2891060411930084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012758702472894897, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8317739367485046, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2381208837032318, + 0.36589449644088745, + 0.38178732991218567, + 0.4816311299800873, + 0.08830660581588745, + 0.06329374760389328, + 0.05036766827106476, + 0.060254089534282684, + 0.1295262575149536, + 0.14677807688713074 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.49075794219970703, + "style_target": 1, + "support_probability": 0.03915722668170929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018597676400217363, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8909385800361633, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2022:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3052387237548828, + 0.35912811756134033, + 0.3810542821884155, + 0.4028487503528595, + 0.09620346873998642, + 0.12024425715208054, + 0.16425222158432007, + 0.12580183148384094, + 0.0957469791173935, + 0.1241636723279953 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2022", + "split": "test", + "style_correct": 0, + "style_probability": 0.7397345900535583, + "style_target": 0, + "support_probability": 0.021898020058870316, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7756004957363913, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3569306433200836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8710249662399292, + 0.31010428071022034, + 0.4007412791252136, + 0.31010428071022034, + 0.09620585292577744 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.9092239141464233, + "style_target": 1, + "support_probability": 0.8271403312683105, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4044456516695334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29204174876213074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2028:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8339986801147461, + 0.3255572021007538, + 0.42172640562057495, + 0.3255572021007538, + 0.04851825162768364 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2028", + "split": "test", + "style_correct": 1, + "style_probability": 0.39292097091674805, + "style_target": 0, + "support_probability": 0.8545359969139099, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6222816201725933, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3773999810218811, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5718247890472412, + 0.47206005454063416, + 0.45152074098587036, + 0.9639185070991516, + 0.06025667116045952, + 0.06448636949062347 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.8294957876205444, + "style_target": 1, + "support_probability": 0.46137648820877075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07373556276837957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4290628135204315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2031:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6462512016296387, + 0.5259341597557068, + 0.44301337003707886, + 0.9778131246566772, + 0.09632855653762817, + 0.1524256020784378 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2031", + "split": "test", + "style_correct": 1, + "style_probability": 0.2267814576625824, + "style_target": 0, + "support_probability": 0.44341975450515747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3134932169387188, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26455891132354736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5626327991485596, + 0.20849989354610443, + 0.19864588975906372, + 0.8727810978889465, + 0.0787007212638855, + 0.059997063130140305, + 0.09508620202541351, + 0.07566051930189133, + 0.08170576393604279 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 1, + "style_probability": 0.5259708166122437, + "style_target": 1, + "support_probability": 0.6090503931045532, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07883390667071444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4214770495891571, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2035:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4941307008266449, + 0.16222012042999268, + 0.16150693595409393, + 0.8145516514778137, + 0.14361083507537842, + 0.10939335823059082, + 0.0897146612405777, + 0.33834725618362427, + 0.22860261797904968 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2035", + "split": "test", + "style_correct": 0, + "style_probability": 0.6213193535804749, + "style_target": 0, + "support_probability": 0.19040125608444214, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01686340682151344, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.653997004032135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28762176632881165, + 0.19163332879543304, + 0.5040215849876404, + 0.28762176632881165, + 0.18337491154670715, + 0.0522916242480278, + 0.05209001153707504, + 0.047932159155607224, + 0.09658937901258469, + 0.11378523707389832 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.8848038911819458, + "style_target": 1, + "support_probability": 0.04624062404036522, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00397650240011435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7467470169067383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2050:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2984887659549713, + 0.21122555434703827, + 0.5120670199394226, + 0.2984887659549713, + 0.15086735785007477, + 0.06470649689435959, + 0.05821632221341133, + 0.1384533792734146, + 0.14232544600963593, + 0.11543186008930206 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2050", + "split": "test", + "style_correct": 1, + "style_probability": 0.25170809030532837, + "style_target": 0, + "support_probability": 0.039551619440317154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2249443708741694, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20279505848884583, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20446567237377167, + 0.5334086418151855, + 0.6439357995986938, + 0.6388640403747559, + 0.1175917536020279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 1, + "style_probability": 0.6299642324447632, + "style_target": 1, + "support_probability": 0.5910101532936096, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21442369624153734, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2216404229402542, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2054:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16247636079788208, + 0.566336452960968, + 0.7113538384437561, + 0.6683847308158875, + 0.16067615151405334 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2054", + "split": "test", + "style_correct": 1, + "style_probability": 0.4969766438007355, + "style_target": 0, + "support_probability": 0.5416121482849121, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07586694034794574, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3522723615169525, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28780344128608704, + 0.3226398229598999, + 0.33021029829978943, + 0.7901750802993774, + 0.21520181000232697, + 0.10695020854473114 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 0, + "style_probability": 0.4877558648586273, + "style_target": 1, + "support_probability": 0.11018934100866318, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008532084069706404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5336666107177734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2059:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24664385616779327, + 0.3240863084793091, + 0.2709762752056122, + 0.8289028406143188, + 0.08091569691896439, + 0.08380970358848572 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2059", + "split": "test", + "style_correct": 1, + "style_probability": 0.14306510984897614, + "style_target": 0, + "support_probability": 0.1549696922302246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5928108789985167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028771258890628815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15822674334049225, + 0.381792813539505, + 0.9837607741355896, + 0.3368341028690338, + 0.0918736383318901, + 0.06068303436040878, + 0.12177760154008865, + 0.10789086669683456, + 0.08123430609703064 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 1, + "style_probability": 0.5450924634933472, + "style_target": 1, + "support_probability": 0.49204444885253906, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5851673659336569, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05236046016216278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2062:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18950384855270386, + 0.39020875096321106, + 0.9881545305252075, + 0.32723045349121094, + 0.04407922923564911, + 0.08022309094667435, + 0.15846651792526245, + 0.3390428125858307, + 0.05342628434300423 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2062", + "split": "test", + "style_correct": 0, + "style_probability": 0.839682400226593, + "style_target": 0, + "support_probability": 0.30154842138290405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03734881108548694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7761790752410889, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3716210126876831, + 0.8357719779014587, + 0.37330111861228943, + 0.420137494802475, + 0.1714736670255661, + 0.08361683040857315, + 0.07436244934797287, + 0.11734692752361298, + 0.13961327075958252, + 0.10204483568668365 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 1, + "style_probability": 0.6461475491523743, + "style_target": 1, + "support_probability": 0.05691641569137573, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05550617424218762, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.586614191532135, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2065:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4321076571941376, + 0.8820586800575256, + 0.3492467999458313, + 0.45008742809295654, + 0.15391014516353607, + 0.12424210458993912, + 0.1739489585161209, + 0.14427821338176727, + 0.11676393449306488, + 0.1407211720943451 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2065", + "split": "test", + "style_correct": 1, + "style_probability": 0.47675395011901855, + "style_target": 0, + "support_probability": 0.12179756909608841, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.701071852438929, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4205944240093231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14615480601787567, + 0.982200026512146, + 0.3636724352836609, + 0.14615480601787567, + 0.14545650780200958 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 1, + "style_probability": 0.827019214630127, + "style_target": 1, + "support_probability": 0.5183883309364319, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4795355991973338, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.40866127610206604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2077:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1718587428331375, + 0.945098876953125, + 0.3695319890975952, + 0.1718587428331375, + 0.1787790209054947 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2077", + "split": "test", + "style_correct": 0, + "style_probability": 0.6880015134811401, + "style_target": 0, + "support_probability": 0.4701688587665558, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7859786982932064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07533741742372513, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6904727816581726, + 0.19969764351844788, + 0.26900529861450195, + 0.9368651509284973, + 0.03744203969836235, + 0.059396322816610336 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.7622933387756348, + "style_target": 1, + "support_probability": 0.8985484838485718, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6222315653360511, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06873748451471329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2079:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6720787882804871, + 0.2598963677883148, + 0.2526395916938782, + 0.9132874011993408, + 0.044023383408784866, + 0.06515476852655411 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2079", + "split": "test", + "style_correct": 1, + "style_probability": 0.34738320112228394, + "style_target": 0, + "support_probability": 0.906808614730835, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6625744103915281, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34577181935310364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6417583227157593, + 0.47039222717285156, + 0.39417365193367004, + 0.8236838579177856, + 0.12921707332134247, + 0.06407002359628677, + 0.08358926326036453, + 0.05985088273882866, + 0.09927283972501755 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.8630305528640747, + "style_target": 1, + "support_probability": 0.8386776447296143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012261516790829763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28887730836868286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6532613635063171, + 0.557616114616394, + 0.44500160217285156, + 0.901923656463623, + 0.1086844801902771, + 0.0786721482872963, + 0.10058113187551498, + 0.09818006306886673, + 0.1265517622232437 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2083", + "split": "test", + "style_correct": 1, + "style_probability": 0.0791628435254097, + "style_target": 0, + "support_probability": 0.6815069913864136, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6727063081941083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2645427882671356, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3893802762031555, + 0.4037055969238281, + 0.8524330854415894, + 0.3330647051334381, + 0.05888446420431137, + 0.08773934096097946, + 0.03315171226859093, + 0.09265371412038803, + 0.05303541198372841, + 0.0975760817527771 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 1, + "style_probability": 0.6505430340766907, + "style_target": 1, + "support_probability": 0.6923904418945312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7055499101288376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2265951782464981, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2086:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3135332465171814, + 0.3530980050563812, + 0.8289196491241455, + 0.27173736691474915, + 0.08386510610580444, + 0.036424893885850906, + 0.04439340531826019, + 0.059535250067710876, + 0.1010235846042633, + 0.09645853191614151 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2086", + "split": "test", + "style_correct": 0, + "style_probability": 0.5484831929206848, + "style_target": 0, + "support_probability": 0.7906283736228943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011583264837174092, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7899643778800964, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24943818151950836, + 0.19319339096546173, + 0.2772485315799713, + 0.6670531034469604, + 0.296937495470047 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 1, + "style_probability": 0.5294513702392578, + "style_target": 1, + "support_probability": 0.06219286844134331, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11088525246360702, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6154024004936218, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2091:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2914677560329437, + 0.2298050820827484, + 0.31010183691978455, + 0.7096014618873596, + 0.09693101793527603 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2091", + "split": "test", + "style_correct": 0, + "style_probability": 0.7830406427383423, + "style_target": 0, + "support_probability": 0.16605378687381744, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0010056588144729541, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9096614718437195, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27448296546936035, + 0.4084010720252991, + 0.42989039421081543, + 0.35449492931365967, + 0.10639582574367523, + 0.1041196808218956 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 0, + "style_probability": 0.35017281770706177, + "style_target": 1, + "support_probability": 0.01018847618252039, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013362867910137542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8794620633125305, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2097:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3005385100841522, + 0.38152822852134705, + 0.4274105727672577, + 0.362282395362854, + 0.08968884497880936, + 0.14103388786315918 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2097", + "split": "test", + "style_correct": 1, + "style_probability": 0.13308365643024445, + "style_target": 0, + "support_probability": 0.0158451609313488, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8256180007070668, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06602127850055695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23458270728588104, + 0.23803453147411346, + 0.9673182368278503, + 0.34593960642814636, + 0.08615300059318542, + 0.09226905554533005, + 0.09738416969776154, + 0.14242881536483765, + 0.10879959166049957 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 1, + "style_probability": 0.5978570580482483, + "style_target": 1, + "support_probability": 0.9005672335624695, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8276541885290101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.047791026532649994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2399233728647232, + 0.3336232900619507, + 0.9911292791366577, + 0.46812760829925537, + 0.05819857120513916, + 0.11240982264280319, + 0.06796586513519287, + 0.050740402191877365, + 0.07080616056919098 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2098", + "split": "test", + "style_correct": 0, + "style_probability": 0.5300083160400391, + "style_target": 0, + "support_probability": 0.9263002276420593, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.048076432433683536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7002326250076294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8925181031227112, + 0.2795003056526184, + 0.2544305920600891, + 0.2795003056526184, + 0.09895062446594238, + 0.08511310070753098, + 0.18468531966209412, + 0.14062698185443878, + 0.15634194016456604, + 0.1242644339799881, + 0.37456098198890686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 1, + "style_probability": 0.6363763213157654, + "style_target": 1, + "support_probability": 0.06964316964149475, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20351799918251512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6524762511253357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2108:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8866559863090515, + 0.28934934735298157, + 0.18256184458732605, + 0.28934934735298157, + 0.07985950261354446, + 0.12075485289096832, + 0.053032759577035904, + 0.08962681144475937, + 0.10767880827188492, + 0.1119614914059639 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2108", + "split": "test", + "style_correct": 0, + "style_probability": 0.922309935092926, + "style_target": 0, + "support_probability": 0.11246202141046524, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8825669965249006, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0493975430727005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.707126796245575, + 0.3791797161102295, + 0.9907845258712769, + 0.4520868957042694, + 0.06451548635959625 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.8186245560646057, + "style_target": 1, + "support_probability": 0.8686254024505615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5868312126966089, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05555693432688713, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2122:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.734684944152832, + 0.45984962582588196, + 0.9927668571472168, + 0.5024760365486145, + 0.10112353414297104 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2122", + "split": "test", + "style_correct": 1, + "style_probability": 0.49446436762809753, + "style_target": 0, + "support_probability": 0.884939432144165, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8055614301107781, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0627320185303688, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3223751485347748, + 0.4908043444156647, + 0.8609744310379028, + 0.29089200496673584, + 0.1213751882314682, + 0.06382210552692413 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 1, + "style_probability": 0.5833569169044495, + "style_target": 1, + "support_probability": 0.9667063355445862, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6932359163107478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06023788079619408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2130:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31078508496284485, + 0.4722309112548828, + 0.7795511484146118, + 0.2916429042816162, + 0.060340940952301025, + 0.21864883601665497 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2130", + "split": "test", + "style_correct": 1, + "style_probability": 0.4624737501144409, + "style_target": 0, + "support_probability": 0.9450161457061768, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4784942942990185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16176113486289978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25782594084739685, + 0.24612928926944733, + 0.41377803683280945, + 0.8899585008621216, + 0.11689390987157822, + 0.1436474770307541, + 0.09240853786468506, + 0.10898035019636154, + 0.08765348792076111 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.7990622520446777, + "style_target": 1, + "support_probability": 0.3477274179458618, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09809677673453267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2543353736400604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23071898519992828, + 0.24495038390159607, + 0.42221444845199585, + 0.863726794719696, + 0.12236510962247849, + 0.19657748937606812, + 0.14781315624713898, + 0.3605061173439026, + 0.19498910009860992 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2131", + "split": "test", + "style_correct": 1, + "style_probability": 0.2697124779224396, + "style_target": 0, + "support_probability": 0.22047042846679688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011439338842983987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6708716154098511, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23555351793766022, + 0.704778254032135, + 0.3835688531398773, + 0.23555351793766022, + 0.11331728100776672, + 0.07786095142364502, + 0.054571483284235, + 0.08050335198640823, + 0.04675423353910446, + 0.04026718810200691, + 0.07179940491914749 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.7328941226005554, + "style_target": 1, + "support_probability": 0.01910514570772648, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013515238956043338, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7580710053443909, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2133:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2753562927246094, + 0.7242498397827148, + 0.37438133358955383, + 0.2753562927246094, + 0.06425362080335617, + 0.039467405527830124, + 0.062214601784944534, + 0.09477569162845612, + 0.07071799039840698, + 0.09653015434741974 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2133", + "split": "test", + "style_correct": 1, + "style_probability": 0.2943729758262634, + "style_target": 0, + "support_probability": 0.028637001290917397, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6352490436870113, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16873353719711304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3059523105621338, + 0.17609383165836334, + 0.3045991063117981, + 0.9407403469085693, + 0.04138875752687454 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 1, + "style_probability": 0.8119374513626099, + "style_target": 1, + "support_probability": 0.6575350165367126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5417393995229745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1729975938796997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2147:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35063984990119934, + 0.203168585896492, + 0.29315218329429626, + 0.9132604002952576, + 0.10846390575170517 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2147", + "split": "test", + "style_correct": 0, + "style_probability": 0.6066197752952576, + "style_target": 0, + "support_probability": 0.6766760349273682, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6799760668357919, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3164461851119995, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39549246430397034, + 0.8578392267227173, + 0.38004347681999207, + 0.32063236832618713, + 0.05249840393662453, + 0.036147378385066986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.8824310898780823, + "style_target": 1, + "support_probability": 0.7235299944877625, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32773319344896207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3023636043071747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2149:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4045797288417816, + 0.8385420441627502, + 0.3465101718902588, + 0.3495308756828308, + 0.20392805337905884, + 0.14712071418762207 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2149", + "split": "test", + "style_correct": 1, + "style_probability": 0.3709980249404907, + "style_target": 0, + "support_probability": 0.7392110228538513, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04779102286334737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7718483805656433, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7004197239875793, + 0.37532564997673035, + 0.34886419773101807, + 0.2455366998910904, + 0.06675810366868973, + 0.06087186932563782, + 0.06235780939459801, + 0.08894655108451843, + 0.08739569038152695 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.7092502117156982, + "style_target": 1, + "support_probability": 0.1365424245595932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007055115171752612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7425668239593506, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2156:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7651646137237549, + 0.37547147274017334, + 0.3265969157218933, + 0.2774921655654907, + 0.08690428733825684, + 0.08812771737575531, + 0.07762663811445236, + 0.071198470890522, + 0.10547725856304169 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2156", + "split": "test", + "style_correct": 1, + "style_probability": 0.2217050939798355, + "style_target": 0, + "support_probability": 0.23334762454032898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17412573257293393, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30317145586013794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7544034719467163, + 0.2835778295993805, + 0.6521160006523132, + 0.16326314210891724, + 0.08416189253330231, + 0.08112066984176636, + 0.05848899856209755, + 0.06667497009038925, + 0.05814291164278984, + 0.09084390848875046, + 0.05756088346242905 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.8064382076263428, + "style_target": 1, + "support_probability": 0.35350504517555237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.055756556828694404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28787946701049805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2168:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7026907801628113, + 0.26594287157058716, + 0.6034293174743652, + 0.12111129611730576, + 0.08283602446317673, + 0.0917929857969284, + 0.062261637300252914, + 0.15864259004592896, + 0.11335575580596924, + 0.07801038771867752, + 0.06639622896909714 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2168", + "split": "test", + "style_correct": 1, + "style_probability": 0.3788573145866394, + "style_target": 0, + "support_probability": 0.37088122963905334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6218465145620867, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2548411786556244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3789116442203522, + 0.27849170565605164, + 0.9220339059829712, + 0.32760149240493774, + 0.12532402575016022 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.8465403318405151, + "style_target": 1, + "support_probability": 0.48509207367897034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2517040428115642, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22455255687236786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2170:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3968225419521332, + 0.28917670249938965, + 0.9094377160072327, + 0.303897500038147, + 0.127101868391037 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2170", + "split": "test", + "style_correct": 1, + "style_probability": 0.4279993772506714, + "style_target": 0, + "support_probability": 0.5037246346473694, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9137184272041701, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3461971879005432, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15485651791095734, + 0.35556405782699585, + 0.986936628818512, + 0.45089027285575867, + 0.08424142748117447, + 0.11818304657936096 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.8611487746238708, + "style_target": 1, + "support_probability": 0.8626569509506226, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6513446006507309, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2769802212715149, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2178:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1638101488351822, + 0.45760250091552734, + 0.994512677192688, + 0.5260313153266907, + 0.05029734596610069, + 0.10057200491428375 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2178", + "split": "test", + "style_correct": 1, + "style_probability": 0.3341752290725708, + "style_target": 0, + "support_probability": 0.9403451085090637, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6918189160701793, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1342458873987198, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38512465357780457, + 0.12612542510032654, + 0.32088202238082886, + 0.9923684000968933, + 0.0564296618103981, + 0.1546427607536316, + 0.08620879799127579, + 0.08178623020648956, + 0.07747610658407211 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 1, + "style_probability": 0.6302677392959595, + "style_target": 1, + "support_probability": 0.8279811143875122, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4833795572437083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20137083530426025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2179:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3524116575717926, + 0.13366277515888214, + 0.29530203342437744, + 0.9873610734939575, + 0.09666983038187027, + 0.042061012238264084, + 0.07792797684669495, + 0.12681005895137787, + 0.13639983534812927 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2179", + "split": "test", + "style_correct": 1, + "style_probability": 0.35378506779670715, + "style_target": 0, + "support_probability": 0.8699123859405518, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8441133732226546, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09280578792095184, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20109178125858307, + 0.40210676193237305, + 0.3629024624824524, + 0.9518074989318848, + 0.05080293491482735, + 0.1062924861907959, + 0.08088846504688263, + 0.197710782289505, + 0.12597957253456116, + 0.07465921342372894, + 0.08803685009479523 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.8186125755310059, + "style_target": 1, + "support_probability": 0.6928465962409973, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6628874125041371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05802352726459503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2183:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21496079862117767, + 0.3875364661216736, + 0.354690283536911, + 0.9508448243141174, + 0.15033337473869324, + 0.12310728430747986, + 0.0675206258893013, + 0.1066041812300682, + 0.07343486696481705, + 0.058427341282367706, + 0.15711058676242828 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2183", + "split": "test", + "style_correct": 1, + "style_probability": 0.3821331262588501, + "style_target": 0, + "support_probability": 0.7997174859046936, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8539414879807764, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.09235712140798569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25224635004997253, + 0.23665116727352142, + 0.2663203477859497, + 0.9214465618133545, + 0.03885248303413391 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.9068031907081604, + "style_target": 1, + "support_probability": 0.7774872183799744, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04140066442406054, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14756560325622559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2195:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2777693271636963, + 0.2337370067834854, + 0.3142608106136322, + 0.9633811712265015, + 0.10136225074529648 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2195", + "split": "test", + "style_correct": 1, + "style_probability": 0.14956973493099213, + "style_target": 0, + "support_probability": 0.6159366965293884, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.753128542899411, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08771020919084549, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6005952954292297, + 0.9300277829170227, + 0.5761790871620178, + 0.6005952954292297, + 0.04288461431860924, + 0.025771547108888626 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 1, + "style_probability": 0.7868048548698425, + "style_target": 1, + "support_probability": 0.7978765964508057, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7223247370991075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08913686126470566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2197:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6442521810531616, + 0.8675186038017273, + 0.5158988833427429, + 0.6442521810531616, + 0.07342956960201263, + 0.09937895089387894 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2197", + "split": "test", + "style_correct": 0, + "style_probability": 0.6494082808494568, + "style_target": 0, + "support_probability": 0.8330295085906982, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9723574151286093, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20676682889461517, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:2209:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40403902530670166, + 0.9955275654792786, + 0.4942466914653778, + 0.30267849564552307, + 0.0933021530508995, + 0.07414878159761429, + 0.06031256914138794, + 0.04297901317477226, + 0.04869113862514496 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.8746787905693054, + "style_target": 1, + "support_probability": 0.9414011240005493, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6632881463090925, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19544507563114166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2209:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4034954011440277, + 0.9859822988510132, + 0.46983200311660767, + 0.28205835819244385, + 0.11418861150741577, + 0.10528343915939331, + 0.08675834536552429, + 0.05534743517637253, + 0.17777809500694275 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2209", + "split": "test", + "style_correct": 1, + "style_probability": 0.21194422245025635, + "style_target": 0, + "support_probability": 0.8909662961959839, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7701818178007187, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20771165192127228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9656897783279419, + 0.3224901854991913, + 0.370513379573822, + 0.5137671232223511, + 0.03289424628019333, + 0.06750723719596863, + 0.03894966095685959, + 0.041645992547273636, + 0.06056889519095421, + 0.08214670419692993, + 0.0706535130739212 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.7244070172309875, + "style_target": 1, + "support_probability": 0.6199735999107361, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.26421193680561395, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1733793169260025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2216:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9887420535087585, + 0.3877149522304535, + 0.4853416383266449, + 0.5834959149360657, + 0.09024827927350998, + 0.08331109583377838, + 0.04668206349015236, + 0.08296419680118561, + 0.07161486148834229, + 0.11045356094837189 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2216", + "split": "test", + "style_correct": 1, + "style_probability": 0.19387099146842957, + "style_target": 0, + "support_probability": 0.6942941546440125, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8701476460936419, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1288372129201889, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31761959195137024, + 0.987360954284668, + 0.1727389395236969, + 0.07002704590559006, + 0.05710329860448837 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 1, + "style_probability": 0.75627201795578, + "style_target": 1, + "support_probability": 0.8903593420982361, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7729638892569142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15850043296813965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2221:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27236300706863403, + 0.955115795135498, + 0.18189525604248047, + 0.08836764097213745, + 0.1264403909444809 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2221", + "split": "test", + "style_correct": 0, + "style_probability": 0.5958154201507568, + "style_target": 0, + "support_probability": 0.8534828424453735, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8559215960158325, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04279732331633568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.96856689453125, + 0.33291158080101013, + 0.7377358078956604, + 0.7138331532478333, + 0.05932149663567543, + 0.09179366379976273 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.6258990168571472, + "style_target": 1, + "support_probability": 0.8912107944488525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6402793468229362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.050583627074956894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2260:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9394019842147827, + 0.3248794376850128, + 0.6797406077384949, + 0.6480397582054138, + 0.18322502076625824, + 0.12704405188560486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2260", + "split": "test", + "style_correct": 1, + "style_probability": 0.25220564007759094, + "style_target": 0, + "support_probability": 0.8705838918685913, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005491294292649135, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9035205245018005, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16059426963329315, + 0.8139639496803284, + 0.444193959236145, + 0.34201303124427795, + 0.0836566612124443, + 0.10756316781044006, + 0.096058189868927, + 0.07296924293041229, + 0.08214690536260605 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.7893574237823486, + "style_target": 1, + "support_probability": 0.014503437094390392, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012494461090997997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8452283143997192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16574737429618835, + 0.8260420560836792, + 0.518024742603302, + 0.3669309616088867, + 0.07008075714111328, + 0.21813583374023438, + 0.13768933713436127, + 0.11564945429563522, + 0.18437381088733673 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2277", + "split": "test", + "style_correct": 1, + "style_probability": 0.2505183815956116, + "style_target": 0, + "support_probability": 0.04482926055788994, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6240807869382365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4447917640209198, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12717388570308685, + 0.2916959226131439, + 0.9403700828552246, + 0.2268911600112915, + 0.10250362753868103, + 0.05045368894934654, + 0.08914113789796829, + 0.05896943807601929, + 0.06424979865550995, + 0.0436822883784771, + 0.05508866906166077 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 1, + "style_probability": 0.7661266326904297, + "style_target": 1, + "support_probability": 0.40323406457901, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19099082865001282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6706563234329224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2282:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1542665958404541, + 0.2856261432170868, + 0.8723375201225281, + 0.24019883573055267, + 0.055351536720991135, + 0.05312459543347359, + 0.10329422354698181, + 0.06620046496391296, + 0.07882439345121384, + 0.06881968677043915, + 0.049631647765636444 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2282", + "split": "test", + "style_correct": 0, + "style_probability": 0.7510692477226257, + "style_target": 0, + "support_probability": 0.1400669366121292, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07258376788491462, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3882730007171631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8541362285614014, + 0.7498877644538879, + 0.3532857298851013, + 0.33025118708610535, + 0.07707211375236511 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.43289750814437866, + "style_target": 1, + "support_probability": 0.20643198490142822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2168778003080574, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.516480565071106, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2288:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8975111842155457, + 0.8259995579719543, + 0.2943476140499115, + 0.3139936029911041, + 0.35948899388313293 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2288", + "split": "test", + "style_correct": 0, + "style_probability": 0.8578767776489258, + "style_target": 0, + "support_probability": 0.30734673142433167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045533030798288195, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8368096351623535, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5534182786941528, + 0.31114038825035095, + 0.5534182786941528, + 0.3348064124584198, + 0.06934139877557755, + 0.08542720973491669 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 0, + "style_probability": 0.43463000655174255, + "style_target": 1, + "support_probability": 0.22388865053653717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000287061193933777, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9805750846862793, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5209900140762329, + 0.24590250849723816, + 0.5209900140762329, + 0.3107403516769409, + 0.092902272939682, + 0.13603352010250092 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2291", + "split": "test", + "style_correct": 1, + "style_probability": 0.11098272353410721, + "style_target": 0, + "support_probability": 0.029227187857031822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3997224617892386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09335280954837799, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1804724782705307, + 0.9521518349647522, + 0.1342121958732605, + 0.21035313606262207, + 0.06298410147428513, + 0.05924887955188751, + 0.08864638209342957, + 0.12849268317222595, + 0.17204400897026062 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 0, + "style_probability": 0.43436774611473083, + "style_target": 1, + "support_probability": 0.6276658773422241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33166258657055364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21794524788856506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2293:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1711391657590866, + 0.9415766596794128, + 0.15564468502998352, + 0.26114436984062195, + 0.07703600078821182, + 0.1342204511165619, + 0.09326107800006866, + 0.07092858105897903, + 0.15364426374435425 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2293", + "split": "test", + "style_correct": 0, + "style_probability": 0.62890625, + "style_target": 0, + "support_probability": 0.39967799186706543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008191041241804144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5827845931053162, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1942347288131714, + 0.1632172018289566, + 0.8456288576126099, + 0.269444078207016, + 0.057848669588565826, + 0.06982871145009995, + 0.05771195515990257, + 0.08407750725746155, + 0.09222275763750076, + 0.17250485718250275, + 0.17920537292957306 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.22666755318641663, + "style_target": 1, + "support_probability": 0.029101775959134102, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09977677627087189, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48101603984832764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2294:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21120482683181763, + 0.1455952376127243, + 0.8681091070175171, + 0.30442237854003906, + 0.08465342223644257, + 0.10946564376354218, + 0.11609748750925064, + 0.15286466479301453, + 0.09156353771686554, + 0.09317677468061447, + 0.16622212529182434 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2294", + "split": "test", + "style_correct": 0, + "style_probability": 0.8335562348365784, + "style_target": 0, + "support_probability": 0.06480377167463303, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8640223312283979, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07469562441110611, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1974944770336151, + 0.13438068330287933, + 0.2758173644542694, + 0.9903907775878906, + 0.1976480931043625 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.5550892949104309, + "style_target": 1, + "support_probability": 0.9114617109298706, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5826733621413283, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08810216188430786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2303:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20908166468143463, + 0.11375447362661362, + 0.2767457067966461, + 0.9889179468154907, + 0.19023707509040833 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2303", + "split": "test", + "style_correct": 1, + "style_probability": 0.308399498462677, + "style_target": 0, + "support_probability": 0.8898806571960449, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5858237862775795, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24416901171207428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3845978379249573, + 0.21811798214912415, + 0.850294291973114, + 0.3465859889984131, + 0.13549616932868958, + 0.19597461819648743 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 1, + "style_probability": 0.631279706954956, + "style_target": 1, + "support_probability": 0.8005074262619019, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06606266860534671, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37658441066741943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38707464933395386, + 0.25691691040992737, + 0.9704827070236206, + 0.4621772766113281, + 0.08456899225711823, + 0.16180509328842163 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2318", + "split": "test", + "style_correct": 1, + "style_probability": 0.15689586102962494, + "style_target": 0, + "support_probability": 0.6548075079917908, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006489098588299273, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7286537289619446, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2921566367149353, + 0.22333259880542755, + 0.5294835567474365, + 0.2921566367149353, + 0.12237507849931717, + 0.10035731643438339, + 0.09023253619670868, + 0.19419652223587036, + 0.1636703908443451 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 1, + "style_probability": 0.5781376361846924, + "style_target": 1, + "support_probability": 0.020307032391428947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010526770912615224, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7104065418243408, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2330:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23746809363365173, + 0.21180152893066406, + 0.5806694626808167, + 0.23746809363365173, + 0.07418271154165268, + 0.05018620565533638, + 0.13079968094825745, + 0.08504381775856018, + 0.10039255768060684 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2330", + "split": "test", + "style_correct": 0, + "style_probability": 0.8519216775894165, + "style_target": 0, + "support_probability": 0.027113480493426323, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2171907912017659, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.64312344789505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23690472543239594, + 0.19653448462486267, + 0.7248014807701111, + 0.25764816999435425, + 0.09221424162387848, + 0.10613191872835159, + 0.06782615929841995, + 0.08534225076436996, + 0.07153879851102829, + 0.13754621148109436, + 0.16019080579280853 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 0, + "style_probability": 0.45168620347976685, + "style_target": 1, + "support_probability": 0.5354164242744446, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19519881786677828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.400836706161499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2334:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24619603157043457, + 0.18551450967788696, + 0.7661975622177124, + 0.25666892528533936, + 0.056133341044187546, + 0.13798299431800842, + 0.0708165317773819, + 0.07113276422023773, + 0.05131014809012413, + 0.04550345614552498, + 0.16841956973075867 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2334", + "split": "test", + "style_correct": 1, + "style_probability": 0.21916750073432922, + "style_target": 0, + "support_probability": 0.6496204137802124, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23629883398001542, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7696936726570129, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.765790581703186, + 0.46916529536247253, + 0.432804137468338, + 0.3321092128753662, + 0.15912888944149017 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 1, + "style_probability": 0.8815715312957764, + "style_target": 1, + "support_probability": 0.21697038412094116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14573001621781856, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7010660767555237, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6741900444030762, + 0.47646692395210266, + 0.4196760654449463, + 0.2996682822704315, + 0.11581484228372574 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2340", + "split": "test", + "style_correct": 0, + "style_probability": 0.6771381497383118, + "style_target": 0, + "support_probability": 0.304472416639328, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6639724750304055, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.045549176633358, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9940887689590454, + 0.31179818511009216, + 0.4687027931213379, + 0.31179818511009216, + 0.12416122108697891, + 0.10509859025478363 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.7289674282073975, + "style_target": 1, + "support_probability": 0.6121107339859009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4919856299563535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.036371681839227676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2348:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9838060140609741, + 0.28370973467826843, + 0.4350658059120178, + 0.28370973467826843, + 0.060958828777074814, + 0.104168601334095 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2348", + "split": "test", + "style_correct": 1, + "style_probability": 0.4109717309474945, + "style_target": 0, + "support_probability": 0.6778441667556763, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8969365997968666, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35636380314826965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27304694056510925, + 0.27873125672340393, + 0.27304694056510925, + 0.9558092355728149, + 0.035488102585077286, + 0.05896981433033943, + 0.10499942302703857, + 0.30163317918777466, + 0.07835163176059723 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.9323887228965759, + "style_target": 1, + "support_probability": 0.8124369978904724, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19334000973768362, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12673325836658478, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35719436407089233, + 0.270306259393692, + 0.35719436407089233, + 0.9870138764381409, + 0.05368625372648239, + 0.21367543935775757, + 0.15395857393741608, + 0.06940607726573944, + 0.11632198095321655 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2355", + "split": "test", + "style_correct": 1, + "style_probability": 0.19315758347511292, + "style_target": 0, + "support_probability": 0.8127620220184326, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2149310911262852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5479239821434021, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.44313913583755493, + 0.2041264772415161, + 0.36870092153549194, + 0.7914315462112427, + 0.057695746421813965, + 0.06992847472429276, + 0.048945024609565735, + 0.06559151411056519, + 0.11111125349998474, + 0.21505115926265717, + 0.08211248368024826 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 1, + "style_probability": 0.8785405158996582, + "style_target": 1, + "support_probability": 0.09764502197504044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12042832319367312, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6495106220245361, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2359:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39567506313323975, + 0.17403803765773773, + 0.38707494735717773, + 0.776375412940979, + 0.058029647916555405, + 0.09391216188669205, + 0.06311789155006409, + 0.06284984201192856, + 0.12631738185882568, + 0.18924371898174286, + 0.185345858335495 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2359", + "split": "test", + "style_correct": 0, + "style_probability": 0.6280277967453003, + "style_target": 0, + "support_probability": 0.0800396278500557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13691361724644047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4187237620353699, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3508647680282593, + 0.3799906373023987, + 0.3508647680282593, + 0.9006250500679016, + 0.15112681686878204 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 1, + "style_probability": 0.5157892107963562, + "style_target": 1, + "support_probability": 0.16206857562065125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11009308830172418, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.37944212555885315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40173810720443726, + 0.37427788972854614, + 0.40173810720443726, + 0.9366236329078674, + 0.06335574388504028 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2367", + "split": "test", + "style_correct": 0, + "style_probability": 0.7152252197265625, + "style_target": 0, + "support_probability": 0.11441495269536972, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6549810241360063, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11265291273593903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9169405102729797, + 0.5401538610458374, + 0.11976170539855957, + 0.32623976469039917, + 0.1395310014486313, + 0.09954457730054855 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 0, + "style_probability": 0.43182507157325745, + "style_target": 1, + "support_probability": 0.7505563497543335, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30315931372735433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10652455687522888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2380:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8993555903434753, + 0.4872834384441376, + 0.11064032465219498, + 0.27240458130836487, + 0.0656583234667778, + 0.11335606127977371 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2380", + "split": "test", + "style_correct": 1, + "style_probability": 0.17948821187019348, + "style_target": 0, + "support_probability": 0.6681106686592102, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7101853510796197, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15946191549301147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36623114347457886, + 0.4091129004955292, + 0.9411394000053406, + 0.2076391875743866, + 0.10598528385162354, + 0.06917758285999298, + 0.08577394485473633, + 0.05809500440955162, + 0.03766800835728645 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.7574465274810791, + "style_target": 1, + "support_probability": 0.7555991411209106, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.269207837992953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08539305627346039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2382:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.341948002576828, + 0.45310473442077637, + 0.9819847941398621, + 0.26181620359420776, + 0.047777388244867325, + 0.07713088393211365, + 0.16796836256980896, + 0.08164254575967789, + 0.11783295124769211 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2382", + "split": "test", + "style_correct": 1, + "style_probability": 0.26378461718559265, + "style_target": 0, + "support_probability": 0.7939340472221375, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8817411396154995, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03563254326581955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2617012858390808, + 0.41016751527786255, + 0.2617012858390808, + 0.9814246892929077, + 0.0643627867102623, + 0.09320778399705887, + 0.12254805117845535, + 0.11389706283807755, + 0.1288563311100006, + 0.06940308958292007, + 0.046824321150779724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 1, + "style_probability": 0.8000828623771667, + "style_target": 1, + "support_probability": 0.9114495515823364, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.931786254762109, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02698562853038311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2383:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2671973407268524, + 0.39569517970085144, + 0.2671973407268524, + 0.9942504167556763, + 0.10947791486978531, + 0.061784856021404266, + 0.07009205967187881, + 0.05201677605509758, + 0.04269632697105408, + 0.03141898289322853 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2383", + "split": "test", + "style_correct": 0, + "style_probability": 0.8921916484832764, + "style_target": 0, + "support_probability": 0.8798272013664246, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6746920759908228, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06539326161146164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5523147583007812, + 0.17892758548259735, + 0.8389007449150085, + 0.44547584652900696, + 0.125279039144516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 0, + "style_probability": 0.3012728989124298, + "style_target": 1, + "support_probability": 0.9627839922904968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040916727405502236, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14717188477516174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6880910992622375, + 0.16029994189739227, + 0.8925634622573853, + 0.4257241487503052, + 0.12240102142095566 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2404", + "split": "test", + "style_correct": 1, + "style_probability": 0.06263604760169983, + "style_target": 0, + "support_probability": 0.8890058994293213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3197163689485194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1527663916349411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1467708945274353, + 0.2623794972896576, + 0.35792699456214905, + 0.8769795298576355, + 0.1188575029373169, + 0.1348818838596344 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 0, + "style_probability": 0.42933306097984314, + "style_target": 1, + "support_probability": 0.35706040263175964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2331128260237847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2617831528186798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13353875279426575, + 0.2923099994659424, + 0.30401408672332764, + 0.8269864320755005, + 0.05237356573343277, + 0.10796521604061127 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2411", + "split": "test", + "style_correct": 0, + "style_probability": 0.5528789162635803, + "style_target": 0, + "support_probability": 0.21845652163028717, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025190602862930733, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8543718457221985, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7595999836921692, + 0.5295281410217285, + 0.5295281410217285, + 0.27312391996383667, + 0.22174926102161407, + 0.15305981040000916, + 0.06248039752244949, + 0.12165496498346329, + 0.23096713423728943 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.30802610516548157, + "style_target": 1, + "support_probability": 0.17768073081970215, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021304382883145845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9440491199493408, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2412:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7302872538566589, + 0.5476839542388916, + 0.5476839542388916, + 0.2264648824930191, + 0.07612229883670807, + 0.06090817227959633, + 0.1078437864780426, + 0.07108468562364578, + 0.04754899814724922 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2412", + "split": "test", + "style_correct": 0, + "style_probability": 0.7294058799743652, + "style_target": 0, + "support_probability": 0.1139882430434227, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4458009057329182, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19817285239696503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14028026163578033, + 0.9251853823661804, + 0.5237685441970825, + 0.31255868077278137, + 0.04266214743256569, + 0.07095402479171753, + 0.146339550614357, + 0.12907586991786957, + 0.030838118866086006, + 0.09041568636894226, + 0.17882059514522552 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 0, + "style_probability": 0.45858076214790344, + "style_target": 1, + "support_probability": 0.6094082593917847, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06771378356041424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33892586827278137, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2413:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.08390074223279953, + 0.9584012031555176, + 0.5141956806182861, + 0.3372947573661804, + 0.10840180516242981, + 0.08613087236881256, + 0.06759336590766907, + 0.0554552786052227, + 0.032701969146728516, + 0.09700261056423187 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2413", + "split": "test", + "style_correct": 1, + "style_probability": 0.249021977186203, + "style_target": 0, + "support_probability": 0.47892045974731445, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25575754841499787, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2227867692708969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1428012251853943, + 0.952348530292511, + 0.5767252445220947, + 0.1428012251853943, + 0.053744349628686905 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 1, + "style_probability": 0.7006042003631592, + "style_target": 1, + "support_probability": 0.34249556064605713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2288835803858689, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22473207116127014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15169180929660797, + 0.9065965414047241, + 0.5688313841819763, + 0.15169180929660797, + 0.05573559179902077 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2425", + "split": "test", + "style_correct": 0, + "style_probability": 0.5899744033813477, + "style_target": 0, + "support_probability": 0.39833512902259827, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6981018420864018, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10962099581956863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7393131852149963, + 0.16295383870601654, + 0.7165626287460327, + 0.2930658459663391, + 0.03932845965027809, + 0.04821501672267914 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.7980375289916992, + "style_target": 1, + "support_probability": 0.9489654898643494, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5255725634644826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08139859139919281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2440:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.677479088306427, + 0.19770820438861847, + 0.7044802308082581, + 0.27086150646209717, + 0.03554248437285423, + 0.04872007668018341 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2440", + "split": "test", + "style_correct": 1, + "style_probability": 0.27418753504753113, + "style_target": 0, + "support_probability": 0.9410231709480286, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016548048183831212, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9914804697036743, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23763275146484375, + 0.5638338923454285, + 0.41224294900894165, + 0.23763275146484375, + 0.11420243978500366, + 0.06678025424480438, + 0.07723037153482437, + 0.05686497315764427, + 0.10311691462993622 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.7367637753486633, + "style_target": 1, + "support_probability": 0.00675597507506609, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008218027592012331, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9594128727912903, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2441:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26771607995033264, + 0.6076012253761292, + 0.40715667605400085, + 0.26771607995033264, + 0.098182313144207, + 0.3769919276237488, + 0.13611792027950287, + 0.11881214380264282, + 0.14256784319877625 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2441", + "split": "test", + "style_correct": 1, + "style_probability": 0.26896971464157104, + "style_target": 0, + "support_probability": 0.039729926735162735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7825430494888829, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2090064436197281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.373382568359375, + 0.38792720437049866, + 0.15619634091854095, + 0.9925227761268616, + 0.03879590705037117, + 0.03839442878961563, + 0.04947153106331825, + 0.05864924564957619, + 0.04755108058452606, + 0.04060615226626396, + 0.03219045326113701 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 1, + "style_probability": 0.8770527839660645, + "style_target": 1, + "support_probability": 0.6603550910949707, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6357792572107428, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17961731553077698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2459:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3624742329120636, + 0.37108564376831055, + 0.19070348143577576, + 0.9872283935546875, + 0.05689709261059761, + 0.0670333206653595, + 0.03569868579506874, + 0.0767780989408493, + 0.03786151111125946, + 0.06520070880651474, + 0.060169849544763565 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2459", + "split": "test", + "style_correct": 0, + "style_probability": 0.7172781229019165, + "style_target": 0, + "support_probability": 0.6970744729042053, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13557794708015788, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5337353944778442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19904807209968567, + 0.9030358791351318, + 0.2067902386188507, + 0.3215070366859436, + 0.03782477602362633 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.8613144755363464, + "style_target": 1, + "support_probability": 0.1935364454984665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07052538117456565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3024863302707672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2461:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21866731345653534, + 0.9102326035499573, + 0.1687334179878235, + 0.44499820470809937, + 0.06756710261106491 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2461", + "split": "test", + "style_correct": 1, + "style_probability": 0.36397162079811096, + "style_target": 0, + "support_probability": 0.2277405709028244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007519033207735056, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9264459609985352, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.198382169008255, + 0.7250103950500488, + 0.3440510332584381, + 0.3272413909435272, + 0.08983108401298523, + 0.03749540075659752 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.6304420828819275, + "style_target": 1, + "support_probability": 0.06054104492068291, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002302057551129679, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9399957060813904, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2465:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24062485992908478, + 0.8369316458702087, + 0.3354310393333435, + 0.3758375942707062, + 0.13175906240940094, + 0.13197337090969086 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2465", + "split": "test", + "style_correct": 1, + "style_probability": 0.2341277003288269, + "style_target": 0, + "support_probability": 0.06863993406295776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029603856308161536, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7467960715293884, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21051575243473053, + 0.21051575243473053, + 0.37670618295669556, + 0.25016942620277405, + 0.13801465928554535, + 0.05078370124101639, + 0.044069092720746994, + 0.08211663365364075, + 0.04695810005068779 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.8768879175186157, + "style_target": 1, + "support_probability": 0.05843991041183472, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00651165891015326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7607489824295044, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19708454608917236, + 0.19708454608917236, + 0.3037986159324646, + 0.2552693784236908, + 0.1192190870642662, + 0.12028218805789948, + 0.2126912623643875, + 0.10843552649021149, + 0.3155215084552765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2490", + "split": "test", + "style_correct": 1, + "style_probability": 0.4941825866699219, + "style_target": 0, + "support_probability": 0.04724762961268425, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7650325118341073, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21601951122283936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33160609006881714, + 0.15232251584529877, + 0.33160609006881714, + 0.9796130657196045, + 0.03805365413427353, + 0.04742954671382904, + 0.0988236665725708, + 0.03739499673247337, + 0.04850533604621887, + 0.0680806040763855, + 0.035369694232940674 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 1, + "style_probability": 0.8337602019309998, + "style_target": 1, + "support_probability": 0.8387763500213623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3682117404538499, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22775417566299438, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2491:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2731005549430847, + 0.17068839073181152, + 0.2731005549430847, + 0.9568765759468079, + 0.22777718305587769, + 0.07427389919757843, + 0.1334347426891327, + 0.12260576337575912, + 0.10639877617359161, + 0.08505924046039581 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2491", + "split": "test", + "style_correct": 1, + "style_probability": 0.497433066368103, + "style_target": 0, + "support_probability": 0.7834311127662659, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02809627427541272, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2909495234489441, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3064698576927185, + 0.844482421875, + 0.32559293508529663, + 0.36283835768699646, + 0.07244430482387543 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 0, + "style_probability": 0.4697950482368469, + "style_target": 1, + "support_probability": 0.06509335339069366, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028446170242638624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18924985826015472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33009594678878784, + 0.8614429831504822, + 0.350039541721344, + 0.3159787058830261, + 0.14720110595226288 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2493", + "split": "test", + "style_correct": 1, + "style_probability": 0.3318595290184021, + "style_target": 0, + "support_probability": 0.09442156553268433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07321054435597539, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3417554199695587, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46747714281082153, + 0.5043957233428955, + 0.6949710249900818, + 0.26304566860198975, + 0.09468591958284378, + 0.04727344959974289 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 0, + "style_probability": 0.4592461585998535, + "style_target": 1, + "support_probability": 0.16349242627620697, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0074494825620341504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4877000153064728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2494:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47690272331237793, + 0.5478675365447998, + 0.8030493259429932, + 0.22993342578411102, + 0.07204367220401764, + 0.12530481815338135 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2494", + "split": "test", + "style_correct": 1, + "style_probability": 0.17448922991752625, + "style_target": 0, + "support_probability": 0.18582630157470703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004654279080719304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7290857434272766, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2854215204715729, + 0.2892965078353882, + 0.20176388323307037, + 0.32558080554008484, + 0.12152037769556046, + 0.11552567034959793, + 0.11783373355865479, + 0.14002355933189392, + 0.07005912065505981 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 1, + "style_probability": 0.526593804359436, + "style_target": 1, + "support_probability": 0.01805262267589569, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013589419613885042, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6148086786270142, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2499:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31758999824523926, + 0.34150993824005127, + 0.24288266897201538, + 0.3691040277481079, + 0.0629686713218689, + 0.15897800028324127, + 0.13447940349578857, + 0.16156421601772308, + 0.20337186753749847 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2499", + "split": "test", + "style_correct": 0, + "style_probability": 0.8060544729232788, + "style_target": 0, + "support_probability": 0.022150682285428047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32633586945816795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3232313394546509, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8355821967124939, + 0.19125868380069733, + 0.49613964557647705, + 0.5389622449874878, + 0.05256844311952591, + 0.06785688549280167, + 0.0780165046453476, + 0.06572923809289932, + 0.1074003055691719, + 0.0438416562974453 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 1, + "style_probability": 0.6554277539253235, + "style_target": 1, + "support_probability": 0.305410772562027, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19443900884191476, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4474359452724457, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2500:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8486435413360596, + 0.20080982148647308, + 0.4404850900173187, + 0.5235424637794495, + 0.09443314373493195, + 0.14152131974697113, + 0.11025369167327881, + 0.14250877499580383, + 0.08124811947345734, + 0.07605547457933426, + 0.12765520811080933 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2500", + "split": "test", + "style_correct": 0, + "style_probability": 0.5410569310188293, + "style_target": 0, + "support_probability": 0.15988411009311676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5364463780166915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.43285807967185974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28999534249305725, + 0.43699243664741516, + 0.4887836277484894, + 0.7812710404396057, + 0.061457280069589615 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 1, + "style_probability": 0.9223290681838989, + "style_target": 1, + "support_probability": 0.39788538217544556, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12195093476333164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5081633925437927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2503:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21424216032028198, + 0.35801181197166443, + 0.4538984000682831, + 0.7365558743476868, + 0.26819586753845215 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2503", + "split": "test", + "style_correct": 0, + "style_probability": 0.5510719418525696, + "style_target": 0, + "support_probability": 0.30461639165878296, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7477965585523734, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22017565369606018, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.357340008020401, + 0.357340008020401, + 0.8917434811592102, + 0.2775923013687134, + 0.04444769397377968, + 0.043578214943408966 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 1, + "style_probability": 0.9071065783500671, + "style_target": 1, + "support_probability": 0.6866084933280945, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47062965187878447, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19693343341350555, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2506:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32553714513778687, + 0.32553714513778687, + 0.8641615509986877, + 0.2525213360786438, + 0.07264788448810577, + 0.13301919400691986 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2506", + "split": "test", + "style_correct": 0, + "style_probability": 0.57459956407547, + "style_target": 0, + "support_probability": 0.7406647205352783, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5702021237426003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28962042927742004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38732603192329407, + 0.25543639063835144, + 0.08769994229078293, + 0.8243134617805481, + 0.054573312401771545, + 0.05802363157272339, + 0.02461881749331951, + 0.04106704518198967, + 0.06776852160692215 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.7203056812286377, + "style_target": 1, + "support_probability": 0.6310640573501587, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010191352615737332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5971314311027527, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2507:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49528807401657104, + 0.2665195167064667, + 0.10444065183401108, + 0.8397822380065918, + 0.11041821539402008, + 0.2184857875108719, + 0.06897956877946854, + 0.06032206118106842, + 0.08831000328063965 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2507", + "split": "test", + "style_correct": 1, + "style_probability": 0.09609651565551758, + "style_target": 0, + "support_probability": 0.42768993973731995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20942561308133634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7293825149536133, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8853344917297363, + 0.10653109848499298, + 0.4449438452720642, + 0.22077208757400513, + 0.06632255017757416, + 0.04645749181509018, + 0.028240125626325607, + 0.08670800924301147, + 0.05529911816120148, + 0.07733376324176788, + 0.06525110453367233 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 1, + "style_probability": 0.8343316912651062, + "style_target": 1, + "support_probability": 0.22736971080303192, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07862643491340914, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7248380780220032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2512:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8276099562644958, + 0.12664148211479187, + 0.38830646872520447, + 0.22676904499530792, + 0.08430878072977066, + 0.0701611191034317, + 0.18356873095035553, + 0.06967824697494507, + 0.0740756094455719, + 0.24564290046691895 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2512", + "split": "test", + "style_correct": 0, + "style_probability": 0.5453382730484009, + "style_target": 0, + "support_probability": 0.2216518670320511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7182385939478237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1966204047203064, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6134740710258484, + 0.49914535880088806, + 0.7433823347091675, + 0.613413393497467, + 0.09962505102157593 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.8361223340034485, + "style_target": 1, + "support_probability": 0.914919912815094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4200641314239526, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30472806096076965, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5588446259498596, + 0.40677687525749207, + 0.6833123564720154, + 0.5896970629692078, + 0.17889711260795593 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2518", + "split": "test", + "style_correct": 1, + "style_probability": 0.3315507471561432, + "style_target": 0, + "support_probability": 0.8339323997497559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.049125749442485045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8205398917198181, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32071730494499207, + 0.1881536841392517, + 0.35731756687164307, + 0.8561737537384033, + 0.0716550424695015, + 0.12020906805992126 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.7540437579154968, + "style_target": 1, + "support_probability": 0.04533553123474121, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017442254829082287, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.92696213722229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2984783351421356, + 0.226643368601799, + 0.3019076883792877, + 0.8743304014205933, + 0.09101621806621552, + 0.1375572383403778 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2523", + "split": "test", + "style_correct": 1, + "style_probability": 0.1930713951587677, + "style_target": 0, + "support_probability": 0.02338729240000248, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0125343505428156, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9495142698287964, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3451220393180847, + 0.45604726672172546, + 0.3451220393180847, + 0.5253682732582092, + 0.07021501660346985, + 0.06271374225616455, + 0.07730812579393387, + 0.16420429944992065, + 0.11519595235586166 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 1, + "style_probability": 0.8608401417732239, + "style_target": 1, + "support_probability": 0.02711358666419983, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03331491557633436, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8973657488822937, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2531:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3309246301651001, + 0.4710501432418823, + 0.3309246301651001, + 0.6409456133842468, + 0.08115144073963165, + 0.06134071573615074, + 0.03852209448814392, + 0.044308654963970184, + 0.06091050058603287 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2531", + "split": "test", + "style_correct": 0, + "style_probability": 0.6602600812911987, + "style_target": 0, + "support_probability": 0.05598781257867813, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13857128584049858, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7736437320709229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1854182481765747, + 0.5303764343261719, + 0.3668120801448822, + 0.3730170726776123, + 0.05646101385354996, + 0.08635719865560532, + 0.04882849380373955, + 0.04079553112387657, + 0.05203225091099739, + 0.06389044225215912, + 0.13400831818580627 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.9199102520942688, + "style_target": 1, + "support_probability": 0.07140209525823593, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006994383298519479, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8729299306869507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2535:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22218704223632812, + 0.48405906558036804, + 0.37158122658729553, + 0.3157477080821991, + 0.1572650521993637, + 0.05337673798203468, + 0.05016527697443962, + 0.06598365306854248, + 0.04441281035542488, + 0.1570935845375061, + 0.19252197444438934 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2535", + "split": "test", + "style_correct": 1, + "style_probability": 0.23882941901683807, + "style_target": 0, + "support_probability": 0.0314883328974247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014610051704649685, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8686265349388123, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3868267238140106, + 0.36849814653396606, + 0.25923529267311096, + 0.38940972089767456, + 0.24279482662677765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.7948293089866638, + "style_target": 1, + "support_probability": 0.046672359108924866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0010444000729904132, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8776156306266785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2547:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4258403182029724, + 0.4541788399219513, + 0.3007620573043823, + 0.4266774654388428, + 0.21106533706188202 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2547", + "split": "test", + "style_correct": 1, + "style_probability": 0.2790506184101105, + "style_target": 0, + "support_probability": 0.0602821409702301, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05534049942135244, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45292091369628906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26245713233947754, + 0.32508906722068787, + 0.5690888166427612, + 0.39073941111564636, + 0.07439091056585312, + 0.11847266554832458 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.8142749071121216, + "style_target": 1, + "support_probability": 0.16157139837741852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010220668006118424, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7278854250907898, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2558:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2444162517786026, + 0.3387377858161926, + 0.5225638747215271, + 0.31785643100738525, + 0.16391712427139282, + 0.10886093974113464 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2558", + "split": "test", + "style_correct": 1, + "style_probability": 0.47119608521461487, + "style_target": 0, + "support_probability": 0.06913759559392929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22541640833792803, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5283153057098389, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7178361415863037, + 0.08020529896020889, + 0.5256379842758179, + 0.7178361415863037, + 0.07573175430297852, + 0.17704856395721436, + 0.08973207324743271, + 0.061524756252765656, + 0.06717433780431747 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.8745043873786926, + "style_target": 1, + "support_probability": 0.31448894739151, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.051604452897251456, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44799932837486267, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2566:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6823015213012695, + 0.08910603076219559, + 0.4559355676174164, + 0.6823015213012695, + 0.04094192013144493, + 0.030103204771876335, + 0.09439144283533096, + 0.05261087045073509, + 0.09958726167678833 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2566", + "split": "test", + "style_correct": 1, + "style_probability": 0.3131881058216095, + "style_target": 0, + "support_probability": 0.3443669080734253, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016452370472582612, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7481698989868164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3666638135910034, + 0.4398363530635834, + 0.16874735057353973, + 0.077634796500206, + 0.10502772033214569, + 0.07984893769025803, + 0.06405872851610184, + 0.14133059978485107, + 0.10890494287014008, + 0.041916798800230026, + 0.11960907280445099 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.6064338684082031, + "style_target": 1, + "support_probability": 0.0049420311115682125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004868286140995092, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7502507567405701, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2569:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4159514605998993, + 0.47661057114601135, + 0.2303282618522644, + 0.09287036210298538, + 0.10752367973327637, + 0.08251336216926575, + 0.1872447282075882, + 0.07939237356185913, + 0.11309701204299927, + 0.06717744469642639 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2569", + "split": "test", + "style_correct": 1, + "style_probability": 0.2699976861476898, + "style_target": 0, + "support_probability": 0.007934757508337498, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.40257008839471814, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6619974970817566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30203402042388916, + 0.24514755606651306, + 0.3874247074127197, + 0.9196898341178894, + 0.10295901447534561 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 1, + "style_probability": 0.8275892734527588, + "style_target": 1, + "support_probability": 0.1724863350391388, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1648063817066001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7456939220428467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2571:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2683016359806061, + 0.22727210819721222, + 0.36561617255210876, + 0.8843786120414734, + 0.12776432931423187 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2571", + "split": "test", + "style_correct": 1, + "style_probability": 0.4589739441871643, + "style_target": 0, + "support_probability": 0.18079392611980438, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8935809388310929, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05012790113687515, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6320899724960327, + 0.9431166648864746, + 0.5979037880897522, + 0.6888847351074219, + 0.06164253503084183, + 0.03840123489499092 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.7897112965583801, + "style_target": 1, + "support_probability": 0.9165351390838623, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5620905684509836, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08623667806386948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5559768676757812, + 0.9119861125946045, + 0.5603318810462952, + 0.5946934223175049, + 0.05989409610629082, + 0.07808326184749603 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2577", + "split": "test", + "style_correct": 1, + "style_probability": 0.37685316801071167, + "style_target": 0, + "support_probability": 0.8437448143959045, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15386478605942955, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1320950835943222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7540637850761414, + 0.22072875499725342, + 0.27731481194496155, + 0.2826826572418213, + 0.01923537440598011, + 0.11852363497018814, + 0.13805553317070007, + 0.08167784661054611, + 0.04696331545710564 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.6497806310653687, + "style_target": 1, + "support_probability": 0.546027660369873, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0052887051167792845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16444216668605804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2592:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8139048218727112, + 0.23479683697223663, + 0.3091532588005066, + 0.35879576206207275, + 0.041802406311035156, + 0.12075452506542206, + 0.067717544734478, + 0.04496997594833374, + 0.04578804224729538 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2592", + "split": "test", + "style_correct": 1, + "style_probability": 0.08516476303339005, + "style_target": 0, + "support_probability": 0.5493047833442688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7514653862578768, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13344496488571167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7693428993225098, + 0.2681080102920532, + 0.4707459509372711, + 0.7895481586456299, + 0.0741397961974144, + 0.0682314783334732, + 0.03682360798120499, + 0.12238936126232147, + 0.08849731087684631, + 0.1308443248271942, + 0.04809441417455673 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.7773378491401672, + "style_target": 1, + "support_probability": 0.9261717796325684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5674808843943886, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2236342877149582, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2600:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6679765582084656, + 0.3154878616333008, + 0.44351091980934143, + 0.645105242729187, + 0.12116272002458572, + 0.05868815630674362, + 0.046636756509542465, + 0.13233040273189545, + 0.04292959347367287, + 0.046089258044958115, + 0.05952669307589531 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2600", + "split": "test", + "style_correct": 1, + "style_probability": 0.4653749167919159, + "style_target": 0, + "support_probability": 0.8855538368225098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030160153409586773, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7023212909698486, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2611590325832367, + 0.860901951789856, + 0.2783868908882141, + 0.3214775025844574, + 0.12438886612653732 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.6957480311393738, + "style_target": 1, + "support_probability": 0.12238456308841705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005944808178268699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8128731846809387, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2605:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24866454303264618, + 0.8007678985595703, + 0.22776396572589874, + 0.3124297559261322, + 0.09848981350660324 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2605", + "split": "test", + "style_correct": 1, + "style_probability": 0.4675996005535126, + "style_target": 0, + "support_probability": 0.037480250000953674, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6048117251054173, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15306992828845978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37563446164131165, + 0.9830058217048645, + 0.2670794725418091, + 0.1251181662082672, + 0.06275021284818649, + 0.08012771606445312 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.7918266654014587, + "style_target": 1, + "support_probability": 0.6933473348617554, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02724609712050219, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25489890575408936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2609:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4968322515487671, + 0.992021381855011, + 0.35282593965530396, + 0.16905958950519562, + 0.05276993289589882, + 0.08133213967084885 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2609", + "split": "test", + "style_correct": 1, + "style_probability": 0.14852729439735413, + "style_target": 0, + "support_probability": 0.5675011277198792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05715619498759228, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8564287424087524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28170037269592285, + 0.5725676417350769, + 0.40247586369514465, + 0.25885871052742004, + 0.16125541925430298, + 0.10689564049243927, + 0.07729173451662064, + 0.04988798126578331, + 0.10137335956096649 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 1, + "style_probability": 0.6961560249328613, + "style_target": 1, + "support_probability": 0.18381871283054352, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04970430186190456, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8117254376411438, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2614:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25043052434921265, + 0.5280164480209351, + 0.3842109441757202, + 0.24349945783615112, + 0.08256015926599503, + 0.2309117168188095, + 0.06484584510326385, + 0.28934264183044434, + 0.09647718816995621 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2614", + "split": "test", + "style_correct": 0, + "style_probability": 0.5376465320587158, + "style_target": 0, + "support_probability": 0.17547902464866638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0952535748270229, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7760160565376282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5391326546669006, + 0.30340296030044556, + 0.336273193359375, + 0.5875820517539978, + 0.10344939678907394, + 0.04780241474509239, + 0.15461799502372742, + 0.13413794338703156, + 0.07678558677434921, + 0.04695384204387665, + 0.12582693994045258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.7696796655654907, + "style_target": 1, + "support_probability": 0.154333233833313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02834360677409941, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7602421641349792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2616:negative", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.5148887038230896, + 0.3202694058418274, + 0.33029937744140625, + 0.5602039098739624, + 0.23958063125610352, + 0.06056256964802742, + 0.12698869407176971, + 0.046834226697683334, + 0.10072881728410721, + 0.10438841581344604, + 0.13462090492248535, + 0.1432238519191742, + 0.1579507440328598, + 0.10189390182495117 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2616", + "split": "test", + "style_correct": 1, + "style_probability": 0.3741327226161957, + "style_target": 0, + "support_probability": 0.18871240317821503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15821616757007465, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6332446932792664, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.774329423904419, + 0.22580575942993164, + 0.5199210047721863, + 0.14174622297286987, + 0.118700310587883 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.8638606071472168, + "style_target": 1, + "support_probability": 0.2945725917816162, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005509360329029934, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5489746332168579, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2630:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9017672538757324, + 0.2750399112701416, + 0.6599308252334595, + 0.18794986605644226, + 0.12114559859037399 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2630", + "split": "test", + "style_correct": 1, + "style_probability": 0.15424157679080963, + "style_target": 0, + "support_probability": 0.23937101662158966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030591219431980732, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.41591328382492065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29677891731262207, + 0.5369774699211121, + 0.25378450751304626, + 0.989433228969574, + 0.08662966638803482, + 0.0996350422501564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 1, + "style_probability": 0.7933992147445679, + "style_target": 1, + "support_probability": 0.05303115397691727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03542466352285811, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.32271525263786316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2643:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28407981991767883, + 0.532149612903595, + 0.2848736643791199, + 0.9786816239356995, + 0.15324853360652924, + 0.11260203272104263 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2643", + "split": "test", + "style_correct": 0, + "style_probability": 0.6387492418289185, + "style_target": 0, + "support_probability": 0.1103256493806839, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.482809928689683, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4668537676334381, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2043386846780777, + 0.7903487086296082, + 0.2144751250743866, + 0.7763038277626038, + 0.07289819419384003, + 0.09632593393325806, + 0.05346491187810898, + 0.06389191001653671, + 0.09499336034059525 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 1, + "style_probability": 0.8647411465644836, + "style_target": 1, + "support_probability": 0.5588335394859314, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3377761524141647, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.261574387550354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2675:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2253493219614029, + 0.7761592864990234, + 0.22886894643306732, + 0.7664479613304138, + 0.0595167838037014, + 0.079885333776474 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2675", + "split": "test", + "style_correct": 0, + "style_probability": 0.5187961459159851, + "style_target": 0, + "support_probability": 0.6970275640487671, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047780292470795516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8540249466896057, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.309591680765152, + 0.13496339321136475, + 0.46874719858169556, + 0.13209572434425354, + 0.03343896195292473, + 0.04575048387050629, + 0.03840160742402077, + 0.060584958642721176, + 0.057613931596279144, + 0.08289623260498047, + 0.14514505863189697 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.8629956841468811, + "style_target": 1, + "support_probability": 0.09498373419046402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0012664690010714544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9616348147392273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2676:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.31254953145980835, + 0.20892645418643951, + 0.45547741651535034, + 0.18098895251750946, + 0.1877361387014389, + 0.08638647198677063, + 0.15831699967384338, + 0.08962085843086243, + 0.05741792917251587, + 0.15401476621627808 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2676", + "split": "test", + "style_correct": 1, + "style_probability": 0.2602969706058502, + "style_target": 0, + "support_probability": 0.05266662314534187, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7758065178806817, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3971951901912689, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8543068170547485, + 0.5045437812805176, + 0.5128402709960938, + 0.19852283596992493, + 0.03482400253415108 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.8762705326080322, + "style_target": 1, + "support_probability": 0.8360662460327148, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6757991245822126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21742206811904907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2684:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8355233073234558, + 0.46280086040496826, + 0.5118287801742554, + 0.19250918924808502, + 0.18449833989143372 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2684", + "split": "test", + "style_correct": 1, + "style_probability": 0.4779292345046997, + "style_target": 0, + "support_probability": 0.8537781238555908, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.47949659075567297, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18304291367530823, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2124425321817398, + 0.2651034891605377, + 0.9935542345046997, + 0.335501104593277, + 0.10499543696641922, + 0.06044765189290047 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.5080178380012512, + "style_target": 1, + "support_probability": 0.5236001014709473, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3861202908401395, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13730023801326752, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2686:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20718063414096832, + 0.25237178802490234, + 0.9884445667266846, + 0.3451697528362274, + 0.09965398907661438 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2686", + "split": "test", + "style_correct": 1, + "style_probability": 0.28569602966308594, + "style_target": 0, + "support_probability": 0.5880947113037109, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3652978917403403, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6441434025764465, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2726302146911621, + 0.4258976876735687, + 0.5694394111633301, + 0.4850733280181885, + 0.036002811044454575, + 0.029611077159643173, + 0.06319032609462738, + 0.07981450855731964, + 0.04431363195180893 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.8262879848480225, + "style_target": 1, + "support_probability": 0.5878506898880005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005887142135222403, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7178800702095032, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40200069546699524, + 0.616973876953125, + 0.6171494126319885, + 0.5842443108558655, + 0.06066996604204178, + 0.04756845906376839, + 0.08571124076843262, + 0.05254023149609566, + 0.12336556613445282 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2690", + "split": "test", + "style_correct": 1, + "style_probability": 0.07431408762931824, + "style_target": 0, + "support_probability": 0.5289488434791565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011737242191844915, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.93828946352005, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6541767716407776, + 0.6429705023765564, + 0.30938127636909485, + 0.6261888742446899, + 0.08790189772844315, + 0.05643557757139206, + 0.12832921743392944, + 0.12078295648097992, + 0.10946094989776611, + 0.07814431935548782 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.5475817918777466, + "style_target": 1, + "support_probability": 0.0726761445403099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020999727270038408, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8260862827301025, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2700:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6601375937461853, + 0.6225696206092834, + 0.27401238679885864, + 0.6459265351295471, + 0.06729583442211151, + 0.10124200582504272, + 0.07759778946638107, + 0.13254593312740326, + 0.06894055008888245, + 0.05199103429913521, + 0.08402886986732483 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2700", + "split": "test", + "style_correct": 1, + "style_probability": 0.33793315291404724, + "style_target": 0, + "support_probability": 0.16454890370368958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.422069184463453, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10246970504522324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.379183828830719, + 0.6142162084579468, + 0.22362074255943298, + 0.9443016052246094, + 0.06135380268096924 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 1, + "style_probability": 0.8868055939674377, + "style_target": 1, + "support_probability": 0.47888699173927307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36690720391380444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0790352150797844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2711:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36742472648620605, + 0.5809125900268555, + 0.2303505539894104, + 0.9175785779953003, + 0.07025102525949478, + 0.05251726880669594, + 0.20317423343658447, + 0.10169725865125656, + 0.11697152256965637, + 0.09347973018884659, + 0.1023503914475441 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2711", + "split": "test", + "style_correct": 0, + "style_probability": 0.6591747403144836, + "style_target": 0, + "support_probability": 0.5340954661369324, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.78441269227989, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2448679506778717, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9315037131309509, + 0.30425652861595154, + 0.43377822637557983, + 0.42963728308677673, + 0.039234355092048645, + 0.0373581200838089 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.7036502957344055, + "style_target": 1, + "support_probability": 0.7781389355659485, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0447094970562778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.383976548910141, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9634450078010559, + 0.3726920485496521, + 0.38487938046455383, + 0.4580625593662262, + 0.08067532628774643, + 0.10639079660177231 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2712", + "split": "test", + "style_correct": 1, + "style_probability": 0.10383691638708115, + "style_target": 0, + "support_probability": 0.5778022408485413, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4501920446802288, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6717731952667236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1516699343919754, + 0.2803700864315033, + 0.2808600962162018, + 0.7756551504135132, + 0.07417969405651093, + 0.2625921070575714, + 0.10813412815332413, + 0.05998943746089935, + 0.10858957469463348 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.910099983215332, + "style_target": 1, + "support_probability": 0.19809138774871826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1315167684856462, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6068164110183716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2723:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17439547181129456, + 0.2749314308166504, + 0.2677348256111145, + 0.7566474080085754, + 0.07698929309844971, + 0.14304237067699432, + 0.0755990669131279, + 0.08137206733226776, + 0.06344295293092728 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2723", + "split": "test", + "style_correct": 1, + "style_probability": 0.45735740661621094, + "style_target": 0, + "support_probability": 0.28541332483291626, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8555533806741379, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08199314028024673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1665397733449936, + 0.3072577118873596, + 0.3156527280807495, + 0.9011826515197754, + 0.03170071914792061, + 0.021884165704250336, + 0.047834496945142746, + 0.11587966978549957, + 0.06244324892759323, + 0.03083028644323349, + 0.035182978957891464 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.7689357995986938, + "style_target": 1, + "support_probability": 0.9200606346130371, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6396289582831969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09225284308195114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2727:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18474218249320984, + 0.28152358531951904, + 0.3071967363357544, + 0.8741552829742432, + 0.08243957906961441, + 0.04028228297829628, + 0.07857346534729004, + 0.11550246924161911, + 0.04790431261062622 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2727", + "split": "test", + "style_correct": 1, + "style_probability": 0.27207863330841064, + "style_target": 0, + "support_probability": 0.9055687189102173, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21436749148580866, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.790115475654602, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24359771609306335, + 0.4363459050655365, + 0.1693509817123413, + 0.43553897738456726, + 0.0726071149110794 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.9465739130973816, + "style_target": 1, + "support_probability": 0.27605774998664856, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02510384874460225, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8428269028663635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2419433891773224, + 0.567848801612854, + 0.15856829285621643, + 0.5900514721870422, + 0.182748481631279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2731", + "split": "test", + "style_correct": 1, + "style_probability": 0.43432244658470154, + "style_target": 0, + "support_probability": 0.24014553427696228, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7873112528281646, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11929097026586533, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9022013545036316, + 0.6056687831878662, + 0.24537397921085358, + 0.2980726659297943, + 0.10696649551391602, + 0.06946911662817001 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 1, + "style_probability": 0.9454473257064819, + "style_target": 1, + "support_probability": 0.4203094244003296, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7883596099104189, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02305659092962742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2736:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9306355118751526, + 0.6738293170928955, + 0.2605246603488922, + 0.33979326486587524, + 0.07691371440887451, + 0.21294701099395752 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2736", + "split": "test", + "style_correct": 0, + "style_probability": 0.5733819603919983, + "style_target": 0, + "support_probability": 0.8074976205825806, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6924176850522556, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1613648384809494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9670518636703491, + 0.29738810658454895, + 0.28448551893234253, + 0.24950070679187775, + 0.037545133382081985, + 0.04016381502151489, + 0.06557676941156387, + 0.11245747655630112, + 0.037644609808921814 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.8075119853019714, + "style_target": 1, + "support_probability": 0.5026777982711792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46415010374767, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1708814799785614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2740:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9544677734375, + 0.2988828718662262, + 0.2883850336074829, + 0.2496260404586792, + 0.1050291657447815, + 0.034893106669187546 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2740", + "split": "test", + "style_correct": 1, + "style_probability": 0.44597482681274414, + "style_target": 0, + "support_probability": 0.5689250230789185, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.831694427928369, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08184445649385452, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2119024097919464, + 0.4524178206920624, + 0.9240837693214417, + 0.42557796835899353, + 0.053414151072502136, + 0.05197768658399582, + 0.05640877038240433, + 0.0591622032225132, + 0.055243879556655884, + 0.029602665454149246 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.6629500389099121, + "style_target": 1, + "support_probability": 0.9282283186912537, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10180025117323102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14727599918842316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2742:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24130158126354218, + 0.5023276805877686, + 0.9508154988288879, + 0.5749452710151672, + 0.06396139413118362, + 0.07331932336091995, + 0.02866559848189354, + 0.05082143470644951, + 0.047718122601509094, + 0.08047150075435638 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2742", + "split": "test", + "style_correct": 1, + "style_probability": 0.0806083232164383, + "style_target": 0, + "support_probability": 0.8855385184288025, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47080143509477773, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11191752552986145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28956839442253113, + 0.054489023983478546, + 0.3039509952068329, + 0.9731534719467163, + 0.0437861829996109 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 1, + "style_probability": 0.8134310245513916, + "style_target": 1, + "support_probability": 0.3874582052230835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3031475520557745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0794905349612236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2747:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3138837218284607, + 0.057700660079717636, + 0.3008978068828583, + 0.9627724289894104, + 0.05336630716919899 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2747", + "split": "test", + "style_correct": 0, + "style_probability": 0.6121883988380432, + "style_target": 0, + "support_probability": 0.41432589292526245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5999694545780585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31084492802619934, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5047925710678101, + 0.9077305793762207, + 0.22996552288532257, + 0.2541007399559021, + 0.038788460195064545, + 0.05743398144841194 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.8092218637466431, + "style_target": 1, + "support_probability": 0.7023823261260986, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21192397670731294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24874049425125122, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2757:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43492111563682556, + 0.8897708058357239, + 0.2391275018453598, + 0.2370724380016327, + 0.0736602246761322 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2757", + "split": "test", + "style_correct": 1, + "style_probability": 0.3266945779323578, + "style_target": 0, + "support_probability": 0.6043955087661743, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10136708204522284, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8284648656845093, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.618638813495636, + 0.2709547281265259, + 0.3531956374645233, + 0.3205512762069702, + 0.10477665811777115, + 0.07371020317077637, + 0.04595334827899933, + 0.06309151649475098, + 0.0707106813788414 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.8327688574790955, + "style_target": 1, + "support_probability": 0.13788704574108124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02745098498093946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7027645707130432, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2760:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7226516008377075, + 0.3161754012107849, + 0.40423640608787537, + 0.3216361701488495, + 0.1078905239701271, + 0.22766004502773285, + 0.1343199759721756, + 0.10779835283756256, + 0.16058915853500366 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2760", + "split": "test", + "style_correct": 1, + "style_probability": 0.18380418419837952, + "style_target": 0, + "support_probability": 0.4099515676498413, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8316110987260253, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10686864703893661, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27857986092567444, + 0.9343687891960144, + 0.307422935962677, + 0.6749123930931091, + 0.06001320853829384, + 0.16739821434020996, + 0.04666272550821304, + 0.04282765835523605, + 0.040413081645965576, + 0.0682075172662735, + 0.04308311268687248 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 1, + "style_probability": 0.7345055341720581, + "style_target": 1, + "support_probability": 0.7902417182922363, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6240330405537263, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23034659028053284, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2761:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.286480575799942, + 0.8888539671897888, + 0.3272996246814728, + 0.5768607258796692, + 0.06176392361521721, + 0.10001686215400696, + 0.0605887733399868, + 0.14732049405574799, + 0.12379702180624008, + 0.16048745810985565, + 0.10427354276180267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2761", + "split": "test", + "style_correct": 0, + "style_probability": 0.5767402052879333, + "style_target": 0, + "support_probability": 0.7075427174568176, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020930965443101805, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.891076922416687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6407340168952942, + 0.3862665593624115, + 0.2554852366447449, + 0.17670989036560059, + 0.09208568930625916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 1, + "style_probability": 0.7471110224723816, + "style_target": 1, + "support_probability": 0.043880365788936615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14217711270056113, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8536133766174316, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2764:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.635206401348114, + 0.3687591850757599, + 0.2121226191520691, + 0.18466834723949432, + 0.07804985344409943 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2764", + "split": "test", + "style_correct": 0, + "style_probability": 0.9518424272537231, + "style_target": 0, + "support_probability": 0.10449378192424774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11468772522066573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24556659162044525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7765035033226013, + 0.6299506425857544, + 0.41656625270843506, + 0.2902357280254364, + 0.1977756917476654, + 0.17774398624897003 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 0, + "style_probability": 0.4592280089855194, + "style_target": 1, + "support_probability": 0.4011000394821167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05667041352668711, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21842853724956512, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2772:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7670595645904541, + 0.5680610537528992, + 0.4603496789932251, + 0.3466269075870514, + 0.1500529646873474 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2772", + "split": "test", + "style_correct": 1, + "style_probability": 0.30866143107414246, + "style_target": 0, + "support_probability": 0.44332996010780334, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.21403358322870325, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4638645350933075, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40404149889945984, + 0.22715404629707336, + 0.8292377591133118, + 0.16195927560329437, + 0.10993435978889465, + 0.09390641748905182, + 0.08403708785772324, + 0.21454161405563354, + 0.11396051198244095 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 0, + "style_probability": 0.40575820207595825, + "style_target": 1, + "support_probability": 0.5824856162071228, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03891842064652451, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3193974494934082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2778:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2862931787967682, + 0.2599733769893646, + 0.9456688761711121, + 0.21413271129131317, + 0.14391177892684937, + 0.1801174283027649, + 0.14197979867458344, + 0.17046424746513367, + 0.13744106888771057 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2778", + "split": "test", + "style_correct": 1, + "style_probability": 0.09715574979782104, + "style_target": 0, + "support_probability": 0.6616122126579285, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029426308134854032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9255125522613525, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24561680853366852, + 0.23822623491287231, + 0.30964139103889465, + 0.8489552736282349, + 0.11080154031515121, + 0.2353927046060562, + 0.203754723072052, + 0.3061186671257019, + 0.16852085292339325, + 0.24743567407131195, + 0.24876530468463898 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.3869349956512451, + "style_target": 1, + "support_probability": 0.1561097651720047, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01246344696346491, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9544358253479004, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2779:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2934347689151764, + 0.26898112893104553, + 0.2991313338279724, + 0.8016343712806702, + 0.2314910590648651, + 0.09758235514163971, + 0.15675684809684753, + 0.1720799207687378, + 0.15252864360809326, + 0.14143715798854828, + 0.14783628284931183 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2779", + "split": "test", + "style_correct": 0, + "style_probability": 0.7633084058761597, + "style_target": 0, + "support_probability": 0.047367967665195465, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8442573820865107, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07142757624387741, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6331220865249634, + 0.7876594662666321, + 0.9204116463661194, + 0.4017418622970581, + 0.20393285155296326 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.806178867816925, + "style_target": 1, + "support_probability": 0.8991987705230713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06800711944515436, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13251852989196777, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2781:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7374083399772644, + 0.8362255096435547, + 0.9619813561439514, + 0.5091255307197571, + 0.18272557854652405 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:2781", + "split": "test", + "style_correct": 1, + "style_probability": 0.08836664259433746, + "style_target": 0, + "support_probability": 0.8204783797264099, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09673259095409394, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9088793992996216, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21322937309741974, + 0.6970565319061279, + 0.6112130284309387, + 0.3160327970981598, + 0.18467499315738678, + 0.10905102640390396 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 1, + "style_probability": 0.725134551525116, + "style_target": 1, + "support_probability": 0.19258183240890503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03148726138747038, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8959332704544067, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2797:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2141067385673523, + 0.5310469269752502, + 0.5732740759849548, + 0.3502190113067627, + 0.062022071331739426, + 0.08881008625030518 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2797", + "split": "test", + "style_correct": 1, + "style_probability": 0.49618637561798096, + "style_target": 0, + "support_probability": 0.12275365740060806, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8642536754813791, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06308431923389435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1488926261663437, + 0.6156523823738098, + 0.2201572060585022, + 0.9940874576568604, + 0.13454940915107727, + 0.04475092887878418, + 0.2039373219013214, + 0.08193635195493698, + 0.12375583499670029 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.7584971785545349, + "style_target": 1, + "support_probability": 0.7550768256187439, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5611308880202444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.052501339465379715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2807:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18131904304027557, + 0.5902863144874573, + 0.2384287416934967, + 0.988628625869751, + 0.0654478445649147, + 0.09466037899255753 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2807", + "split": "test", + "style_correct": 1, + "style_probability": 0.31575074791908264, + "style_target": 0, + "support_probability": 0.7377665638923645, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3657475353170003, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49663251638412476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33607861399650574, + 0.5165879130363464, + 0.33607861399650574, + 0.9436842799186707, + 0.08189056068658829, + 0.1374216377735138, + 0.09679719805717468, + 0.08670520037412643, + 0.08540862053632736, + 0.29059043526649475 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.6906388401985168, + "style_target": 1, + "support_probability": 0.3628321588039398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044174408785406356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6364632248878479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2815:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3114291727542877, + 0.4626755714416504, + 0.3114291727542877, + 0.9420644044876099, + 0.18396055698394775, + 0.10825658589601517, + 0.12020527571439743, + 0.09176965802907944, + 0.09461157023906708, + 0.19563506543636322 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2815", + "split": "test", + "style_correct": 1, + "style_probability": 0.20686383545398712, + "style_target": 0, + "support_probability": 0.3327392339706421, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2257141193083143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5776485204696655, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24919454753398895, + 0.29192811250686646, + 0.24919454753398895, + 0.7743024230003357, + 0.12142819911241531 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.8103440403938293, + "style_target": 1, + "support_probability": 0.21534839272499084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0849021981736371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5629528760910034, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24214030802249908, + 0.27646714448928833, + 0.24214030802249908, + 0.7695522308349609, + 0.2454828917980194 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2819", + "split": "test", + "style_correct": 1, + "style_probability": 0.45995429158210754, + "style_target": 0, + "support_probability": 0.20616856217384338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4546327665864922, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3609517514705658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8864487409591675, + 0.7831612825393677, + 0.29632288217544556, + 0.3357074558734894, + 0.08838771283626556, + 0.17922525107860565 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 1, + "style_probability": 0.8630719780921936, + "style_target": 1, + "support_probability": 0.4108709692955017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2700848710227035, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25837644934654236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2820:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8714113235473633, + 0.7456368207931519, + 0.2985537648200989, + 0.32441815733909607, + 0.15669551491737366 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:2820", + "split": "test", + "style_correct": 1, + "style_probability": 0.47138455510139465, + "style_target": 0, + "support_probability": 0.5458276271820068, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6203296391257851, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5709949135780334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3693624436855316, + 0.9518265128135681, + 0.683531641960144, + 0.1891988217830658, + 0.06720324605703354, + 0.11910303682088852, + 0.0637759119272232, + 0.07669872790575027, + 0.17471641302108765 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.8907883167266846, + "style_target": 1, + "support_probability": 0.4565874934196472, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01651140323924439, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5395262837409973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4250670373439789, + 0.987127423286438, + 0.7071853280067444, + 0.23825928568840027, + 0.09160017967224121, + 0.09386056661605835, + 0.14262902736663818, + 0.06650891900062561, + 0.15218514204025269 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:2821", + "split": "test", + "style_correct": 1, + "style_probability": 0.10922219604253769, + "style_target": 0, + "support_probability": 0.4660740792751312, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8747217352452344, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11886050552129745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20021328330039978, + 0.37208959460258484, + 0.8992851376533508, + 0.30740582942962646, + 0.03798840567469597, + 0.039761509746313095, + 0.06303063780069351, + 0.029672810807824135, + 0.03529118001461029, + 0.07313216477632523 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 1, + "style_probability": 0.8905448317527771, + "style_target": 1, + "support_probability": 0.9622493386268616, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7390190175563305, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36110150814056396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2826:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20999020338058472, + 0.4195992350578308, + 0.8137935400009155, + 0.31994614005088806, + 0.05298595875501633, + 0.027242464944720268, + 0.06153179705142975, + 0.02504647709429264, + 0.08349581807851791, + 0.2639848291873932 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2826", + "split": "test", + "style_correct": 0, + "style_probability": 0.5995190143585205, + "style_target": 0, + "support_probability": 0.9515474438667297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2035643208038094, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04790187627077103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1459851861000061, + 0.8705962300300598, + 0.2570517957210541, + 0.39635005593299866, + 0.08905603736639023 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.25956711173057556, + "style_target": 1, + "support_probability": 0.559055745601654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30947127742649627, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.060852546244859695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1536511480808258, + 0.8893424272537231, + 0.2689843475818634, + 0.3863693177700043, + 0.020380988717079163 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2829", + "split": "test", + "style_correct": 0, + "style_probability": 0.6012479066848755, + "style_target": 0, + "support_probability": 0.451419860124588, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16507498153644826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20433276891708374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27734145522117615, + 0.8299329876899719, + 0.40733346343040466, + 0.20671983063220978, + 0.20304323732852936, + 0.279410719871521 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 0, + "style_probability": 0.4588736593723297, + "style_target": 1, + "support_probability": 0.29603642225265503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18415673048884426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10399522632360458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2841:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20344950258731842, + 0.9370277523994446, + 0.33840247988700867, + 0.19905950129032135, + 0.13483524322509766, + 0.1713833063840866 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2841", + "split": "test", + "style_correct": 1, + "style_probability": 0.2968549430370331, + "style_target": 0, + "support_probability": 0.5967541933059692, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032679795026445475, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6956420540809631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3111887574195862, + 0.26578041911125183, + 0.828485369682312, + 0.407545804977417, + 0.13774167001247406, + 0.11559009552001953, + 0.14282743632793427, + 0.23640136420726776, + 0.1717081516981125 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.26895779371261597, + "style_target": 1, + "support_probability": 0.17709596455097198, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05560062937151856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7968546748161316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2846:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39705371856689453, + 0.2514193654060364, + 0.8464846611022949, + 0.4188897907733917, + 0.1991046965122223, + 0.11409980058670044, + 0.2274530529975891, + 0.08433529734611511, + 0.21995429694652557 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2846", + "split": "test", + "style_correct": 0, + "style_probability": 0.575168788433075, + "style_target": 0, + "support_probability": 0.1360759735107422, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04189029841768365, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5939599871635437, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.49710091948509216, + 0.24807797372341156, + 0.405727744102478, + 0.24688374996185303, + 0.05662703886628151, + 0.10875417292118073, + 0.056558143347501755, + 0.06695876270532608, + 0.04870753735303879, + 0.08664301037788391, + 0.08395065367221832 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.2692517042160034, + "style_target": 1, + "support_probability": 0.37025007605552673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06866539322672516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6888819932937622, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2861:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5294729471206665, + 0.2331627905368805, + 0.3850076496601105, + 0.21698233485221863, + 0.03402591124176979, + 0.03265518322587013, + 0.07487499713897705, + 0.05832716450095177, + 0.08868363499641418, + 0.04987221211194992 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2861", + "split": "test", + "style_correct": 0, + "style_probability": 0.603874683380127, + "style_target": 0, + "support_probability": 0.4482409954071045, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22943619676590998, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.44250816106796265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27099186182022095, + 0.5854536890983582, + 0.5022087097167969, + 0.46192166209220886, + 0.035108424723148346 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.7520339488983154, + "style_target": 1, + "support_probability": 0.3092930316925049, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016865252409526965, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4192497134208679, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2869:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3180740773677826, + 0.6934298276901245, + 0.602462649345398, + 0.5454752445220947, + 0.12565962970256805 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2869", + "split": "test", + "style_correct": 1, + "style_probability": 0.15582749247550964, + "style_target": 0, + "support_probability": 0.40295740962028503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6226418288189706, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23585179448127747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45104607939720154, + 0.8020491600036621, + 0.5408986210823059, + 0.8147522807121277, + 0.07054711878299713, + 0.06581325083971024 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 1, + "style_probability": 0.8698248863220215, + "style_target": 1, + "support_probability": 0.7737017273902893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23708409313814727, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39332959055900574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2877:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42304685711860657, + 0.7727895975112915, + 0.5696541666984558, + 0.6839578151702881, + 0.1660720407962799, + 0.08244927227497101 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2877", + "split": "test", + "style_correct": 0, + "style_probability": 0.5435789227485657, + "style_target": 0, + "support_probability": 0.6037622690200806, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05045698428066459, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9321699142456055, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39833104610443115, + 0.48628371953964233, + 0.5262064933776855, + 0.35918286442756653, + 0.09365397691726685, + 0.24253728985786438, + 0.0598832443356514, + 0.09839406609535217, + 0.061308201402425766 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.8694320321083069, + "style_target": 1, + "support_probability": 0.06399553269147873, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003064684090965608, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9372259378433228, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2886:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3139254152774811, + 0.5599652528762817, + 0.5149602890014648, + 0.31615889072418213, + 0.16254080832004547, + 0.23772059381008148 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2886", + "split": "test", + "style_correct": 1, + "style_probability": 0.22345352172851562, + "style_target": 0, + "support_probability": 0.01492813415825367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0038282851880501725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9911226034164429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24193766713142395, + 0.419325053691864, + 0.17901189625263214, + 0.61920565366745, + 0.061513811349868774, + 0.09122974425554276, + 0.11197895556688309, + 0.08652787655591965, + 0.09478367120027542, + 0.061892468482255936, + 0.05233948677778244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.7640693187713623, + "style_target": 1, + "support_probability": 0.00783049687743187, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005100611368081054, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9859769344329834, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3262586295604706, + 0.3896561563014984, + 0.23838239908218384, + 0.6396910548210144, + 0.15016144514083862, + 0.11931057274341583, + 0.17007727921009064, + 0.16283337771892548, + 0.13735343515872955, + 0.12569132447242737 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2891", + "split": "test", + "style_correct": 1, + "style_probability": 0.1695389598608017, + "style_target": 0, + "support_probability": 0.024972284212708473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8710138317199032, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10986301302909851, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.597926139831543, + 0.9980903267860413, + 0.22407346963882446, + 0.20229016244411469, + 0.12419064342975616 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 1, + "style_probability": 0.6861889362335205, + "style_target": 1, + "support_probability": 0.682996392250061, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7678548260585778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10854505747556686, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2893:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5587969422340393, + 0.9924119114875793, + 0.23081867396831512, + 0.24261066317558289, + 0.07389937341213226 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2893", + "split": "test", + "style_correct": 0, + "style_probability": 0.6552304625511169, + "style_target": 0, + "support_probability": 0.6250762939453125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46412366138719463, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3702610433101654, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9665355086326599, + 0.2971147298812866, + 0.5104985237121582, + 0.4717899262905121, + 0.36397460103034973, + 0.12889347970485687 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.8553788661956787, + "style_target": 1, + "support_probability": 0.249435156583786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13406429445666923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33774426579475403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9608193039894104, + 0.32361900806427, + 0.4384300410747528, + 0.4145083725452423, + 0.2326827496290207 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2896", + "split": "test", + "style_correct": 1, + "style_probability": 0.2803306579589844, + "style_target": 0, + "support_probability": 0.3368516266345978, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8043839431306843, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.313586950302124, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8901503682136536, + 0.4601014256477356, + 0.14793671667575836, + 0.38474446535110474, + 0.08698256313800812, + 0.12053085118532181, + 0.036762356758117676, + 0.2473306804895401, + 0.12836021184921265 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.9513720870018005, + "style_target": 1, + "support_probability": 0.7038165330886841, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05139979576468346, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36965954303741455, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2900:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9529866576194763, + 0.5485495328903198, + 0.20614217221736908, + 0.4629254937171936, + 0.08815998584032059, + 0.21126346290111542, + 0.0979129821062088, + 0.06532517820596695, + 0.08423738181591034 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2900", + "split": "test", + "style_correct": 1, + "style_probability": 0.1619473099708557, + "style_target": 0, + "support_probability": 0.6890441179275513, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.0705235345535762, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5897554159164429, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.257110595703125, + 0.20252297818660736, + 0.8497726321220398, + 0.7290410995483398, + 0.12010806798934937, + 0.07139138132333755, + 0.07392618060112, + 0.10908770561218262, + 0.2116757035255432, + 0.37248867750167847, + 0.18646100163459778 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 1, + "style_probability": 0.7815487384796143, + "style_target": 1, + "support_probability": 0.07830465584993362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.059427200366850556, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5907526016235352, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2902:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2807655930519104, + 0.2208225280046463, + 0.8052247762680054, + 0.6476865410804749, + 0.08908551931381226, + 0.06305333971977234, + 0.14989522099494934, + 0.1248457282781601, + 0.12042152881622314, + 0.05820654332637787, + 0.14311924576759338 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2902", + "split": "test", + "style_correct": 0, + "style_probability": 0.5254570841789246, + "style_target": 0, + "support_probability": 0.0979529544711113, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4006583897898075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.31848520040512085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2745121717453003, + 0.12566329538822174, + 0.4767792224884033, + 0.83013516664505, + 0.12991803884506226 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 1, + "style_probability": 0.6038860082626343, + "style_target": 1, + "support_probability": 0.439730703830719, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5370117978419843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24568875133991241, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2911:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.272724986076355, + 0.10352818667888641, + 0.5696790218353271, + 0.8947709202766418, + 0.13157445192337036 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2911", + "split": "test", + "style_correct": 0, + "style_probability": 0.6650416851043701, + "style_target": 0, + "support_probability": 0.5208236575126648, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4777029367964936, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13164249062538147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9952612519264221, + 0.19138799607753754, + 0.46820804476737976, + 0.44911810755729675, + 0.06780482828617096, + 0.15142101049423218 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.6752634048461914, + "style_target": 1, + "support_probability": 0.3633003532886505, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4102683634080008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08040988445281982, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2920:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9984543323516846, + 0.18743175268173218, + 0.5010445713996887, + 0.40352949500083923, + 0.12249203026294708 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2920", + "split": "test", + "style_correct": 1, + "style_probability": 0.3398095667362213, + "style_target": 0, + "support_probability": 0.5842291116714478, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4241061359546627, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26796475052833557, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6080827713012695, + 0.1680639237165451, + 0.8950895071029663, + 0.3207317590713501, + 0.11069194972515106, + 0.11024954915046692, + 0.08177272975444794, + 0.08858514577150345, + 0.1354721486568451 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.5561313629150391, + "style_target": 1, + "support_probability": 0.44093436002731323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06672053858378034, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29874512553215027, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2922:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.646689236164093, + 0.22337378561496735, + 0.9751410484313965, + 0.3363838791847229, + 0.08588806539773941, + 0.16865690052509308, + 0.12009677290916443, + 0.14785513281822205, + 0.28256523609161377 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2922", + "split": "test", + "style_correct": 1, + "style_probability": 0.14567923545837402, + "style_target": 0, + "support_probability": 0.4950988292694092, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.026616641443794945, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7333031296730042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24500374495983124, + 0.44928985834121704, + 0.4425220489501953, + 0.5788857340812683, + 0.04557539522647858, + 0.05225128307938576, + 0.08052868396043777, + 0.06496573239564896, + 0.08424116671085358, + 0.03297296166419983, + 0.13944219052791595 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 1, + "style_probability": 0.7005382776260376, + "style_target": 1, + "support_probability": 0.06646253168582916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.044395995466232734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6231165528297424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2931:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2569653391838074, + 0.46981146931648254, + 0.48922157287597656, + 0.6476115584373474, + 0.04506246745586395, + 0.03684593737125397, + 0.05138732120394707, + 0.05536426976323128, + 0.05535317584872246, + 0.10762940347194672, + 0.21700982749462128 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2931", + "split": "test", + "style_correct": 0, + "style_probability": 0.8675973415374756, + "style_target": 0, + "support_probability": 0.08746405690908432, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25612358081389175, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7154918313026428, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44258007407188416, + 0.36712661385536194, + 0.5192346572875977, + 0.3429982364177704, + 0.08378498256206512 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 1, + "style_probability": 0.6179884076118469, + "style_target": 1, + "support_probability": 0.49761131405830383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019427724863495748, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7425145506858826, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2934:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5314930081367493, + 0.4905448853969574, + 0.5136759281158447, + 0.33661866188049316, + 0.10906492173671722, + 0.2761021852493286, + 0.049451299011707306, + 0.1352997124195099, + 0.32315513491630554, + 0.14679576456546783, + 0.0706719160079956 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2934", + "split": "test", + "style_correct": 1, + "style_probability": 0.41703757643699646, + "style_target": 0, + "support_probability": 0.15141817927360535, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7728676944800483, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14394403994083405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5809526443481445, + 0.5628510117530823, + 0.32576489448547363, + 0.9261179566383362, + 0.08576697111129761, + 0.06694631278514862 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.7019627690315247, + "style_target": 1, + "support_probability": 0.7711489796638489, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.041371619415748295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3027957081794739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2943:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6349470615386963, + 0.6527080535888672, + 0.45047444105148315, + 0.9720682501792908, + 0.13387252390384674, + 0.21165935695171356 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:2943", + "split": "test", + "style_correct": 1, + "style_probability": 0.0629500150680542, + "style_target": 0, + "support_probability": 0.75559002161026, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07406101615031833, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6274856925010681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7567184567451477, + 0.24882856011390686, + 0.5783318281173706, + 0.25035011768341064, + 0.09608302265405655, + 0.10720539093017578, + 0.10439415276050568, + 0.10842064023017883, + 0.12295068055391312 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 1, + "style_probability": 0.6489367485046387, + "style_target": 1, + "support_probability": 0.17035160958766937, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49998591273016757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21944761276245117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2944:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7777048945426941, + 0.25893786549568176, + 0.6098873019218445, + 0.1814691722393036, + 0.04563430696725845, + 0.14900751411914825, + 0.16652877628803253, + 0.03452596440911293, + 0.04778943955898285 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:2944", + "split": "test", + "style_correct": 0, + "style_probability": 0.6988329887390137, + "style_target": 0, + "support_probability": 0.7292539477348328, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8823244292586878, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0698259249329567, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1707749366760254, + 0.41595691442489624, + 0.911602258682251, + 0.43278199434280396, + 0.08753275871276855, + 0.06783328950405121, + 0.08684135973453522, + 0.07390005141496658, + 0.06110944598913193, + 0.05439761281013489, + 0.10896261036396027 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 1, + "style_probability": 0.6848735809326172, + "style_target": 1, + "support_probability": 0.9620832800865173, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8421733744179108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08311961591243744, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2946:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18147674202919006, + 0.497537761926651, + 0.9326663017272949, + 0.5240024924278259, + 0.039573490619659424, + 0.07834012806415558, + 0.10937745124101639, + 0.10535391420125961, + 0.08208303898572922 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2946", + "split": "test", + "style_correct": 0, + "style_probability": 0.5377902388572693, + "style_target": 0, + "support_probability": 0.9265174269676208, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4396578390492003, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.621442973613739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23436160385608673, + 0.7832445502281189, + 0.1759413480758667, + 0.23436160385608673, + 0.04997553676366806 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 1, + "style_probability": 0.8929410576820374, + "style_target": 1, + "support_probability": 0.4559228718280792, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.355101000940099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5602368116378784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2957:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2347206026315689, + 0.7649186253547668, + 0.16515511274337769, + 0.2347206026315689, + 0.06655342876911163 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2957", + "split": "test", + "style_correct": 0, + "style_probability": 0.6192851066589355, + "style_target": 0, + "support_probability": 0.5856156349182129, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008855933417687199, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5079834461212158, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47113651037216187, + 0.5127675533294678, + 0.28343701362609863, + 0.2359125316143036, + 0.0946258082985878, + 0.09427225589752197 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 1, + "style_probability": 0.622725784778595, + "style_target": 1, + "support_probability": 0.03177863731980324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0032617707758208496, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5869684815406799, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2972:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43931931257247925, + 0.47612178325653076, + 0.2723170518875122, + 0.24201422929763794, + 0.07104117423295975 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2972", + "split": "test", + "style_correct": 1, + "style_probability": 0.25700050592422485, + "style_target": 0, + "support_probability": 0.029143134132027626, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18857078740526492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34127241373062134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2240946888923645, + 0.29165610671043396, + 0.892998456954956, + 0.2240946888923645, + 0.0651584342122078, + 0.04089232161641121, + 0.053559768944978714, + 0.04630929231643677, + 0.07302956283092499 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.7976653575897217, + "style_target": 1, + "support_probability": 0.1346057653427124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007802262827177167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4079991579055786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2978:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21973881125450134, + 0.37501880526542664, + 0.9428848624229431, + 0.21973881125450134, + 0.07028177380561829, + 0.04356703907251358, + 0.04916252940893173, + 0.04879862442612648, + 0.06283356994390488 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:2978", + "split": "test", + "style_correct": 1, + "style_probability": 0.14320848882198334, + "style_target": 0, + "support_probability": 0.17050065100193024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002343742855092923, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9537975192070007, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5841543078422546, + 0.258259117603302, + 0.2924845516681671, + 0.16981883347034454, + 0.0676674023270607, + 0.0764845460653305, + 0.06251303106546402, + 0.05307482182979584, + 0.053021810948848724, + 0.07438699156045914, + 0.04999323934316635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 1, + "style_probability": 0.6684410572052002, + "style_target": 1, + "support_probability": 0.007336413487792015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006580550426788513, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9264984726905823, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5477485060691833, + 0.26368796825408936, + 0.3196139335632324, + 0.17110808193683624, + 0.08960560709238052, + 0.0502198226749897, + 0.15275509655475616, + 0.04990718513727188, + 0.0968710407614708, + 0.1477014422416687, + 0.07744485139846802 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2984", + "split": "test", + "style_correct": 0, + "style_probability": 0.5777300000190735, + "style_target": 0, + "support_probability": 0.01643643155694008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030316839218907923, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8822697997093201, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3842657506465912, + 0.24475030601024628, + 0.24475030601024628, + 0.3279065191745758, + 0.07340974360704422 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.4300960898399353, + "style_target": 1, + "support_probability": 0.031206712126731873, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014444379242270487, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9762347936630249, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31396517157554626, + 0.23497259616851807, + 0.23497259616851807, + 0.26646965742111206, + 0.0630204826593399 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2988", + "split": "test", + "style_correct": 0, + "style_probability": 0.937522292137146, + "style_target": 0, + "support_probability": 0.01578054018318653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06696404414530122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29123759269714355, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37830543518066406, + 0.8231210708618164, + 0.2680884003639221, + 0.23786713182926178, + 0.05885875225067139, + 0.10575079172849655 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 0, + "style_probability": 0.49427884817123413, + "style_target": 1, + "support_probability": 0.19113872945308685, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.048460793092453, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.30803000926971436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2989:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3500444293022156, + 0.7761925458908081, + 0.24134817719459534, + 0.21166488528251648, + 0.11717888712882996 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:2989", + "split": "test", + "style_correct": 1, + "style_probability": 0.3056304454803467, + "style_target": 0, + "support_probability": 0.22697973251342773, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030465197692336474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.911568284034729, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2379310429096222, + 0.3041130304336548, + 0.1936967670917511, + 0.2080220878124237, + 0.11319609731435776, + 0.13842511177062988, + 0.16776521503925323, + 0.21382613480091095, + 0.15710332989692688 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 0, + "style_probability": 0.42105594277381897, + "style_target": 1, + "support_probability": 0.03358650207519531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003755906878979214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9395166635513306, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:2993:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29234081506729126, + 0.41487136483192444, + 0.2741115093231201, + 0.21582698822021484, + 0.10444090515375137, + 0.1911574900150299, + 0.19572708010673523, + 0.13103292882442474, + 0.12331786751747131 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2993", + "split": "test", + "style_correct": 1, + "style_probability": 0.18690498173236847, + "style_target": 0, + "support_probability": 0.037697359919548035, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3104516397092425, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.159979447722435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17902109026908875, + 0.625764012336731, + 0.3742980659008026, + 0.2039453089237213, + 0.06470178812742233, + 0.11745011806488037, + 0.06103108078241348, + 0.12003292888402939, + 0.03940397873520851, + 0.0463046170771122, + 0.049727912992239 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 1, + "style_probability": 0.5832032561302185, + "style_target": 1, + "support_probability": 0.6646379828453064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3584336710464342, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23618821799755096, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:2997:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20291125774383545, + 0.6349387764930725, + 0.32558882236480713, + 0.1585213840007782, + 0.04433796554803848, + 0.08313050121068954, + 0.14522185921669006, + 0.11160874366760254, + 0.06554025411605835, + 0.03269337862730026, + 0.034668050706386566 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:2997", + "split": "test", + "style_correct": 0, + "style_probability": 0.7484855055809021, + "style_target": 0, + "support_probability": 0.6079908013343811, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9118926912803573, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10370136797428131, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3011:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2985345423221588, + 0.8028380274772644, + 0.2985345423221588, + 0.9411731958389282, + 0.060109611600637436 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.848118007183075, + "style_target": 1, + "support_probability": 0.9310357570648193, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21497133900775015, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16301535069942474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3011:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3282700479030609, + 0.8425765633583069, + 0.3282700479030609, + 0.9705191850662231, + 0.11237229406833649 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3011", + "split": "test", + "style_correct": 1, + "style_probability": 0.12320777028799057, + "style_target": 0, + "support_probability": 0.9201440215110779, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27191031941026544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16855761408805847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4734291732311249, + 0.41763898730278015, + 0.601967453956604, + 0.9693707823753357, + 0.07633336633443832, + 0.03372947499155998 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 1, + "style_probability": 0.6738072633743286, + "style_target": 1, + "support_probability": 0.47226208448410034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32121535457102723, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13652655482292175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3015:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46246713399887085, + 0.39995452761650085, + 0.5687015056610107, + 0.9574988484382629, + 0.04331328719854355, + 0.048911333084106445 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3015", + "split": "test", + "style_correct": 0, + "style_probability": 0.5974010229110718, + "style_target": 0, + "support_probability": 0.5030084848403931, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7180569075243923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08076158910989761, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15171582996845245, + 0.2234998494386673, + 0.30234888195991516, + 0.984840452671051, + 0.05438194423913956, + 0.10735069215297699, + 0.07064121216535568, + 0.06055312231183052, + 0.02801152504980564 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.7626814842224121, + "style_target": 1, + "support_probability": 0.8458403944969177, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4760772828118149, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05669359117746353, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3019:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15061055123806, + 0.2301541268825531, + 0.3584708869457245, + 0.9770954847335815, + 0.055782631039619446, + 0.10062836855649948 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3019", + "split": "test", + "style_correct": 1, + "style_probability": 0.3296729028224945, + "style_target": 0, + "support_probability": 0.9007524251937866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00771113996150463, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9441486597061157, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16936920583248138, + 0.16998831927776337, + 0.21793487668037415, + 0.2896457314491272, + 0.13484561443328857, + 0.09337089210748672, + 0.09570048004388809, + 0.12132041901350021, + 0.10653086751699448, + 0.14556176960468292 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.8821219205856323, + "style_target": 1, + "support_probability": 0.039369724690914154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0020705554199966507, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9230086207389832, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3035:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18868549168109894, + 0.19358722865581512, + 0.34039732813835144, + 0.43592017889022827, + 0.1715504676103592, + 0.0899161547422409, + 0.1278771460056305, + 0.06598975509405136, + 0.08268538862466812, + 0.1437484622001648 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3035", + "split": "test", + "style_correct": 1, + "style_probability": 0.21429690718650818, + "style_target": 0, + "support_probability": 0.1725061982870102, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1891953410360081, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5337733626365662, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2783089578151703, + 0.5229488015174866, + 0.38115960359573364, + 0.3070732355117798, + 0.14161750674247742 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 1, + "style_probability": 0.7271243929862976, + "style_target": 1, + "support_probability": 0.5554563403129578, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01885908739525322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.628294050693512, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3041:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26199084520339966, + 0.7476071119308472, + 0.5107207298278809, + 0.46052825450897217, + 0.14890238642692566 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3041", + "split": "test", + "style_correct": 1, + "style_probability": 0.2585029602050781, + "style_target": 0, + "support_probability": 0.6419969201087952, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6742142826650976, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45576977729797363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9247528910636902, + 0.1914546638727188, + 0.40443313121795654, + 0.4878481924533844, + 0.1048944815993309, + 0.15508419275283813 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 1, + "style_probability": 0.6848031878471375, + "style_target": 1, + "support_probability": 0.5889959931373596, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.806580362036371, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.383354514837265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3048:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9706196188926697, + 0.19034118950366974, + 0.4002947509288788, + 0.5451041460037231, + 0.09530662000179291, + 0.15366221964359283 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3048", + "split": "test", + "style_correct": 0, + "style_probability": 0.8097336888313293, + "style_target": 0, + "support_probability": 0.6653394103050232, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8230173257274487, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13487768173217773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19242380559444427, + 0.9299277663230896, + 0.23856686055660248, + 0.18328511714935303, + 0.05880783870816231, + 0.07251428067684174, + 0.07782169431447983, + 0.1030879020690918, + 0.14779473841190338 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 1, + "style_probability": 0.5382259488105774, + "style_target": 1, + "support_probability": 0.874426543712616, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8414898636419216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05417937412858009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3053:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19862933456897736, + 0.9483063817024231, + 0.2775512933731079, + 0.15786416828632355, + 0.04224788770079613, + 0.06517324596643448 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3053", + "split": "test", + "style_correct": 1, + "style_probability": 0.35182610154151917, + "style_target": 0, + "support_probability": 0.9353441596031189, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014396513660843052, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9168047904968262, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3985773026943207, + 0.2154010534286499, + 0.34775298833847046, + 0.40504851937294006, + 0.08951175212860107, + 0.2236100137233734, + 0.20954039692878723, + 0.14183005690574646, + 0.14586660265922546, + 0.24864788353443146 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 1, + "style_probability": 0.6911097168922424, + "style_target": 1, + "support_probability": 0.043929241597652435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026059195933963437, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9678871035575867, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3060:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.50859534740448, + 0.2107953131198883, + 0.3598374128341675, + 0.40860235691070557, + 0.1727961301803589, + 0.14519277215003967, + 0.10685981810092926, + 0.09728268533945084, + 0.0773092657327652, + 0.198045015335083 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3060", + "split": "test", + "style_correct": 1, + "style_probability": 0.31954634189605713, + "style_target": 0, + "support_probability": 0.055921316146850586, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9045813885534669, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0739361047744751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6904644966125488, + 0.22348780930042267, + 0.9270116090774536, + 0.18550175428390503, + 0.0487334281206131 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 1, + "style_probability": 0.8715609908103943, + "style_target": 1, + "support_probability": 0.8692180514335632, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7182862137666177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16441616415977478, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3062:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6540140509605408, + 0.2366955280303955, + 0.8991937041282654, + 0.20814929902553558, + 0.20262384414672852, + 0.1411619633436203, + 0.12262922525405884, + 0.11971868574619293, + 0.10183592140674591, + 0.07037236541509628, + 0.11834582686424255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3062", + "split": "test", + "style_correct": 1, + "style_probability": 0.41684404015541077, + "style_target": 0, + "support_probability": 0.7591450810432434, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.052158467746863835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8232011795043945, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9345607757568359, + 0.11791414022445679, + 0.5418198704719543, + 0.25314030051231384, + 0.1123204156756401, + 0.038359157741069794 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.8104144334793091, + "style_target": 1, + "support_probability": 0.1090584471821785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008297825091211016, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8691660165786743, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3068:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.957662045955658, + 0.16343049705028534, + 0.6332109570503235, + 0.3018767237663269, + 0.05313966050744057, + 0.11377532035112381 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3068", + "split": "test", + "style_correct": 1, + "style_probability": 0.383796364068985, + "style_target": 0, + "support_probability": 0.11587461084127426, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5743324439834296, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2527928054332733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36950039863586426, + 0.23760902881622314, + 0.9740588068962097, + 0.3044303357601166, + 0.18773189187049866, + 0.04516090452671051, + 0.04008689150214195, + 0.17545311152935028, + 0.037633176892995834 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 1, + "style_probability": 0.7292978763580322, + "style_target": 1, + "support_probability": 0.4781331419944763, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4283289269015657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36885038018226624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39787545800209045, + 0.21845915913581848, + 0.9633557200431824, + 0.2916988134384155, + 0.08489622920751572, + 0.09460573643445969, + 0.18482157588005066, + 0.06614743918180466, + 0.07027346640825272 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3074", + "split": "test", + "style_correct": 0, + "style_probability": 0.5304092764854431, + "style_target": 0, + "support_probability": 0.4674619138240814, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12255805475724915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48378515243530273, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2861022353172302, + 0.531951904296875, + 0.9538602828979492, + 0.2861022353172302, + 0.06602244079113007, + 0.05839545652270317, + 0.16380013525485992, + 0.05916556343436241, + 0.02761046029627323, + 0.1182994693517685, + 0.04176252707839012 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 1, + "style_probability": 0.7845202684402466, + "style_target": 1, + "support_probability": 0.1793227642774582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04651744611619969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6321160793304443, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3078:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30742892622947693, + 0.5160373449325562, + 0.9408823847770691, + 0.30742892622947693, + 0.06306876242160797, + 0.06763175129890442, + 0.0584288127720356, + 0.05742804333567619, + 0.07905488461256027 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3078", + "split": "test", + "style_correct": 0, + "style_probability": 0.5323889255523682, + "style_target": 0, + "support_probability": 0.11612915247678757, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3715390336171503, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14482931792736053, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12671610713005066, + 0.9402226209640503, + 0.1423114389181137, + 0.12946638464927673, + 0.08012760430574417 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 1, + "style_probability": 0.6308274865150452, + "style_target": 1, + "support_probability": 0.35272708535194397, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22372246048857036, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10155302286148071, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3081:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1254948377609253, + 0.941839873790741, + 0.1693664789199829, + 0.12449590861797333, + 0.06231445074081421, + 0.03755665943026543, + 0.05599047243595123, + 0.06740505993366241, + 0.06993326544761658, + 0.06512255221605301, + 0.05893278867006302 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3081", + "split": "test", + "style_correct": 1, + "style_probability": 0.45791199803352356, + "style_target": 0, + "support_probability": 0.4512891471385956, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1814408173991584, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5584572553634644, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5206495523452759, + 0.2292153239250183, + 0.6839414238929749, + 0.5206495523452759, + 0.08337608724832535, + 0.13769836723804474 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 0, + "style_probability": 0.4953155815601349, + "style_target": 1, + "support_probability": 0.2850116491317749, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034301859533876207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9250705242156982, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3086:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5533870458602905, + 0.14698617160320282, + 0.8791275024414062, + 0.5533870458602905, + 0.1003296971321106, + 0.09426140785217285 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3086", + "split": "test", + "style_correct": 1, + "style_probability": 0.1308930516242981, + "style_target": 0, + "support_probability": 0.06390649080276489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032230774319826905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7181698083877563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5474687218666077, + 0.2885069251060486, + 0.7868251204490662, + 0.40391039848327637, + 0.07707523554563522, + 0.09332627803087234, + 0.2633442282676697, + 0.08455909788608551, + 0.11787397414445877 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 1, + "style_probability": 0.5325784683227539, + "style_target": 1, + "support_probability": 0.09734907746315002, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3216258931250753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4901743233203888, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5305303335189819, + 0.2805303633213043, + 0.8479718565940857, + 0.4070846140384674, + 0.12579017877578735, + 0.09440866112709045, + 0.046390555799007416, + 0.07828427106142044, + 0.07920821756124496 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3090", + "split": "test", + "style_correct": 0, + "style_probability": 0.8899567723274231, + "style_target": 0, + "support_probability": 0.202730193734169, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2383490290009789, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.482198566198349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7884024381637573, + 0.2994990050792694, + 0.4447091817855835, + 0.5619655847549438, + 0.08119429647922516, + 0.08373435586690903, + 0.06855757534503937, + 0.05058974027633667, + 0.1419789046049118, + 0.10802158713340759, + 0.050525471568107605 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 1, + "style_probability": 0.7517893314361572, + "style_target": 1, + "support_probability": 0.3234694004058838, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15911938203350218, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3443812429904938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3100:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8085341453552246, + 0.34990981221199036, + 0.45838096737861633, + 0.5232782959938049, + 0.04251673072576523, + 0.049253061413764954, + 0.07276154309511185, + 0.04143252968788147, + 0.14033879339694977 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3100", + "split": "test", + "style_correct": 0, + "style_probability": 0.5353075265884399, + "style_target": 0, + "support_probability": 0.39931491017341614, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05453226763103557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6434361338615417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4068131744861603, + 0.25902676582336426, + 0.4735568165779114, + 0.7549875974655151, + 0.04834204912185669 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 1, + "style_probability": 0.5908783674240112, + "style_target": 1, + "support_probability": 0.1048627719283104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010505860028035661, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8323856592178345, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3103:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38400572538375854, + 0.323422908782959, + 0.5221750736236572, + 0.6137226819992065, + 0.16227936744689941 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3103", + "split": "test", + "style_correct": 0, + "style_probability": 0.5478391647338867, + "style_target": 0, + "support_probability": 0.03115701861679554, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7596763858134068, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11946964263916016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9758920669555664, + 0.6247082948684692, + 0.41513705253601074, + 0.32283636927604675, + 0.06637737154960632, + 0.09875746071338654 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.70503169298172, + "style_target": 1, + "support_probability": 0.5317875146865845, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5432006805382059, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08089202642440796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3104:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9603261351585388, + 0.5813291072845459, + 0.40360909700393677, + 0.27704474329948425, + 0.2453889399766922 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3104", + "split": "test", + "style_correct": 1, + "style_probability": 0.2738189995288849, + "style_target": 0, + "support_probability": 0.6297931671142578, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.36198885022812094, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3262220621109009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5294669270515442, + 0.31187596917152405, + 0.9578456282615662, + 0.28293630480766296, + 0.06611523032188416, + 0.09848049283027649, + 0.055150941014289856, + 0.06165493652224541, + 0.1336277276277542 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.7770336866378784, + "style_target": 1, + "support_probability": 0.3289848268032074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02805666320888866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3336428701877594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3106:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5509219169616699, + 0.30860766768455505, + 0.9800874590873718, + 0.2840767204761505, + 0.09994924068450928, + 0.1596180498600006, + 0.18593746423721313, + 0.20050251483917236, + 0.21895112097263336 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3106", + "split": "test", + "style_correct": 1, + "style_probability": 0.13914941251277924, + "style_target": 0, + "support_probability": 0.29146113991737366, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9145340487213929, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19007541239261627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5595552325248718, + 0.257097452878952, + 0.9528400897979736, + 0.12370286881923676, + 0.032784536480903625, + 0.04518925026059151, + 0.06128338351845741, + 0.060343239456415176, + 0.041962992399930954, + 0.06254549324512482, + 0.06840324401855469 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 1, + "style_probability": 0.8388826251029968, + "style_target": 1, + "support_probability": 0.9586963057518005, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8106610686261461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19268865883350372, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3110:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5290777087211609, + 0.26967963576316833, + 0.9354493021965027, + 0.15254369378089905, + 0.07978767156600952, + 0.08024211972951889, + 0.05101744458079338, + 0.10334806889295578, + 0.051184892654418945, + 0.06555923819541931, + 0.07782215625047684 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3110", + "split": "test", + "style_correct": 1, + "style_probability": 0.482878178358078, + "style_target": 0, + "support_probability": 0.943489670753479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27097254647979163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17079611122608185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2912123501300812, + 0.747643768787384, + 0.5431543588638306, + 0.2817080616950989, + 0.057628534734249115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 1, + "style_probability": 0.5137434601783752, + "style_target": 1, + "support_probability": 0.43551450967788696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6421108045876521, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11557437479496002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26436442136764526, + 0.8308109641075134, + 0.5376533269882202, + 0.1953848898410797, + 0.06821084767580032 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3113", + "split": "test", + "style_correct": 0, + "style_probability": 0.7810200452804565, + "style_target": 0, + "support_probability": 0.605236828327179, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09950968940553828, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4920995235443115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3748362362384796, + 0.46640336513519287, + 0.6061715483665466, + 0.40838855504989624, + 0.1836836040019989, + 0.07877419143915176 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 1, + "style_probability": 0.5875365734100342, + "style_target": 1, + "support_probability": 0.22189299762248993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04838253438267781, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.36285385489463806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3118:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.386263370513916, + 0.47905996441841125, + 0.5888315439224243, + 0.3688497543334961, + 0.1345377117395401 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3118", + "split": "test", + "style_correct": 1, + "style_probability": 0.21105335652828217, + "style_target": 0, + "support_probability": 0.3202165365219116, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6255995440237996, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2817400097846985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4011264443397522, + 0.2692753076553345, + 0.5127313137054443, + 0.8775622844696045, + 0.1829783022403717, + 0.30169743299484253, + 0.14734789729118347, + 0.13853052258491516, + 0.14099597930908203 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 1, + "style_probability": 0.529506266117096, + "style_target": 1, + "support_probability": 0.8057445287704468, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.023911098495972286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4163738787174225, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3119:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4699837863445282, + 0.2648603320121765, + 0.45208582282066345, + 0.9405994415283203, + 0.15390804409980774, + 0.19021038711071014, + 0.12279454618692398, + 0.1014660894870758, + 0.1990787237882614 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3119", + "split": "test", + "style_correct": 1, + "style_probability": 0.10158666223287582, + "style_target": 0, + "support_probability": 0.6544095277786255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.048956602691880846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8442313075065613, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26850107312202454, + 0.5334938764572144, + 0.6562693119049072, + 0.26850107312202454, + 0.12476485222578049, + 0.23574504256248474, + 0.10481633245944977, + 0.1934209167957306, + 0.14391925930976868, + 0.25488588213920593, + 0.09412852674722672 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 1, + "style_probability": 0.5842649936676025, + "style_target": 1, + "support_probability": 0.10445312410593033, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2368550316604452, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5199649333953857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3122:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27706006169319153, + 0.5644623041152954, + 0.7169745564460754, + 0.27706006169319153, + 0.10084006935358047, + 0.06889815628528595, + 0.052741505205631256, + 0.10989339649677277, + 0.21865622699260712, + 0.0752447247505188, + 0.0754927396774292 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3122", + "split": "test", + "style_correct": 0, + "style_probability": 0.7403665781021118, + "style_target": 0, + "support_probability": 0.2921542525291443, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01348700565866557, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8224279284477234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4501408636569977, + 0.5833397507667542, + 0.8929867744445801, + 0.4378534257411957, + 0.10914216190576553 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.7135043144226074, + "style_target": 1, + "support_probability": 0.05476069822907448, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015285866667904403, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8832014203071594, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3126:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4815533459186554, + 0.658210039138794, + 0.9091347455978394, + 0.44709473848342896, + 0.14770150184631348 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3126", + "split": "test", + "style_correct": 1, + "style_probability": 0.2419191300868988, + "style_target": 0, + "support_probability": 0.045008961111307144, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02532050789341267, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5988014340400696, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.332437127828598, + 0.8564731478691101, + 0.569886326789856, + 0.3896160423755646, + 0.0409243144094944, + 0.08805691450834274 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 1, + "style_probability": 0.5891034007072449, + "style_target": 1, + "support_probability": 0.088066965341568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05883017572873861, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5007336735725403, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3141:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3474166989326477, + 0.8157570362091064, + 0.539030134677887, + 0.3722240924835205, + 0.0653800293803215, + 0.06434719264507294 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3141", + "split": "test", + "style_correct": 0, + "style_probability": 0.5583176016807556, + "style_target": 0, + "support_probability": 0.1573607474565506, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20367139780936755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13962091505527496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25663647055625916, + 0.3862081468105316, + 0.417639821767807, + 0.9350517988204956, + 0.03313874453306198, + 0.04357033595442772, + 0.0265343114733696, + 0.08146258443593979, + 0.07877711206674576 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.5537292957305908, + "style_target": 1, + "support_probability": 0.3524034321308136, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.152218067288187, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11196151375770569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3143:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2550157308578491, + 0.3193069100379944, + 0.416713684797287, + 0.9322630167007446, + 0.06831707805395126, + 0.08461542427539825 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3143", + "split": "test", + "style_correct": 1, + "style_probability": 0.2728375494480133, + "style_target": 0, + "support_probability": 0.4864546060562134, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19714873718767834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09454996883869171, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6018932461738586, + 0.5216674208641052, + 0.803734540939331, + 0.3900616466999054, + 0.06412285566329956, + 0.1332656592130661, + 0.04866962507367134, + 0.04552850127220154, + 0.03190470114350319, + 0.04840498045086861, + 0.11478924751281738 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 1, + "style_probability": 0.6366352438926697, + "style_target": 1, + "support_probability": 0.2987809479236603, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026283821348204166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10529796779155731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3146:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6766393184661865, + 0.5789276361465454, + 0.8902546763420105, + 0.43484964966773987, + 0.08453450351953506, + 0.06398473680019379, + 0.1258573979139328, + 0.1263904720544815, + 0.12133470922708511, + 0.1217244416475296 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3146", + "split": "test", + "style_correct": 1, + "style_probability": 0.14663292467594147, + "style_target": 0, + "support_probability": 0.3528725802898407, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022277475383936537, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4298021197319031, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.228325754404068, + 0.6607719659805298, + 0.5475500226020813, + 0.1947074830532074, + 0.15848422050476074 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 0, + "style_probability": 0.42726337909698486, + "style_target": 1, + "support_probability": 0.05836288630962372, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026844807949890193, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43574318289756775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3149:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22774188220500946, + 0.6490040421485901, + 0.5588185787200928, + 0.19240495562553406, + 0.1718108355998993 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3149", + "split": "test", + "style_correct": 0, + "style_probability": 0.5657460689544678, + "style_target": 0, + "support_probability": 0.07652740925550461, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.10046925081859116, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.42754843831062317, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17193034291267395, + 0.2313523292541504, + 0.515556812286377, + 0.9578481316566467, + 0.23106083273887634, + 0.14894534647464752 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 0, + "style_probability": 0.34949031472206116, + "style_target": 1, + "support_probability": 0.5644416809082031, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12065284893461835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5824620723724365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3151:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18346050381660461, + 0.2363581508398056, + 0.5616730451583862, + 0.9658864736557007, + 0.08158286660909653, + 0.1874961107969284 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3151", + "split": "test", + "style_correct": 0, + "style_probability": 0.6349563002586365, + "style_target": 0, + "support_probability": 0.47138577699661255, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02204301682325438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2940589189529419, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21010707318782806, + 0.20292672514915466, + 0.1343153715133667, + 0.8411657810211182, + 0.06691239029169083, + 0.07454011589288712, + 0.05958844721317291, + 0.1508299708366394, + 0.04748298227787018 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 0, + "style_probability": 0.383038729429245, + "style_target": 1, + "support_probability": 0.13584759831428528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006600675895802295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28865525126457214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3163:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2494749128818512, + 0.3159220516681671, + 0.1748950481414795, + 0.8683659434318542, + 0.05413755401968956, + 0.04249931871891022, + 0.05993122607469559, + 0.052020516246557236, + 0.08155450224876404 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3163", + "split": "test", + "style_correct": 1, + "style_probability": 0.2811093330383301, + "style_target": 0, + "support_probability": 0.1846148669719696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010734815973875152, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5856179594993591, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2689758837223053, + 0.12442542612552643, + 0.6691657900810242, + 0.23824000358581543, + 0.07636962085962296, + 0.1261545866727829, + 0.18812896311283112, + 0.04589703306555748, + 0.0967043936252594, + 0.13303212821483612, + 0.07681035250425339 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 0, + "style_probability": 0.4021027088165283, + "style_target": 1, + "support_probability": 0.04254906252026558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013224223516365097, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6838012337684631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3174:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3003917932510376, + 0.1269254833459854, + 0.587117612361908, + 0.26191937923431396, + 0.050771210342645645, + 0.09980668127536774, + 0.2097962200641632, + 0.06055045127868652, + 0.21205416321754456, + 0.07742290198802948, + 0.0865069180727005 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3174", + "split": "test", + "style_correct": 1, + "style_probability": 0.3999321758747101, + "style_target": 0, + "support_probability": 0.05763624608516693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08382037747284654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05990070104598999, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3660152852535248, + 0.3159525394439697, + 0.9404452443122864, + 0.4113481044769287, + 0.032471127808094025 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.5972566604614258, + "style_target": 1, + "support_probability": 0.3144383430480957, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02793344288406164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057460084557533264, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3178:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34228357672691345, + 0.3022468090057373, + 0.9140247106552124, + 0.3765658438205719, + 0.048321083188056946, + 0.07036034762859344, + 0.12712998688220978, + 0.21515703201293945, + 0.08935194462537766, + 0.08257804065942764, + 0.05570276454091072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3178", + "split": "test", + "style_correct": 1, + "style_probability": 0.22859856486320496, + "style_target": 0, + "support_probability": 0.31876739859580994, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7149302867448313, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3354186415672302, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8837310075759888, + 0.4806387424468994, + 0.5253685712814331, + 0.5909460783004761, + 0.047502025961875916, + 0.052961450070142746 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.7614359259605408, + "style_target": 1, + "support_probability": 0.8360668420791626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.043738647606020635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49133560061454773, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3192:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8941627144813538, + 0.47206827998161316, + 0.4978284239768982, + 0.6194288730621338, + 0.05345100909471512, + 0.1243404895067215 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3192", + "split": "test", + "style_correct": 1, + "style_probability": 0.12250300496816635, + "style_target": 0, + "support_probability": 0.6572466492652893, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5297443715262204, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3812007009983063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6945450305938721, + 0.3635125756263733, + 0.6736999154090881, + 0.445634126663208, + 0.11971305310726166, + 0.14218692481517792, + 0.03270110860466957, + 0.04419515281915665, + 0.03761603310704231 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 1, + "style_probability": 0.9007900953292847, + "style_target": 1, + "support_probability": 0.5042166113853455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09934303040824942, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6009745001792908, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.613280713558197, + 0.3341074287891388, + 0.6588668823242188, + 0.4702669680118561, + 0.09161277115345001, + 0.06450199335813522, + 0.15859365463256836, + 0.2605799436569214, + 0.09867025166749954 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3200", + "split": "test", + "style_correct": 0, + "style_probability": 0.5073344707489014, + "style_target": 0, + "support_probability": 0.28784435987472534, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17779573436191495, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5318731069564819, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4496723413467407, + 0.6889876127243042, + 0.10178890079259872, + 0.18323755264282227, + 0.0539502389729023, + 0.06040109694004059, + 0.07804472744464874, + 0.0815577358007431, + 0.07137574255466461, + 0.05847959592938423, + 0.05642954260110855 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.7887570858001709, + "style_target": 1, + "support_probability": 0.19512958824634552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02627491442233598, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5881518721580505, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3217:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49096789956092834, + 0.6556839942932129, + 0.12175267189741135, + 0.20604781806468964, + 0.11313693970441818, + 0.2817639112472534, + 0.15481117367744446, + 0.07108431309461594, + 0.2570767402648926 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3217", + "split": "test", + "style_correct": 1, + "style_probability": 0.3565244674682617, + "style_target": 0, + "support_probability": 0.1536339819431305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09623705449928367, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7868435382843018, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30335015058517456, + 0.6226297616958618, + 0.45167094469070435, + 0.6711819171905518, + 0.0642363503575325 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.7739317417144775, + "style_target": 1, + "support_probability": 0.2274019867181778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031806240505977135, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8492717742919922, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3449351191520691, + 0.630124568939209, + 0.4612778127193451, + 0.6850330829620361, + 0.1484615057706833 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3225", + "split": "test", + "style_correct": 1, + "style_probability": 0.15397237241268158, + "style_target": 0, + "support_probability": 0.15851299464702606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6633307556311472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14738601446151733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6627746820449829, + 0.6389318108558655, + 0.9562415480613708, + 0.24509747326374054, + 0.15020683407783508, + 0.07023869454860687 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 1, + "style_probability": 0.5045856833457947, + "style_target": 1, + "support_probability": 0.5762826800346375, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.73037938640951, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13890007138252258, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3226:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6610119938850403, + 0.6002601981163025, + 0.9269404411315918, + 0.24191589653491974, + 0.16345489025115967, + 0.20330460369586945 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3226", + "split": "test", + "style_correct": 0, + "style_probability": 0.6270011067390442, + "style_target": 0, + "support_probability": 0.6426005363464355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002859726826620792, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9731113314628601, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8461928963661194, + 0.23017781972885132, + 0.2473469078540802, + 0.25713062286376953, + 0.07145970314741135, + 0.0775739923119545, + 0.1030811071395874, + 0.09968637675046921, + 0.08390581607818604 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.7206429243087769, + "style_target": 1, + "support_probability": 0.005310217384248972, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006704779672220831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9218471646308899, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3236:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8131058216094971, + 0.21463017165660858, + 0.24704338610172272, + 0.26769521832466125, + 0.19043050706386566, + 0.1311560571193695 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3236", + "split": "test", + "style_correct": 1, + "style_probability": 0.2748687267303467, + "style_target": 0, + "support_probability": 0.022394435480237007, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.061651016426291796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3882119357585907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8472004532814026, + 0.3409501612186432, + 0.24609065055847168, + 0.36193716526031494, + 0.09111307561397552, + 0.05283878743648529, + 0.06613517552614212, + 0.06804587692022324, + 0.052746567875146866, + 0.18855276703834534, + 0.04780406877398491 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.7268223166465759, + "style_target": 1, + "support_probability": 0.0738786980509758, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0039183130099675325, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35735198855400085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3244:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8782542943954468, + 0.353077232837677, + 0.29963892698287964, + 0.4290638566017151, + 0.15920116007328033, + 0.11226836591959, + 0.13733425736427307, + 0.12385561317205429, + 0.08160663396120071, + 0.159508615732193 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3244", + "split": "test", + "style_correct": 1, + "style_probability": 0.09064966440200806, + "style_target": 0, + "support_probability": 0.2384367138147354, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9186256210548827, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10785543918609619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17527493834495544, + 0.2730673849582672, + 0.21147838234901428, + 0.9393951296806335, + 0.07254917919635773 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 1, + "style_probability": 0.9231459498405457, + "style_target": 1, + "support_probability": 0.792356014251709, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7796684205419594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.0959601104259491, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3247:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2036340981721878, + 0.2561775743961334, + 0.2227071076631546, + 0.9134697318077087, + 0.15021534264087677 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3247", + "split": "test", + "style_correct": 0, + "style_probability": 0.6263425946235657, + "style_target": 0, + "support_probability": 0.7297879457473755, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.776458631061729, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3336673974990845, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8881440162658691, + 0.6299472451210022, + 0.30279672145843506, + 0.7584316730499268, + 0.09091998636722565, + 0.10341876745223999 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.8537210822105408, + "style_target": 1, + "support_probability": 0.793715238571167, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3151222089314558, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.361476331949234, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3252:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.860480546951294, + 0.5948686599731445, + 0.2656770944595337, + 0.7137638926506042, + 0.19899693131446838 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3252", + "split": "test", + "style_correct": 1, + "style_probability": 0.34544628858566284, + "style_target": 0, + "support_probability": 0.7464444637298584, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2305759666150777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3164023160934448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5740041732788086, + 0.5148926377296448, + 0.9461345672607422, + 0.33127713203430176, + 0.07298730313777924, + 0.12553180754184723, + 0.19102071225643158, + 0.11563578248023987, + 0.13323964178562164 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.8257172107696533, + "style_target": 1, + "support_probability": 0.22314338386058807, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006262136341461277, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.591214656829834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3254:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6439865827560425, + 0.42594125866889954, + 0.9536479711532593, + 0.3865540623664856, + 0.08670857548713684, + 0.23682375252246857, + 0.2358175665140152, + 0.14989076554775238, + 0.17157486081123352 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3254", + "split": "test", + "style_correct": 1, + "style_probability": 0.18956099450588226, + "style_target": 0, + "support_probability": 0.14572899043560028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6503303788292243, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.030577104538679123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22790198028087616, + 0.545556366443634, + 0.9857782125473022, + 0.18306539952754974, + 0.10143020004034042, + 0.05895709618926048, + 0.04070901870727539, + 0.06237335875630379, + 0.06281033903360367, + 0.04508915916085243, + 0.06013915687799454 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 1, + "style_probability": 0.5908268690109253, + "style_target": 1, + "support_probability": 0.6944028735160828, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7360547385830998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.049070410430431366, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3258:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2302485853433609, + 0.5698599219322205, + 0.952398419380188, + 0.18983733654022217, + 0.08888249844312668, + 0.057574257254600525, + 0.046350616961717606, + 0.11418142914772034, + 0.10264971107244492, + 0.06750690937042236, + 0.08162630349397659 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3258", + "split": "test", + "style_correct": 0, + "style_probability": 0.5628013014793396, + "style_target": 0, + "support_probability": 0.7653099894523621, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5499829005117433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36208340525627136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8570898771286011, + 0.21517938375473022, + 0.3871361315250397, + 0.14340224862098694, + 0.05840253084897995 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.9150087237358093, + "style_target": 1, + "support_probability": 0.5457504391670227, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09352217782578622, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4964219033718109, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3264:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7918875217437744, + 0.22327695786952972, + 0.36280834674835205, + 0.16415631771087646, + 0.10173787921667099, + 0.06855382770299911, + 0.1595826894044876, + 0.08741753548383713, + 0.09577842056751251, + 0.08539970219135284, + 0.1582120805978775 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3264", + "split": "test", + "style_correct": 1, + "style_probability": 0.4158948063850403, + "style_target": 0, + "support_probability": 0.3064456880092621, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07312816338756933, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7733829021453857, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20179039239883423, + 0.3138912320137024, + 0.08924303203821182, + 0.8139973282814026, + 0.08318690955638885, + 0.08701684325933456 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.8215926289558411, + "style_target": 1, + "support_probability": 0.1745220273733139, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00025331725547220164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9807733297348022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2705671489238739, + 0.3703923523426056, + 0.11029499769210815, + 0.7227805852890015, + 0.10203271359205246, + 0.0822497010231018 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3271", + "split": "test", + "style_correct": 1, + "style_probability": 0.13616132736206055, + "style_target": 0, + "support_probability": 0.027289757505059242, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09997942719214238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29703113436698914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7139634490013123, + 0.1531699150800705, + 0.3121090829372406, + 0.25995704531669617, + 0.053357116878032684, + 0.058214686810970306, + 0.06660035252571106, + 0.07104628533124924, + 0.09975866228342056 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 1, + "style_probability": 0.7801948189735413, + "style_target": 1, + "support_probability": 0.16469696164131165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04791184882398271, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42084628343582153, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6579271554946899, + 0.18672339618206024, + 0.3209650218486786, + 0.2539907991886139, + 0.10234393179416656, + 0.05291513353586197, + 0.13212241232395172, + 0.183790922164917, + 0.12039855122566223 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3276", + "split": "test", + "style_correct": 0, + "style_probability": 0.5381336808204651, + "style_target": 0, + "support_probability": 0.08716922998428345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12631800770991933, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7986762523651123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25880590081214905, + 0.1577230989933014, + 0.47039860486984253, + 0.5308059453964233, + 0.035767849534749985, + 0.052499920129776, + 0.04083489999175072, + 0.05928438901901245, + 0.062257979065179825, + 0.03997424617409706, + 0.09804029017686844 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 1, + "style_probability": 0.9500448107719421, + "style_target": 1, + "support_probability": 0.13622361421585083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.037661067696533886, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7204081416130066, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3287:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25057950615882874, + 0.15015728771686554, + 0.4426161050796509, + 0.476250559091568, + 0.04687649756669998, + 0.03624037653207779, + 0.08386057615280151, + 0.07884589582681656, + 0.06331969052553177 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3287", + "split": "test", + "style_correct": 0, + "style_probability": 0.6360960602760315, + "style_target": 0, + "support_probability": 0.11102686077356339, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5382653612111241, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1170203685760498, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44697657227516174, + 0.8582679629325867, + 0.12258864939212799, + 0.6254706382751465, + 0.15137526392936707 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.7023654580116272, + "style_target": 1, + "support_probability": 0.5083678960800171, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04144667311536154, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.20711152255535126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3289:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5169748663902283, + 0.8998314738273621, + 0.15410539507865906, + 0.7117569446563721, + 0.09004697948694229 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:3289", + "split": "test", + "style_correct": 1, + "style_probability": 0.1761397421360016, + "style_target": 0, + "support_probability": 0.45247432589530945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6903230260076967, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11264064908027649, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9977017045021057, + 0.4365389347076416, + 0.5712160468101501, + 0.4405919909477234, + 0.04416709393262863, + 0.13507822155952454 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 1, + "style_probability": 0.6394903063774109, + "style_target": 1, + "support_probability": 0.685495138168335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6378657307538553, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09175697714090347, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3292:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.991510808467865, + 0.46366414427757263, + 0.5113034844398499, + 0.4174918234348297, + 0.1040489599108696, + 0.09348569065332413 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3292", + "split": "test", + "style_correct": 0, + "style_probability": 0.6742401719093323, + "style_target": 0, + "support_probability": 0.6328085660934448, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011242453147030202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9584269523620605, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23673194646835327, + 0.30881381034851074, + 0.7589902281761169, + 0.21045896410942078, + 0.06131031736731529, + 0.044454626739025116, + 0.06438224017620087, + 0.04958261922001839, + 0.14350609481334686 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.8957934975624084, + "style_target": 1, + "support_probability": 0.010537579655647278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003351221978369235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9599640965461731, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3302:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24847695231437683, + 0.2798447906970978, + 0.703879177570343, + 0.1921195536851883, + 0.08507250249385834, + 0.18683692812919617 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3302", + "split": "test", + "style_correct": 1, + "style_probability": 0.3474670648574829, + "style_target": 0, + "support_probability": 0.00902046263217926, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0671768972260658, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7131580710411072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.34961768984794617, + 0.5271406173706055, + 0.6843782663345337, + 0.3590465188026428, + 0.06117287650704384, + 0.07110021263360977, + 0.05175180733203888, + 0.07061978429555893, + 0.05086889490485191, + 0.13416381180286407 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.8506165742874146, + "style_target": 1, + "support_probability": 0.0962178185582161, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004210178057510396, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5395182371139526, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4312102794647217, + 0.532617449760437, + 0.7697778344154358, + 0.42504167556762695, + 0.06949308514595032, + 0.06569972634315491, + 0.0888519436120987, + 0.09839075803756714, + 0.07217377424240112, + 0.15405170619487762 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3310", + "split": "test", + "style_correct": 1, + "style_probability": 0.1421487033367157, + "style_target": 0, + "support_probability": 0.35485604405403137, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20167863981694015, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2427302747964859, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23779331147670746, + 0.5413576364517212, + 0.2821415662765503, + 0.8961516618728638, + 0.11716064065694809 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 0, + "style_probability": 0.48945873975753784, + "style_target": 1, + "support_probability": 0.4298092722892761, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02713826007657006, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.17886322736740112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23133057355880737, + 0.53968745470047, + 0.27571752667427063, + 0.9825100302696228, + 0.176916241645813 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3315", + "split": "test", + "style_correct": 1, + "style_probability": 0.15903513133525848, + "style_target": 0, + "support_probability": 0.3759883940219879, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02282444423827801, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7599083185195923, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32304224371910095, + 0.8370020985603333, + 0.2559047341346741, + 0.32304224371910095, + 0.1407451629638672, + 0.12130587548017502 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 0, + "style_probability": 0.45165887475013733, + "style_target": 1, + "support_probability": 0.055282317101955414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015032447429880524, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8329559564590454, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3317:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27097904682159424, + 0.9000462889671326, + 0.20056283473968506, + 0.27097904682159424, + 0.10112868994474411, + 0.1018393263220787 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3317", + "split": "test", + "style_correct": 0, + "style_probability": 0.7209265828132629, + "style_target": 0, + "support_probability": 0.03539661318063736, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5002192266572258, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18275722861289978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7351669669151306, + 0.4737969934940338, + 0.4996115565299988, + 0.47114354372024536, + 0.07991489768028259, + 0.08082887530326843, + 0.10454189032316208, + 0.159019336104393, + 0.12757189571857452 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 1, + "style_probability": 0.5690729022026062, + "style_target": 1, + "support_probability": 0.7417292594909668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3719288452858365, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1432162970304489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3324:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6861332654953003, + 0.48151636123657227, + 0.5028117895126343, + 0.4778447151184082, + 0.0645507201552391, + 0.18395842611789703 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3324", + "split": "test", + "style_correct": 1, + "style_probability": 0.38201650977134705, + "style_target": 0, + "support_probability": 0.7777135372161865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23675553806645772, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4199129641056061, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3078369200229645, + 0.5834694504737854, + 0.5868226289749146, + 0.3583551347255707, + 0.19827334582805634, + 0.07755383849143982, + 0.03339936211705208, + 0.04502025619149208, + 0.08810961246490479, + 0.0665111392736435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 1, + "style_probability": 0.5534155368804932, + "style_target": 1, + "support_probability": 0.570975661277771, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023550787673123356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49298691749572754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3333:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6055736541748047, + 0.7452988624572754, + 0.6495944261550903, + 0.37907931208610535, + 0.12286265194416046, + 0.058866843581199646, + 0.04941602796316147, + 0.10903998464345932, + 0.10830625891685486, + 0.09660680592060089 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3333", + "split": "test", + "style_correct": 1, + "style_probability": 0.08221408724784851, + "style_target": 0, + "support_probability": 0.6468547582626343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09604672510250989, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6767849326133728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23317795991897583, + 0.3001706898212433, + 0.3203577697277069, + 0.556873619556427, + 0.03378954529762268 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.8512685298919678, + "style_target": 1, + "support_probability": 0.12549002468585968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0161869531753498, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5079758763313293, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3335:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20070771872997284, + 0.29963555932044983, + 0.3085010349750519, + 0.6331605315208435, + 0.06295059621334076, + 0.05098211020231247, + 0.03591246157884598, + 0.06448239833116531, + 0.24326062202453613, + 0.07426819205284119, + 0.2132946401834488 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3335", + "split": "test", + "style_correct": 1, + "style_probability": 0.29732879996299744, + "style_target": 0, + "support_probability": 0.09888266772031784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7585933604011431, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43602925539016724, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5662519335746765, + 0.357286661863327, + 0.8054879307746887, + 0.26782307028770447, + 0.09208375960588455, + 0.02524193935096264 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.9129599928855896, + "style_target": 1, + "support_probability": 0.7718717455863953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2604288322236492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42040178179740906, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3338:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6140074133872986, + 0.4523555636405945, + 0.8892577886581421, + 0.27012771368026733, + 0.042495254427194595, + 0.1345476359128952 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3338", + "split": "test", + "style_correct": 1, + "style_probability": 0.29227936267852783, + "style_target": 0, + "support_probability": 0.7181641459465027, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6809577111810157, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2471204400062561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9718905687332153, + 0.35248470306396484, + 0.35566115379333496, + 0.1962592899799347, + 0.06532370299100876, + 0.11387255042791367, + 0.16504739224910736, + 0.12166795134544373, + 0.0468515045940876 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.8227097988128662, + "style_target": 1, + "support_probability": 0.679901659488678, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5181904174511942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22511355578899384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3344:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9398684501647949, + 0.3616315424442291, + 0.24974800646305084, + 0.24667391180992126, + 0.07161909341812134, + 0.10386279225349426, + 0.0589645691215992, + 0.14707022905349731, + 0.05506039038300514 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3344", + "split": "test", + "style_correct": 1, + "style_probability": 0.48210951685905457, + "style_target": 0, + "support_probability": 0.7928444147109985, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.20789899569947143, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7448169589042664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4060414731502533, + 0.22573614120483398, + 0.7169144749641418, + 0.3449704647064209, + 0.10930634289979935, + 0.08659964054822922, + 0.05325912684202194, + 0.08418740332126617, + 0.12988343834877014, + 0.08573831617832184, + 0.04336046054959297 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.9289373755455017, + "style_target": 1, + "support_probability": 0.13267207145690918, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12187610248636817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3457832634449005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3346:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3993390202522278, + 0.2177358865737915, + 0.687722384929657, + 0.29587769508361816, + 0.5688963532447815, + 0.23508258163928986, + 0.1022845208644867, + 0.07306798547506332, + 0.10524023324251175 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3346", + "split": "test", + "style_correct": 1, + "style_probability": 0.30243271589279175, + "style_target": 0, + "support_probability": 0.39256423711776733, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06216831865497509, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6305810809135437, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3469700217247009, + 0.35656026005744934, + 0.1617891937494278, + 0.6630401015281677, + 0.1842816323041916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 0, + "style_probability": 0.3229769766330719, + "style_target": 1, + "support_probability": 0.20441484451293945, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08525988368973536, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.33849525451660156, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3351:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43063637614250183, + 0.5268203020095825, + 0.23964712023735046, + 0.8462709784507751, + 0.09034369885921478 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3351", + "split": "test", + "style_correct": 1, + "style_probability": 0.23273052275180817, + "style_target": 0, + "support_probability": 0.651819109916687, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5931302141359538, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.195855051279068, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.566602349281311, + 0.8246676921844482, + 0.36657729744911194, + 0.5918713808059692, + 0.08135196566581726, + 0.06171750649809837 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 0, + "style_probability": 0.3429482579231262, + "style_target": 1, + "support_probability": 0.8415903449058533, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5713134655129295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30965492129325867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3353:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.489688903093338, + 0.7765176296234131, + 0.4119575023651123, + 0.5577779412269592, + 0.05585802346467972, + 0.05963389202952385 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3353", + "split": "test", + "style_correct": 1, + "style_probability": 0.4763484001159668, + "style_target": 0, + "support_probability": 0.7940632700920105, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13486809011531076, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3562915027141571, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7338168025016785, + 0.27910134196281433, + 0.5411317944526672, + 0.9356306195259094, + 0.039321206510066986, + 0.08894063532352448, + 0.12407802790403366, + 0.06135553866624832, + 0.15200746059417725 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 0, + "style_probability": 0.2687280774116516, + "style_target": 1, + "support_probability": 0.40275630354881287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22776492936975504, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30237480998039246, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7520723938941956, + 0.30561766028404236, + 0.5645937323570251, + 0.9497909545898438, + 0.030341506004333496, + 0.027342259883880615, + 0.026284165680408478, + 0.03946894034743309, + 0.06897196173667908 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3355", + "split": "test", + "style_correct": 1, + "style_probability": 0.48992571234703064, + "style_target": 0, + "support_probability": 0.5203914642333984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1669788784740014, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18934613466262817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33724501729011536, + 0.560731053352356, + 0.6836860775947571, + 0.8541916608810425, + 0.05750371515750885, + 0.1293095201253891, + 0.08613546937704086, + 0.1885814070701599, + 0.12235979735851288, + 0.07110978662967682, + 0.06081670895218849 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 1, + "style_probability": 0.5067787170410156, + "style_target": 1, + "support_probability": 0.2819977402687073, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05885059442882668, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14248552918434143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3359:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.442924439907074, + 0.5960200428962708, + 0.7623346447944641, + 0.8962072134017944, + 0.1170036718249321, + 0.14055898785591125, + 0.11900803446769714, + 0.09478788822889328, + 0.08779515326023102, + 0.11391666531562805 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3359", + "split": "test", + "style_correct": 1, + "style_probability": 0.4543611407279968, + "style_target": 0, + "support_probability": 0.3052322268486023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21362772201230665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.20368202030658722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23837806284427643, + 0.19066086411476135, + 0.23837806284427643, + 0.9003198742866516, + 0.055128928273916245 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 1, + "style_probability": 0.6827490329742432, + "style_target": 1, + "support_probability": 0.3917352557182312, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15196872264920724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1966620534658432, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3367:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2278444916009903, + 0.1728324145078659, + 0.2278444916009903, + 0.9513232111930847, + 0.12236116826534271 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3367", + "split": "test", + "style_correct": 0, + "style_probability": 0.682040810585022, + "style_target": 0, + "support_probability": 0.48736804723739624, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.570444192706745, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1708872765302658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30584073066711426, + 0.8067336678504944, + 0.530100405216217, + 0.2697533071041107, + 0.1551312655210495, + 0.08068423718214035 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 0, + "style_probability": 0.45010071992874146, + "style_target": 1, + "support_probability": 0.783294141292572, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.866384603287866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08639296889305115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3369:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38211938738822937, + 0.8838252425193787, + 0.6124142408370972, + 0.2778644561767578, + 0.04188910871744156, + 0.051051873713731766 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3369", + "split": "test", + "style_correct": 0, + "style_probability": 0.8758261203765869, + "style_target": 0, + "support_probability": 0.9264840483665466, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.479313856959692, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21230386197566986, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2578088343143463, + 0.4287630319595337, + 0.4166153371334076, + 0.9498019218444824, + 0.054025642573833466, + 0.03310897573828697, + 0.07242463529109955, + 0.08859872072935104, + 0.048131123185157776 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 1, + "style_probability": 0.6359057426452637, + "style_target": 1, + "support_probability": 0.45803990960121155, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23855694040552503, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2530760169029236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25722765922546387, + 0.4017447829246521, + 0.3580802381038666, + 0.9443415999412537, + 0.0611935518682003, + 0.09231165796518326 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3371", + "split": "test", + "style_correct": 1, + "style_probability": 0.49977394938468933, + "style_target": 0, + "support_probability": 0.38286125659942627, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013671460283632088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8760244846343994, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2785041332244873, + 0.8055648803710938, + 0.33224907517433167, + 0.2785041332244873, + 0.18330329656600952, + 0.14908476173877716, + 0.26312899589538574, + 0.21302999556064606, + 0.1646241992712021, + 0.09071656316518784, + 0.1489502638578415 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 0, + "style_probability": 0.4913117587566376, + "style_target": 1, + "support_probability": 0.052742768079042435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050635979883452815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4994296431541443, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3373:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.353440523147583, + 0.8886421918869019, + 0.45681679248809814, + 0.353440523147583, + 0.16454946994781494, + 0.07786749303340912, + 0.1874585896730423, + 0.10102929919958115, + 0.07887215912342072, + 0.1333935707807541 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3373", + "split": "test", + "style_correct": 1, + "style_probability": 0.1620393544435501, + "style_target": 0, + "support_probability": 0.6218845248222351, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1683118203949414, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2623392641544342, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30982038378715515, + 0.3869745433330536, + 0.9753348231315613, + 0.3817618787288666, + 0.10619431734085083 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.6295056343078613, + "style_target": 1, + "support_probability": 0.41716012358665466, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26187297562221096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17942816019058228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3382:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3242250978946686, + 0.3952986001968384, + 0.9609189033508301, + 0.30211684107780457, + 0.14614087343215942, + 0.09438479691743851, + 0.11358737200498581, + 0.11637319624423981, + 0.09510362148284912, + 0.07237137854099274, + 0.25262251496315 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3382", + "split": "test", + "style_correct": 1, + "style_probability": 0.399821013212204, + "style_target": 0, + "support_probability": 0.5416361093521118, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39463056635469584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15753790736198425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6945658326148987, + 0.22177402675151825, + 0.6945658326148987, + 0.8634434938430786, + 0.11329861730337143, + 0.10608090460300446 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.7789179086685181, + "style_target": 1, + "support_probability": 0.39815619587898254, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03295854161640599, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21633391082286835, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3383:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7625295519828796, + 0.2558278739452362, + 0.7625295519828796, + 0.9074636101722717, + 0.04788056015968323, + 0.10830305516719818 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3383", + "split": "test", + "style_correct": 1, + "style_probability": 0.2518874704837799, + "style_target": 0, + "support_probability": 0.3734588027000427, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7428918342409645, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10342849791049957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5462480783462524, + 0.6027729511260986, + 0.5477217435836792, + 0.7527079582214355, + 0.0305115208029747, + 0.030023038387298584, + 0.04741077125072479, + 0.03173026069998741, + 0.041911277920007706 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 1, + "style_probability": 0.9131093621253967, + "style_target": 1, + "support_probability": 0.9174647927284241, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6679124579125961, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1818966120481491, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3395:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5514323115348816, + 0.478763610124588, + 0.5357925891876221, + 0.6956163048744202, + 0.02936670370399952, + 0.04821295663714409, + 0.03888902813196182, + 0.04181186109781265, + 0.040508877485990524 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3395", + "split": "test", + "style_correct": 0, + "style_probability": 0.7550899386405945, + "style_target": 0, + "support_probability": 0.9266283512115479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5196690351594171, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4597584903240204, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3418326675891876, + 0.40700116753578186, + 0.3418326675891876, + 0.6916393041610718, + 0.07541998475790024, + 0.07879270613193512, + 0.03537628799676895, + 0.04110271856188774, + 0.03682449460029602, + 0.03403268754482269, + 0.033909693360328674 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.9004547595977783, + "style_target": 1, + "support_probability": 0.6126484274864197, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2678516969050832, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3886488676071167, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3399:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3969830274581909, + 0.4286037087440491, + 0.3969830274581909, + 0.6672695875167847, + 0.2582971751689911, + 0.056517116725444794, + 0.02735489420592785, + 0.14524772763252258, + 0.14123490452766418 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3399", + "split": "test", + "style_correct": 1, + "style_probability": 0.40475285053253174, + "style_target": 0, + "support_probability": 0.6829293370246887, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.053125264346902686, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6975604295730591, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2500317096710205, + 0.8820005059242249, + 0.379148930311203, + 0.5201528072357178, + 0.07092611491680145 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 1, + "style_probability": 0.5645534992218018, + "style_target": 1, + "support_probability": 0.1350094974040985, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00928316543885027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7906924486160278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3185959458351135, + 0.8916621804237366, + 0.36266255378723145, + 0.5123934149742126, + 0.12198860943317413 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3409", + "split": "test", + "style_correct": 1, + "style_probability": 0.4478689134120941, + "style_target": 0, + "support_probability": 0.1162961795926094, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4007808312677863, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15453451871871948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5451183319091797, + 0.39184996485710144, + 0.8433871865272522, + 0.6560922861099243, + 0.07333976775407791, + 0.07015655934810638 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 0, + "style_probability": 0.3427466154098511, + "style_target": 1, + "support_probability": 0.8143253326416016, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4389609190003725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29158443212509155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5131015777587891, + 0.2648650109767914, + 0.8517728447914124, + 0.6787320971488953, + 0.022919710725545883, + 0.06101329252123833 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3430", + "split": "test", + "style_correct": 1, + "style_probability": 0.44515538215637207, + "style_target": 0, + "support_probability": 0.8162673115730286, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1464230120191985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33932438492774963, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3864381015300751, + 0.39203137159347534, + 0.31808218359947205, + 0.9320149421691895, + 0.06968498229980469, + 0.10498557984828949, + 0.09874304383993149, + 0.10356912761926651, + 0.17187929153442383 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.28220877051353455, + "style_target": 1, + "support_probability": 0.5872791409492493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.33369673470007477, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4358333349227905, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3431:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4103909730911255, + 0.3796253204345703, + 0.2784889340400696, + 0.9542621970176697, + 0.07969831675291061, + 0.05964062735438347, + 0.04901955649256706, + 0.11446431279182434, + 0.07677352428436279 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3431", + "split": "test", + "style_correct": 0, + "style_probability": 0.7560899257659912, + "style_target": 0, + "support_probability": 0.5744786262512207, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.0044599595491190405, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9309312105178833, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28022849559783936, + 0.33642467856407166, + 0.16179892420768738, + 0.6685529947280884, + 0.17155544459819794, + 0.1560780555009842, + 0.15264026820659637, + 0.31401294469833374, + 0.20404089987277985, + 0.27297908067703247, + 0.3014657497406006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 0, + "style_probability": 0.19164904952049255, + "style_target": 1, + "support_probability": 0.026981033384799957, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015145562624145568, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8840935230255127, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3439:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3506453037261963, + 0.4739578366279602, + 0.17749349772930145, + 0.8371203541755676, + 0.25894975662231445, + 0.11016276478767395, + 0.2989569902420044, + 0.09292811900377274, + 0.07777412235736847, + 0.10379427671432495 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3439", + "split": "test", + "style_correct": 1, + "style_probability": 0.3190244734287262, + "style_target": 0, + "support_probability": 0.16571177542209625, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8816932459909061, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03372887521982193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4829678535461426, + 0.36819911003112793, + 0.9536399245262146, + 0.4829678535461426, + 0.10479792952537537 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 1, + "style_probability": 0.8421550393104553, + "style_target": 1, + "support_probability": 0.9234384298324585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4403312041172817, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11054534465074539, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3442:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4301186800003052, + 0.3061539828777313, + 0.8971139192581177, + 0.4301186800003052, + 0.12113270163536072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3442", + "split": "test", + "style_correct": 0, + "style_probability": 0.5374376773834229, + "style_target": 0, + "support_probability": 0.7524458169937134, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.9174071897542467, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11736032366752625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23308877646923065, + 0.23897844552993774, + 0.9987851977348328, + 0.08274177461862564, + 0.20716983079910278, + 0.08504042774438858 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.7965240478515625, + "style_target": 1, + "support_probability": 0.8824350833892822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7259273709255147, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08746982365846634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3446:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24730414152145386, + 0.2466251701116562, + 0.9963793158531189, + 0.09163860976696014, + 0.07929182797670364 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3446", + "split": "test", + "style_correct": 1, + "style_probability": 0.20081569254398346, + "style_target": 0, + "support_probability": 0.9160311222076416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8107749743099077, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11895252019166946, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15956340730190277, + 0.9696593284606934, + 0.36167263984680176, + 0.21777546405792236, + 0.031197765842080116, + 0.04890841618180275, + 0.05873292684555054, + 0.08464819192886353, + 0.12041101604700089 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.7453233003616333, + "style_target": 1, + "support_probability": 0.8003067970275879, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4078763957289197, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15387366712093353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3449:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1854521632194519, + 0.9812394380569458, + 0.38380199670791626, + 0.23783427476882935, + 0.044654570519924164, + 0.047413185238838196, + 0.0704713687300682, + 0.07247298210859299, + 0.09839823096990585 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3449", + "split": "test", + "style_correct": 1, + "style_probability": 0.3055703341960907, + "style_target": 0, + "support_probability": 0.8029510378837585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11984342675182802, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8321710824966431, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4050246477127075, + 0.8293576240539551, + 0.2450617253780365, + 0.4499158561229706, + 0.03476788476109505, + 0.048652585595846176, + 0.101921945810318, + 0.05230114981532097, + 0.061784084886312485, + 0.32170525193214417, + 0.1331581175327301 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 1, + "style_probability": 0.8125786781311035, + "style_target": 1, + "support_probability": 0.13800343871116638, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.047245898387849206, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8489306569099426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3463:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.44798743724823, + 0.8175714612007141, + 0.26340019702911377, + 0.5218148827552795, + 0.04380318522453308, + 0.048332180827856064, + 0.10940288752317429, + 0.11395648866891861, + 0.07865136861801147, + 0.08660248667001724, + 0.17980097234249115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3463", + "split": "test", + "style_correct": 0, + "style_probability": 0.5928504467010498, + "style_target": 0, + "support_probability": 0.08145647495985031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13874781027815253, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.49640128016471863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47386690974235535, + 0.7084554433822632, + 0.5448188781738281, + 0.33711767196655273, + 0.052044861018657684 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.8546377420425415, + "style_target": 1, + "support_probability": 0.08430210500955582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006881405503890715, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6762514114379883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3469:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.508105456829071, + 0.6943607926368713, + 0.5032697319984436, + 0.32710838317871094, + 0.06161733716726303, + 0.08207547664642334, + 0.055830080062150955, + 0.07109137624502182, + 0.07985904812812805, + 0.07361922413110733 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3469", + "split": "test", + "style_correct": 1, + "style_probability": 0.3629312217235565, + "style_target": 0, + "support_probability": 0.01593596115708351, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7403628882666168, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15735512971878052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3391248285770416, + 0.29327645897865295, + 0.9373974800109863, + 0.3682903051376343, + 0.261879563331604, + 0.05074716731905937 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.7184494733810425, + "style_target": 1, + "support_probability": 0.6475927233695984, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23843702835379685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1595744490623474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3474:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3429943919181824, + 0.36956119537353516, + 0.9660741090774536, + 0.4260784983634949, + 0.058629635721445084, + 0.0664447769522667 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3474", + "split": "test", + "style_correct": 1, + "style_probability": 0.2336878776550293, + "style_target": 0, + "support_probability": 0.7094923257827759, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6478547619294659, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27291861176490784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5680791139602661, + 0.9400310516357422, + 0.3490236699581146, + 0.3897949755191803, + 0.03997412696480751, + 0.06288094073534012, + 0.10643365979194641, + 0.15954475104808807, + 0.049122147262096405 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 1, + "style_probability": 0.7911390662193298, + "style_target": 1, + "support_probability": 0.6312355399131775, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.615726724356378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.379543274641037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3477:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4705951511859894, + 0.913868248462677, + 0.36546388268470764, + 0.38151228427886963, + 0.1237756684422493, + 0.05517297238111496, + 0.12997238337993622, + 0.10710463672876358, + 0.15496006608009338 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3477", + "split": "test", + "style_correct": 0, + "style_probability": 0.6224677562713623, + "style_target": 0, + "support_probability": 0.6173032522201538, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18330179880617692, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8264300227165222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:positive", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.43494367599487305, + 0.5107457041740417, + 0.6132981181144714, + 0.3618558645248413, + 0.07307982444763184, + 0.09517905116081238, + 0.060459643602371216, + 0.03722234442830086, + 0.09923601150512695, + 0.12059124559164047, + 0.2997787892818451, + 0.06987638771533966 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.9172671437263489, + "style_target": 1, + "support_probability": 0.09498320519924164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022926990262538412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7824469208717346, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4222700893878937, + 0.448019415140152, + 0.5652199387550354, + 0.3669893741607666, + 0.24275250732898712, + 0.04764305427670479, + 0.1610185205936432, + 0.2480703443288803, + 0.10486036539077759 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3478", + "split": "test", + "style_correct": 1, + "style_probability": 0.4416809678077698, + "style_target": 0, + "support_probability": 0.08745044469833374, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5955944790906642, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09662476181983948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7124803066253662, + 0.7451558113098145, + 0.7180124521255493, + 0.5696478486061096, + 0.06965702772140503 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 0, + "style_probability": 0.30558153986930847, + "style_target": 1, + "support_probability": 0.9375548362731934, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04603620920324336, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14038346707820892, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7611672878265381, + 0.8383249640464783, + 0.7734473347663879, + 0.5709059834480286, + 0.09506737440824509 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3481", + "split": "test", + "style_correct": 1, + "style_probability": 0.0731603130698204, + "style_target": 0, + "support_probability": 0.8793259859085083, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5418597515468875, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0779547244310379, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6538987159729004, + 0.9446991086006165, + 0.31081223487854004, + 0.22493720054626465, + 0.06744375079870224, + 0.05812043696641922 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 1, + "style_probability": 0.5670314431190491, + "style_target": 1, + "support_probability": 0.6114071607589722, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6471992730933778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11206429451704025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3505:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6408095359802246, + 0.9285595417022705, + 0.27958714962005615, + 0.23627299070358276, + 0.12366780638694763, + 0.04872529208660126 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3505", + "split": "test", + "style_correct": 0, + "style_probability": 0.7199900150299072, + "style_target": 0, + "support_probability": 0.5541542768478394, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09548603005127011, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7429569959640503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09037385135889053, + 0.446432501077652, + 0.7647086381912231, + 0.15627536177635193, + 0.08334124833345413, + 0.1345372349023819, + 0.10860886424779892, + 0.18421421945095062, + 0.16009025275707245 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 1, + "style_probability": 0.5077187418937683, + "style_target": 1, + "support_probability": 0.1887306123971939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2759414107152942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6949511766433716, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3510:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.08212042599916458, + 0.47252124547958374, + 0.7947821021080017, + 0.14544647932052612, + 0.1497383564710617, + 0.09063663333654404, + 0.04772321507334709, + 0.118638776242733, + 0.06361062079668045 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3510", + "split": "test", + "style_correct": 0, + "style_probability": 0.7542679905891418, + "style_target": 0, + "support_probability": 0.2726200520992279, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6529662017820783, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2326277792453766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14241552352905273, + 0.12514227628707886, + 0.357975572347641, + 0.9540047645568848, + 0.10311836749315262, + 0.1411200910806656, + 0.052696116268634796, + 0.10658915340900421, + 0.07410694658756256, + 0.13842999935150146, + 0.13782642781734467 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 0, + "style_probability": 0.41468262672424316, + "style_target": 1, + "support_probability": 0.7692973017692566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11015894039657637, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33124083280563354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3511:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15002046525478363, + 0.1345575600862503, + 0.3889788091182709, + 0.9855656027793884, + 0.0911414623260498, + 0.08276243507862091, + 0.05802014842629433, + 0.08955196291208267, + 0.05657243728637695, + 0.10539333522319794 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3511", + "split": "test", + "style_correct": 1, + "style_probability": 0.15925240516662598, + "style_target": 0, + "support_probability": 0.6087977886199951, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8946284627715784, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12408575415611267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29250568151474, + 0.31273701786994934, + 0.986088216304779, + 0.36303582787513733, + 0.08088470995426178 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 1, + "style_probability": 0.9221025109291077, + "style_target": 1, + "support_probability": 0.7599673867225647, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6980820542205706, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07691863924264908, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3009631335735321, + 0.3785431385040283, + 0.9843696355819702, + 0.3449159264564514, + 0.10406859219074249 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3518", + "split": "test", + "style_correct": 1, + "style_probability": 0.4640503227710724, + "style_target": 0, + "support_probability": 0.8758893609046936, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03393698777682986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9092010259628296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09823452681303024, + 0.09823452681303024, + 0.5132268071174622, + 0.4258762001991272, + 0.12574826180934906, + 0.089564748108387 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.8213579058647156, + "style_target": 1, + "support_probability": 0.06142306700348854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01941182526410323, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8519831895828247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3531:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12221749871969223, + 0.12221749871969223, + 0.490771621465683, + 0.3927866816520691, + 0.06481818109750748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3531", + "split": "test", + "style_correct": 1, + "style_probability": 0.3252752721309662, + "style_target": 0, + "support_probability": 0.09294553846120834, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05877777349655843, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6909258365631104, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5254167318344116, + 0.7013626098632812, + 0.13347773253917694, + 0.6083155274391174, + 0.0473988801240921, + 0.06379551440477371, + 0.029045304283499718, + 0.13523156940937042, + 0.06268753856420517 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.5797452926635742, + "style_target": 1, + "support_probability": 0.12665066123008728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006529882575195822, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6940541863441467, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3533:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6386817097663879, + 0.7754181027412415, + 0.19374167919158936, + 0.6845523118972778, + 0.06500465422868729, + 0.20642556250095367, + 0.11661752313375473, + 0.07135190069675446, + 0.10881583392620087 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3533", + "split": "test", + "style_correct": 1, + "style_probability": 0.11357426643371582, + "style_target": 0, + "support_probability": 0.12143748253583908, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8883968232276162, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04969729855656624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5526604056358337, + 0.751611053943634, + 0.9076050519943237, + 0.33080482482910156, + 0.1246543750166893, + 0.044942572712898254, + 0.050645437091588974, + 0.022273123264312744, + 0.12738114595413208, + 0.0842338278889656 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.8405776023864746, + "style_target": 1, + "support_probability": 0.9565925598144531, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7630044761002317, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06803125888109207, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3534:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5635927319526672, + 0.6303778290748596, + 0.8379008173942566, + 0.35411298274993896, + 0.03751702979207039, + 0.03389499709010124, + 0.05330438166856766, + 0.03366100415587425, + 0.07067227363586426, + 0.03414495661854744, + 0.03967295214533806 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3534", + "split": "test", + "style_correct": 1, + "style_probability": 0.4607383608818054, + "style_target": 0, + "support_probability": 0.9475903511047363, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06422531575622159, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7915652394294739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5664255023002625, + 0.3821260929107666, + 0.5492979288101196, + 0.5664255023002625, + 0.046982284635305405 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 1, + "style_probability": 0.6731687784194946, + "style_target": 1, + "support_probability": 0.09540161490440369, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.043411138279875416, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.883394181728363, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3537:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5899914503097534, + 0.49737998843193054, + 0.6012978553771973, + 0.5899914503097534, + 0.07642492651939392 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3537", + "split": "test", + "style_correct": 0, + "style_probability": 0.8443628549575806, + "style_target": 0, + "support_probability": 0.04243040084838867, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23599297653629492, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5524335503578186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4934063255786896, + 0.44989505410194397, + 0.6777229309082031, + 0.4934063255786896, + 0.13669353723526, + 0.03783736005425453 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 1, + "style_probability": 0.5903841853141785, + "style_target": 1, + "support_probability": 0.2679518759250641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17764067724960242, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.39987125992774963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3542:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48380154371261597, + 0.45767590403556824, + 0.7122685313224792, + 0.48380154371261597, + 0.05260038375854492 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3542", + "split": "test", + "style_correct": 1, + "style_probability": 0.38913586735725403, + "style_target": 0, + "support_probability": 0.358369916677475, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6513356105467665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13072477281093597, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8260784149169922, + 0.33026233315467834, + 0.3380516469478607, + 0.33026233315467834, + 0.17645269632339478, + 0.04858553409576416, + 0.07698646932840347, + 0.18890753388404846, + 0.08847110718488693 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.6552743911743164, + "style_target": 1, + "support_probability": 0.6877596378326416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.275758007745198, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054727308452129364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3560:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9359567761421204, + 0.3357475996017456, + 0.34175801277160645, + 0.3357475996017456, + 0.060147132724523544, + 0.07815917581319809, + 0.0960114598274231, + 0.10959530621767044, + 0.1495407223701477 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3560", + "split": "test", + "style_correct": 1, + "style_probability": 0.20361663401126862, + "style_target": 0, + "support_probability": 0.8385534882545471, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4928319917311228, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11758048832416534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3566:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3112717270851135, + 0.47692885994911194, + 0.9594037532806396, + 0.3112717270851135, + 0.04332084581255913, + 0.0460229218006134, + 0.07643144577741623, + 0.026425857096910477, + 0.08515757322311401, + 0.08924565464258194, + 0.05027027800679207 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.36221781373023987, + "style_target": 1, + "support_probability": 0.8187360763549805, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8626160781323904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1418451964855194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3566:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.389083594083786, + 0.4215320646762848, + 0.9674359560012817, + 0.389083594083786, + 0.03833477571606636, + 0.04287467151880264, + 0.08592382073402405, + 0.09389283508062363, + 0.04273027181625366, + 0.06949012726545334 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3566", + "split": "test", + "style_correct": 0, + "style_probability": 0.878241777420044, + "style_target": 0, + "support_probability": 0.8612204194068909, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42815515509229835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16161637008190155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4122384786605835, + 0.5285413861274719, + 0.807601273059845, + 0.4636896252632141, + 0.06877125054597855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.8565302491188049, + "style_target": 1, + "support_probability": 0.48182013630867004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0105774996109142, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27931180596351624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3570:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45893487334251404, + 0.5846859216690063, + 0.8307563662528992, + 0.5367850065231323, + 0.08851876109838486 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3570", + "split": "test", + "style_correct": 1, + "style_probability": 0.1501370221376419, + "style_target": 0, + "support_probability": 0.31542646884918213, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14823376073704875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21864835917949677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5636260509490967, + 0.9226760268211365, + 0.4974144399166107, + 0.4171000123023987, + 0.04079778864979744, + 0.07411938160657883 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6520988941192627, + "style_target": 1, + "support_probability": 0.43214476108551025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17175033064197986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12674356997013092, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5471528768539429, + 0.8698663115501404, + 0.5162813067436218, + 0.4217735230922699, + 0.11636997759342194, + 0.11314460635185242 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3577", + "split": "test", + "style_correct": 0, + "style_probability": 0.547467052936554, + "style_target": 0, + "support_probability": 0.47303974628448486, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6997170987316252, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07742959260940552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43906596302986145, + 0.6897839903831482, + 0.9141591787338257, + 0.06858834624290466, + 0.1091923639178276, + 0.023895075544714928, + 0.08160120993852615, + 0.04543273150920868, + 0.10065560787916183 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.6849479675292969, + "style_target": 1, + "support_probability": 0.8265155553817749, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47143760306383337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.040765780955553055, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4065588712692261, + 0.6950804591178894, + 0.8796776533126831, + 0.1023375615477562, + 0.10122140496969223, + 0.07153863459825516 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3578", + "split": "test", + "style_correct": 1, + "style_probability": 0.2774803340435028, + "style_target": 0, + "support_probability": 0.8788630366325378, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007432251320758087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9325007796287537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7506155967712402, + 0.2991708517074585, + 0.185174360871315, + 0.19353607296943665, + 0.09977111220359802, + 0.0647982507944107, + 0.06184760853648186, + 0.08133434504270554, + 0.07897300273180008, + 0.12079951167106628 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.876420795917511, + "style_target": 1, + "support_probability": 0.019526230171322823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006128410609243182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8859594464302063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3580:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8577500581741333, + 0.3490479588508606, + 0.19500291347503662, + 0.21160458028316498, + 0.09527407586574554, + 0.06852655112743378, + 0.20082257688045502, + 0.08779576420783997, + 0.06628618389368057, + 0.1016324982047081 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3580", + "split": "test", + "style_correct": 1, + "style_probability": 0.2370401918888092, + "style_target": 0, + "support_probability": 0.12390502542257309, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36159541991368366, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22492387890815735, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32562750577926636, + 0.6393024921417236, + 0.26848047971725464, + 0.13874505460262299, + 0.09108997881412506 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 0, + "style_probability": 0.396867036819458, + "style_target": 1, + "support_probability": 0.736100435256958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3988421587223883, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29483553767204285, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33787015080451965, + 0.5860669016838074, + 0.29485461115837097, + 0.09787493944168091, + 0.06209693104028702 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3597", + "split": "test", + "style_correct": 0, + "style_probability": 0.5512540340423584, + "style_target": 0, + "support_probability": 0.7996553182601929, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005432796632844904, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8698899745941162, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4268455505371094, + 0.5935406684875488, + 0.20121149718761444, + 0.4896997809410095, + 0.09828788787126541, + 0.12490057200193405 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 0, + "style_probability": 0.17967326939105988, + "style_target": 1, + "support_probability": 0.02942134067416191, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008456804658199814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9326769113540649, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3603:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45710667967796326, + 0.6447932124137878, + 0.18020930886268616, + 0.5488388538360596, + 0.09823136776685715, + 0.2589724063873291 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3603", + "split": "test", + "style_correct": 1, + "style_probability": 0.4949949383735657, + "style_target": 0, + "support_probability": 0.04077592492103577, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.10843681837727015, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6648509502410889, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44050920009613037, + 0.6535190939903259, + 0.565100908279419, + 0.4119838774204254, + 0.08610748499631882, + 0.16026946902275085, + 0.056796155869960785, + 0.13030102849006653, + 0.0983128696680069 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 0, + "style_probability": 0.3356669843196869, + "style_target": 1, + "support_probability": 0.2058567851781845, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006936619993775925, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8638138771057129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3605:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4604050815105438, + 0.8450043797492981, + 0.7301246523857117, + 0.617489755153656, + 0.0959276407957077, + 0.25414469838142395, + 0.07681646943092346, + 0.0959199070930481, + 0.13680268824100494 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:3605", + "split": "test", + "style_correct": 1, + "style_probability": 0.12129577249288559, + "style_target": 0, + "support_probability": 0.08370856195688248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4405306245794129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13908155262470245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.48028796911239624, + 0.36589524149894714, + 0.48028796911239624, + 0.9467231631278992, + 0.07049256563186646, + 0.1409149020910263, + 0.09360592812299728, + 0.10490880906581879, + 0.19863606989383698, + 0.09711192548274994 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 1, + "style_probability": 0.5586661696434021, + "style_target": 1, + "support_probability": 0.6148496866226196, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5131127496190402, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2239960879087448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3619:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.476081907749176, + 0.3689863383769989, + 0.476081907749176, + 0.9396305084228516, + 0.056305501610040665, + 0.1234576553106308, + 0.04929335042834282, + 0.08645232021808624, + 0.09619447588920593, + 0.11044041812419891, + 0.08058593422174454 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3619", + "split": "test", + "style_correct": 0, + "style_probability": 0.730057954788208, + "style_target": 0, + "support_probability": 0.5194541811943054, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015219746937504722, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7540011405944824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2846907079219818, + 0.6793642640113831, + 0.278058260679245, + 0.23959171772003174, + 0.053302157670259476 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.8646401166915894, + "style_target": 1, + "support_probability": 0.029035186395049095, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012646333133213167, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6662216782569885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3621:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2779860198497772, + 0.6400622129440308, + 0.27489587664604187, + 0.2469438910484314, + 0.09103385359048843, + 0.1565355658531189, + 0.1147354245185852, + 0.10422562062740326, + 0.1733582317829132, + 0.12402569502592087, + 0.14580267667770386 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3621", + "split": "test", + "style_correct": 1, + "style_probability": 0.429500013589859, + "style_target": 0, + "support_probability": 0.03191275894641876, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016849498623805807, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.674547016620636, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45466622710227966, + 0.3372694253921509, + 0.5214483141899109, + 0.6254892945289612, + 0.06832338869571686, + 0.07936698943376541 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.6962428092956543, + "style_target": 1, + "support_probability": 0.11256808787584305, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001303421998009938, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7423657178878784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3631:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.540507972240448, + 0.3783493638038635, + 0.5309276580810547, + 0.6364797949790955, + 0.06989789754152298, + 0.21170787513256073 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3631", + "split": "test", + "style_correct": 1, + "style_probability": 0.13560694456100464, + "style_target": 0, + "support_probability": 0.17974533140659332, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0010983521576524162, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9847434759140015, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5333929061889648, + 0.22480754554271698, + 0.21293707191944122, + 0.6216096878051758, + 0.04377235099673271, + 0.09319167584180832, + 0.13329645991325378, + 0.09649097174406052, + 0.12415575236082077 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 1, + "style_probability": 0.6125327348709106, + "style_target": 1, + "support_probability": 0.01200183667242527, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0025378830541426542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9821390509605408, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3642:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4594520628452301, + 0.24450287222862244, + 0.27007782459259033, + 0.6055585145950317, + 0.21160662174224854, + 0.08059658855199814, + 0.13777998089790344, + 0.06713907420635223, + 0.09924042224884033 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3642", + "split": "test", + "style_correct": 0, + "style_probability": 0.6492156982421875, + "style_target": 0, + "support_probability": 0.00898791290819645, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3677418786523816, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4556436240673065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3285582959651947, + 0.3100408613681793, + 0.3966776728630066, + 0.3285582959651947, + 0.27803510427474976, + 0.053495075553655624, + 0.04612349346280098, + 0.030032720416784286, + 0.06464990973472595, + 0.05475955829024315, + 0.05355928838253021 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.9111559391021729, + "style_target": 1, + "support_probability": 0.7183022499084473, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30546835865040656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13952426612377167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3650:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32612869143486023, + 0.2944900393486023, + 0.49259689450263977, + 0.32612869143486023, + 0.056214384734630585, + 0.12726284563541412, + 0.05978839099407196, + 0.03864211216568947, + 0.13285550475120544 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3650", + "split": "test", + "style_correct": 1, + "style_probability": 0.2992525100708008, + "style_target": 0, + "support_probability": 0.8822053074836731, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5837596537341484, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21508201956748962, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21729014813899994, + 0.8149096965789795, + 0.21174468100070953, + 0.31349149346351624, + 0.05693298578262329 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 1, + "style_probability": 0.6471487879753113, + "style_target": 1, + "support_probability": 0.8375730514526367, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5796290853067774, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16512583196163177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3653:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2109566628932953, + 0.828916072845459, + 0.24895602464675903, + 0.34484565258026123, + 0.07013686746358871, + 0.07618820667266846, + 0.13214567303657532, + 0.20943547785282135, + 0.12343008816242218, + 0.06091004237532616, + 0.17044217884540558 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3653", + "split": "test", + "style_correct": 0, + "style_probability": 0.5344558358192444, + "style_target": 0, + "support_probability": 0.8268464803695679, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18657557860340293, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5334584712982178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28656405210494995, + 0.37091031670570374, + 0.22746260464191437, + 0.8157128095626831, + 0.2103147804737091, + 0.15559834241867065 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.6854689717292786, + "style_target": 1, + "support_probability": 0.42419204115867615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.033233842516011254, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.36532101035118103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3659:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28903764486312866, + 0.39463767409324646, + 0.18660010397434235, + 0.910946249961853, + 0.10502425581216812, + 0.17487822473049164 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3659", + "split": "test", + "style_correct": 1, + "style_probability": 0.27858975529670715, + "style_target": 0, + "support_probability": 0.6934946179389954, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01624936228118279, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7385153770446777, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1408131867647171, + 0.5664392709732056, + 0.15109004080295563, + 0.5499892234802246, + 0.10349148511886597, + 0.1378197968006134, + 0.3208957314491272, + 0.16409622132778168, + 0.08821353316307068 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 1, + "style_probability": 0.5178813338279724, + "style_target": 1, + "support_probability": 0.1006380021572113, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13383088943023758, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6984726190567017, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3668:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.14878810942173004, + 0.6711313128471375, + 0.13787813484668732, + 0.6351546049118042, + 0.18490533530712128, + 0.09366857260465622, + 0.08642174303531647, + 0.16563889384269714, + 0.3293446898460388 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3668", + "split": "test", + "style_correct": 0, + "style_probability": 0.8569462299346924, + "style_target": 0, + "support_probability": 0.2953941226005554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08196932246734522, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.751875638961792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1767929047346115, + 0.14086614549160004, + 0.8574091196060181, + 0.2327379435300827, + 0.05300503224134445, + 0.1334238350391388, + 0.048814140260219574, + 0.08326023817062378, + 0.08691806346178055, + 0.17978796362876892, + 0.09168760478496552 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 1, + "style_probability": 0.5767179131507874, + "style_target": 1, + "support_probability": 0.16545429825782776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06629931462871587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49282407760620117, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3690:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18201982975006104, + 0.12846101820468903, + 0.8755596876144409, + 0.20581048727035522, + 0.08799365162849426, + 0.06922487914562225, + 0.10035628825426102, + 0.08282541483640671, + 0.09254491329193115 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3690", + "split": "test", + "style_correct": 1, + "style_probability": 0.279763787984848, + "style_target": 0, + "support_probability": 0.376456081867218, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02560548968719267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6851755380630493, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2723597288131714, + 0.504723846912384, + 0.15731526911258698, + 0.5312409996986389, + 0.12477704137563705 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 0, + "style_probability": 0.28981903195381165, + "style_target": 1, + "support_probability": 0.11383894830942154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02514641005210816, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.784223198890686, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2795396149158478, + 0.43352437019348145, + 0.14776071906089783, + 0.4966685175895691, + 0.10472239553928375 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3712", + "split": "test", + "style_correct": 1, + "style_probability": 0.4524006247520447, + "style_target": 0, + "support_probability": 0.09469785541296005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020396671551431433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4158766567707062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24459628760814667, + 0.8268684148788452, + 0.30414366722106934, + 0.3399601876735687, + 0.061047185212373734, + 0.1411387324333191 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.2743072509765625, + "style_target": 1, + "support_probability": 0.15074685215950012, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028133522573910334, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4905920624732971, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3729:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22281745076179504, + 0.8559034466743469, + 0.3212411403656006, + 0.34640881419181824, + 0.03714046999812126, + 0.24212907254695892 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3729", + "split": "test", + "style_correct": 0, + "style_probability": 0.5512406826019287, + "style_target": 0, + "support_probability": 0.12032772600650787, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03704722776875369, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8962451219558716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6822990775108337, + 0.26223716139793396, + 0.3982900083065033, + 0.32917889952659607, + 0.100411556661129, + 0.046445801854133606, + 0.06654340028762817, + 0.07094045728445053, + 0.07828623056411743 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 1, + "style_probability": 0.5614410638809204, + "style_target": 1, + "support_probability": 0.08300372958183289, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.034924214253429404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7709987759590149, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3732:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8199114799499512, + 0.391920268535614, + 0.622201144695282, + 0.3959949314594269, + 0.056136712431907654, + 0.09143230319023132, + 0.0635143369436264, + 0.0513504296541214, + 0.14539378881454468 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3732", + "split": "test", + "style_correct": 1, + "style_probability": 0.373477041721344, + "style_target": 0, + "support_probability": 0.26795750856399536, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03949500235860625, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8381302952766418, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34442049264907837, + 0.34442049264907837, + 0.5442933440208435, + 0.4939523935317993, + 0.0548844113945961, + 0.09262281656265259, + 0.04212266951799393, + 0.09128371626138687, + 0.11265633255243301, + 0.052739255130290985, + 0.141217902302742 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 0, + "style_probability": 0.28288692235946655, + "style_target": 1, + "support_probability": 0.29210594296455383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10156069626527219, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7589757442474365, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3738:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31106895208358765, + 0.31106895208358765, + 0.5683561563491821, + 0.6062154173851013, + 0.06596743315458298, + 0.05502226948738098, + 0.042445968836545944, + 0.07259844243526459, + 0.04356438294053078, + 0.054605256766080856, + 0.09671682864427567 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3738", + "split": "test", + "style_correct": 1, + "style_probability": 0.46549689769744873, + "style_target": 0, + "support_probability": 0.44000518321990967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.853906708790138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06656252592802048, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9286182522773743, + 0.5761346220970154, + 0.192841038107872, + 0.22535090148448944, + 0.10777013003826141 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.8271344304084778, + "style_target": 1, + "support_probability": 0.8951144218444824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6632150036113416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054411631077528, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3740:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.877324640750885, + 0.5545416474342346, + 0.20468463003635406, + 0.2727363407611847, + 0.11285419762134552, + 0.05104782432317734, + 0.048136595636606216, + 0.049747150391340256, + 0.057758595794439316, + 0.24156072735786438, + 0.06293229758739471 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3740", + "split": "test", + "style_correct": 1, + "style_probability": 0.33061879873275757, + "style_target": 0, + "support_probability": 0.8998993635177612, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003144478354446644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8397230505943298, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.48227953910827637, + 0.3962458670139313, + 0.3412391245365143, + 0.5576946139335632, + 0.08858264982700348, + 0.29516172409057617 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.869826078414917, + "style_target": 1, + "support_probability": 0.006457197479903698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00048163981903075876, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8158665895462036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3751:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5613240003585815, + 0.4201015532016754, + 0.3909975290298462, + 0.560187816619873, + 0.06538853049278259, + 0.08609167486429214 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3751", + "split": "test", + "style_correct": 1, + "style_probability": 0.26963621377944946, + "style_target": 0, + "support_probability": 0.012109760195016861, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6160278543736125, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2990953028202057, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8982740640640259, + 0.2731740176677704, + 0.3141707181930542, + 0.7218272089958191, + 0.08958485722541809, + 0.1210440993309021, + 0.09966684132814407, + 0.05845115706324577, + 0.10603786259889603 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 1, + "style_probability": 0.8237884640693665, + "style_target": 1, + "support_probability": 0.4980977773666382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5070915512770924, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25130346417427063, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8551633954048157, + 0.22296740114688873, + 0.31160447001457214, + 0.7902228236198425, + 0.05140328034758568, + 0.06461311131715775, + 0.04398474097251892, + 0.05211523175239563, + 0.037266943603754044 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:3756", + "split": "test", + "style_correct": 0, + "style_probability": 0.6292961239814758, + "style_target": 0, + "support_probability": 0.6073713898658752, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32951466697916487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46820974349975586, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5106214284896851, + 0.18103834986686707, + 0.8520859479904175, + 0.2988583445549011, + 0.09577581286430359, + 0.15561294555664062, + 0.13071022927761078, + 0.13873182237148285, + 0.06468403339385986, + 0.1183541938662529 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.8388376235961914, + "style_target": 1, + "support_probability": 0.22424861788749695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1603821498791591, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2518899738788605, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3758:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5167351365089417, + 0.18904109299182892, + 0.848349392414093, + 0.32434970140457153, + 0.09068524837493896, + 0.17837479710578918, + 0.07792891561985016, + 0.10251080244779587, + 0.17220701277256012 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3758", + "split": "test", + "style_correct": 1, + "style_probability": 0.34640395641326904, + "style_target": 0, + "support_probability": 0.4565032124519348, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.22580599518333597, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2741875648498535, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6042702794075012, + 0.4760551154613495, + 0.8734135627746582, + 0.45950648188591003, + 0.19315294921398163 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.568905234336853, + "style_target": 1, + "support_probability": 0.510330080986023, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.011222403092196466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3605455756187439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3762:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.693579912185669, + 0.4223150610923767, + 0.923707902431488, + 0.4642297327518463, + 0.11982898414134979 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3762", + "split": "test", + "style_correct": 1, + "style_probability": 0.11098037660121918, + "style_target": 0, + "support_probability": 0.5394512414932251, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05300336763977143, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7262542247772217, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27382028102874756, + 0.49209368228912354, + 0.6298801898956299, + 0.37945619225502014, + 0.10670366138219833, + 0.07404055446386337 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 1, + "style_probability": 0.8030733466148376, + "style_target": 1, + "support_probability": 0.1268300861120224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023521874057319543, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7694469690322876, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3764:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.272000789642334, + 0.43044665455818176, + 0.6547212600708008, + 0.2985854744911194, + 0.05845426395535469, + 0.1130145713686943 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3764", + "split": "test", + "style_correct": 0, + "style_probability": 0.6715728044509888, + "style_target": 0, + "support_probability": 0.05184333398938179, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2653420313399124, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7686083912849426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21063436567783356, + 0.6085652112960815, + 0.46490657329559326, + 0.7288675308227539, + 0.056177813559770584, + 0.07896411418914795, + 0.10056954622268677, + 0.10065743327140808, + 0.05693867430090904 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.8579279780387878, + "style_target": 1, + "support_probability": 0.32575738430023193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08772057802858325, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7485465407371521, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3774:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18761318922042847, + 0.5836511850357056, + 0.47928735613822937, + 0.7388650178909302, + 0.07127887010574341, + 0.11979331076145172 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3774", + "split": "test", + "style_correct": 1, + "style_probability": 0.3287711441516876, + "style_target": 0, + "support_probability": 0.3576712906360626, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6609209990676135, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15522964298725128, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9520557522773743, + 0.3823796510696411, + 0.12293075770139694, + 0.19469498097896576, + 0.17065338790416718, + 0.0743899941444397, + 0.0882532149553299, + 0.07888086885213852, + 0.0814121887087822, + 0.343849241733551, + 0.12956717610359192 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.7882744073867798, + "style_target": 1, + "support_probability": 0.5845984816551208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17002563877828614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12155670672655106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3776:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9809547662734985, + 0.45824724435806274, + 0.11524464935064316, + 0.20654526352882385, + 0.11599569022655487, + 0.04963919520378113, + 0.05986162647604942, + 0.11922822892665863, + 0.10015975683927536, + 0.13435611128807068 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3776", + "split": "test", + "style_correct": 1, + "style_probability": 0.11577054858207703, + "style_target": 0, + "support_probability": 0.7573016285896301, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29804623930839824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.22178100049495697, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33761847019195557, + 0.21335797011852264, + 0.23548543453216553, + 0.8167509436607361, + 0.11709140241146088 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.7541479468345642, + "style_target": 1, + "support_probability": 0.34881484508514404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11190763665733795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.32793891429901123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3795:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33937016129493713, + 0.20619146525859833, + 0.23090943694114685, + 0.7532082200050354, + 0.10056523233652115 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3795", + "split": "test", + "style_correct": 1, + "style_probability": 0.3789418339729309, + "style_target": 0, + "support_probability": 0.3307664394378662, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7994470933329438, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.47524574398994446, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15483218431472778, + 0.14585080742835999, + 0.7622448801994324, + 0.9180475473403931, + 0.033798784017562866, + 0.15517973899841309 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 1, + "style_probability": 0.9566880464553833, + "style_target": 1, + "support_probability": 0.6171141266822815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5258798104554501, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4472183585166931, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3819:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17784850299358368, + 0.17110544443130493, + 0.7423827648162842, + 0.8936028480529785, + 0.052018798887729645 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:3819", + "split": "test", + "style_correct": 0, + "style_probability": 0.6697003841400146, + "style_target": 0, + "support_probability": 0.5928042531013489, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.987034491640742, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.005230461712926626, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:3820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9968544840812683, + 0.8713943958282471, + 0.15200379490852356, + 0.10792646557092667, + 0.05949912965297699, + 0.17305319011211395, + 0.07811184972524643, + 0.05511108413338661, + 0.09625307470560074 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.9017350077629089, + "style_target": 1, + "support_probability": 0.9876842498779297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6359809853481657, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0038537010550498962, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9997722506523132, + 0.9392301440238953, + 0.2429337203502655, + 0.1746755838394165, + 0.10501820594072342, + 0.06335345655679703, + 0.12396698445081711, + 0.09377217292785645, + 0.12026426941156387 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:3820", + "split": "test", + "style_correct": 1, + "style_probability": 0.11675433069467545, + "style_target": 0, + "support_probability": 0.9868174195289612, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031775510965328948, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9751855134963989, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3572019040584564, + 0.36562660336494446, + 0.3572019040584564, + 0.871487021446228, + 0.16263550519943237, + 0.09547385573387146, + 0.08955954015254974, + 0.1916104406118393, + 0.17478477954864502, + 0.08064734190702438, + 0.181182861328125 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 1, + "style_probability": 0.6694005131721497, + "style_target": 1, + "support_probability": 0.012088894844055176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038326930518630675, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8084913492202759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3827:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38917914032936096, + 0.35555073618888855, + 0.38917914032936096, + 0.8834576606750488, + 0.13257896900177002, + 0.17925381660461426, + 0.11834406852722168, + 0.11199922859668732, + 0.2881850600242615, + 0.2514617443084717, + 0.14081981778144836 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3827", + "split": "test", + "style_correct": 0, + "style_probability": 0.5863455533981323, + "style_target": 0, + "support_probability": 0.10548344254493713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08049461519009336, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1164507120847702, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2772654891014099, + 0.861323893070221, + 0.3119584023952484, + 0.41113904118537903, + 0.07711520791053772 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 0, + "style_probability": 0.4243467450141907, + "style_target": 1, + "support_probability": 0.23604826629161835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42156694767900044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0856812596321106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3829:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29783767461776733, + 0.9130061268806458, + 0.31095683574676514, + 0.48098868131637573, + 0.027060305699706078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3829", + "split": "test", + "style_correct": 0, + "style_probability": 0.9070412516593933, + "style_target": 0, + "support_probability": 0.1944824755191803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013758545240676438, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9174742102622986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24873362481594086, + 0.23393689095973969, + 0.36849191784858704, + 0.24873362481594086, + 0.2206653654575348, + 0.11399753391742706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 0, + "style_probability": 0.3607938289642334, + "style_target": 1, + "support_probability": 0.07100686430931091, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031770700633299676, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8671066164970398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3834:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4242710471153259, + 0.3606593906879425, + 0.6418434977531433, + 0.4242710471153259, + 0.08208267390727997, + 0.18927061557769775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3834", + "split": "test", + "style_correct": 1, + "style_probability": 0.44760212302207947, + "style_target": 0, + "support_probability": 0.1410849243402481, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1255711379240303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3513447642326355, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3265059292316437, + 0.36913198232650757, + 0.8272315263748169, + 0.2540903389453888, + 0.10127802193164825, + 0.1397356390953064, + 0.14820893108844757, + 0.11354313045740128, + 0.23092694580554962 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 0, + "style_probability": 0.3927323520183563, + "style_target": 1, + "support_probability": 0.349455326795578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.134167952304769, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5576959252357483, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3453041613101959, + 0.35656917095184326, + 0.796712338924408, + 0.261731892824173, + 0.06616757810115814, + 0.17698360979557037, + 0.12991464138031006, + 0.0767800435423851, + 0.08235707879066467 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3838", + "split": "test", + "style_correct": 0, + "style_probability": 0.594542384147644, + "style_target": 0, + "support_probability": 0.2240787297487259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03884075906509343, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8704450726509094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5945615768432617, + 0.5048031806945801, + 0.7340217232704163, + 0.5945615768432617, + 0.06431876122951508, + 0.047436028718948364, + 0.08160598576068878, + 0.09477893263101578, + 0.06678114831447601, + 0.037990279495716095, + 0.2134196162223816 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.42714864015579224, + "style_target": 1, + "support_probability": 0.10618191957473755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0414001373653159, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7651883363723755, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3842:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5837056636810303, + 0.5509046912193298, + 0.7301763296127319, + 0.5837056636810303, + 0.04030022397637367, + 0.06839783489704132, + 0.04393792897462845, + 0.09631799161434174, + 0.3451928198337555, + 0.06579544395208359, + 0.03688759729266167 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3842", + "split": "test", + "style_correct": 0, + "style_probability": 0.7470909357070923, + "style_target": 0, + "support_probability": 0.11005032807588577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.79869516531177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0668434277176857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.895890474319458, + 0.24164891242980957, + 0.13476641476154327, + 0.5030652284622192, + 0.0733729749917984 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.84326171875, + "style_target": 1, + "support_probability": 0.8677873611450195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04958201274464535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11374805867671967, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3844:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9184420704841614, + 0.3397490680217743, + 0.16258060932159424, + 0.6576310992240906, + 0.12466315925121307 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3844", + "split": "test", + "style_correct": 1, + "style_probability": 0.14057236909866333, + "style_target": 0, + "support_probability": 0.7678651213645935, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4987008969563789, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5416727662086487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46573859453201294, + 0.3615092635154724, + 0.848461389541626, + 0.22064568102359772, + 0.047422364354133606, + 0.0682111531496048 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 1, + "style_probability": 0.828563392162323, + "style_target": 1, + "support_probability": 0.45411884784698486, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2102703762353908, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6532720923423767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3846:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5213810801506042, + 0.40903469920158386, + 0.7965442538261414, + 0.20031090080738068, + 0.11688199639320374, + 0.13933882117271423 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3846", + "split": "test", + "style_correct": 0, + "style_probability": 0.6571393609046936, + "style_target": 0, + "support_probability": 0.25098907947540283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11411666553682666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6351983547210693, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2910368740558624, + 0.3871554434299469, + 0.28382739424705505, + 0.9193668365478516, + 0.17458899319171906, + 0.07525185495615005, + 0.12743216753005981, + 0.10047027468681335, + 0.24727299809455872 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.8004409670829773, + "style_target": 1, + "support_probability": 0.17777466773986816, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013677519717364198, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7218034267425537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3855:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28611496090888977, + 0.3994390666484833, + 0.27506065368652344, + 0.8755440711975098, + 0.14938820898532867, + 0.1773928850889206 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3855", + "split": "test", + "style_correct": 1, + "style_probability": 0.23478655517101288, + "style_target": 0, + "support_probability": 0.14156095683574677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03964039411788466, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8021066784858704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5678732395172119, + 0.3763737678527832, + 0.7530674338340759, + 0.43632495403289795, + 0.1422988921403885, + 0.095908522605896, + 0.07022583484649658, + 0.09652139991521835, + 0.06435523182153702, + 0.0620105154812336, + 0.126495823264122 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.8502686619758606, + "style_target": 1, + "support_probability": 0.04072326794266701, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0043113578907710845, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8203112483024597, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3860:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7504464387893677, + 0.5125634074211121, + 0.7518747448921204, + 0.5389593839645386, + 0.23734936118125916, + 0.08045691251754761, + 0.19947366416454315, + 0.05446980521082878, + 0.0594087652862072, + 0.09283433109521866 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3860", + "split": "test", + "style_correct": 1, + "style_probability": 0.20131197571754456, + "style_target": 0, + "support_probability": 0.1124735176563263, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.744639845030477, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.062496647238731384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5972490906715393, + 0.9658663272857666, + 0.39635762572288513, + 0.28172430396080017, + 0.19155488908290863 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.21517615020275116, + "style_target": 1, + "support_probability": 0.819212019443512, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8000977381594332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10012254118919373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3865:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6423466801643372, + 0.9489384293556213, + 0.4133523404598236, + 0.34056657552719116, + 0.061139270663261414 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3865", + "split": "test", + "style_correct": 0, + "style_probability": 0.5286503434181213, + "style_target": 0, + "support_probability": 0.7445972561836243, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014503312184859373, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9088190793991089, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21662233769893646, + 0.44596168398857117, + 0.30707496404647827, + 0.5644487738609314, + 0.054630763828754425, + 0.10774236917495728 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.3181895613670349, + "style_target": 1, + "support_probability": 0.010264202952384949, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009659925293507673, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9228478074073792, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3867:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2373291552066803, + 0.4136642515659332, + 0.2789558172225952, + 0.5859894156455994, + 0.2153940200805664, + 0.1718757003545761 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3867", + "split": "test", + "style_correct": 0, + "style_probability": 0.6720572710037231, + "style_target": 0, + "support_probability": 0.021284380927681923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012344439947487018, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8134455680847168, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5270174741744995, + 0.30811527371406555, + 0.22669462859630585, + 0.6348893046379089, + 0.05607125163078308, + 0.06529638916254044, + 0.08912482112646103, + 0.045908164232969284, + 0.12440633773803711 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 0, + "style_probability": 0.30814000964164734, + "style_target": 1, + "support_probability": 0.06510569155216217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02490834205755732, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7848221659660339, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3868:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6257440447807312, + 0.26137688755989075, + 0.2820103168487549, + 0.7214517593383789, + 0.03576647490262985, + 0.10799254477024078, + 0.07142633199691772, + 0.05809948965907097, + 0.07258839905261993 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3868", + "split": "test", + "style_correct": 1, + "style_probability": 0.462255597114563, + "style_target": 0, + "support_probability": 0.18326374888420105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1862243860697177, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6159002184867859, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.19847196340560913, + 0.31378212571144104, + 0.7736763954162598, + 0.26252129673957825, + 0.0953541100025177, + 0.04684075340628624, + 0.11305931955575943, + 0.1156461089849472, + 0.049963582307100296, + 0.14409692585468292 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 0, + "style_probability": 0.423015296459198, + "style_target": 1, + "support_probability": 0.3588910698890686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2932809218070709, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.624540388584137, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3870:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22960872948169708, + 0.3488197922706604, + 0.7113239169120789, + 0.26477062702178955, + 0.036774035543203354, + 0.0571918785572052, + 0.04820358380675316, + 0.047135476022958755, + 0.08720941841602325, + 0.054392289370298386, + 0.0718366727232933 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3870", + "split": "test", + "style_correct": 0, + "style_probability": 0.6954056024551392, + "style_target": 0, + "support_probability": 0.40390658378601074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7097649794501031, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2597931921482086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18879558145999908, + 0.38140758872032166, + 0.3620959222316742, + 0.8953439593315125, + 0.24119649827480316 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 1, + "style_probability": 0.508836030960083, + "style_target": 1, + "support_probability": 0.8526713252067566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8650493893559279, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23673409223556519, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17334012687206268, + 0.3755754232406616, + 0.33961910009384155, + 0.9396976828575134, + 0.24044696986675262 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3899", + "split": "test", + "style_correct": 0, + "style_probability": 0.8218579292297363, + "style_target": 0, + "support_probability": 0.8423293232917786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7915960069896676, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029362812638282776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9604330658912659, + 0.38065919280052185, + 0.7060830593109131, + 0.3806084990501404, + 0.07287725061178207, + 0.22833478450775146 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 1, + "style_probability": 0.6240168809890747, + "style_target": 1, + "support_probability": 0.6757468581199646, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5106001370428288, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02380002848803997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3912:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9670307636260986, + 0.3877120614051819, + 0.6814547181129456, + 0.35765114426612854, + 0.05550652742385864 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:3912", + "split": "test", + "style_correct": 1, + "style_probability": 0.4182995557785034, + "style_target": 0, + "support_probability": 0.613753616809845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031312504206987553, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9602287411689758, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35463106632232666, + 0.34351012110710144, + 0.3610583245754242, + 0.24748213589191437, + 0.12447060644626617, + 0.14997167885303497, + 0.10336937010288239, + 0.17687158286571503, + 0.2357352375984192 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 0, + "style_probability": 0.3485563397407532, + "style_target": 1, + "support_probability": 0.036746080964803696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004977951303005085, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9657343626022339, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3915:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38154760003089905, + 0.3669039011001587, + 0.34116801619529724, + 0.22663605213165283, + 0.12434421479701996, + 0.22360162436962128, + 0.1461455374956131, + 0.10474557429552078, + 0.1666470170021057 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3915", + "split": "test", + "style_correct": 1, + "style_probability": 0.09008929878473282, + "style_target": 0, + "support_probability": 0.056306637823581696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17272937642151834, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5918017625808716, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23910143971443176, + 0.3210285007953644, + 0.6033225059509277, + 0.7361034154891968, + 0.06546987593173981, + 0.07538871467113495, + 0.17714011669158936, + 0.09112926572561264, + 0.0992106944322586, + 0.1349029392004013, + 0.13435098528862 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.47054505348205566, + "style_target": 1, + "support_probability": 0.40612611174583435, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23056714967022887, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8296555280685425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3943:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20636405050754547, + 0.31857019662857056, + 0.5314774513244629, + 0.7611863613128662, + 0.08478234708309174, + 0.0898328423500061, + 0.08110881596803665, + 0.10682141780853271, + 0.11255697160959244, + 0.24461546540260315 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3943", + "split": "test", + "style_correct": 0, + "style_probability": 0.8357989192008972, + "style_target": 0, + "support_probability": 0.22602273523807526, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7942416490177067, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07646641880273819, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8771002888679504, + 0.34835854172706604, + 0.7384006381034851, + 0.1871250718832016, + 0.2111714780330658 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.8628514409065247, + "style_target": 1, + "support_probability": 0.7346639037132263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42961845083511113, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06191364303231239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3956:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9101523160934448, + 0.41892382502555847, + 0.8115611672401428, + 0.1694307029247284, + 0.09056377410888672 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:3956", + "split": "test", + "style_correct": 1, + "style_probability": 0.4194706678390503, + "style_target": 0, + "support_probability": 0.7888399958610535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031163307098358795, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7950730323791504, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3183654546737671, + 0.24342435598373413, + 0.5272255539894104, + 0.3183654546737671, + 0.06344959884881973, + 0.1346580684185028 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 1, + "style_probability": 0.6972272992134094, + "style_target": 1, + "support_probability": 0.11069110780954361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04962787349249176, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8281940221786499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31259724497795105, + 0.3030478060245514, + 0.41155093908309937, + 0.31259724497795105, + 0.1785866618156433, + 0.13928064703941345 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3958", + "split": "test", + "style_correct": 0, + "style_probability": 0.6167195439338684, + "style_target": 0, + "support_probability": 0.13453362882137299, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01871675872198608, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9012255072593689, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31032827496528625, + 0.5998361110687256, + 0.18433897197246552, + 0.512322187423706, + 0.07566963136196136, + 0.1336415559053421, + 0.09177663177251816, + 0.07827027142047882, + 0.15905475616455078 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.864378809928894, + "style_target": 1, + "support_probability": 0.06825514137744904, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01588534346604642, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7970137000083923, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3969:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2704100012779236, + 0.6236220002174377, + 0.15810967981815338, + 0.5334285497665405, + 0.17888881266117096, + 0.16775928437709808 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3969", + "split": "test", + "style_correct": 1, + "style_probability": 0.3400157690048218, + "style_target": 0, + "support_probability": 0.1265655755996704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016716194319044142, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9698062539100647, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1341370940208435, + 0.23838084936141968, + 0.6340226531028748, + 0.12151626497507095, + 0.04771950840950012, + 0.2027120292186737, + 0.06375884264707565, + 0.10074717551469803, + 0.08564084768295288, + 0.08389637619256973, + 0.11729742586612701 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.5062278509140015, + "style_target": 1, + "support_probability": 0.00931611005216837, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00023734162986659835, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9677327275276184, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:3974:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16327939927577972, + 0.2800308167934418, + 0.6515654921531677, + 0.13582658767700195, + 0.11498203128576279, + 0.08329296112060547, + 0.26677241921424866, + 0.1290682703256607, + 0.08771846443414688, + 0.11803890764713287 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:3974", + "split": "test", + "style_correct": 1, + "style_probability": 0.13167282938957214, + "style_target": 0, + "support_probability": 0.011875380761921406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7621570726244897, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14538554847240448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22972221672534943, + 0.33373281359672546, + 0.468722939491272, + 0.9027533531188965, + 0.11122239381074905 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.5589252710342407, + "style_target": 1, + "support_probability": 0.8871570825576782, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08147133542843799, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2993577718734741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29302555322647095, + 0.4332074820995331, + 0.5363617539405823, + 0.9673896431922913, + 0.13605666160583496 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:3975", + "split": "test", + "style_correct": 1, + "style_probability": 0.16355867683887482, + "style_target": 0, + "support_probability": 0.697245180606842, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.48449828011225193, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24016131460666656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26316747069358826, + 0.12383627891540527, + 0.9112568497657776, + 0.17076772451400757, + 0.10854313522577286, + 0.08094900101423264 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 0, + "style_probability": 0.46254536509513855, + "style_target": 1, + "support_probability": 0.8087190389633179, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7307615089432851, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1413489282131195, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3982:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27979183197021484, + 0.0954664871096611, + 0.9519520401954651, + 0.12317633628845215, + 0.10216174274682999, + 0.13162563741207123 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:3982", + "split": "test", + "style_correct": 0, + "style_probability": 0.6745321750640869, + "style_target": 0, + "support_probability": 0.8901060819625854, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7779866461894471, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.018000777810811996, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4133531153202057, + 0.9258548617362976, + 0.21640470623970032, + 0.31133973598480225, + 0.03613471984863281, + 0.04551509767770767, + 0.07391541451215744, + 0.053745366632938385, + 0.08398083597421646 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 1, + "style_probability": 0.5877086520195007, + "style_target": 1, + "support_probability": 0.9046986103057861, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5998707698916235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010543029755353928, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:3997:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4175407290458679, + 0.9302564263343811, + 0.2263822853565216, + 0.29860758781433105, + 0.0706363245844841, + 0.06464143842458725 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:3997", + "split": "test", + "style_correct": 1, + "style_probability": 0.38860225677490234, + "style_target": 0, + "support_probability": 0.9120739698410034, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12935868881059506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6413292288780212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20704935491085052, + 0.7437062859535217, + 0.17841440439224243, + 0.3105110824108124, + 0.09470519423484802, + 0.22541537880897522, + 0.07671570032835007, + 0.10587757080793381, + 0.15347325801849365, + 0.11056455224752426, + 0.12612061202526093 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 1, + "style_probability": 0.5146520137786865, + "style_target": 1, + "support_probability": 0.38874396681785583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0703180681136182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5149776935577393, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4009:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26335981488227844, + 0.9133946299552917, + 0.226457417011261, + 0.42022570967674255, + 0.15037140250205994, + 0.11553647369146347, + 0.19975589215755463, + 0.15251129865646362, + 0.08982793241739273, + 0.14473043382167816 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4009", + "split": "test", + "style_correct": 1, + "style_probability": 0.3168058395385742, + "style_target": 0, + "support_probability": 0.571064829826355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48743497857515194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29039186239242554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2357015758752823, + 0.259080171585083, + 0.7479500770568848, + 0.2357015758752823, + 0.14092706143856049 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 1, + "style_probability": 0.7496351599693298, + "style_target": 1, + "support_probability": 0.46411949396133423, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27823444961973287, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6210725903511047, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4010:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22384032607078552, + 0.24141137301921844, + 0.7297080159187317, + 0.22384032607078552, + 0.15624067187309265 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4010", + "split": "test", + "style_correct": 0, + "style_probability": 0.896926760673523, + "style_target": 0, + "support_probability": 0.1450677365064621, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13756389269807826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.642544150352478, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5834266543388367, + 0.4175589084625244, + 0.43788963556289673, + 0.4759114682674408, + 0.11572583019733429, + 0.04737367480993271 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.6422418355941772, + "style_target": 1, + "support_probability": 0.2715308368206024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10641120782615497, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37334388494491577, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4020:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5937290787696838, + 0.4693082571029663, + 0.4231317937374115, + 0.5024769306182861, + 0.06730629503726959 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4020", + "split": "test", + "style_correct": 1, + "style_probability": 0.37302306294441223, + "style_target": 0, + "support_probability": 0.39885157346725464, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15075549135413982, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5572630763053894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9157158136367798, + 0.3922768533229828, + 0.32964950799942017, + 0.3922768533229828, + 0.08233461529016495, + 0.09381280094385147, + 0.24567966163158417, + 0.17658573389053345, + 0.09692467749118805 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 0, + "style_probability": 0.4572412371635437, + "style_target": 1, + "support_probability": 0.2430606335401535, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2695150724876427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13067634403705597, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9720247983932495, + 0.39044269919395447, + 0.3321039080619812, + 0.39044269919395447, + 0.05352161452174187, + 0.11211053282022476, + 0.12276553362607956, + 0.07207781821489334, + 0.13174375891685486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4036", + "split": "test", + "style_correct": 1, + "style_probability": 0.09170962125062943, + "style_target": 0, + "support_probability": 0.9311562776565552, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06721618783610593, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7992030382156372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22173842787742615, + 0.22173842787742615, + 0.7105041742324829, + 0.2889390289783478, + 0.06788866966962814, + 0.03914420306682587, + 0.028989452868700027, + 0.23326840996742249, + 0.07489563524723053, + 0.04795156419277191 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.375517874956131, + "style_target": 1, + "support_probability": 0.13268102705478668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05164635811192307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8417315483093262, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4038:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20950114727020264, + 0.20950114727020264, + 0.7118946313858032, + 0.2646719813346863, + 0.06648263335227966, + 0.038635559380054474, + 0.06927785277366638, + 0.06260214000940323, + 0.08185216039419174, + 0.07514302432537079, + 0.046984538435935974 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4038", + "split": "test", + "style_correct": 0, + "style_probability": 0.6433331966400146, + "style_target": 0, + "support_probability": 0.11291681230068207, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.47578536160375506, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5453453063964844, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1878228634595871, + 0.17467722296714783, + 0.396852046251297, + 0.956274688243866, + 0.05738641321659088 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.8056224584579468, + "style_target": 1, + "support_probability": 0.36229732632637024, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07213659410652708, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5232859253883362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4039:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22107325494289398, + 0.21293659508228302, + 0.5242858529090881, + 0.9802850484848022, + 0.09872981905937195 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4039", + "split": "test", + "style_correct": 1, + "style_probability": 0.20420421659946442, + "style_target": 0, + "support_probability": 0.5085117220878601, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7051444291696782, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1620761901140213, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17813079059123993, + 0.5079678893089294, + 0.9688382148742676, + 0.17813079059123993, + 0.09433948248624802, + 0.08029365539550781 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 1, + "style_probability": 0.7524219751358032, + "style_target": 1, + "support_probability": 0.5812267661094666, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6648422012778319, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12267421185970306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2277504950761795, + 0.487411767244339, + 0.9362371563911438, + 0.2277504950761795, + 0.047876447439193726, + 0.06861545890569687 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4042", + "split": "test", + "style_correct": 0, + "style_probability": 0.6438024640083313, + "style_target": 0, + "support_probability": 0.5907695293426514, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006794008462228661, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9931285381317139, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.185445174574852, + 0.350791871547699, + 0.12571465969085693, + 0.6790693402290344, + 0.0702609121799469, + 0.11740812659263611, + 0.10296281427145004, + 0.13176706433296204, + 0.07873279601335526 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.7584457397460938, + "style_target": 1, + "support_probability": 0.003952702973037958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001257718448567438, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9713841080665588, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4055:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20057471096515656, + 0.3514743447303772, + 0.12282906472682953, + 0.6586927175521851, + 0.19691678881645203, + 0.21626102924346924 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4055", + "split": "test", + "style_correct": 1, + "style_probability": 0.2375604659318924, + "style_target": 0, + "support_probability": 0.009442112408578396, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010394162740253675, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8732179999351501, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3485783636569977, + 0.7850897908210754, + 0.7226090431213379, + 0.2710895240306854, + 0.11591753363609314, + 0.08598966151475906, + 0.04362422600388527, + 0.1391645222902298, + 0.12947706878185272, + 0.09244818240404129 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.8871199488639832, + "style_target": 1, + "support_probability": 0.01604369841516018, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002654275818972486, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7198026180267334, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4062:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4348464012145996, + 0.8642051815986633, + 0.8375675678253174, + 0.3384193778038025, + 0.10765557736158371, + 0.08116205781698227, + 0.3263867199420929, + 0.14357800781726837, + 0.0900234803557396, + 0.13663643598556519 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:4062", + "split": "test", + "style_correct": 1, + "style_probability": 0.1898840069770813, + "style_target": 0, + "support_probability": 0.11073258519172668, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4762537246927537, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06420721858739853, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2858591377735138, + 0.9680861830711365, + 0.18635833263397217, + 0.1161992996931076, + 0.02423872798681259 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.3305380046367645, + "style_target": 1, + "support_probability": 0.7503670454025269, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6354892769697571, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11729087680578232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4073:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29747986793518066, + 0.9568527936935425, + 0.20903897285461426, + 0.130708709359169, + 0.18849113583564758 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4073", + "split": "test", + "style_correct": 0, + "style_probability": 0.6027472019195557, + "style_target": 0, + "support_probability": 0.6189413666725159, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01677160176846481, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8431493043899536, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4721876382827759, + 0.3748186230659485, + 0.4602168798446655, + 0.6098393797874451, + 0.12308803200721741, + 0.03034653700888157 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.3743664622306824, + "style_target": 1, + "support_probability": 0.20530818402767181, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08472722310956372, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8544127345085144, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4074:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5095963478088379, + 0.37161698937416077, + 0.5083336234092712, + 0.6138908267021179, + 0.031395334750413895, + 0.06052467226982117 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4074", + "split": "test", + "style_correct": 0, + "style_probability": 0.8513435125350952, + "style_target": 0, + "support_probability": 0.2530370354652405, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008797791592599412, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6112719774246216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.523910641670227, + 0.3786308765411377, + 0.32599517703056335, + 0.5256195664405823, + 0.08156155794858932, + 0.22667694091796875, + 0.1908463090658188, + 0.20174983143806458, + 0.13283945620059967 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 0, + "style_probability": 0.3473806083202362, + "style_target": 1, + "support_probability": 0.051486361771821976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0723651046073215, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22135969996452332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4075:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7276358604431152, + 0.5059645771980286, + 0.4174247086048126, + 0.7267030477523804, + 0.10056333243846893, + 0.10964349657297134, + 0.15398254990577698, + 0.07340306788682938, + 0.1452939510345459 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4075", + "split": "test", + "style_correct": 1, + "style_probability": 0.4577632248401642, + "style_target": 0, + "support_probability": 0.4553871154785156, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.48111076534922503, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1478358507156372, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8126179575920105, + 0.5048056840896606, + 0.5790098905563354, + 0.5370497107505798, + 0.09668051451444626, + 0.1016218289732933, + 0.057292867451906204, + 0.11547217518091202, + 0.11534461379051208, + 0.13276046514511108, + 0.07297394424676895 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 0, + "style_probability": 0.2533918619155884, + "style_target": 1, + "support_probability": 0.900343656539917, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.676225108715677, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19060690701007843, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4076:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8154277205467224, + 0.6166261434555054, + 0.6292127966880798, + 0.5300766825675964, + 0.16600683331489563, + 0.12967011332511902, + 0.094429612159729, + 0.10866522043943405, + 0.12396181374788284, + 0.10623469948768616, + 0.12322942167520523 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4076", + "split": "test", + "style_correct": 1, + "style_probability": 0.44639497995376587, + "style_target": 0, + "support_probability": 0.9330763220787048, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44770043756017947, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1360984444618225, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3403041660785675, + 0.28086021542549133, + 0.9406315684318542, + 0.24331489205360413, + 0.06694549322128296 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.7284643054008484, + "style_target": 1, + "support_probability": 0.49183303117752075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2269930189169553, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10716858506202698, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4078:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30594348907470703, + 0.28457343578338623, + 0.9254651665687561, + 0.26601171493530273, + 0.1311374455690384, + 0.07508835196495056, + 0.08774825185537338, + 0.13996455073356628, + 0.08826044946908951, + 0.08033333718776703, + 0.1357446014881134 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4078", + "split": "test", + "style_correct": 1, + "style_probability": 0.3577899932861328, + "style_target": 0, + "support_probability": 0.46319031715393066, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8636966416235161, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2613173723220825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4071056842803955, + 0.5405157804489136, + 0.9494633078575134, + 0.32914695143699646, + 0.04454540088772774, + 0.10000604391098022 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.8506988883018494, + "style_target": 1, + "support_probability": 0.6396692991256714, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46672447908258086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2767867147922516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47701123356819153, + 0.6504224538803101, + 0.9772880673408508, + 0.3690323233604431, + 0.07430972903966904, + 0.151373028755188 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4082", + "split": "test", + "style_correct": 1, + "style_probability": 0.38345879316329956, + "style_target": 0, + "support_probability": 0.7462661266326904, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5341359791546694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5604907274246216, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4465727210044861, + 0.30828461050987244, + 0.16466982662677765, + 0.8432899117469788, + 0.06513005495071411, + 0.0255326796323061, + 0.04088037088513374, + 0.04282185435295105, + 0.03267379105091095 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 1, + "style_probability": 0.8474302887916565, + "style_target": 1, + "support_probability": 0.5164207220077515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4442851802612502, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5267559289932251, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4083:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.400539755821228, + 0.3297176957130432, + 0.18212181329727173, + 0.7840678095817566, + 0.050575532019138336, + 0.0639646053314209, + 0.04441358894109726, + 0.1097990944981575, + 0.05414111167192459 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4083", + "split": "test", + "style_correct": 0, + "style_probability": 0.6568021178245544, + "style_target": 0, + "support_probability": 0.6693180203437805, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4991475180324514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5660097002983093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4162900447845459, + 0.2792002260684967, + 0.8585672974586487, + 0.4162900447845459, + 0.07586973160505295, + 0.056276727467775345, + 0.06203663721680641, + 0.0516701340675354, + 0.09359680116176605, + 0.050998251885175705, + 0.04981720820069313 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.9091478586196899, + "style_target": 1, + "support_probability": 0.3655770421028137, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26232472732933587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44306686520576477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4090:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4196033477783203, + 0.24894388020038605, + 0.8462242484092712, + 0.4196033477783203, + 0.15374405682086945, + 0.10359033942222595, + 0.12756071984767914, + 0.06592259556055069, + 0.06491005420684814 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4090", + "split": "test", + "style_correct": 1, + "style_probability": 0.37630903720855713, + "style_target": 0, + "support_probability": 0.5964891910552979, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.14386286397141745, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.79445880651474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7563222646713257, + 0.10741279274225235, + 0.5243550539016724, + 0.29611900448799133, + 0.046223048120737076 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.776150643825531, + "style_target": 1, + "support_probability": 0.3035977780818939, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00286165399338001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7485038042068481, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8218805193901062, + 0.14258012175559998, + 0.5073909163475037, + 0.352970153093338, + 0.12153682857751846 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4096", + "split": "test", + "style_correct": 1, + "style_probability": 0.08713552355766296, + "style_target": 0, + "support_probability": 0.28394192457199097, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8249298795060298, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07486093789339066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4101:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33186203241348267, + 0.9806032776832581, + 0.47172266244888306, + 0.33186203241348267, + 0.10701622068881989, + 0.08981122821569443 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 1, + "style_probability": 0.6251217126846313, + "style_target": 1, + "support_probability": 0.9420521855354309, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7677532323963163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09060350805521011, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4101:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3249267637729645, + 0.9710382223129272, + 0.4643585681915283, + 0.3249267637729645, + 0.17034587264060974, + 0.3449268043041229 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4101", + "split": "test", + "style_correct": 0, + "style_probability": 0.5271171927452087, + "style_target": 0, + "support_probability": 0.9238085746765137, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9214014940317128, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.028788039460778236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9858254790306091, + 0.48543354868888855, + 0.34626084566116333, + 0.32598841190338135, + 0.10823943465948105, + 0.08908634632825851, + 0.05415415018796921, + 0.0539773553609848, + 0.08167771995067596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.8775079846382141, + "style_target": 1, + "support_probability": 0.69432133436203, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.673894189793316, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031202085316181183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4104:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9828705787658691, + 0.4811347723007202, + 0.32116419076919556, + 0.33889228105545044, + 0.11771731078624725, + 0.07242626696825027 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4104", + "split": "test", + "style_correct": 1, + "style_probability": 0.42612195014953613, + "style_target": 0, + "support_probability": 0.6555009484291077, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4696420456725719, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044887199997901917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.34062817692756653, + 0.9467164278030396, + 0.6217760443687439, + 0.22469303011894226, + 0.05812942609190941, + 0.18788614869117737, + 0.07796840369701385, + 0.05315837636590004, + 0.07995613664388657, + 0.11956478655338287 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 1, + "style_probability": 0.6445459127426147, + "style_target": 1, + "support_probability": 0.5228545069694519, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09631749767922672, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04059693589806557, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4105:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.43456169962882996, + 0.965522050857544, + 0.6988975405693054, + 0.2708679437637329, + 0.08387642353773117, + 0.08214226365089417, + 0.09656911343336105, + 0.11164838820695877, + 0.14140072464942932, + 0.0910775437951088 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4105", + "split": "test", + "style_correct": 1, + "style_probability": 0.2472306489944458, + "style_target": 0, + "support_probability": 0.5184560418128967, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8311430436824594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.27315446734428406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41577666997909546, + 0.4456973075866699, + 0.4078149199485779, + 0.9161669611930847, + 0.08229251950979233 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 1, + "style_probability": 0.8610007166862488, + "style_target": 1, + "support_probability": 0.5550134778022766, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6726440755901422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16467316448688507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4107:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43694809079170227, + 0.4979057312011719, + 0.46960729360580444, + 0.9102315902709961, + 0.09455960243940353 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4107", + "split": "test", + "style_correct": 0, + "style_probability": 0.5014477968215942, + "style_target": 0, + "support_probability": 0.6920881271362305, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.710455709256408, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0599374920129776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3411998152732849, + 0.9277259111404419, + 0.20796534419059753, + 0.30219411849975586, + 0.0484718419611454, + 0.036685504019260406 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.7954073548316956, + "style_target": 1, + "support_probability": 0.5286836624145508, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27324605432512783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03268001973628998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33958831429481506, + 0.92796790599823, + 0.24554094672203064, + 0.2989763021469116, + 0.04871119558811188 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4113", + "split": "test", + "style_correct": 1, + "style_probability": 0.1486123502254486, + "style_target": 0, + "support_probability": 0.6540576815605164, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28460581003663776, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40336355566978455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1806727945804596, + 0.23584426939487457, + 0.9294209480285645, + 0.1797621101140976, + 0.1170758306980133, + 0.0441817082464695, + 0.14210212230682373, + 0.10479893535375595, + 0.06159813329577446 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.7735716700553894, + "style_target": 1, + "support_probability": 0.28685563802719116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04317620497004482, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.345019668340683, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23180654644966125, + 0.30002135038375854, + 0.9605621099472046, + 0.20663101971149445, + 0.09304647892713547, + 0.11608508229255676, + 0.1083037257194519, + 0.08961685001850128, + 0.13730749487876892 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4118", + "split": "test", + "style_correct": 1, + "style_probability": 0.3167033791542053, + "style_target": 0, + "support_probability": 0.3541639447212219, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8621517115592391, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020976968109607697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.425626665353775, + 0.5397406220436096, + 0.22035714983940125, + 0.9989311099052429, + 0.05074870213866234, + 0.059342361986637115, + 0.0839400589466095, + 0.04314159229397774, + 0.06297390162944794, + 0.04176755249500275, + 0.042855218052864075 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 1, + "style_probability": 0.682723343372345, + "style_target": 1, + "support_probability": 0.8888923525810242, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7039165584664957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03407943248748779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4119:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45274099707603455, + 0.49254482984542847, + 0.19947275519371033, + 0.9928625226020813, + 0.14239181578159332, + 0.03744896501302719, + 0.08761663734912872, + 0.07672275602817535, + 0.10134965181350708, + 0.11266179382801056, + 0.09855548292398453 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4119", + "split": "test", + "style_correct": 0, + "style_probability": 0.5688406825065613, + "style_target": 0, + "support_probability": 0.8378878235816956, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6230094023154464, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2842777669429779, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.40993019938468933, + 0.8148326277732849, + 0.17284847795963287, + 0.26098331809043884, + 0.11367862671613693 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.4214304983615875, + "style_target": 1, + "support_probability": 0.8513686060905457, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8719437936153972, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09432116895914078, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4133:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4224614202976227, + 0.9356350302696228, + 0.11564701795578003, + 0.16751915216445923, + 0.047369785606861115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4133", + "split": "test", + "style_correct": 0, + "style_probability": 0.7094193696975708, + "style_target": 0, + "support_probability": 0.9247841835021973, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.21940854186176395, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1614466905593872, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2649267017841339, + 0.1955111175775528, + 0.2649267017841339, + 0.7398096323013306, + 0.08247461915016174, + 0.04061814397573471 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 0, + "style_probability": 0.3222663104534149, + "style_target": 1, + "support_probability": 0.636412501335144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13419894842599867, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1241392269730568, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4139:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24445202946662903, + 0.2150808423757553, + 0.24445202946662903, + 0.8164757490158081, + 0.08550001680850983 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4139", + "split": "test", + "style_correct": 1, + "style_probability": 0.1599661111831665, + "style_target": 0, + "support_probability": 0.6789739727973938, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6477663281161199, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031054718419909477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26783522963523865, + 0.14322112500667572, + 0.8472467660903931, + 0.33833184838294983, + 0.07273458689451218, + 0.08595987409353256, + 0.07686257362365723, + 0.08539967983961105, + 0.06596080958843231 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 1, + "style_probability": 0.6425207257270813, + "style_target": 1, + "support_probability": 0.8135253190994263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41533160992806906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.022375527769327164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4162:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4335066080093384, + 0.1406489759683609, + 0.9215709567070007, + 0.4678100645542145, + 0.06030416861176491, + 0.06363992393016815, + 0.07110057771205902, + 0.05815264210104942, + 0.10993605107069016 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4162", + "split": "test", + "style_correct": 1, + "style_probability": 0.3433275818824768, + "style_target": 0, + "support_probability": 0.9225419163703918, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5471179067745027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02975725755095482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3772673010826111, + 0.233191579580307, + 0.39597028493881226, + 0.8907123804092407, + 0.06940475851297379, + 0.06782076507806778, + 0.058766305446624756, + 0.06944485008716583, + 0.08242999017238617, + 0.09954684227705002, + 0.06096946448087692 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 1, + "style_probability": 0.7162472605705261, + "style_target": 1, + "support_probability": 0.6297078728675842, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5604187481630944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03916852921247482, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4163:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3755117952823639, + 0.24354217946529388, + 0.4185405969619751, + 0.8784155249595642, + 0.04566704481840134, + 0.025040872395038605, + 0.03406115993857384, + 0.03290078043937683, + 0.048655200749635696, + 0.07936989516019821, + 0.046060118824243546 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4163", + "split": "test", + "style_correct": 0, + "style_probability": 0.9113057851791382, + "style_target": 0, + "support_probability": 0.49459126591682434, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5065395443883887, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.11125078052282333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.284006804227829, + 0.9688766598701477, + 0.21807506680488586, + 0.4722742438316345, + 0.05660925805568695 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 0, + "style_probability": 0.2761593461036682, + "style_target": 1, + "support_probability": 0.5500743389129639, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15948648550364108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23713725805282593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4169:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31523123383522034, + 0.9913217425346375, + 0.251720130443573, + 0.5271246433258057, + 0.1123519316315651 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4169", + "split": "test", + "style_correct": 1, + "style_probability": 0.15796972811222076, + "style_target": 0, + "support_probability": 0.615196943283081, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00825402500145861, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4741377532482147, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43114620447158813, + 0.6291379332542419, + 0.3249039053916931, + 0.576794445514679, + 0.0548546239733696, + 0.06775709241628647 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.46261516213417053, + "style_target": 1, + "support_probability": 0.11093276739120483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009436482443891945, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6758930683135986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4177:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3605422377586365, + 0.5357531309127808, + 0.3362957537174225, + 0.562894344329834, + 0.07031809538602829, + 0.09588441997766495 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4177", + "split": "test", + "style_correct": 0, + "style_probability": 0.6665238738059998, + "style_target": 0, + "support_probability": 0.04516499862074852, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2891121165877273, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12784665822982788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4186:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4383087456226349, + 0.24475564062595367, + 0.9797505140304565, + 0.2704952657222748, + 0.1241205483675003, + 0.10666421800851822, + 0.07035601139068604, + 0.08409295231103897, + 0.0687350183725357 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.4345889389514923, + "style_target": 1, + "support_probability": 0.6166141033172607, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7829331906084747, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13438774645328522, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4186:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4541078805923462, + 0.2472197711467743, + 0.9908026456832886, + 0.28746679425239563, + 0.07165896892547607, + 0.03960787504911423, + 0.08306524902582169, + 0.07782850414514542, + 0.10102332383394241 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4186", + "split": "test", + "style_correct": 0, + "style_probability": 0.8747609853744507, + "style_target": 0, + "support_probability": 0.6300852298736572, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2296820014779755, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3902297914028168, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45841020345687866, + 0.11956454068422318, + 0.7229203581809998, + 0.8485392332077026, + 0.09829103946685791, + 0.1282980889081955, + 0.13187597692012787, + 0.11699937283992767, + 0.10738483816385269, + 0.07153214514255524, + 0.10952547937631607 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 0, + "style_probability": 0.38253262639045715, + "style_target": 1, + "support_probability": 0.5981101393699646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02103632169706604, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42529296875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.543301522731781, + 0.09522055834531784, + 0.8253818154335022, + 0.8874714970588684, + 0.09284129738807678, + 0.14101754128932953, + 0.15666663646697998, + 0.08589835464954376, + 0.05491367727518082, + 0.11322738230228424 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4195", + "split": "test", + "style_correct": 1, + "style_probability": 0.1199798583984375, + "style_target": 0, + "support_probability": 0.5880631804466248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7696822045231038, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0712043046951294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9296965003013611, + 0.09206519275903702, + 0.13808582723140717, + 0.3406229615211487, + 0.0463164858520031 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 1, + "style_probability": 0.7936878204345703, + "style_target": 1, + "support_probability": 0.8065034747123718, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.708071064707891, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0522371381521225, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4200:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8961166739463806, + 0.10991345345973969, + 0.15181268751621246, + 0.4418187737464905, + 0.14366719126701355 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4200", + "split": "test", + "style_correct": 0, + "style_probability": 0.6095616221427917, + "style_target": 0, + "support_probability": 0.8524818420410156, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06586556379846797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5971906781196594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32111984491348267, + 0.11422151327133179, + 0.8010826110839844, + 0.3608850836753845, + 0.07733923941850662, + 0.05561969056725502 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.7020590901374817, + "style_target": 1, + "support_probability": 0.18988476693630219, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036366247415171575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.531013548374176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4202:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36534324288368225, + 0.1209181398153305, + 0.7818800210952759, + 0.35950198769569397, + 0.2345770299434662 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4202", + "split": "test", + "style_correct": 1, + "style_probability": 0.4036670923233032, + "style_target": 0, + "support_probability": 0.23342861235141754, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11041641617462394, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6131923794746399, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37562668323516846, + 0.24391180276870728, + 0.48594793677330017, + 0.48994573950767517, + 0.04523360729217529, + 0.05482640862464905, + 0.24823114275932312, + 0.04756060615181923, + 0.0217258483171463 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.6580389738082886, + "style_target": 1, + "support_probability": 0.18083907663822174, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007570463852351938, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6997860670089722, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4210:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3297061026096344, + 0.28750234842300415, + 0.5423610210418701, + 0.44202369451522827, + 0.047515515238046646, + 0.049141157418489456, + 0.09669871628284454, + 0.048758964985609055, + 0.08807425945997238 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4210", + "split": "test", + "style_correct": 1, + "style_probability": 0.16262662410736084, + "style_target": 0, + "support_probability": 0.2070397436618805, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03821968906744355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38094231486320496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28078973293304443, + 0.7980549931526184, + 0.39477255940437317, + 0.12387888133525848, + 0.15542489290237427, + 0.09419091790914536, + 0.11118974536657333, + 0.0591450110077858, + 0.09684392064809799, + 0.10323306173086166, + 0.043114498257637024 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 0, + "style_probability": 0.4792231619358063, + "style_target": 1, + "support_probability": 0.18340541422367096, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13613180495383403, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3106444180011749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4217:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.308612585067749, + 0.7646278142929077, + 0.3151613771915436, + 0.13228662312030792, + 0.03724508732557297, + 0.07912877202033997, + 0.04096062108874321, + 0.03392121568322182, + 0.03229321911931038, + 0.14401714503765106 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4217", + "split": "test", + "style_correct": 1, + "style_probability": 0.4855442941188812, + "style_target": 0, + "support_probability": 0.24800662696361542, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.458617968461752, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1261061131954193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19736413657665253, + 0.16495607793331146, + 0.27630698680877686, + 0.9049749970436096, + 0.14367182552814484 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.3096415400505066, + "style_target": 1, + "support_probability": 0.7854790687561035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8552167092136286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.21791037917137146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4231:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.16680730879306793, + 0.1892484426498413, + 0.2906455099582672, + 0.9098431468009949, + 0.1427319347858429 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4231", + "split": "test", + "style_correct": 0, + "style_probability": 0.9272804260253906, + "style_target": 0, + "support_probability": 0.8211050629615784, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001568855230857439, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.604388952255249, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2580789625644684, + 0.6638297438621521, + 0.5702333450317383, + 0.40350812673568726, + 0.08177968114614487, + 0.09765233099460602 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 0, + "style_probability": 0.28071314096450806, + "style_target": 1, + "support_probability": 0.010746152140200138, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003753658969959939, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7170326113700867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4233:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2176642119884491, + 0.6633970737457275, + 0.5884600877761841, + 0.4380972981452942, + 0.09752801060676575, + 0.19726626574993134 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4233", + "split": "test", + "style_correct": 1, + "style_probability": 0.0997094064950943, + "style_target": 0, + "support_probability": 0.013421187177300453, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.39934065562282406, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11119190603494644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4167135953903198, + 0.41154372692108154, + 0.490388423204422, + 0.41154372692108154, + 0.05902866646647453, + 0.04876260459423065, + 0.03678608685731888, + 0.06840290129184723, + 0.03456728160381317 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.4595664143562317, + "style_target": 1, + "support_probability": 0.9060035347938538, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4097250463036061, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16903680562973022, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4240:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4302634000778198, + 0.4327215254306793, + 0.46151086688041687, + 0.4327215254306793, + 0.050388965755701065, + 0.03397894650697708, + 0.06309084594249725, + 0.06900857388973236, + 0.040823910385370255 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4240", + "split": "test", + "style_correct": 0, + "style_probability": 0.6395953297615051, + "style_target": 0, + "support_probability": 0.915857195854187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04655797227062109, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06180145964026451, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3371674120426178, + 0.2710122764110565, + 0.3371674120426178, + 0.9528514742851257, + 0.10571091622114182, + 0.14603690803050995, + 0.13344964385032654, + 0.0784943699836731, + 0.08445969223976135, + 0.05168019235134125, + 0.1176600232720375 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.2467566281557083, + "style_target": 1, + "support_probability": 0.23117388784885406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19541418896608587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10403238236904144, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4243:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35354912281036377, + 0.2413315623998642, + 0.35354912281036377, + 0.9608541131019592, + 0.04519427940249443, + 0.06200789660215378, + 0.08101816475391388, + 0.049698200076818466, + 0.045604411512613297, + 0.08388849347829819, + 0.06746942549943924 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4243", + "split": "test", + "style_correct": 0, + "style_probability": 0.823616087436676, + "style_target": 0, + "support_probability": 0.1739656925201416, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7151737774106337, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.06950580328702927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36409926414489746, + 0.24780413508415222, + 0.12790705263614655, + 0.990770161151886, + 0.062285955995321274 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.5765396952629089, + "style_target": 1, + "support_probability": 0.8661094903945923, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17732569826945976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08449268341064453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4251:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42888087034225464, + 0.32921648025512695, + 0.16392570734024048, + 0.9946708083152771, + 0.13316644728183746 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4251", + "split": "test", + "style_correct": 1, + "style_probability": 0.18489237129688263, + "style_target": 0, + "support_probability": 0.8682827949523926, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7062418706761129, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07623866200447083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.869779646396637, + 0.2886667549610138, + 0.3929787576198578, + 0.4987834692001343, + 0.1339188665151596, + 0.07889029383659363 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.8307937979698181, + "style_target": 1, + "support_probability": 0.6742931604385376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22431650880995768, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16759595274925232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4252:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8014594912528992, + 0.331993967294693, + 0.3380086123943329, + 0.45080265402793884, + 0.1391567587852478, + 0.10579438507556915 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4252", + "split": "test", + "style_correct": 1, + "style_probability": 0.31743350625038147, + "style_target": 0, + "support_probability": 0.544696033000946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13931386601188578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8931472897529602, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21069581806659698, + 0.2961408197879791, + 0.9092165231704712, + 0.37878498435020447, + 0.05372334271669388, + 0.0908212661743164, + 0.07759510725736618, + 0.26265648007392883, + 0.12332998961210251 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.7522766590118408, + "style_target": 1, + "support_probability": 0.17193254828453064, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0873301136504905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7832301259040833, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4258:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20609334111213684, + 0.26762673258781433, + 0.8957237601280212, + 0.3800460696220398, + 0.11977653205394745, + 0.15145213901996613 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4258", + "split": "test", + "style_correct": 1, + "style_probability": 0.24648389220237732, + "style_target": 0, + "support_probability": 0.3471894860267639, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6027335123417998, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.052663564682006836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3992544710636139, + 0.6648105978965759, + 0.34731554985046387, + 0.9820773601531982, + 0.020658714696764946, + 0.024510527029633522, + 0.03299400210380554, + 0.030929123982787132, + 0.06716541945934296, + 0.08026020973920822, + 0.06793361157178879 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 1, + "style_probability": 0.524940550327301, + "style_target": 1, + "support_probability": 0.841966450214386, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13735209219441025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10568547993898392, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4267:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4150325357913971, + 0.6361172199249268, + 0.3801804780960083, + 0.9741331338882446, + 0.0764220729470253, + 0.06575837731361389, + 0.06534676998853683, + 0.06770111620426178, + 0.038044314831495285, + 0.06902438402175903 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4267", + "split": "test", + "style_correct": 1, + "style_probability": 0.3070647418498993, + "style_target": 0, + "support_probability": 0.7175333499908447, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6090869249188486, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23742938041687012, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.728373646736145, + 0.7101792097091675, + 0.513625979423523, + 0.728373646736145, + 0.11694308370351791 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 1, + "style_probability": 0.625122606754303, + "style_target": 1, + "support_probability": 0.9362378716468811, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1519256619123217, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1581016629934311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4273:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8729657530784607, + 0.8868564963340759, + 0.5711554884910583, + 0.8729657530784607, + 0.1310158669948578 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4273", + "split": "test", + "style_correct": 1, + "style_probability": 0.12142087519168854, + "style_target": 0, + "support_probability": 0.9191798567771912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00038572972046380116, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9868910908699036, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19435907900333405, + 0.39463120698928833, + 0.36465364694595337, + 0.25032496452331543, + 0.07950620353221893, + 0.18692362308502197 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 1, + "style_probability": 0.5611075758934021, + "style_target": 1, + "support_probability": 0.003898977767676115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030312702304337576, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.972609281539917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4281:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18125298619270325, + 0.36907142400741577, + 0.37260061502456665, + 0.25273415446281433, + 0.1691434383392334, + 0.17976994812488556 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4281", + "split": "test", + "style_correct": 0, + "style_probability": 0.8568949103355408, + "style_target": 0, + "support_probability": 0.011931576766073704, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3907022974720036, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22484928369522095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37631702423095703, + 0.577060878276825, + 0.5964802503585815, + 0.37631702423095703, + 0.08233585208654404, + 0.09947223961353302, + 0.10049805790185928, + 0.10713543742895126, + 0.04962190240621567 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 1, + "style_probability": 0.629914402961731, + "style_target": 1, + "support_probability": 0.6928762793540955, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24729197806174066, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2737525701522827, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4290:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3595336675643921, + 0.6008002758026123, + 0.6572892069816589, + 0.3595336675643921, + 0.06558629125356674, + 0.10060875862836838 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4290", + "split": "test", + "style_correct": 1, + "style_probability": 0.4488118588924408, + "style_target": 0, + "support_probability": 0.5434731245040894, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004134177515379699, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9247905611991882, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26999765634536743, + 0.41192400455474854, + 0.17522414028644562, + 0.14364421367645264, + 0.06176891550421715, + 0.09711071103811264, + 0.08488224446773529, + 0.07123565673828125, + 0.0431717224419117, + 0.07409052550792694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 1, + "style_probability": 0.6122837066650391, + "style_target": 1, + "support_probability": 0.030809950083494186, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005357941203790851, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.860797643661499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4293:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32091018557548523, + 0.4687277674674988, + 0.2067207247018814, + 0.15483462810516357, + 0.08625657856464386, + 0.08266425132751465, + 0.06494573503732681, + 0.07767249643802643, + 0.03581755608320236, + 0.08416764438152313 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4293", + "split": "test", + "style_correct": 0, + "style_probability": 0.6805196404457092, + "style_target": 0, + "support_probability": 0.06414550542831421, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7296778377780129, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.1571369767189026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48777180910110474, + 0.34506961703300476, + 0.17073571681976318, + 0.9725148677825928, + 0.053312405943870544 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 1, + "style_probability": 0.7173294425010681, + "style_target": 1, + "support_probability": 0.7745221257209778, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5631584248390027, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11079427599906921, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4295:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4791325032711029, + 0.31800660490989685, + 0.18513572216033936, + 0.9654216766357422, + 0.07951491326093674, + 0.03269018977880478, + 0.028778282925486565, + 0.047017063945531845, + 0.057988833636045456, + 0.05265047773718834, + 0.10300746560096741 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4295", + "split": "test", + "style_correct": 1, + "style_probability": 0.3739258646965027, + "style_target": 0, + "support_probability": 0.8429141044616699, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9167684028642817, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2101086527109146, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3474467396736145, + 0.959340512752533, + 0.4451565444469452, + 0.42569735646247864, + 0.11963997781276703, + 0.07981666177511215 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.924410879611969, + "style_target": 1, + "support_probability": 0.9014195799827576, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11257377423407933, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15079446136951447, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4297:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.304858535528183, + 0.9784820675849915, + 0.5093579292297363, + 0.5013389587402344, + 0.0758749321103096, + 0.15860845148563385 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4297", + "split": "test", + "style_correct": 1, + "style_probability": 0.14490467309951782, + "style_target": 0, + "support_probability": 0.8726895451545715, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6096988532897427, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12341993302106857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7673240303993225, + 0.8469103574752808, + 0.14327628910541534, + 0.7673240303993225, + 0.05096152424812317, + 0.17334924638271332, + 0.11992739140987396, + 0.1101943701505661, + 0.11648615449666977 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 1, + "style_probability": 0.7342261075973511, + "style_target": 1, + "support_probability": 0.5732520222663879, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45537588705300885, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19722777605056763, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4313:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6955674886703491, + 0.7993809580802917, + 0.19437584280967712, + 0.6955674886703491, + 0.11950188875198364, + 0.11139795184135437, + 0.049273498356342316, + 0.0702335387468338, + 0.09889942407608032 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4313", + "split": "test", + "style_correct": 0, + "style_probability": 0.5971193313598633, + "style_target": 0, + "support_probability": 0.5152153968811035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.053587433338222475, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6257666945457458, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6059443950653076, + 0.46246907114982605, + 0.6059443950653076, + 0.7983057498931885, + 0.11608149856328964, + 0.05317552760243416, + 0.16662882268428802, + 0.0852108970284462, + 0.11940981447696686, + 0.12256459146738052 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.6799287796020508, + "style_target": 1, + "support_probability": 0.08175724744796753, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025283479386142286, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6206700801849365, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4319:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5533175468444824, + 0.4336681663990021, + 0.5533175468444824, + 0.7696741819381714, + 0.1546241194009781, + 0.07476237416267395, + 0.06060335040092468, + 0.18139362335205078, + 0.07574416697025299 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4319", + "split": "test", + "style_correct": 1, + "style_probability": 0.39741796255111694, + "style_target": 0, + "support_probability": 0.08655940741300583, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10985147085293434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3456592261791229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2257603257894516, + 0.3740749955177307, + 0.22685584425926208, + 0.47089916467666626, + 0.09711507707834244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.8918336033821106, + "style_target": 1, + "support_probability": 0.14617958664894104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017450201374025207, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5958118438720703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4335:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2831263840198517, + 0.3843649625778198, + 0.2112116813659668, + 0.5018311738967896, + 0.06505265831947327 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4335", + "split": "test", + "style_correct": 1, + "style_probability": 0.1629892736673355, + "style_target": 0, + "support_probability": 0.1173766702413559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17960888440113632, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6985296607017517, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7655280232429504, + 0.10073162615299225, + 0.44404008984565735, + 0.47926992177963257, + 0.1339465230703354, + 0.1460106521844864 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 1, + "style_probability": 0.8176246285438538, + "style_target": 1, + "support_probability": 0.29495084285736084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2043890385856386, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44683754444122314, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4344:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6935241222381592, + 0.11486075073480606, + 0.44615039229393005, + 0.5055168867111206, + 0.18139545619487762, + 0.16702714562416077 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4344", + "split": "test", + "style_correct": 0, + "style_probability": 0.6053866744041443, + "style_target": 0, + "support_probability": 0.4806501269340515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007523240185794919, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4636598527431488, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40907368063926697, + 0.8615161776542664, + 0.376600444316864, + 0.5482087731361389, + 0.08514325320720673, + 0.10540466010570526, + 0.0706167072057724, + 0.0407961830496788, + 0.06675329059362411 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.5940220952033997, + "style_target": 1, + "support_probability": 0.019040938466787338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031593223297632375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5535138249397278, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4349:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41474273800849915, + 0.8256133198738098, + 0.3428986668586731, + 0.5436387658119202, + 0.06829668581485748, + 0.08136030286550522 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4349", + "split": "test", + "style_correct": 1, + "style_probability": 0.26639917492866516, + "style_target": 0, + "support_probability": 0.007554101292043924, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06135059966847666, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.822753369808197, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11887770146131516, + 0.29733946919441223, + 0.8266605138778687, + 0.626602292060852, + 0.07832425832748413, + 0.1848565638065338, + 0.08839455246925354, + 0.1273944228887558, + 0.09020577371120453, + 0.08812308311462402 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.7983355522155762, + "style_target": 1, + "support_probability": 0.06886367499828339, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0039261198497572165, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8006686568260193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4366:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18423938751220703, + 0.3597398102283478, + 0.9163560271263123, + 0.7228209972381592, + 0.11484060436487198, + 0.060866717249155045, + 0.12553563714027405, + 0.08867599070072174, + 0.06870801001787186, + 0.11574822664260864 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:4366", + "split": "test", + "style_correct": 1, + "style_probability": 0.09950456023216248, + "style_target": 0, + "support_probability": 0.11967875808477402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11761622616461763, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.22225156426429749, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23985779285430908, + 0.1904352903366089, + 0.30218085646629333, + 0.7970212697982788, + 0.08653254806995392 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.5756994485855103, + "style_target": 1, + "support_probability": 0.2840604782104492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004214459336549793, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.40496957302093506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4375:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2070533186197281, + 0.24968542158603668, + 0.32202261686325073, + 0.8719419240951538, + 0.13213679194450378 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4375", + "split": "test", + "style_correct": 1, + "style_probability": 0.14571727812290192, + "style_target": 0, + "support_probability": 0.22860725224018097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6715128111822821, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1557880938053131, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8372825384140015, + 0.7586743831634521, + 0.3683421015739441, + 0.7021160125732422, + 0.07631991803646088, + 0.02918488346040249 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 0, + "style_probability": 0.4534701704978943, + "style_target": 1, + "support_probability": 0.8904609680175781, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8329028328888376, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11552213877439499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4379:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.848045825958252, + 0.8533547520637512, + 0.4244263172149658, + 0.6816576719284058, + 0.08179650455713272, + 0.05229044333100319 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:4379", + "split": "test", + "style_correct": 0, + "style_probability": 0.9024724960327148, + "style_target": 0, + "support_probability": 0.9322985410690308, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.559164124771101, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0890512615442276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5110417008399963, + 0.49734172224998474, + 0.6144517660140991, + 0.5509066581726074, + 0.04035413637757301, + 0.10237733274698257, + 0.04375757277011871, + 0.16786153614521027, + 0.03849335387349129 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 1, + "style_probability": 0.674303412437439, + "style_target": 1, + "support_probability": 0.9160491228103638, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4423706882623222, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10056523233652115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4382:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5137732625007629, + 0.44741520285606384, + 0.6202820539474487, + 0.5800751447677612, + 0.1653052568435669, + 0.09166167676448822 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4382", + "split": "test", + "style_correct": 1, + "style_probability": 0.4762020707130432, + "style_target": 0, + "support_probability": 0.8294340968132019, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.33378191102357846, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2952774465084076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45507705211639404, + 0.8605241775512695, + 0.2089916318655014, + 0.2614465653896332, + 0.042632997035980225, + 0.11091584712266922, + 0.046691082417964935, + 0.12285332381725311, + 0.1003597304224968, + 0.17445339262485504, + 0.05709607154130936 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 0, + "style_probability": 0.4518675208091736, + "style_target": 1, + "support_probability": 0.6814169883728027, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04081212736977946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26064586639404297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.616127073764801, + 0.9550104141235352, + 0.32738426327705383, + 0.3251558542251587, + 0.0894487202167511, + 0.09608869254589081, + 0.08396288007497787, + 0.11335863918066025, + 0.05747981742024422, + 0.15334740281105042 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4389", + "split": "test", + "style_correct": 1, + "style_probability": 0.14056570827960968, + "style_target": 0, + "support_probability": 0.7404575347900391, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3974300783734108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14816948771476746, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28774937987327576, + 0.3754577040672302, + 0.7390549778938293, + 0.13844120502471924, + 0.042169295251369476 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 1, + "style_probability": 0.5585775375366211, + "style_target": 1, + "support_probability": 0.4505261182785034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.47066120676266365, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12449732422828674, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4394:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28794407844543457, + 0.4108451306819916, + 0.8240671753883362, + 0.12179809808731079, + 0.04204476997256279 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4394", + "split": "test", + "style_correct": 0, + "style_probability": 0.8171321749687195, + "style_target": 0, + "support_probability": 0.3739210069179535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6472034123471175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1424802839756012, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9830828905105591, + 0.3069651424884796, + 0.3548450171947479, + 0.40386733412742615, + 0.06284588575363159, + 0.09579534828662872 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.5524886846542358, + "style_target": 1, + "support_probability": 0.6264632940292358, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44179851261808517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11916154623031616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4404:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9873784184455872, + 0.28355273604393005, + 0.3263842463493347, + 0.3294454514980316, + 0.11185052990913391 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4404", + "split": "test", + "style_correct": 1, + "style_probability": 0.4276690185070038, + "style_target": 0, + "support_probability": 0.6568204760551453, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6338648424943649, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05891445651650429, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8774450421333313, + 0.3146727681159973, + 0.24955697357654572, + 0.15423014760017395, + 0.05601918697357178, + 0.05556869134306908, + 0.05303340032696724, + 0.09750117361545563, + 0.07090631872415543 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 0, + "style_probability": 0.18647964298725128, + "style_target": 1, + "support_probability": 0.7929620742797852, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16475125577080085, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07172467559576035, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4408:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9670212864875793, + 0.31597593426704407, + 0.41492146253585815, + 0.18464608490467072, + 0.1386263221502304, + 0.09512809664011002, + 0.0706411674618721, + 0.05702492967247963, + 0.06798974424600601 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4408", + "split": "test", + "style_correct": 1, + "style_probability": 0.05729985982179642, + "style_target": 0, + "support_probability": 0.8290318250656128, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006292044223804692, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5995030999183655, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2424371838569641, + 0.3075474500656128, + 0.3527899384498596, + 0.2547300457954407, + 0.04929168522357941, + 0.1043759360909462, + 0.09838754683732986, + 0.06710615009069443, + 0.07406916469335556, + 0.08285564184188843, + 0.09440404921770096 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.41774043440818787, + "style_target": 1, + "support_probability": 0.04633781313896179, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011151951212597712, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7122384309768677, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4409:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2572212219238281, + 0.2866622507572174, + 0.32474878430366516, + 0.2901747524738312, + 0.10860932618379593, + 0.09571225196123123, + 0.06928177177906036, + 0.036910489201545715, + 0.06550297886133194, + 0.088339664041996 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4409", + "split": "test", + "style_correct": 0, + "style_probability": 0.8649486899375916, + "style_target": 0, + "support_probability": 0.025960629805922508, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06650032054514865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5317868590354919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2321227490901947, + 0.2661278545856476, + 0.4470445215702057, + 0.3367987871170044, + 0.04112110286951065 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 0, + "style_probability": 0.42608898878097534, + "style_target": 1, + "support_probability": 0.46769022941589355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023765290768796524, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.635501503944397, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2307766079902649, + 0.26102349162101746, + 0.4102780520915985, + 0.3611122965812683, + 0.06146986782550812, + 0.1297961175441742, + 0.13818751275539398, + 0.08667360246181488, + 0.05045752972364426, + 0.10451337695121765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4410", + "split": "test", + "style_correct": 1, + "style_probability": 0.33264124393463135, + "style_target": 0, + "support_probability": 0.1229909136891365, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6117131224508512, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2376597821712494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23491571843624115, + 0.26301833987236023, + 0.23491571843624115, + 0.9466254115104675, + 0.06762154400348663, + 0.08574235439300537 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 1, + "style_probability": 0.6429987549781799, + "style_target": 1, + "support_probability": 0.6304358839988708, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11982109111429651, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3744623064994812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4411:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30503568053245544, + 0.3277393579483032, + 0.30503568053245544, + 0.9731206893920898, + 0.053114164620637894, + 0.10065992176532745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4411", + "split": "test", + "style_correct": 1, + "style_probability": 0.33751508593559265, + "style_target": 0, + "support_probability": 0.5460984706878662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009466593038004067, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6258139610290527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21266867220401764, + 0.7871580719947815, + 0.3870132565498352, + 0.6998947262763977, + 0.08788809925317764, + 0.054967716336250305, + 0.18562494218349457, + 0.0612218976020813, + 0.09495281428098679 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 1, + "style_probability": 0.5627887845039368, + "style_target": 1, + "support_probability": 0.015961967408657074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0056998432003478585, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6302761435508728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4419:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2142709493637085, + 0.8482118248939514, + 0.3402700424194336, + 0.7572397589683533, + 0.09081496298313141, + 0.03565443307161331, + 0.03973940387368202, + 0.049498435109853745, + 0.04038751497864723 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:4419", + "split": "test", + "style_correct": 0, + "style_probability": 0.5482838749885559, + "style_target": 0, + "support_probability": 0.014953426085412502, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0037940434173104596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8298566937446594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7186067700386047, + 0.40514257550239563, + 0.17248183488845825, + 0.19972938299179077, + 0.14692480862140656, + 0.12250082939863205, + 0.10004184395074844, + 0.09140362590551376, + 0.12938269972801208, + 0.09511084109544754, + 0.07780663669109344 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.6776393055915833, + "style_target": 1, + "support_probability": 0.004828834906220436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008941581661833505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6411045789718628, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4428:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7587918639183044, + 0.3801139295101166, + 0.18256773054599762, + 0.17841140925884247, + 0.08091667294502258, + 0.14543254673480988, + 0.13144132494926453, + 0.18982745707035065, + 0.16039273142814636 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4428", + "split": "test", + "style_correct": 1, + "style_probability": 0.4296005070209503, + "style_target": 0, + "support_probability": 0.017065325751900673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003579272649175679, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9795896410942078, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22236067056655884, + 0.5532929301261902, + 0.23459209501743317, + 0.37529435753822327, + 0.07144902646541595 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.7009350657463074, + "style_target": 1, + "support_probability": 0.014059535227715969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005313910681195741, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9651308059692383, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4435:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2502066195011139, + 0.6208698153495789, + 0.26709336042404175, + 0.3974131941795349, + 0.08536092191934586 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4435", + "split": "test", + "style_correct": 1, + "style_probability": 0.40510261058807373, + "style_target": 0, + "support_probability": 0.038546305149793625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22307483748769918, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10898412019014359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19938033819198608, + 0.5297777652740479, + 0.3172195255756378, + 0.8994897603988647, + 0.10541678220033646, + 0.04621879383921623 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 1, + "style_probability": 0.611676037311554, + "style_target": 1, + "support_probability": 0.33887356519699097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1239182503894698, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07334577292203903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4439:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18411263823509216, + 0.5001513361930847, + 0.3173602223396301, + 0.8758496642112732, + 0.15213434398174286 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4439", + "split": "test", + "style_correct": 1, + "style_probability": 0.2382344901561737, + "style_target": 0, + "support_probability": 0.4276009500026703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24663129599629124, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7261890172958374, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5240421295166016, + 0.2730708420276642, + 0.2685958445072174, + 0.9554621577262878, + 0.10927636176347733, + 0.07742828130722046, + 0.06786081939935684, + 0.07814177125692368, + 0.06424462050199509 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.6015316843986511, + "style_target": 1, + "support_probability": 0.2690752446651459, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022181664182377322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7714285254478455, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4443:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6315955519676208, + 0.36075663566589355, + 0.3599107563495636, + 0.9752901196479797, + 0.09274297207593918, + 0.14404049515724182, + 0.1447138488292694, + 0.07921002805233002, + 0.18703480064868927 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4443", + "split": "test", + "style_correct": 1, + "style_probability": 0.09830448776483536, + "style_target": 0, + "support_probability": 0.19654670357704163, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8384687144844634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09786465764045715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.925568699836731, + 0.5543425679206848, + 0.17774944007396698, + 0.2178019881248474, + 0.039069581776857376, + 0.09865280240774155, + 0.11380106955766678, + 0.06825603544712067, + 0.04514496400952339, + 0.058476824313402176, + 0.055577605962753296 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 1, + "style_probability": 0.8259894847869873, + "style_target": 1, + "support_probability": 0.6935170292854309, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7325860841227083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09507489949464798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4444:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9171015024185181, + 0.5409443974494934, + 0.16821224987506866, + 0.23098374903202057, + 0.05631718039512634, + 0.12182549387216568, + 0.09444966167211533, + 0.03773629292845726, + 0.07731761783361435, + 0.10338291525840759 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4444", + "split": "test", + "style_correct": 0, + "style_probability": 0.5392767190933228, + "style_target": 0, + "support_probability": 0.7593202590942383, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.084222930211868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4704011380672455, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2075830101966858, + 0.8597221970558167, + 0.29344093799591064, + 0.4721706211566925, + 0.283988356590271 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.8667047023773193, + "style_target": 1, + "support_probability": 0.08466246724128723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011080355541383426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6204800009727478, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4445:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.16451163589954376, + 0.8395001888275146, + 0.27239418029785156, + 0.466130793094635, + 0.15296006202697754, + 0.09211961925029755, + 0.17434993386268616, + 0.15682874619960785, + 0.3639185428619385, + 0.22621014714241028 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4445", + "split": "test", + "style_correct": 1, + "style_probability": 0.15124142169952393, + "style_target": 0, + "support_probability": 0.04764913395047188, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7835257778789355, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25815850496292114, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3608435392379761, + 0.3465787172317505, + 0.9245128631591797, + 0.27053341269493103, + 0.05125822499394417, + 0.05907383933663368 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.8288865685462952, + "style_target": 1, + "support_probability": 0.6951125264167786, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0682854178945913, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.41099271178245544, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3748806416988373, + 0.36151570081710815, + 0.9558508396148682, + 0.29447808861732483, + 0.09031951427459717, + 0.17072221636772156 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4450", + "split": "test", + "style_correct": 1, + "style_probability": 0.13388292491436005, + "style_target": 0, + "support_probability": 0.5818051695823669, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9551828993030149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04080118611454964, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4459:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35234954953193665, + 0.5204554200172424, + 0.4307388365268707, + 0.9943183064460754, + 0.05029415339231491, + 0.06373137980699539, + 0.07621464133262634, + 0.06184251606464386, + 0.07418164610862732 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 1, + "style_probability": 0.8503385186195374, + "style_target": 1, + "support_probability": 0.9341849088668823, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8899162772679466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04316248372197151, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4459:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3946210443973541, + 0.5368604063987732, + 0.4204801619052887, + 0.9879372715950012, + 0.10588863492012024, + 0.1659763604402542, + 0.08586307615041733, + 0.06679856032133102, + 0.14324666559696198 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4459", + "split": "test", + "style_correct": 0, + "style_probability": 0.5116591453552246, + "style_target": 0, + "support_probability": 0.949478268623352, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6955709931493245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23504333198070526, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.907261312007904, + 0.2900531589984894, + 0.20308642089366913, + 0.16817116737365723, + 0.061982087790966034, + 0.05279222130775452, + 0.06461117416620255, + 0.07910999655723572, + 0.08711247146129608, + 0.09872633218765259 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 1, + "style_probability": 0.9195294380187988, + "style_target": 1, + "support_probability": 0.6012969017028809, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4284722397986905, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21110253036022186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.890450119972229, + 0.28557464480400085, + 0.21029064059257507, + 0.16538211703300476, + 0.13526611030101776, + 0.06210269033908844, + 0.07548370212316513, + 0.12336626648902893, + 0.07537625730037689 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4460", + "split": "test", + "style_correct": 0, + "style_probability": 0.5876989960670471, + "style_target": 0, + "support_probability": 0.654323160648346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10496075529626395, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.35869044065475464, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6280046105384827, + 0.5835407376289368, + 0.5416747331619263, + 0.27678313851356506, + 0.11211016029119492 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 0, + "style_probability": 0.479115754365921, + "style_target": 1, + "support_probability": 0.3631996214389801, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06299502826870727, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36864006519317627, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4462:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6693567633628845, + 0.6001479625701904, + 0.5404196977615356, + 0.25377020239830017, + 0.10472188144922256, + 0.06390870362520218, + 0.04388931766152382, + 0.047810837626457214, + 0.23429551720619202, + 0.1093016192317009 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4462", + "split": "test", + "style_correct": 1, + "style_probability": 0.3896007835865021, + "style_target": 0, + "support_probability": 0.2309727519750595, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06395926206236435, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6290426254272461, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5073117613792419, + 0.3117775022983551, + 0.4357469081878662, + 0.3117775022983551, + 0.06172940880060196, + 0.13614602386951447 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 1, + "style_probability": 0.6537423729896545, + "style_target": 1, + "support_probability": 0.2050880491733551, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01771686807497818, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4601152539253235, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6473128795623779, + 0.26043903827667236, + 0.55214524269104, + 0.26043903827667236, + 0.052241962403059006, + 0.13079358637332916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4464", + "split": "test", + "style_correct": 1, + "style_probability": 0.3213532269001007, + "style_target": 0, + "support_probability": 0.3128816485404968, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.27620191714746767, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09716783463954926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27043601870536804, + 0.17893071472644806, + 0.9249619245529175, + 0.27043601870536804, + 0.10835236310958862, + 0.07502038776874542, + 0.05367949604988098, + 0.039124272763729095, + 0.10273585468530655 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 1, + "style_probability": 0.521273136138916, + "style_target": 1, + "support_probability": 0.5602388978004456, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5869601114321092, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09220226854085922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4470:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31030407547950745, + 0.1994386613368988, + 0.9448813796043396, + 0.31030407547950745, + 0.03952345997095108, + 0.055351391434669495, + 0.03182500973343849, + 0.07601083815097809, + 0.05625983700156212 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4470", + "split": "test", + "style_correct": 0, + "style_probability": 0.8611198663711548, + "style_target": 0, + "support_probability": 0.5177578330039978, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.40374927568237595, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18410509824752808, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21451395750045776, + 0.8959679007530212, + 0.3263796865940094, + 0.3206294775009155, + 0.08943995833396912, + 0.08665300905704498, + 0.11468152701854706, + 0.10278832912445068, + 0.09175674617290497, + 0.1924418807029724 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 1, + "style_probability": 0.5683691501617432, + "style_target": 1, + "support_probability": 0.6595481038093567, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3383941342238721, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09176266193389893, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4473:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24991914629936218, + 0.9334408640861511, + 0.36470016837120056, + 0.3408682942390442, + 0.11512870341539383, + 0.11000735312700272, + 0.11236292868852615, + 0.16940277814865112, + 0.16706067323684692 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4473", + "split": "test", + "style_correct": 1, + "style_probability": 0.32692262530326843, + "style_target": 0, + "support_probability": 0.7913026213645935, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8690330297283566, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07821230590343475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.945757269859314, + 0.2854064702987671, + 0.6213482022285461, + 0.31317850947380066, + 0.22059178352355957 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 1, + "style_probability": 0.6509565711021423, + "style_target": 1, + "support_probability": 0.921457052230835, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8380211101295743, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1003018394112587, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4476:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9122726321220398, + 0.3230748176574707, + 0.5690187811851501, + 0.3409355878829956, + 0.14573170244693756 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4476", + "split": "test", + "style_correct": 1, + "style_probability": 0.46876993775367737, + "style_target": 0, + "support_probability": 0.9467802047729492, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6978226820902194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.056272394955158234, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3011718988418579, + 0.9894932508468628, + 0.5148301720619202, + 0.45208749175071716, + 0.11569386720657349, + 0.12403787672519684 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 1, + "style_probability": 0.9046272039413452, + "style_target": 1, + "support_probability": 0.37685543298721313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27410539275948415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07550989091396332, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4481:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3091857135295868, + 0.9756449460983276, + 0.5119745135307312, + 0.40836063027381897, + 0.22921492159366608 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4481", + "split": "test", + "style_correct": 0, + "style_probability": 0.5395620465278625, + "style_target": 0, + "support_probability": 0.30811038613319397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37724592449520783, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.734916090965271, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43872547149658203, + 0.35359856486320496, + 0.971053421497345, + 0.43386876583099365, + 0.05848334729671478, + 0.03917592391371727, + 0.052256930619478226, + 0.10363404452800751, + 0.18203087151050568 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.7626243233680725, + "style_target": 1, + "support_probability": 0.49162811040878296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08472992331630015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7601364850997925, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4486:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4507916569709778, + 0.38782230019569397, + 0.9807604551315308, + 0.5002954602241516, + 0.08165714144706726, + 0.06785519421100616, + 0.07385318726301193, + 0.05702256038784981, + 0.14148585498332977 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4486", + "split": "test", + "style_correct": 1, + "style_probability": 0.3332468867301941, + "style_target": 0, + "support_probability": 0.5748141407966614, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.198250926279, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2572859823703766, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9351557493209839, + 0.6163711547851562, + 0.5610108971595764, + 0.48451992869377136, + 0.036038950085639954, + 0.07980142533779144, + 0.08189203590154648, + 0.08502037078142166, + 0.15019240975379944, + 0.05245009437203407, + 0.14366284012794495 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 1, + "style_probability": 0.5011208057403564, + "style_target": 1, + "support_probability": 0.41416144371032715, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.23714024761535232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3029290437698364, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8812848925590515, + 0.5949148535728455, + 0.571313202381134, + 0.5036908388137817, + 0.08017253875732422, + 0.13777850568294525, + 0.15699955821037292, + 0.055214546620845795, + 0.0760197788476944, + 0.10432424396276474 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4492", + "split": "test", + "style_correct": 1, + "style_probability": 0.49126341938972473, + "style_target": 0, + "support_probability": 0.35129040479660034, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07213734655687888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.747875988483429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37175458669662476, + 0.521411657333374, + 0.7158455848693848, + 0.2400771826505661, + 0.10701286792755127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 1, + "style_probability": 0.6717299818992615, + "style_target": 1, + "support_probability": 0.21332044899463654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019394900280166102, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8407438397407532, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4494:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3819490373134613, + 0.45126983523368835, + 0.698371410369873, + 0.24369320273399353, + 0.09903613477945328, + 0.08874437212944031, + 0.10536685585975647, + 0.152960404753685, + 0.22553271055221558, + 0.07127907872200012, + 0.10098963975906372 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4494", + "split": "test", + "style_correct": 1, + "style_probability": 0.29975876212120056, + "style_target": 0, + "support_probability": 0.10615307092666626, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6973083694558326, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18710942566394806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7726006507873535, + 0.49944522976875305, + 0.7946545481681824, + 0.6305036544799805, + 0.11938786506652832, + 0.06595157086849213 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.6936626434326172, + "style_target": 1, + "support_probability": 0.9178752899169922, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.039119666114985074, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27666619420051575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4502:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7698003053665161, + 0.4909578561782837, + 0.836397111415863, + 0.7422351837158203, + 0.05827917903661728, + 0.09506737440824509 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:4502", + "split": "test", + "style_correct": 1, + "style_probability": 0.06471215188503265, + "style_target": 0, + "support_probability": 0.8789608478546143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2153248941265673, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16438648104667664, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.45619168877601624, + 0.45619168877601624, + 0.430724173784256, + 0.8494257926940918, + 0.07804451882839203, + 0.040567778050899506, + 0.11536227911710739, + 0.1455947905778885, + 0.05013516545295715 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.5743662118911743, + "style_target": 1, + "support_probability": 0.28852027654647827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15572101455553877, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2214125394821167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4503:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4389027953147888, + 0.4389027953147888, + 0.4602057933807373, + 0.7798564434051514, + 0.062383297830820084, + 0.1087101474404335, + 0.05441559478640556, + 0.11263443529605865, + 0.2148185521364212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4503", + "split": "test", + "style_correct": 1, + "style_probability": 0.3413504958152771, + "style_target": 0, + "support_probability": 0.36608728766441345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01014224002724218, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9201743006706238, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21217970550060272, + 0.5167182087898254, + 0.43888217210769653, + 0.6905102729797363, + 0.14701050519943237, + 0.16864357888698578, + 0.08564446866512299, + 0.10235834866762161, + 0.16646136343479156, + 0.19046802818775177, + 0.08024828135967255 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.7363575100898743, + "style_target": 1, + "support_probability": 0.04003110155463219, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007149326092176442, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8521261215209961, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4505:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21987177431583405, + 0.5142492651939392, + 0.4287319481372833, + 0.6751247644424438, + 0.1105462908744812, + 0.09224144369363785, + 0.07302048802375793, + 0.09052859246730804, + 0.13987305760383606 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4505", + "split": "test", + "style_correct": 1, + "style_probability": 0.3906567096710205, + "style_target": 0, + "support_probability": 0.05159492790699005, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6719738577099008, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14152927696704865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1963064819574356, + 0.4823364317417145, + 0.1963064819574356, + 0.9796540141105652, + 0.1130516529083252 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 1, + "style_probability": 0.7201465368270874, + "style_target": 1, + "support_probability": 0.717241108417511, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6877447601798963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07396703213453293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4515:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1743251085281372, + 0.5458095669746399, + 0.1743251085281372, + 0.9906647205352783, + 0.1530156284570694, + 0.0856509804725647, + 0.122298963367939, + 0.10813497751951218, + 0.08493897318840027, + 0.08049997687339783, + 0.13546282052993774 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4515", + "split": "test", + "style_correct": 0, + "style_probability": 0.524151086807251, + "style_target": 0, + "support_probability": 0.8455845713615417, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8435629799268582, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16243986785411835, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3425796926021576, + 0.333866685628891, + 0.27701857686042786, + 0.9188023805618286, + 0.055627088993787766, + 0.030119270086288452 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 1, + "style_probability": 0.5263128876686096, + "style_target": 1, + "support_probability": 0.8947767615318298, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2622659819319573, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16450336575508118, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4523:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43741151690483093, + 0.4091053903102875, + 0.23960764706134796, + 0.9593355059623718, + 0.06582576036453247, + 0.12543441355228424 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4523", + "split": "test", + "style_correct": 1, + "style_probability": 0.21058283746242523, + "style_target": 0, + "support_probability": 0.824729859828949, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03929318289424976, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8071406483650208, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2956925630569458, + 0.17264358699321747, + 0.47017592191696167, + 0.5252484083175659, + 0.12755879759788513, + 0.03967045247554779, + 0.10975116491317749, + 0.1066613718867302, + 0.06512795388698578 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.4625254273414612, + "style_target": 1, + "support_probability": 0.1447625309228897, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08986748029025547, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8171730041503906, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4535:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4504763185977936, + 0.14619962871074677, + 0.5803447365760803, + 0.6684520244598389, + 0.026980670168995857, + 0.18728730082511902, + 0.16337022185325623, + 0.2752017080783844, + 0.09675145149230957 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4535", + "split": "test", + "style_correct": 0, + "style_probability": 0.7394664287567139, + "style_target": 0, + "support_probability": 0.31845298409461975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11678632986881608, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1603354662656784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35278603434562683, + 0.5826125144958496, + 0.38732990622520447, + 0.387361079454422, + 0.03394390270113945, + 0.05628054961562157, + 0.0458926260471344, + 0.034049756824970245, + 0.09746536612510681, + 0.048190657049417496, + 0.025841975584626198 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 0, + "style_probability": 0.39966684579849243, + "style_target": 1, + "support_probability": 0.4121095538139343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08330113486998149, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11596980690956116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4555:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3744460344314575, + 0.6103861927986145, + 0.45636454224586487, + 0.396351158618927, + 0.0656891018152237, + 0.1098620593547821, + 0.12340809404850006, + 0.08107506483793259, + 0.12930987775325775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4555", + "split": "test", + "style_correct": 1, + "style_probability": 0.35658955574035645, + "style_target": 0, + "support_probability": 0.366759330034256, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1530826970610022, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7075526714324951, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8097052574157715, + 0.40833646059036255, + 0.2564426064491272, + 0.40132370591163635, + 0.0590217150747776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 0, + "style_probability": 0.40113553404808044, + "style_target": 1, + "support_probability": 0.31353873014450073, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06796792761049096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7179142832756042, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4560:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8972484469413757, + 0.5055748224258423, + 0.2728709578514099, + 0.5169284343719482, + 0.0950918197631836 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4560", + "split": "test", + "style_correct": 1, + "style_probability": 0.41841745376586914, + "style_target": 0, + "support_probability": 0.23930682241916656, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.39404732643134105, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3355112671852112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5216847658157349, + 0.43345439434051514, + 0.6641762256622314, + 0.848558783531189, + 0.12830734252929688, + 0.056800421327352524 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 0, + "style_probability": 0.45305243134498596, + "style_target": 1, + "support_probability": 0.5979431867599487, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.39586501420567544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3743043541908264, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4571:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5150137543678284, + 0.42362740635871887, + 0.6813016533851624, + 0.9190565943717957, + 0.10982189327478409, + 0.04308411478996277 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4571", + "split": "test", + "style_correct": 0, + "style_probability": 0.5439731478691101, + "style_target": 0, + "support_probability": 0.5191301107406616, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4846324405800715, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04073748365044594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.443526953458786, + 0.2622319161891937, + 0.8812756538391113, + 0.32802513241767883, + 0.0971427857875824, + 0.06089380756020546, + 0.07556983083486557, + 0.10895334929227829, + 0.05283546820282936 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 0, + "style_probability": 0.435420423746109, + "style_target": 1, + "support_probability": 0.6826499700546265, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3077002779465232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04105917736887932, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4578:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37124869227409363, + 0.30466896295547485, + 0.8757337331771851, + 0.2854524254798889, + 0.06145595386624336, + 0.08369752019643784 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4578", + "split": "test", + "style_correct": 1, + "style_probability": 0.2834796905517578, + "style_target": 0, + "support_probability": 0.6810892224311829, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01716886192724099, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9737846255302429, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27568745613098145, + 0.3576406240463257, + 0.829006016254425, + 0.24306125938892365, + 0.10381688922643661, + 0.08489654213190079, + 0.14781777560710907, + 0.09009172022342682, + 0.10406111925840378, + 0.12553663551807404 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.6702308654785156, + "style_target": 1, + "support_probability": 0.061612796038389206, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005581922869312138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9568869471549988, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4586:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3263908326625824, + 0.3723938465118408, + 0.8933478593826294, + 0.29264429211616516, + 0.2679712474346161, + 0.11163560301065445, + 0.22406063973903656, + 0.1314557045698166, + 0.10773219913244247, + 0.26579219102859497 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4586", + "split": "test", + "style_correct": 1, + "style_probability": 0.2885693609714508, + "style_target": 0, + "support_probability": 0.15356940031051636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030244147501523592, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7564564943313599, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20816586911678314, + 0.4323630630970001, + 0.8902639150619507, + 0.33069294691085815, + 0.20540206134319305 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.29913944005966187, + "style_target": 1, + "support_probability": 0.04149025306105614, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019381126225049176, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9143660664558411, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4590:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2273319512605667, + 0.4318251609802246, + 0.9042916297912598, + 0.3026057481765747, + 0.1363658308982849 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4590", + "split": "test", + "style_correct": 0, + "style_probability": 0.7547546625137329, + "style_target": 0, + "support_probability": 0.012358458712697029, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8118395724167158, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.009846898727118969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5848025679588318, + 0.2242690920829773, + 0.2082473486661911, + 0.9694929122924805, + 0.04760504886507988, + 0.0666845515370369 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 0, + "style_probability": 0.19250716269016266, + "style_target": 1, + "support_probability": 0.9836128950119019, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.503872884171443, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.02306338958442211, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4591:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7348284721374512, + 0.2561526298522949, + 0.28361082077026367, + 0.9924852252006531, + 0.04771694168448448, + 0.11628128588199615 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4591", + "split": "test", + "style_correct": 1, + "style_probability": 0.10789474844932556, + "style_target": 0, + "support_probability": 0.9866618514060974, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.45668866323773294, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08268338441848755, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29791459441185, + 0.2091347873210907, + 0.1662738174200058, + 0.9551803469657898, + 0.05170850083231926, + 0.15684650838375092, + 0.06977152079343796, + 0.05053110048174858, + 0.17923139035701752 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 0, + "style_probability": 0.2626081705093384, + "style_target": 1, + "support_probability": 0.7516227960586548, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3967727348765564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13993079960346222, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4595:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28686603903770447, + 0.17706139385700226, + 0.21365764737129211, + 0.9301702976226807, + 0.0566868856549263, + 0.03129538148641586, + 0.059252019971609116, + 0.062310703098773956, + 0.09161745756864548 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4595", + "split": "test", + "style_correct": 1, + "style_probability": 0.4815204441547394, + "style_target": 0, + "support_probability": 0.6043192148208618, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07035394360697377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2747582793235779, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7799548506736755, + 0.09616032242774963, + 0.42164692282676697, + 0.18886937201023102, + 0.04834369570016861, + 0.0521027147769928, + 0.06590507179498672, + 0.08426138758659363, + 0.05496443063020706, + 0.04013241082429886, + 0.1073082759976387 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.3836851119995117, + "style_target": 1, + "support_probability": 0.21367469429969788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1460905450550385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3475593030452728, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4596:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8012141585350037, + 0.09416250884532928, + 0.3523343801498413, + 0.19753484427928925, + 0.057101067155599594, + 0.06328234076499939, + 0.06346175819635391, + 0.05263159051537514, + 0.03167169168591499, + 0.09432952105998993, + 0.04077215865254402 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4596", + "split": "test", + "style_correct": 0, + "style_probability": 0.8488611578941345, + "style_target": 0, + "support_probability": 0.20057952404022217, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004615260961329887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8688867688179016, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6106533408164978, + 0.21245519816875458, + 0.30452239513397217, + 0.3256532847881317, + 0.08477216958999634 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 1, + "style_probability": 0.6148979663848877, + "style_target": 1, + "support_probability": 0.06816958636045456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000887901382027406, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8590428829193115, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4602:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7025439739227295, + 0.257192999124527, + 0.3241557776927948, + 0.3784204423427582, + 0.1407713145017624 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4602", + "split": "test", + "style_correct": 1, + "style_probability": 0.20096659660339355, + "style_target": 0, + "support_probability": 0.07533585280179977, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5500066705206166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12536661326885223, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18010567128658295, + 0.7677419185638428, + 0.47189730405807495, + 0.3390146493911743, + 0.05669710040092468, + 0.03823619335889816 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.8325175046920776, + "style_target": 1, + "support_probability": 0.6149833798408508, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028739451263448686, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35508158802986145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4605:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20654167234897614, + 0.6771160364151001, + 0.4326029121875763, + 0.33658847212791443, + 0.11236903071403503, + 0.08926916122436523 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4605", + "split": "test", + "style_correct": 1, + "style_probability": 0.44999465346336365, + "style_target": 0, + "support_probability": 0.11535018682479858, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.673400878912009, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14641037583351135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34578514099121094, + 0.8838219046592712, + 0.2198018878698349, + 0.23113493621349335, + 0.033160630613565445, + 0.048446036875247955, + 0.0505734458565712, + 0.0789719820022583, + 0.045178789645433426 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 1, + "style_probability": 0.7850613594055176, + "style_target": 1, + "support_probability": 0.6636624932289124, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3691017160779175, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14785990118980408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34359097480773926, + 0.8768743276596069, + 0.20683656632900238, + 0.23497901856899261, + 0.14562024176120758, + 0.17768241465091705 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4613", + "split": "test", + "style_correct": 1, + "style_probability": 0.39937469363212585, + "style_target": 0, + "support_probability": 0.6504076719284058, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.039917554236976294, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7581790089607239, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.259738028049469, + 0.13655906915664673, + 0.2516270875930786, + 0.31680208444595337, + 0.08081153780221939, + 0.12211382389068604, + 0.026321444660425186, + 0.05282701179385185, + 0.0635363757610321, + 0.08411555737257004, + 0.16720454394817352 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.8850022554397583, + "style_target": 1, + "support_probability": 0.05792096257209778, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000416381827806376, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.95511794090271, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4623:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40022167563438416, + 0.17741912603378296, + 0.30726030468940735, + 0.29259976744651794, + 0.12301034480333328, + 0.05037828907370567, + 0.09599027037620544, + 0.0995689257979393, + 0.06365947425365448, + 0.0653356984257698 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4623", + "split": "test", + "style_correct": 1, + "style_probability": 0.2119755744934082, + "style_target": 0, + "support_probability": 0.018433677032589912, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3126342581266144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4720732867717743, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24539941549301147, + 0.7800649404525757, + 0.17250126600265503, + 0.37790799140930176, + 0.08846137672662735 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.6900767087936401, + "style_target": 1, + "support_probability": 0.601616621017456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034967129544292144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7511768341064453, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4633:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33037126064300537, + 0.8327504396438599, + 0.2816719710826874, + 0.30379727482795715, + 0.16646137833595276 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4633", + "split": "test", + "style_correct": 1, + "style_probability": 0.15547333657741547, + "style_target": 0, + "support_probability": 0.37500473856925964, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10231747259672373, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7740731239318848, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2836580276489258, + 0.15832579135894775, + 0.5059115290641785, + 0.47226855158805847, + 0.1628725379705429, + 0.08918125927448273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 1, + "style_probability": 0.7313795685768127, + "style_target": 1, + "support_probability": 0.17699073255062103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022911161425583915, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8892083764076233, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4634:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2623692750930786, + 0.1601445972919464, + 0.5169805884361267, + 0.5092923045158386, + 0.08356766402721405, + 0.09855624288320541 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4634", + "split": "test", + "style_correct": 0, + "style_probability": 0.8628875613212585, + "style_target": 0, + "support_probability": 0.06225969269871712, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0330802226073048, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.609345555305481, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3918384313583374, + 0.35448014736175537, + 0.9105406403541565, + 0.13604724407196045, + 0.12173973768949509, + 0.04991951212286949, + 0.11401215940713882, + 0.13871945440769196, + 0.07849010825157166 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 1, + "style_probability": 0.6727375388145447, + "style_target": 1, + "support_probability": 0.07424227893352509, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03559817331451942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43930813670158386, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4638:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39185139536857605, + 0.3830069601535797, + 0.9362183213233948, + 0.13754664361476898, + 0.13577741384506226, + 0.29686397314071655 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4638", + "split": "test", + "style_correct": 0, + "style_probability": 0.5591082572937012, + "style_target": 0, + "support_probability": 0.1091524288058281, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7582218931211031, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0323103666305542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33193862438201904, + 0.2237745076417923, + 0.1500197798013687, + 0.9490994811058044, + 0.07875271886587143, + 0.06942111253738403, + 0.09240055084228516, + 0.16456326842308044, + 0.17032678425312042, + 0.06870613247156143 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.5541730523109436, + "style_target": 1, + "support_probability": 0.8166817426681519, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48999523631591657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04635503515601158, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4647:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35836806893348694, + 0.2798978388309479, + 0.1789734959602356, + 0.9884583353996277, + 0.20226512849330902, + 0.07937808334827423, + 0.06873989850282669, + 0.10318573564291, + 0.06542383879423141, + 0.15641960501670837 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4647", + "split": "test", + "style_correct": 1, + "style_probability": 0.20403341948986053, + "style_target": 0, + "support_probability": 0.901136040687561, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8413953886159219, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06128253415226936, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2555713951587677, + 0.20042069256305695, + 0.8775758743286133, + 0.2980942726135254, + 0.17990003526210785 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.8744694590568542, + "style_target": 1, + "support_probability": 0.91631680727005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5398870990130255, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0560956746339798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23912210762500763, + 0.19262069463729858, + 0.822578489780426, + 0.2577151954174042, + 0.03880886361002922, + 0.045208241790533066, + 0.06110261753201485, + 0.1998332142829895, + 0.10821720212697983, + 0.05074242129921913, + 0.24149438738822937 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4650", + "split": "test", + "style_correct": 1, + "style_probability": 0.3657938241958618, + "style_target": 0, + "support_probability": 0.8746718764305115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021649112382262633, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5115017294883728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6819226741790771, + 0.39039942622184753, + 0.24510468542575836, + 0.34144556522369385, + 0.1319781392812729, + 0.07287988066673279 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 1, + "style_probability": 0.5778158903121948, + "style_target": 1, + "support_probability": 0.03434257209300995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015771007882611772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.566443145275116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4656:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7322280406951904, + 0.3639741539955139, + 0.2605802118778229, + 0.2845410704612732, + 0.05096444487571716, + 0.09112183749675751 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4656", + "split": "test", + "style_correct": 1, + "style_probability": 0.1971338391304016, + "style_target": 0, + "support_probability": 0.061889734119176865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24110535412606812, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15523011982440948, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3789319097995758, + 0.3362007737159729, + 0.5043845772743225, + 0.9356099963188171, + 0.08532475680112839, + 0.025898052379488945, + 0.049630917608737946, + 0.08609823882579803, + 0.175898939371109 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 1, + "style_probability": 0.7714765667915344, + "style_target": 1, + "support_probability": 0.13982456922531128, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11676130008268615, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3384385406970978, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36072587966918945, + 0.36372458934783936, + 0.4921499788761139, + 0.8612996339797974, + 0.04670310765504837, + 0.03829418495297432, + 0.043681200593709946, + 0.03263787180185318, + 0.05253031477332115 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4671", + "split": "test", + "style_correct": 0, + "style_probability": 0.7087609767913818, + "style_target": 0, + "support_probability": 0.07018771767616272, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8675724120067905, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09331371635198593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22943153977394104, + 0.9326160550117493, + 0.10374674201011658, + 0.12432128936052322, + 0.09247022122144699, + 0.051008664071559906, + 0.04609038308262825, + 0.04354223981499672, + 0.04081550985574722, + 0.08647359907627106, + 0.1270059198141098 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.8244122266769409, + "style_target": 1, + "support_probability": 0.8583835363388062, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7098687560100602, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06223158910870552, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2022928148508072, + 0.8962010741233826, + 0.11590102314949036, + 0.13645093142986298, + 0.09568244963884354, + 0.09382513910531998, + 0.08814509958028793, + 0.07548493146896362, + 0.08501024544239044 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4681", + "split": "test", + "style_correct": 1, + "style_probability": 0.38785114884376526, + "style_target": 0, + "support_probability": 0.8994715213775635, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.650683229461464, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18943196535110474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7336175441741943, + 0.28315237164497375, + 0.3325439989566803, + 0.28315237164497375, + 0.09213633835315704 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 1, + "style_probability": 0.6090219616889954, + "style_target": 1, + "support_probability": 0.934409499168396, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5922892844473182, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.174801304936409, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4704:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7508226633071899, + 0.33767470717430115, + 0.2907674312591553, + 0.33767470717430115, + 0.05668172612786293, + 0.0788872092962265, + 0.09607637673616409, + 0.070034459233284, + 0.07677609473466873, + 0.04304938763380051, + 0.05667814239859581 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4704", + "split": "test", + "style_correct": 1, + "style_probability": 0.396052747964859, + "style_target": 0, + "support_probability": 0.9027119874954224, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6107252778119943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16125521063804626, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5474151372909546, + 0.4633755385875702, + 0.3348546326160431, + 0.9547466039657593, + 0.23633992671966553, + 0.10584932565689087 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 1, + "style_probability": 0.6627142429351807, + "style_target": 1, + "support_probability": 0.6236973404884338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.075499461697329, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3790527880191803, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4711:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6617802977561951, + 0.4579887390136719, + 0.3535630702972412, + 0.9882453680038452, + 0.1644868403673172, + 0.19901497662067413 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4711", + "split": "test", + "style_correct": 1, + "style_probability": 0.4089215397834778, + "style_target": 0, + "support_probability": 0.43914994597435, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5897910727969702, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21269208192825317, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8660361766815186, + 0.20888200402259827, + 0.44737085700035095, + 0.5024784803390503, + 0.029104111716151237, + 0.06320089846849442, + 0.0939379334449768, + 0.1913338303565979, + 0.13004548847675323 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 1, + "style_probability": 0.6031942367553711, + "style_target": 1, + "support_probability": 0.6530482172966003, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7872151823833207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1597145050764084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9148213267326355, + 0.17260964214801788, + 0.4201961159706116, + 0.4611358940601349, + 0.09996407479047775, + 0.10149095952510834, + 0.09483034163713455, + 0.16449028253555298, + 0.14426632225513458 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4712", + "split": "test", + "style_correct": 0, + "style_probability": 0.8920672535896301, + "style_target": 0, + "support_probability": 0.6223663687705994, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27206677964616865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16629558801651, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3898196220397949, + 0.3928225040435791, + 0.9303392767906189, + 0.19336797297000885, + 0.21215641498565674, + 0.06605595350265503, + 0.07833036780357361, + 0.09031462669372559, + 0.09433585405349731, + 0.1581057906150818, + 0.1247517317533493 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.6595596671104431, + "style_target": 1, + "support_probability": 0.41004809737205505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2234431394731704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11423297226428986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4722:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.407188355922699, + 0.37918755412101746, + 0.9460518956184387, + 0.17986729741096497, + 0.1420147866010666, + 0.057695843279361725, + 0.1657753437757492, + 0.11079441756010056, + 0.13589221239089966 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4722", + "split": "test", + "style_correct": 1, + "style_probability": 0.44527798891067505, + "style_target": 0, + "support_probability": 0.5538887977600098, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8373790127787117, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12843482196331024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4723:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19523297250270844, + 0.29338234663009644, + 0.44405820965766907, + 0.9847266674041748, + 0.15828517079353333 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 1, + "style_probability": 0.7926051616668701, + "style_target": 1, + "support_probability": 0.8661381602287292, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6645932110261192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14416667819023132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19681110978126526, + 0.26213982701301575, + 0.34968531131744385, + 0.9542379975318909, + 0.13319632411003113 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4723", + "split": "test", + "style_correct": 0, + "style_probability": 0.77191561460495, + "style_target": 0, + "support_probability": 0.7279672026634216, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07358891504438113, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.629108190536499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9036721587181091, + 0.31648918986320496, + 0.24155305325984955, + 0.7648935914039612, + 0.12107419222593307, + 0.11895514279603958 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 0, + "style_probability": 0.4482608735561371, + "style_target": 1, + "support_probability": 0.3713071942329407, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032736183929119374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5407557487487793, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4724:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8742930293083191, + 0.2930278480052948, + 0.26706960797309875, + 0.6711217164993286, + 0.1402604579925537 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4724", + "split": "test", + "style_correct": 1, + "style_probability": 0.16692176461219788, + "style_target": 0, + "support_probability": 0.43264514207839966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014283079859232386, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9472975730895996, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2492024451494217, + 0.1845715045928955, + 0.32434138655662537, + 0.6018694043159485, + 0.13581809401512146, + 0.06027959659695625, + 0.1164265051484108, + 0.14272978901863098, + 0.2322039157152176 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.8973233699798584, + "style_target": 1, + "support_probability": 0.013489979319274426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011412002381103734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9458118677139282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4735:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2820696532726288, + 0.20263198018074036, + 0.4262765347957611, + 0.6958844661712646, + 0.08707110583782196, + 0.23381508886814117, + 0.1688048094511032, + 0.09097164869308472, + 0.11782081425189972 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4735", + "split": "test", + "style_correct": 1, + "style_probability": 0.2455613613128662, + "style_target": 0, + "support_probability": 0.018341517075896263, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6130420108637828, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1055695191025734, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9402642846107483, + 0.4181499481201172, + 0.1914730966091156, + 0.3372524082660675, + 0.07949128746986389, + 0.045978840440511703, + 0.04760643094778061, + 0.06479531526565552, + 0.14670568704605103, + 0.1947399079799652 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.8065049648284912, + "style_target": 1, + "support_probability": 0.5616133809089661, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4824388471844294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.055394574999809265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4736:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9499204754829407, + 0.4055042862892151, + 0.15106894075870514, + 0.4045029580593109, + 0.1179775819182396, + 0.08382942527532578, + 0.07220585644245148, + 0.10404592007398605, + 0.24545317888259888, + 0.07441620528697968, + 0.08217358589172363 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4736", + "split": "test", + "style_correct": 1, + "style_probability": 0.49914661049842834, + "style_target": 0, + "support_probability": 0.6705822944641113, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.43542891468183953, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5927727222442627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37959548830986023, + 0.7845141291618347, + 0.48274779319763184, + 0.37959548830986023, + 0.12207154184579849 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.7938959002494812, + "style_target": 1, + "support_probability": 0.5911118984222412, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.33452838864113943, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.492796927690506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4749:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3869140148162842, + 0.733481764793396, + 0.45554134249687195, + 0.3869140148162842, + 0.07161920517683029, + 0.05944898724555969, + 0.054561272263526917, + 0.08712814748287201, + 0.09038545936346054, + 0.07449453324079514, + 0.052969880402088165 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4749", + "split": "test", + "style_correct": 1, + "style_probability": 0.42847880721092224, + "style_target": 0, + "support_probability": 0.6987204551696777, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8623190467024954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10268817096948624, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28013208508491516, + 0.1572222262620926, + 0.9763027429580688, + 0.5010002851486206, + 0.046325985342264175, + 0.0974327027797699 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.867184042930603, + "style_target": 1, + "support_probability": 0.7583203315734863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38997557228273294, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10827358812093735, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4770:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3566257655620575, + 0.20514237880706787, + 0.9834466576576233, + 0.5510749816894531, + 0.04877780005335808, + 0.10436202585697174 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4770", + "split": "test", + "style_correct": 1, + "style_probability": 0.3943493068218231, + "style_target": 0, + "support_probability": 0.7705758810043335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12962605502047977, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49500662088394165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21152624487876892, + 0.9465636610984802, + 0.13474953174591064, + 0.39701271057128906, + 0.08405110985040665, + 0.03445679321885109, + 0.09238546341657639, + 0.07675452530384064, + 0.11538152396678925 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 1, + "style_probability": 0.8597460985183716, + "style_target": 1, + "support_probability": 0.057691238820552826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1621988182355771, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.382747083902359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4785:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22384673357009888, + 0.9283832907676697, + 0.15475459396839142, + 0.3567277193069458, + 0.07602953910827637, + 0.049300335347652435, + 0.06079265847802162, + 0.0929984375834465, + 0.07562558352947235 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4785", + "split": "test", + "style_correct": 0, + "style_probability": 0.5394757390022278, + "style_target": 0, + "support_probability": 0.14021427929401398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0805509277302452, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8627408742904663, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16080127656459808, + 0.2574213147163391, + 0.4712626039981842, + 0.4813629388809204, + 0.06031537428498268, + 0.042315054684877396, + 0.046106964349746704, + 0.032916560769081116, + 0.06527623534202576, + 0.07955058664083481, + 0.045842379331588745 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.8504368662834167, + "style_target": 1, + "support_probability": 0.24789340794086456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012326335043014948, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8816492557525635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4787:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15298618376255035, + 0.2653166353702545, + 0.4372456669807434, + 0.47102782130241394, + 0.09135052561759949, + 0.10017649084329605, + 0.08925007283687592, + 0.028566259890794754, + 0.18449585139751434 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4787", + "split": "test", + "style_correct": 1, + "style_probability": 0.20168468356132507, + "style_target": 0, + "support_probability": 0.18317435681819916, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6134626887440362, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2970377504825592, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.732854962348938, + 0.49098262190818787, + 0.2555534541606903, + 0.3791041076183319, + 0.01776953414082527 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.7755894660949707, + "style_target": 1, + "support_probability": 0.8329513669013977, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015146014743726564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4733206033706665, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4788:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8835397362709045, + 0.5240063667297363, + 0.31968632340431213, + 0.45780909061431885, + 0.09165492653846741 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4788", + "split": "test", + "style_correct": 1, + "style_probability": 0.07128603756427765, + "style_target": 0, + "support_probability": 0.7217031717300415, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13911894798916172, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5744822025299072, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5976571440696716, + 0.9636663794517517, + 0.3484668731689453, + 0.4064864218235016, + 0.0438876673579216, + 0.037470363080501556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 1, + "style_probability": 0.6840278506278992, + "style_target": 1, + "support_probability": 0.22625069320201874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27808926055008953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41357582807540894, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5400383472442627, + 0.9283041954040527, + 0.3685847222805023, + 0.45974889397621155, + 0.06371670216321945, + 0.07779404520988464 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4789", + "split": "test", + "style_correct": 0, + "style_probability": 0.634049117565155, + "style_target": 0, + "support_probability": 0.3855094313621521, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9207775460435385, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.162152960896492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4790:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3743714988231659, + 0.46913549304008484, + 0.9988076686859131, + 0.28131425380706787, + 0.028826292604207993, + 0.04376515373587608, + 0.030851120129227638, + 0.051692161709070206, + 0.04826118424534798 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 1, + "style_probability": 0.6943592429161072, + "style_target": 1, + "support_probability": 0.8750739693641663, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7219057854684934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13316594064235687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4790:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3606848418712616, + 0.46743834018707275, + 0.996819257736206, + 0.2799140512943268, + 0.07489755749702454, + 0.15100665390491486 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4790", + "split": "test", + "style_correct": 1, + "style_probability": 0.38489070534706116, + "style_target": 0, + "support_probability": 0.869164228439331, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013349491825121459, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9373677372932434, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2536695599555969, + 0.5818931460380554, + 0.3639723062515259, + 0.2536695599555969, + 0.08473806828260422, + 0.11885663121938705, + 0.06342004984617233, + 0.08633246272802353, + 0.07537984102964401, + 0.10386897623538971, + 0.08376841247081757 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.7772036790847778, + "style_target": 1, + "support_probability": 0.03717654198408127, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014792201546576234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9120132923126221, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4809:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33315521478652954, + 0.7052186727523804, + 0.4203946888446808, + 0.33315521478652954, + 0.17848585546016693, + 0.06858636438846588, + 0.2868231236934662, + 0.1905871480703354, + 0.09106750786304474, + 0.1597188115119934 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4809", + "split": "test", + "style_correct": 1, + "style_probability": 0.23826612532138824, + "style_target": 0, + "support_probability": 0.0600130520761013, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011163986330133826, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6472879648208618, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45536595582962036, + 0.61570143699646, + 0.42653319239616394, + 0.4945794641971588, + 0.12077251821756363 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 1, + "style_probability": 0.5882071256637573, + "style_target": 1, + "support_probability": 0.02728208713233471, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01563285709123452, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6808747053146362, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4812:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44224539399147034, + 0.5476781725883484, + 0.413912296295166, + 0.4186241328716278, + 0.10019134730100632 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4812", + "split": "test", + "style_correct": 0, + "style_probability": 0.5665985941886902, + "style_target": 0, + "support_probability": 0.028995247557759285, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.13464280338563128, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4397999942302704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5173851251602173, + 0.2558223009109497, + 0.21092848479747772, + 0.2558223009109497, + 0.08083328604698181, + 0.16867060959339142 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 0, + "style_probability": 0.38694390654563904, + "style_target": 1, + "support_probability": 0.5768561959266663, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28777627302055464, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5254843235015869, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4824:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.526181161403656, + 0.23264050483703613, + 0.21417877078056335, + 0.23264050483703613, + 0.08177550137042999, + 0.07609482109546661 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4824", + "split": "test", + "style_correct": 0, + "style_probability": 0.803153395652771, + "style_target": 0, + "support_probability": 0.4666292071342468, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6466002311366026, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.035613033920526505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.330301433801651, + 0.11429479718208313, + 0.9951281547546387, + 0.330301433801651, + 0.04579354450106621, + 0.14091676473617554, + 0.028742024675011635, + 0.04933669790625572, + 0.1047007367014885 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 0, + "style_probability": 0.3815194368362427, + "style_target": 1, + "support_probability": 0.8631206154823303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19200389713660115, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0736854299902916, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37205269932746887, + 0.14197665452957153, + 0.9988622665405273, + 0.37205269932746887, + 0.05042697861790657, + 0.10906325280666351, + 0.06704740971326828, + 0.058022454380989075, + 0.10585866123437881 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4854", + "split": "test", + "style_correct": 1, + "style_probability": 0.20277541875839233, + "style_target": 0, + "support_probability": 0.8165222406387329, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11184947996539131, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.235038161277771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3838536739349365, + 0.8719425797462463, + 0.5394806265830994, + 0.5493797659873962, + 0.07537805289030075, + 0.05203799903392792, + 0.04497649520635605, + 0.16060790419578552, + 0.056587085127830505, + 0.05394936352968216, + 0.0732649564743042 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.3229089081287384, + "style_target": 1, + "support_probability": 0.2986590564250946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3224708816932332, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2592964172363281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4857:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42842352390289307, + 0.8926072716712952, + 0.5065069794654846, + 0.560430109500885, + 0.024228136986494064, + 0.02835419401526451, + 0.16559338569641113, + 0.037374917417764664, + 0.04353036358952522, + 0.050459299236536026, + 0.05322566628456116 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4857", + "split": "test", + "style_correct": 0, + "style_probability": 0.5361196398735046, + "style_target": 0, + "support_probability": 0.4605584442615509, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7526143633798625, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23854485154151917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1736292541027069, + 0.2810034453868866, + 0.38689178228378296, + 0.7659366130828857, + 0.04035664722323418 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.9066945910453796, + "style_target": 1, + "support_probability": 0.9151169657707214, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6622833658163998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2360992431640625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4867:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15264050662517548, + 0.2567552328109741, + 0.3913308084011078, + 0.7486280202865601, + 0.11531024426221848, + 0.05771440640091896, + 0.12197122722864151, + 0.0935002863407135, + 0.12034796178340912, + 0.06286946684122086, + 0.055702224373817444 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4867", + "split": "test", + "style_correct": 1, + "style_probability": 0.4749242961406708, + "style_target": 0, + "support_probability": 0.8869548439979553, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5707055170225175, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29306766390800476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.15621577203273773, + 0.23881258070468903, + 0.9086765050888062, + 0.30147185921669006, + 0.04337722435593605, + 0.050819557160139084 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.8007088899612427, + "style_target": 1, + "support_probability": 0.5265805721282959, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06067713099510863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4067527949810028, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4882:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1873762011528015, + 0.3240046799182892, + 0.951046884059906, + 0.33693036437034607, + 0.055888861417770386, + 0.08819016069173813 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4882", + "split": "test", + "style_correct": 1, + "style_probability": 0.22154757380485535, + "style_target": 0, + "support_probability": 0.49597278237342834, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02316840084796046, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6042625308036804, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26647692918777466, + 0.2683594524860382, + 0.33170267939567566, + 0.24364115297794342, + 0.07393362373113632, + 0.12167292088270187, + 0.08502495288848877, + 0.06447982788085938, + 0.08153143525123596 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 1, + "style_probability": 0.7492170929908752, + "style_target": 1, + "support_probability": 0.04201633855700493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0071554546191261465, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7125778794288635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4884:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3569546043872833, + 0.2965956926345825, + 0.33479541540145874, + 0.26599299907684326, + 0.06994173675775528, + 0.0686914473772049, + 0.058225829154253006, + 0.06820525974035263, + 0.057509418576955795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4884", + "split": "test", + "style_correct": 0, + "style_probability": 0.5389149785041809, + "style_target": 0, + "support_probability": 0.03057505004107952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6482042906659018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42346543073654175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4604000151157379, + 0.3346420228481293, + 0.9779598712921143, + 0.30120980739593506, + 0.09346401691436768, + 0.039463166147470474, + 0.03702728822827339, + 0.1264307200908661, + 0.16450050473213196, + 0.26691940426826477, + 0.08753622323274612 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 1, + "style_probability": 0.78700852394104, + "style_target": 1, + "support_probability": 0.41607680916786194, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18323893177425357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4863373935222626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4886:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42213162779808044, + 0.3173505663871765, + 0.9597469568252563, + 0.2926020324230194, + 0.11003444343805313, + 0.11108700186014175, + 0.4615238308906555, + 0.189787819981575, + 0.24716846644878387 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4886", + "split": "test", + "style_correct": 0, + "style_probability": 0.5120292901992798, + "style_target": 0, + "support_probability": 0.28823190927505493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.452044771337448, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.34065449237823486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31975027918815613, + 0.23782747983932495, + 0.9577869176864624, + 0.1783917099237442, + 0.07193125039339066 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.7954242825508118, + "style_target": 1, + "support_probability": 0.4118361473083496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01970544984861977, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4407010078430176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4890:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37939268350601196, + 0.2175813466310501, + 0.9794841408729553, + 0.18570220470428467, + 0.2528679370880127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4890", + "split": "test", + "style_correct": 1, + "style_probability": 0.1745358258485794, + "style_target": 0, + "support_probability": 0.42057302594184875, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8965219407379265, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07434066385030746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4897:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27362558245658875, + 0.9772562980651855, + 0.24067269265651703, + 0.3317708373069763, + 0.09926823526620865, + 0.12527891993522644 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 1, + "style_probability": 0.8558797836303711, + "style_target": 1, + "support_probability": 0.8222914338111877, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7898600427047882, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07542295753955841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4897:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2876977324485779, + 0.9595982432365417, + 0.20050451159477234, + 0.32009124755859375, + 0.23324744403362274, + 0.13294462859630585 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4897", + "split": "test", + "style_correct": 0, + "style_probability": 0.5964891314506531, + "style_target": 0, + "support_probability": 0.7934533953666687, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10310919634916615, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.660069465637207, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4446861743927002, + 0.7944422364234924, + 0.177885040640831, + 0.4446861743927002, + 0.08851476013660431, + 0.20802219212055206, + 0.05449461564421654, + 0.0690070390701294, + 0.20929376780986786 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.8068585991859436, + "style_target": 1, + "support_probability": 0.17211420834064484, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13463038552729412, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3302690386772156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4905:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.440444678068161, + 0.8110300898551941, + 0.19654691219329834, + 0.440444678068161, + 0.1422208994626999, + 0.14416223764419556 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4905", + "split": "test", + "style_correct": 1, + "style_probability": 0.3852490484714508, + "style_target": 0, + "support_probability": 0.4627152979373932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11712155220055731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3099750280380249, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17375126481056213, + 0.5329745411872864, + 0.5204595923423767, + 0.878619372844696, + 0.11117606610059738, + 0.0816294327378273, + 0.08268892019987106, + 0.10185747593641281, + 0.09512423723936081, + 0.13672037422657013 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.7688432335853577, + "style_target": 1, + "support_probability": 0.2825268507003784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013863628013497114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30308687686920166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4907:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18943525850772858, + 0.5020883083343506, + 0.619402289390564, + 0.9069152474403381, + 0.12669873237609863, + 0.0828942209482193, + 0.09137935191392899, + 0.08395525068044662, + 0.1392129510641098, + 0.23564384877681732 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4907", + "split": "test", + "style_correct": 1, + "style_probability": 0.22204187512397766, + "style_target": 0, + "support_probability": 0.4113198220729828, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002998706952433983, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9157602190971375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18521249294281006, + 0.17117662727832794, + 0.6151127815246582, + 0.330371230840683, + 0.20411577820777893 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 0, + "style_probability": 0.24502310156822205, + "style_target": 1, + "support_probability": 0.023006321862339973, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01411801726224304, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9128713011741638, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4914:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22439789772033691, + 0.1599811315536499, + 0.5696959495544434, + 0.42035332322120667, + 0.1080264002084732 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4914", + "split": "test", + "style_correct": 1, + "style_probability": 0.4724791944026947, + "style_target": 0, + "support_probability": 0.11411023885011673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032664539583751935, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3144521415233612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.386949747800827, + 0.3302469849586487, + 0.4592426121234894, + 0.7340825796127319, + 0.06052227318286896, + 0.12369463592767715 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.2531808018684387, + "style_target": 1, + "support_probability": 0.13435174524784088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18177547398859772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.28356319665908813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4915:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4554533064365387, + 0.35308918356895447, + 0.47236111760139465, + 0.8115847110748291, + 0.08373939990997314, + 0.04972684383392334 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4915", + "split": "test", + "style_correct": 0, + "style_probability": 0.7760679721832275, + "style_target": 0, + "support_probability": 0.21064972877502441, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018098707538355966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7774519920349121, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7742523550987244, + 0.36246779561042786, + 0.533721387386322, + 0.4296676218509674, + 0.08748307079076767, + 0.13159869611263275, + 0.12467312067747116, + 0.11765747517347336, + 0.18878503143787384 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 0, + "style_probability": 0.3055868446826935, + "style_target": 1, + "support_probability": 0.0960991233587265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004869936233887402, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7349966168403625, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4916:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8204222321510315, + 0.410480260848999, + 0.5660440325737, + 0.5100604295730591, + 0.11291354149580002, + 0.12336309999227524, + 0.16627927124500275, + 0.16737015545368195, + 0.10977378487586975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4916", + "split": "test", + "style_correct": 1, + "style_probability": 0.11716752499341965, + "style_target": 0, + "support_probability": 0.1682085245847702, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.071700199836755, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5139069557189941, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7491621971130371, + 0.3150874674320221, + 0.2267432063817978, + 0.3150874674320221, + 0.10552565008401871, + 0.057615965604782104, + 0.06143583357334137, + 0.07840658724308014, + 0.09913239628076553, + 0.0701170340180397 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 1, + "style_probability": 0.5874291658401489, + "style_target": 1, + "support_probability": 0.10992005467414856, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2825975608728726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3418065905570984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4920:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7271673679351807, + 0.34135761857032776, + 0.2539273202419281, + 0.34135761857032776, + 0.08821634948253632, + 0.06021595001220703, + 0.07021854817867279, + 0.12196633219718933, + 0.06661155819892883, + 0.06017071381211281 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4920", + "split": "test", + "style_correct": 0, + "style_probability": 0.7244663834571838, + "style_target": 0, + "support_probability": 0.3926798105239868, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.42368105680195356, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5907231569290161, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31374436616897583, + 0.47582271695137024, + 0.7339776158332825, + 0.31374436616897583, + 0.04912411794066429 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.8936328291893005, + "style_target": 1, + "support_probability": 0.6166177988052368, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07804714489592125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.54823237657547, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4922:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31866779923439026, + 0.47979527711868286, + 0.7153465151786804, + 0.31866779923439026, + 0.06182795390486717, + 0.03252279385924339, + 0.09184239059686661, + 0.46738675236701965, + 0.049054671078920364, + 0.07587499916553497, + 0.20189939439296722 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4922", + "split": "test", + "style_correct": 1, + "style_probability": 0.36230647563934326, + "style_target": 0, + "support_probability": 0.49048057198524475, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.33538380652745836, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45327240228652954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.780281126499176, + 0.23548123240470886, + 0.25177454948425293, + 0.3317318558692932, + 0.22487439215183258, + 0.0616886280477047 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.922300398349762, + "style_target": 1, + "support_probability": 0.1288783997297287, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08171829064906433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21167922019958496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4924:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8966070413589478, + 0.242998406291008, + 0.2074374556541443, + 0.3925183117389679, + 0.06034741923213005, + 0.12720714509487152 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4924", + "split": "test", + "style_correct": 1, + "style_probability": 0.24513927102088928, + "style_target": 0, + "support_probability": 0.5522386431694031, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6815428392522733, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15651555359363556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.466421514749527, + 0.30442506074905396, + 0.854439377784729, + 0.466421514749527, + 0.03271185979247093, + 0.06430873274803162, + 0.03314642608165741, + 0.0697074607014656, + 0.05249045416712761 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 1, + "style_probability": 0.880388081073761, + "style_target": 1, + "support_probability": 0.4887096881866455, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5360858246421785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09785842895507812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4036576449871063, + 0.285195529460907, + 0.8191140294075012, + 0.4036576449871063, + 0.039521582424640656, + 0.04116217792034149, + 0.13784660398960114, + 0.10921876132488251, + 0.23656253516674042 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:4926", + "split": "test", + "style_correct": 0, + "style_probability": 0.5498285889625549, + "style_target": 0, + "support_probability": 0.6130573749542236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018552000738279312, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6785546541213989, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34508898854255676, + 0.28292611241340637, + 0.5601580142974854, + 0.5681740641593933, + 0.04368852078914642, + 0.09832863509654999, + 0.059489745646715164, + 0.11636841297149658, + 0.03187412768602371, + 0.06363847851753235, + 0.07387711852788925 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 1, + "style_probability": 0.5281018018722534, + "style_target": 1, + "support_probability": 0.018079029396176338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004321406121958055, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6700031757354736, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4927:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.29524046182632446, + 0.2838767170906067, + 0.532926619052887, + 0.5900420546531677, + 0.1412205845117569, + 0.12845832109451294, + 0.12429028749465942, + 0.14691919088363647, + 0.12365227937698364 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4927", + "split": "test", + "style_correct": 1, + "style_probability": 0.2891083359718323, + "style_target": 0, + "support_probability": 0.0255732461810112, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5153285887212569, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4642057418823242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8213698267936707, + 0.19344636797904968, + 0.2881056070327759, + 0.3204675018787384, + 0.06478802114725113 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.8513597846031189, + "style_target": 1, + "support_probability": 0.46555155515670776, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006372801849795356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8024416565895081, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4932:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8665817379951477, + 0.24481472373008728, + 0.4156731367111206, + 0.4571542739868164, + 0.14079983532428741 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4932", + "split": "test", + "style_correct": 1, + "style_probability": 0.1287270337343216, + "style_target": 0, + "support_probability": 0.3502315878868103, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9216630418537477, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07415774464607239, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4941:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26350098848342896, + 0.979804277420044, + 0.3537850081920624, + 0.26350098848342896, + 0.05674455314874649, + 0.03275635838508606 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 1, + "style_probability": 0.7890710234642029, + "style_target": 1, + "support_probability": 0.8390564918518066, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9134774345067208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07472489774227142, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4941:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24826587736606598, + 0.9529245495796204, + 0.2957588732242584, + 0.24826587736606598, + 0.06805833429098129, + 0.31639620661735535 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4941", + "split": "test", + "style_correct": 0, + "style_probability": 0.6983746290206909, + "style_target": 0, + "support_probability": 0.8775055408477783, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20913650312873064, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5472897887229919, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.441932737827301, + 0.441932737827301, + 0.3885963261127472, + 0.19939479231834412, + 0.061307720839977264, + 0.07773298770189285, + 0.08224956691265106, + 0.03741743043065071, + 0.03182750567793846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.8351908326148987, + "style_target": 1, + "support_probability": 0.26073968410491943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13354561621925765, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6269758343696594, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47563591599464417, + 0.47563591599464417, + 0.4032497704029083, + 0.22736620903015137, + 0.06321804225444794, + 0.07401420921087265 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4951", + "split": "test", + "style_correct": 1, + "style_probability": 0.44666817784309387, + "style_target": 0, + "support_probability": 0.28886526823043823, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9048756627920439, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15584076941013336, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4953:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4475572109222412, + 0.9714328050613403, + 0.3550790548324585, + 0.5248184204101562, + 0.0394885428249836, + 0.0896473154425621, + 0.06400498747825623, + 0.04682788625359535, + 0.06633813679218292, + 0.03616474196314812 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.8524144291877747, + "style_target": 1, + "support_probability": 0.8117544651031494, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21062740811824202, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14630182087421417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4953:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44228532910346985, + 0.9791727066040039, + 0.3506200909614563, + 0.5132192969322205, + 0.08689535409212112, + 0.10104157030582428, + 0.05556595325469971, + 0.10164424777030945, + 0.05197406932711601, + 0.08986984193325043 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:4953", + "split": "test", + "style_correct": 1, + "style_probability": 0.1958390325307846, + "style_target": 0, + "support_probability": 0.810316264629364, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04765234426478271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3634888827800751, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8013390898704529, + 0.43497762084007263, + 0.22163692116737366, + 0.2730550467967987, + 0.10083894431591034 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 0, + "style_probability": 0.358227014541626, + "style_target": 1, + "support_probability": 0.2480355203151703, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10795880913010514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49718520045280457, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:4968:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7610601782798767, + 0.4555104672908783, + 0.21779301762580872, + 0.33099955320358276, + 0.18735487759113312 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4968", + "split": "test", + "style_correct": 0, + "style_probability": 0.6532894968986511, + "style_target": 0, + "support_probability": 0.22387146949768066, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2882063656144851, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35504865646362305, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7656431198120117, + 0.3443628251552582, + 0.26099181175231934, + 0.5494611263275146, + 0.07430224865674973, + 0.10541579872369766 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.23310521245002747, + "style_target": 1, + "support_probability": 0.6550285220146179, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5178833890836252, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5569552183151245, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4972:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7438661456108093, + 0.31559523940086365, + 0.22546958923339844, + 0.5491235256195068, + 0.07893761992454529, + 0.09973853826522827 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4972", + "split": "test", + "style_correct": 0, + "style_probability": 0.8418688178062439, + "style_target": 0, + "support_probability": 0.4523375928401947, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05937712505749194, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6014501452445984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6900469064712524, + 0.23990289866924286, + 0.3320111632347107, + 0.2486017346382141, + 0.10583779215812683, + 0.13990631699562073, + 0.09718719124794006, + 0.16807769238948822, + 0.2106485813856125 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 0, + "style_probability": 0.36141306161880493, + "style_target": 1, + "support_probability": 0.09291023761034012, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01311343548893407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7299422025680542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8627603650093079, + 0.35144126415252686, + 0.4044749140739441, + 0.2684141993522644, + 0.08975935727357864, + 0.05127394571900368, + 0.09552983194589615, + 0.05238962918519974, + 0.08043917268514633 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4976", + "split": "test", + "style_correct": 1, + "style_probability": 0.17562910914421082, + "style_target": 0, + "support_probability": 0.12050063163042068, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5713420997066158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03623330965638161, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8957634568214417, + 0.13836519420146942, + 0.16457289457321167, + 0.2904818058013916, + 0.0484607107937336, + 0.10765980929136276, + 0.15010641515254974, + 0.13374948501586914, + 0.07532449066638947, + 0.08078034967184067, + 0.20650993287563324 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.39385583996772766, + "style_target": 1, + "support_probability": 0.8353292942047119, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5492302265940481, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08431299775838852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4980:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.848638653755188, + 0.1629524677991867, + 0.1830741912126541, + 0.2867191731929779, + 0.10002713650465012, + 0.07286890596151352, + 0.09911663830280304, + 0.12139474600553513, + 0.027488525956869125, + 0.19684769213199615, + 0.19836831092834473 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4980", + "split": "test", + "style_correct": 0, + "style_probability": 0.6029515862464905, + "style_target": 0, + "support_probability": 0.703137993812561, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6825506018556524, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40292081236839294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.877432107925415, + 0.2419726699590683, + 0.11969322711229324, + 0.17489825189113617, + 0.15338823199272156 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 1, + "style_probability": 0.9047267436981201, + "style_target": 1, + "support_probability": 0.4656957983970642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44189396795251307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3147585988044739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4984:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8699678778648376, + 0.24541659653186798, + 0.12531670928001404, + 0.17277848720550537, + 0.08499643206596375, + 0.08884870260953903, + 0.0972069576382637, + 0.08705899864435196, + 0.206190824508667, + 0.23169001936912537, + 0.16306138038635254 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:4984", + "split": "test", + "style_correct": 1, + "style_probability": 0.45939087867736816, + "style_target": 0, + "support_probability": 0.5662314295768738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46770185016549704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.11934234946966171, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35654157400131226, + 0.14138394594192505, + 0.22419726848602295, + 0.7667868137359619, + 0.14573155343532562, + 0.07535483688116074 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.8259122371673584, + "style_target": 1, + "support_probability": 0.391715943813324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.052755623746158616, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08158000558614731, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:4987:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4886876046657562, + 0.18714837729930878, + 0.348663330078125, + 0.8667360544204712, + 0.08128975331783295, + 0.08612865954637527 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:4987", + "split": "test", + "style_correct": 1, + "style_probability": 0.195638507604599, + "style_target": 0, + "support_probability": 0.5188682675361633, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21485934958665445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05988346412777901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39579102396965027, + 0.9385724663734436, + 0.11124318093061447, + 0.2596437931060791, + 0.1304151862859726, + 0.05969894304871559, + 0.2555685043334961, + 0.09691563248634338, + 0.14167064428329468 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 1, + "style_probability": 0.6816784143447876, + "style_target": 1, + "support_probability": 0.29520928859710693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13044129718939512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20157116651535034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5001:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40650540590286255, + 0.9246593117713928, + 0.15028123557567596, + 0.2530749440193176, + 0.17120659351348877, + 0.17791388928890228, + 0.33810967206954956, + 0.3641444146633148, + 0.1901140958070755 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5001", + "split": "test", + "style_correct": 0, + "style_probability": 0.5928742289543152, + "style_target": 0, + "support_probability": 0.1605713963508606, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8799094236907194, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12902982532978058, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3739809989929199, + 0.2241206020116806, + 0.7305346131324768, + 0.9378476142883301, + 0.0562295988202095, + 0.038070086389780045, + 0.054289091378450394, + 0.06287224590778351, + 0.1182778850197792, + 0.09250320494174957, + 0.06429806351661682 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 1, + "style_probability": 0.867992103099823, + "style_target": 1, + "support_probability": 0.8602215647697449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6322328504102082, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2863413691520691, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5007:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3631375730037689, + 0.21608085930347443, + 0.691484808921814, + 0.907558023929596, + 0.1046195924282074, + 0.07857948541641235, + 0.10798248648643494, + 0.1220860704779625, + 0.21353374421596527 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5007", + "split": "test", + "style_correct": 0, + "style_probability": 0.579140305519104, + "style_target": 0, + "support_probability": 0.6864822506904602, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4586690116385163, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4706154763698578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.15314090251922607, + 0.992752194404602, + 0.30760857462882996, + 0.26504725217819214, + 0.11059077084064484 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.7998257875442505, + "style_target": 1, + "support_probability": 0.4624311327934265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03379267724740065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7422285676002502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18603475391864777, + 0.9971627593040466, + 0.3196738064289093, + 0.32992053031921387, + 0.12939982116222382 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5021", + "split": "test", + "style_correct": 1, + "style_probability": 0.21997644007205963, + "style_target": 0, + "support_probability": 0.32637450098991394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17184488828726963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4119083881378174, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23640896379947662, + 0.18610401451587677, + 0.24549679458141327, + 0.9022073149681091, + 0.07344567775726318, + 0.23179732263088226 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 1, + "style_probability": 0.7388225793838501, + "style_target": 1, + "support_probability": 0.13831737637519836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16351905699927194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27805861830711365, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5023:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2895887494087219, + 0.22419263422489166, + 0.290566086769104, + 0.8437432050704956, + 0.1388804018497467, + 0.057189393788576126 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5023", + "split": "test", + "style_correct": 0, + "style_probability": 0.6317272782325745, + "style_target": 0, + "support_probability": 0.16596676409244537, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5562394621854594, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1126822680234909, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8247299790382385, + 0.2009204924106598, + 0.11456017196178436, + 0.2009204924106598, + 0.06383922696113586, + 0.05824090912938118, + 0.04775295406579971, + 0.04817457124590874, + 0.22731013596057892 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.7757766842842102, + "style_target": 1, + "support_probability": 0.6056935787200928, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.24279317035046155, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11334101855754852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5024:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7620778679847717, + 0.2038489729166031, + 0.12809890508651733, + 0.2038489729166031, + 0.11998587846755981, + 0.09785918891429901 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5024", + "split": "test", + "style_correct": 1, + "style_probability": 0.42227602005004883, + "style_target": 0, + "support_probability": 0.5914574861526489, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15349492742558546, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5908951759338379, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1061769649386406, + 0.33471423387527466, + 0.6854954957962036, + 0.16939668357372284, + 0.06686259061098099, + 0.043723758310079575, + 0.07363922894001007, + 0.04562711343169212, + 0.044497326016426086, + 0.05868949741125107, + 0.07243815064430237 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.8334082961082458, + "style_target": 1, + "support_probability": 0.19709299504756927, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030551118376714865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7600736618041992, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5034:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.12555252015590668, + 0.3682739734649658, + 0.5986696481704712, + 0.19385148584842682, + 0.05495114624500275, + 0.061858560889959335, + 0.038001224398612976, + 0.09047957509756088, + 0.06476785987615585, + 0.09877347946166992 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5034", + "split": "test", + "style_correct": 1, + "style_probability": 0.18415221571922302, + "style_target": 0, + "support_probability": 0.2112528681755066, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9049957006261061, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07834579050540924, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9842635989189148, + 0.6378828883171082, + 0.5267764329910278, + 0.22023582458496094, + 0.1752476841211319 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.6239417195320129, + "style_target": 1, + "support_probability": 0.9750256538391113, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7532928078989762, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04613640904426575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.999480664730072, + 0.6595760583877563, + 0.6921099424362183, + 0.236934095621109, + 0.11918770521879196 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5048", + "split": "test", + "style_correct": 1, + "style_probability": 0.22825153172016144, + "style_target": 0, + "support_probability": 0.9913349747657776, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2337976600811054, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.37606102228164673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3330046236515045, + 0.6191088557243347, + 0.8124474287033081, + 0.3056538999080658, + 0.06904450058937073, + 0.07367308437824249 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 1, + "style_probability": 0.6502660512924194, + "style_target": 1, + "support_probability": 0.35341715812683105, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.35288905827834327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.34097886085510254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5058:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37853294610977173, + 0.6299945712089539, + 0.8512321710586548, + 0.35276731848716736, + 0.06122877821326256, + 0.13605917990207672 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5058", + "split": "test", + "style_correct": 0, + "style_probability": 0.8345973491668701, + "style_target": 0, + "support_probability": 0.33743590116500854, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09376763656688693, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0805954709649086, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23033085465431213, + 0.23994876444339752, + 0.41712695360183716, + 0.874442994594574, + 0.20304548740386963, + 0.07025015354156494, + 0.09190463274717331, + 0.07926089316606522, + 0.11197150498628616 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 1, + "style_probability": 0.6461414694786072, + "style_target": 1, + "support_probability": 0.1273336261510849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0682279096458238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07074124366044998, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5063:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25456738471984863, + 0.23879893124103546, + 0.36816397309303284, + 0.8829951286315918, + 0.12470037490129471, + 0.11358045041561127 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5063", + "split": "test", + "style_correct": 1, + "style_probability": 0.4848123788833618, + "style_target": 0, + "support_probability": 0.13898487389087677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025594291037844833, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.70530104637146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21683746576309204, + 0.6782530546188354, + 0.5020937323570251, + 0.3378921449184418, + 0.07411277294158936, + 0.1993611603975296, + 0.16265349090099335, + 0.1342596858739853, + 0.10000031441450119, + 0.13553230464458466, + 0.2169201523065567 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 1, + "style_probability": 0.6488704085350037, + "style_target": 1, + "support_probability": 0.05398524925112724, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008664798700924401, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8279667496681213, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5074:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1812411993741989, + 0.7230674624443054, + 0.6415339112281799, + 0.3445885181427002, + 0.11156357824802399, + 0.09730785340070724, + 0.0970596969127655, + 0.1373244971036911, + 0.13527561724185944, + 0.15729601681232452 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5074", + "split": "test", + "style_correct": 1, + "style_probability": 0.19175052642822266, + "style_target": 0, + "support_probability": 0.20194590091705322, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016576543116736626, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6809802651405334, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30048105120658875, + 0.3406442403793335, + 0.26263993978500366, + 0.3807525038719177, + 0.320590615272522 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.8834934830665588, + "style_target": 1, + "support_probability": 0.06969519704580307, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003789909282275071, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7752857208251953, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5080:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.300047367811203, + 0.3015066981315613, + 0.21270307898521423, + 0.3102288246154785, + 0.11090631783008575, + 0.08781891316175461, + 0.11693865060806274, + 0.11849352717399597, + 0.09037395566701889, + 0.1370212286710739, + 0.150849848985672, + 0.08663372695446014 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5080", + "split": "test", + "style_correct": 1, + "style_probability": 0.4123050272464752, + "style_target": 0, + "support_probability": 0.03769329562783241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21837317003222445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14694377779960632, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31397920846939087, + 0.3777755796909332, + 0.23279428482055664, + 0.9340184926986694, + 0.14089921116828918, + 0.10664045810699463 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.7136914730072021, + "style_target": 1, + "support_probability": 0.1986500769853592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0266642516578397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1765509396791458, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5087:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3108859062194824, + 0.41939467191696167, + 0.2760814428329468, + 0.9715049266815186, + 0.10886693745851517, + 0.11904938519001007 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5087", + "split": "test", + "style_correct": 1, + "style_probability": 0.13041839003562927, + "style_target": 0, + "support_probability": 0.2844162583351135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8708101349863994, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09317526966333389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28367069363594055, + 0.28367069363594055, + 0.9893994927406311, + 0.30936264991760254, + 0.09638553112745285, + 0.12381460517644882, + 0.10991764813661575, + 0.14317764341831207, + 0.2652372419834137 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 1, + "style_probability": 0.8841146230697632, + "style_target": 1, + "support_probability": 0.692764937877655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6720124182393619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11514285206794739, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5098:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2895530164241791, + 0.2895530164241791, + 0.9801212549209595, + 0.37052011489868164, + 0.1772911548614502, + 0.11234228312969208, + 0.14956772327423096, + 0.2545217275619507, + 0.19064782559871674 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5098", + "split": "test", + "style_correct": 0, + "style_probability": 0.6080171465873718, + "style_target": 0, + "support_probability": 0.6700462698936462, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9052291323262409, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2068815380334854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3011399507522583, + 0.2239585667848587, + 0.3067876100540161, + 0.949155330657959, + 0.06875278055667877, + 0.08017804473638535, + 0.08824458718299866, + 0.1175144836306572, + 0.07437816262245178, + 0.07711491733789444, + 0.21118828654289246 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 1, + "style_probability": 0.9318868517875671, + "style_target": 1, + "support_probability": 0.7010246515274048, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8052148964860066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1211286410689354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5099:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3345123529434204, + 0.21333925426006317, + 0.27180948853492737, + 0.9317319989204407, + 0.27360451221466064, + 0.0940048024058342, + 0.2630620300769806, + 0.14953163266181946, + 0.22864685952663422 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5099", + "split": "test", + "style_correct": 0, + "style_probability": 0.6076918840408325, + "style_target": 0, + "support_probability": 0.8409647941589355, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.018910346818673988, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.31763046979904175, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22894562780857086, + 0.4050304591655731, + 0.6194421052932739, + 0.12712186574935913, + 0.03615015000104904 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.5612041354179382, + "style_target": 1, + "support_probability": 0.10288034379482269, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028082463882355404, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33725303411483765, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5110:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2859015166759491, + 0.43673771619796753, + 0.6830715537071228, + 0.16566811501979828, + 0.13455289602279663 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5110", + "split": "test", + "style_correct": 1, + "style_probability": 0.19125479459762573, + "style_target": 0, + "support_probability": 0.10964450240135193, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8929861293649991, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06917507946491241, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40046602487564087, + 0.9929458498954773, + 0.2299019694328308, + 0.24401965737342834, + 0.0413023941218853, + 0.027080558240413666 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 1, + "style_probability": 0.8373498916625977, + "style_target": 1, + "support_probability": 0.8691545724868774, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8428655479942222, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.058965105563402176, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5137:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4209200441837311, + 0.9844973087310791, + 0.2371007353067398, + 0.2509836256504059, + 0.041881855577230453, + 0.06462540477514267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5137", + "split": "test", + "style_correct": 0, + "style_probability": 0.5440444350242615, + "style_target": 0, + "support_probability": 0.9139598608016968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7790338685886269, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10607912391424179, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6657942533493042, + 0.1675107181072235, + 0.9050756096839905, + 0.6657942533493042, + 0.03370550274848938, + 0.047621022909879684, + 0.08610673993825912, + 0.05177628993988037, + 0.025554874911904335 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 1, + "style_probability": 0.8428645730018616, + "style_target": 1, + "support_probability": 0.49169766902923584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43594469112260725, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14682601392269135, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5142:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6717875003814697, + 0.16009274125099182, + 0.8629804849624634, + 0.6717875003814697, + 0.1828916072845459, + 0.0648726224899292 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5142", + "split": "test", + "style_correct": 0, + "style_probability": 0.5409280061721802, + "style_target": 0, + "support_probability": 0.3301076292991638, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8627595236517536, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03496965393424034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2614898085594177, + 0.5883683562278748, + 0.4601970613002777, + 0.932775616645813, + 0.0890253484249115, + 0.05753082409501076, + 0.029203012585639954, + 0.056292835623025894, + 0.1313094049692154, + 0.1001989096403122, + 0.02952839620411396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.7983385920524597, + "style_target": 1, + "support_probability": 0.8677303791046143, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1939857635338349, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054761338979005814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5147:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.310328871011734, + 0.6448243856430054, + 0.5498053431510925, + 0.9582350254058838, + 0.10298960655927658, + 0.0931176021695137, + 0.12905587255954742, + 0.10439261794090271, + 0.07976821810007095, + 0.12626242637634277 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5147", + "split": "test", + "style_correct": 1, + "style_probability": 0.24890771508216858, + "style_target": 0, + "support_probability": 0.8416017293930054, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005833717983930954, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9392417073249817, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.289625346660614, + 0.367215633392334, + 0.27317389845848083, + 0.13061384856700897, + 0.06229804456233978 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 1, + "style_probability": 0.7406548261642456, + "style_target": 1, + "support_probability": 0.05049466714262962, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005322539583979239, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9008423686027527, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5148:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.226588174700737, + 0.393173485994339, + 0.31315773725509644, + 0.1948775202035904, + 0.14909489452838898 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5148", + "split": "test", + "style_correct": 0, + "style_probability": 0.5269295573234558, + "style_target": 0, + "support_probability": 0.0331757478415966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09152135560697472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10300521552562714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9553021788597107, + 0.24194201827049255, + 0.43612417578697205, + 0.2770841419696808, + 0.1460256725549698, + 0.05112070217728615 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 1, + "style_probability": 0.6087768077850342, + "style_target": 1, + "support_probability": 0.09981179237365723, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05518144865688689, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07429303228855133, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5152:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.934424877166748, + 0.25290656089782715, + 0.42231616377830505, + 0.25904202461242676, + 0.06895775347948074 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5152", + "split": "test", + "style_correct": 1, + "style_probability": 0.33834517002105713, + "style_target": 0, + "support_probability": 0.12838973104953766, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9175815719072489, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08887051790952682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5153:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4857133626937866, + 0.9713993072509766, + 0.2361641228199005, + 0.4857133626937866, + 0.03279365599155426, + 0.08802798390388489, + 0.05375247448682785, + 0.09165512025356293, + 0.06921929866075516 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.8639264106750488, + "style_target": 1, + "support_probability": 0.7145044803619385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2775668939505209, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07641347497701645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5153:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5996795892715454, + 0.9866766929626465, + 0.26622274518013, + 0.5996795892715454, + 0.05527366325259209, + 0.08406538516283035, + 0.13384929299354553, + 0.10422907024621964, + 0.15239359438419342 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5153", + "split": "test", + "style_correct": 1, + "style_probability": 0.24026919901371002, + "style_target": 0, + "support_probability": 0.707233726978302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5547481139889179, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.178011953830719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28032445907592773, + 0.6178030371665955, + 0.4116828143596649, + 0.6864060759544373, + 0.05093976855278015, + 0.052490975707769394, + 0.03601810708642006, + 0.0820673331618309, + 0.04101986065506935, + 0.10102040320634842, + 0.03605299070477486 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 1, + "style_probability": 0.8304893970489502, + "style_target": 1, + "support_probability": 0.6001629829406738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38476507299761303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2123587727546692, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5167:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22540472447872162, + 0.5638721585273743, + 0.39799246191978455, + 0.5806981921195984, + 0.05706500634551048, + 0.044204771518707275, + 0.05568290501832962, + 0.13183213770389557, + 0.08427360653877258, + 0.08614882826805115, + 0.0717688575387001 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5167", + "split": "test", + "style_correct": 0, + "style_probability": 0.5307484269142151, + "style_target": 0, + "support_probability": 0.5921948552131653, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8727909481119768, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30115264654159546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2597525417804718, + 0.3067629635334015, + 0.9122325778007507, + 0.361450731754303, + 0.025462688878178596 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.8651003241539001, + "style_target": 1, + "support_probability": 0.7737396359443665, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6415778376064303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22465205192565918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5170:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23780371248722076, + 0.31128811836242676, + 0.9002668261528015, + 0.35306164622306824, + 0.08987704664468765, + 0.10342774540185928, + 0.05583037808537483, + 0.12697388231754303, + 0.04712070897221565, + 0.10433273017406464, + 0.08021434396505356 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5170", + "split": "test", + "style_correct": 1, + "style_probability": 0.3851868212223053, + "style_target": 0, + "support_probability": 0.6763400435447693, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0015376311771171589, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9634187817573547, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38407182693481445, + 0.47342649102211, + 0.37821146845817566, + 0.3474617600440979, + 0.050051793456077576, + 0.06326555460691452 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.6630041003227234, + "style_target": 1, + "support_probability": 0.009159131906926632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0002338270581425868, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9679536819458008, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5173:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41615980863571167, + 0.47697341442108154, + 0.3983347415924072, + 0.3900359570980072, + 0.06852776557207108, + 0.061283692717552185 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5173", + "split": "test", + "style_correct": 1, + "style_probability": 0.27257239818573, + "style_target": 0, + "support_probability": 0.008263371884822845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10900675606000122, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8724986910820007, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7950050830841064, + 0.49396589398384094, + 0.2208521068096161, + 0.30645060539245605, + 0.11346413940191269, + 0.13415853679180145, + 0.12145565450191498, + 0.076125368475914, + 0.05816212669014931 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.869857132434845, + "style_target": 1, + "support_probability": 0.128151997923851, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05768636827365369, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7455586791038513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8442968726158142, + 0.5784986615180969, + 0.2756935954093933, + 0.25711750984191895, + 0.10334771871566772, + 0.10012765973806381, + 0.0642113909125328, + 0.06464310735464096, + 0.0636458769440651 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5184", + "split": "test", + "style_correct": 1, + "style_probability": 0.4229855537414551, + "style_target": 0, + "support_probability": 0.25886961817741394, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019734264343267416, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9451227188110352, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47195854783058167, + 0.5282720923423767, + 0.45894762873649597, + 0.2587490677833557, + 0.06590909510850906, + 0.11063361167907715, + 0.18862880766391754, + 0.15692779421806335, + 0.10331517457962036, + 0.06915196031332016, + 0.07195592671632767 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.7449923753738403, + "style_target": 1, + "support_probability": 0.06247275695204735, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0037298753755182473, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8603494167327881, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5188:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4629081189632416, + 0.537352979183197, + 0.45818856358528137, + 0.2692188024520874, + 0.2931582033634186, + 0.07619374245405197, + 0.06162815913558006, + 0.1280481517314911, + 0.15932442247867584 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5188", + "split": "test", + "style_correct": 1, + "style_probability": 0.20751158893108368, + "style_target": 0, + "support_probability": 0.0438852533698082, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006460452034430742, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8139280080795288, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.273865669965744, + 0.1428639143705368, + 0.30463355779647827, + 0.40907153487205505, + 0.16692973673343658 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 0, + "style_probability": 0.44661909341812134, + "style_target": 1, + "support_probability": 0.058051299303770065, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003763729448213905, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8385944366455078, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5194:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25894641876220703, + 0.14379778504371643, + 0.3257710337638855, + 0.32424357533454895, + 0.11956628412008286, + 0.1407419592142105, + 0.12601067125797272, + 0.11867185682058334, + 0.059161022305488586, + 0.19057326018810272, + 0.09848655760288239 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5194", + "split": "test", + "style_correct": 1, + "style_probability": 0.2787940800189972, + "style_target": 0, + "support_probability": 0.058439526706933975, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4263366219231273, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37837183475494385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8024803400039673, + 0.13116933405399323, + 0.2739768624305725, + 0.21067166328430176, + 0.2039632350206375, + 0.1548364907503128 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.6023381352424622, + "style_target": 1, + "support_probability": 0.6535398960113525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.038256297657662815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42275357246398926, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5196:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9530163407325745, + 0.10801829397678375, + 0.3380882441997528, + 0.2298678457736969, + 0.08214005082845688, + 0.10122231394052505 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5196", + "split": "test", + "style_correct": 1, + "style_probability": 0.1771077811717987, + "style_target": 0, + "support_probability": 0.6720699667930603, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.113155719336806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3003555238246918, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20692257583141327, + 0.26966118812561035, + 0.9358334541320801, + 0.4428751468658447, + 0.0790620818734169, + 0.10031946003437042, + 0.081536203622818, + 0.1788375973701477, + 0.08771207183599472 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 1, + "style_probability": 0.6219382286071777, + "style_target": 1, + "support_probability": 0.11964541673660278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032154815051823826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49453073740005493, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5198:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2329307347536087, + 0.26539722084999084, + 0.9770594239234924, + 0.36773866415023804, + 0.04299197718501091, + 0.10275768488645554, + 0.12157632410526276, + 0.08941573649644852, + 0.1255594938993454 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5198", + "split": "test", + "style_correct": 1, + "style_probability": 0.4051792621612549, + "style_target": 0, + "support_probability": 0.12824679911136627, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23830457205797909, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4328781068325043, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9235617518424988, + 0.2732744812965393, + 0.1594579517841339, + 0.30461207032203674, + 0.04740016907453537, + 0.041259195655584335, + 0.09070040285587311, + 0.09285669773817062, + 0.04868020489811897, + 0.12999887764453888, + 0.07108550518751144 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.6699318289756775, + "style_target": 1, + "support_probability": 0.2262113392353058, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06989296168826799, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5905081033706665, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5200:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9284610748291016, + 0.24154648184776306, + 0.15911366045475006, + 0.2625065743923187, + 0.17654451727867126, + 0.17953361570835114, + 0.0628540888428688, + 0.22110232710838318, + 0.09420691430568695 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5200", + "split": "test", + "style_correct": 1, + "style_probability": 0.38605955243110657, + "style_target": 0, + "support_probability": 0.1500382125377655, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5301226243714865, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5771205425262451, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3966613709926605, + 0.648358941078186, + 0.47278887033462524, + 0.3966613709926605, + 0.09323485195636749 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 1, + "style_probability": 0.9327448606491089, + "style_target": 1, + "support_probability": 0.5236929059028625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28043347265750285, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.45232725143432617, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5205:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3653494119644165, + 0.6555529832839966, + 0.4629959166049957, + 0.3653494119644165, + 0.23825354874134064 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5205", + "split": "test", + "style_correct": 0, + "style_probability": 0.5851837396621704, + "style_target": 0, + "support_probability": 0.519995391368866, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7467596813464077, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06993067264556885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9570723176002502, + 0.3388998210430145, + 0.375376433134079, + 0.8393146991729736, + 0.06038666144013405, + 0.0586661621928215 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.563185453414917, + "style_target": 1, + "support_probability": 0.9234780669212341, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6669987225991747, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08022141456604004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5220:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9255757927894592, + 0.3275025188922882, + 0.39236998558044434, + 0.8167940974235535, + 0.15822003781795502 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5220", + "split": "test", + "style_correct": 1, + "style_probability": 0.391085147857666, + "style_target": 0, + "support_probability": 0.9317004680633545, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.038510958186908306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8084666728973389, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18086551129817963, + 0.891153872013092, + 0.16334038972854614, + 0.3046726584434509, + 0.0903027355670929, + 0.08589030802249908, + 0.1495787352323532, + 0.09875772893428802, + 0.21859623491764069 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.8505499362945557, + "style_target": 1, + "support_probability": 0.06209084391593933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030569309732839107, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7213291525840759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5233:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19638019800186157, + 0.9088574647903442, + 0.2089952528476715, + 0.35509824752807617, + 0.08539266139268875, + 0.094141386449337, + 0.126450777053833, + 0.09584389626979828, + 0.13356482982635498 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5233", + "split": "test", + "style_correct": 1, + "style_probability": 0.3508780300617218, + "style_target": 0, + "support_probability": 0.06316325068473816, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7625805169411173, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12014913558959961, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28251782059669495, + 0.22707919776439667, + 0.9571666121482849, + 0.5332161784172058, + 0.08772730082273483, + 0.0468602329492569, + 0.043461356312036514, + 0.1175038069486618, + 0.09355176985263824, + 0.17019163072109222 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 1, + "style_probability": 0.8771088123321533, + "style_target": 1, + "support_probability": 0.6353707313537598, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4763197703617621, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18648815155029297, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5234:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2820024788379669, + 0.22058561444282532, + 0.9438706636428833, + 0.4926036596298218, + 0.06103191524744034, + 0.14635510742664337, + 0.12956836819648743, + 0.20276722311973572, + 0.052685610949993134, + 0.22117288410663605 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5234", + "split": "test", + "style_correct": 0, + "style_probability": 0.6466338634490967, + "style_target": 0, + "support_probability": 0.6369249820709229, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7640802798558362, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15734946727752686, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8654641509056091, + 0.3455299735069275, + 0.4292314648628235, + 0.3455299735069275, + 0.04984281212091446 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.716027557849884, + "style_target": 1, + "support_probability": 0.9291613698005676, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40744244893188863, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2996167540550232, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5240:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.815161943435669, + 0.31889140605926514, + 0.4225510358810425, + 0.31889140605926514, + 0.07650873064994812, + 0.0997043326497078, + 0.1487274169921875, + 0.060021232813596725, + 0.16940799355506897, + 0.12557066977024078 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5240", + "split": "test", + "style_correct": 1, + "style_probability": 0.32675689458847046, + "style_target": 0, + "support_probability": 0.761522114276886, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022768265907056055, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8156144618988037, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18721285462379456, + 0.3055107295513153, + 0.18512684106826782, + 0.6233872175216675, + 0.13849560916423798, + 0.1168096587061882 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.8561314344406128, + "style_target": 1, + "support_probability": 0.01703016832470894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003582885194264417, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6188006401062012, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5243:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24972814321517944, + 0.2936929166316986, + 0.1942220777273178, + 0.7308189272880554, + 0.15160612761974335, + 0.21120741963386536 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5243", + "split": "test", + "style_correct": 1, + "style_probability": 0.33424705266952515, + "style_target": 0, + "support_probability": 0.037087298929691315, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7320771680810445, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15713798999786377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17953646183013916, + 0.19234061241149902, + 0.16918468475341797, + 0.999349057674408, + 0.06476271897554398, + 0.15960721671581268, + 0.11609166115522385, + 0.12531955540180206, + 0.12694557011127472 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 1, + "style_probability": 0.5270713567733765, + "style_target": 1, + "support_probability": 0.5599763989448547, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5378822313139544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20554926991462708, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5247:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21609951555728912, + 0.25004318356513977, + 0.17299939692020416, + 0.9902240633964539, + 0.15264739096164703, + 0.05315089225769043, + 0.11705296486616135, + 0.0726240798830986, + 0.13621647655963898 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5247", + "split": "test", + "style_correct": 1, + "style_probability": 0.47182729840278625, + "style_target": 0, + "support_probability": 0.3829491436481476, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6371102808010677, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2669427692890167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41745275259017944, + 0.8735167980194092, + 0.14073887467384338, + 0.41745275259017944, + 0.07222961634397507, + 0.05189130827784538, + 0.046903565526008606, + 0.11329638957977295, + 0.0701645091176033, + 0.13363784551620483, + 0.1358790099620819 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.8746055960655212, + "style_target": 1, + "support_probability": 0.5125628113746643, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2047358635892138, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2277710735797882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40834301710128784, + 0.8736138343811035, + 0.14502359926700592, + 0.40834301710128784, + 0.04639977589249611, + 0.08648017048835754, + 0.08744990080595016, + 0.11724212020635605, + 0.09603293985128403 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5253", + "split": "test", + "style_correct": 1, + "style_probability": 0.33028319478034973, + "style_target": 0, + "support_probability": 0.4894454777240753, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1814294456865753, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5629259943962097, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2293485850095749, + 0.3464055061340332, + 0.8585054874420166, + 0.16196666657924652, + 0.14743486046791077 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.7164846658706665, + "style_target": 1, + "support_probability": 0.1754920780658722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009353000652092932, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8342839479446411, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5254:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22799107432365417, + 0.3100550174713135, + 0.8215014338493347, + 0.1610294133424759, + 0.11689549684524536, + 0.10514821857213974, + 0.07773782312870026, + 0.08166956901550293, + 0.10593754053115845, + 0.18201416730880737, + 0.1010994091629982 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5254", + "split": "test", + "style_correct": 1, + "style_probability": 0.4987398087978363, + "style_target": 0, + "support_probability": 0.03069247491657734, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1815046315156037, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17134910821914673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8463637232780457, + 0.13464675843715668, + 0.2028624713420868, + 0.13464675843715668, + 0.14732643961906433, + 0.052361343055963516 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.6010507345199585, + "style_target": 1, + "support_probability": 0.4457038342952728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01151337721751078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3517805337905884, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5264:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9078307747840881, + 0.13547974824905396, + 0.24110621213912964, + 0.13547974824905396, + 0.046750131994485855, + 0.07981003820896149 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5264", + "split": "test", + "style_correct": 1, + "style_probability": 0.1800205409526825, + "style_target": 0, + "support_probability": 0.45285773277282715, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.046049932627510344, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6782196760177612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19832852482795715, + 0.2885313332080841, + 0.7539163827896118, + 0.19832852482795715, + 0.13446636497974396, + 0.08886298537254333, + 0.06882403045892715, + 0.09215695410966873, + 0.09902121871709824 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 1, + "style_probability": 0.5983326435089111, + "style_target": 1, + "support_probability": 0.09170521050691605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14563639052234123, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49888575077056885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5274:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2236800193786621, + 0.3047091066837311, + 0.8290349245071411, + 0.2236800193786621, + 0.05727436766028404, + 0.08879278600215912, + 0.049767620861530304, + 0.0728556364774704, + 0.06560952961444855 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5274", + "split": "test", + "style_correct": 0, + "style_probability": 0.7352474331855774, + "style_target": 0, + "support_probability": 0.18664373457431793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04801334923860968, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6035279631614685, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3493410050868988, + 0.8048576712608337, + 0.22618195414543152, + 0.26126542687416077, + 0.04408473148941994, + 0.15465675294399261, + 0.15212766826152802, + 0.10264294594526291, + 0.04841616004705429, + 0.08690876513719559 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 1, + "style_probability": 0.5550838708877563, + "style_target": 1, + "support_probability": 0.07031198590993881, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019142963080094777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.714158833026886, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5277:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3230518698692322, + 0.786027193069458, + 0.1968265324831009, + 0.248844176530838, + 0.08485215157270432, + 0.10993395745754242, + 0.11664258688688278, + 0.12073921412229538, + 0.06646568328142166 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5277", + "split": "test", + "style_correct": 1, + "style_probability": 0.39313122630119324, + "style_target": 0, + "support_probability": 0.042637817561626434, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37293544881045193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1883172243833542, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3832862377166748, + 0.5461465120315552, + 0.9002059698104858, + 0.16076329350471497, + 0.08745044469833374 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 1, + "style_probability": 0.6119861602783203, + "style_target": 1, + "support_probability": 0.602956235408783, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42223763710305023, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33903786540031433, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5278:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44981011748313904, + 0.5336278080940247, + 0.870216429233551, + 0.17964357137680054, + 0.07130639255046844 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5278", + "split": "test", + "style_correct": 0, + "style_probability": 0.7560021877288818, + "style_target": 0, + "support_probability": 0.4717700183391571, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12485224628564139, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14927658438682556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.902488112449646, + 0.328307569026947, + 0.4622451663017273, + 0.5479337573051453, + 0.16898560523986816, + 0.1492251604795456 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.3119623064994812, + "style_target": 1, + "support_probability": 0.3202993869781494, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5369079538275514, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07533226162195206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5288:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9311748743057251, + 0.3772846758365631, + 0.4749608337879181, + 0.5774542689323425, + 0.19768781960010529, + 0.09775546938180923 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5288", + "split": "test", + "style_correct": 0, + "style_probability": 0.7147021293640137, + "style_target": 0, + "support_probability": 0.6087076663970947, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040063209723836035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5289226770401001, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.861268162727356, + 0.1852487325668335, + 0.15725453197956085, + 0.17677916586399078, + 0.08418481796979904, + 0.12630581855773926, + 0.1095290556550026, + 0.08318321406841278, + 0.1266186535358429 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 0, + "style_probability": 0.15665452182292938, + "style_target": 1, + "support_probability": 0.1697537899017334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008001136344554516, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4663328528404236, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5308:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9221884608268738, + 0.17084141075611115, + 0.1719234436750412, + 0.24295943975448608, + 0.09157852828502655, + 0.12298416346311569, + 0.0545346662402153, + 0.05583985522389412, + 0.07726725190877914 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5308", + "split": "test", + "style_correct": 1, + "style_probability": 0.0953035056591034, + "style_target": 0, + "support_probability": 0.13619591295719147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08448387707933724, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6459645628929138, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2426147609949112, + 0.2709417939186096, + 0.8884841799736023, + 0.26826614141464233, + 0.047349244356155396, + 0.08211126178503036, + 0.07801166921854019, + 0.11709801852703094, + 0.12697634100914001, + 0.1365412324666977, + 0.13373993337154388 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.27429524064064026, + "style_target": 1, + "support_probability": 0.18323259055614471, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07946181224553238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8196501731872559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5310:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.26080799102783203, + 0.24776121973991394, + 0.8696931004524231, + 0.27019837498664856, + 0.12448393553495407, + 0.043615732342004776, + 0.03199566900730133, + 0.056734420359134674, + 0.035130150616168976, + 0.08929583430290222 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5310", + "split": "test", + "style_correct": 0, + "style_probability": 0.6517608761787415, + "style_target": 0, + "support_probability": 0.09123330563306808, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2917354156641281, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13645373284816742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8964351415634155, + 0.3769158720970154, + 0.13722656667232513, + 0.4423421323299408, + 0.17439384758472443 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 1, + "style_probability": 0.5629498958587646, + "style_target": 1, + "support_probability": 0.21914860606193542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1127510908123606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23725521564483643, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5320:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9146634340286255, + 0.2998172342777252, + 0.11059708148241043, + 0.37872257828712463, + 0.05389412119984627 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5320", + "split": "test", + "style_correct": 0, + "style_probability": 0.6989701986312866, + "style_target": 0, + "support_probability": 0.08937674015760422, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04343068100527247, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7806533575057983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44960832595825195, + 0.5394657850265503, + 0.1841762214899063, + 0.23301789164543152, + 0.05131551995873451, + 0.15049242973327637 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 1, + "style_probability": 0.523736298084259, + "style_target": 1, + "support_probability": 0.14059655368328094, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008511723573974004, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8339905738830566, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5321:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4183540642261505, + 0.5454021692276001, + 0.16079488396644592, + 0.21904443204402924, + 0.21852758526802063 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5321", + "split": "test", + "style_correct": 1, + "style_probability": 0.23154856264591217, + "style_target": 0, + "support_probability": 0.13271595537662506, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.666054457889377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10966084152460098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27543962001800537, + 0.3708864152431488, + 0.6077322959899902, + 0.8559505343437195, + 0.1288040727376938, + 0.060280367732048035, + 0.29291296005249023, + 0.26547762751579285, + 0.08811020851135254 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.6970682144165039, + "style_target": 1, + "support_probability": 0.7838299870491028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13748053328497534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1661713570356369, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5323:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33731094002723694, + 0.4183710515499115, + 0.678368866443634, + 0.943303644657135, + 0.056911662220954895, + 0.09791885316371918, + 0.08294416964054108, + 0.04807021841406822, + 0.11874867975711823 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5323", + "split": "test", + "style_correct": 1, + "style_probability": 0.10800458490848541, + "style_target": 0, + "support_probability": 0.8915548324584961, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37674415604437606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3392086625099182, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9605657458305359, + 0.31315335631370544, + 0.2632048726081848, + 0.12190873175859451, + 0.09114766865968704, + 0.06642718613147736, + 0.1481582522392273, + 0.08253059536218643, + 0.08873188495635986, + 0.08132877200841904, + 0.09702073037624359 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 1, + "style_probability": 0.6408141255378723, + "style_target": 1, + "support_probability": 0.32306429743766785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6859908640078629, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35049906373023987, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5336:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9905204772949219, + 0.24593278765678406, + 0.2016213983297348, + 0.09849961847066879, + 0.12214531749486923, + 0.10064054280519485, + 0.12387941032648087, + 0.12799878418445587, + 0.21968337893486023, + 0.09424865245819092, + 0.09205979853868484 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5336", + "split": "test", + "style_correct": 0, + "style_probability": 0.8012552261352539, + "style_target": 0, + "support_probability": 0.5336224436759949, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4219396889962468, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3256286084651947, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37828630208969116, + 0.37828630208969116, + 0.6398042440414429, + 0.5655068755149841, + 0.06772083789110184 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 1, + "style_probability": 0.6872745752334595, + "style_target": 1, + "support_probability": 0.7690712213516235, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2632685112951627, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34898045659065247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5338:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37132784724235535, + 0.37132784724235535, + 0.6753569841384888, + 0.5476036071777344, + 0.05665159970521927, + 0.06944401562213898, + 0.07976903021335602, + 0.08656612038612366, + 0.06184304133057594, + 0.05433746427297592, + 0.10413382947444916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5338", + "split": "test", + "style_correct": 1, + "style_probability": 0.4230634868144989, + "style_target": 0, + "support_probability": 0.6459941864013672, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014020203661357146, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5438246130943298, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5725739598274231, + 0.4867870807647705, + 0.36764052510261536, + 0.46269455552101135, + 0.1254757195711136, + 0.09673596173524857 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 1, + "style_probability": 0.6623687148094177, + "style_target": 1, + "support_probability": 0.0383913591504097, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008472176367961545, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6011590957641602, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5339:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6388304829597473, + 0.5992122888565063, + 0.3774793744087219, + 0.5637082457542419, + 0.10560766607522964, + 0.18521814048290253 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5339", + "split": "test", + "style_correct": 1, + "style_probability": 0.1552397608757019, + "style_target": 0, + "support_probability": 0.025972900912165642, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0057565790427894226, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.903040885925293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3208051323890686, + 0.30124399065971375, + 0.3208051323890686, + 0.6918759942054749, + 0.06075562536716461, + 0.029946178197860718, + 0.057485632598400116, + 0.10855638980865479, + 0.0563725009560585 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 1, + "style_probability": 0.6830169558525085, + "style_target": 1, + "support_probability": 0.017120443284511566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022915327503965303, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7887390851974487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5351:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31006112694740295, + 0.2709895968437195, + 0.31006112694740295, + 0.7703321576118469, + 0.08066162467002869, + 0.1312435418367386, + 0.03392711281776428, + 0.08306285738945007, + 0.0910588875412941 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5351", + "split": "test", + "style_correct": 0, + "style_probability": 0.8692957758903503, + "style_target": 0, + "support_probability": 0.028541533276438713, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08039475864023295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6275743246078491, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7597870230674744, + 0.38761767745018005, + 0.47874149680137634, + 0.38761767745018005, + 0.05181058496236801, + 0.09950070828199387, + 0.05064317584037781, + 0.03761059418320656, + 0.038096893578767776, + 0.10035400092601776, + 0.06446079909801483 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 1, + "style_probability": 0.6819129586219788, + "style_target": 1, + "support_probability": 0.18794767558574677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06113355612262917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6052053570747375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5360:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7778052687644958, + 0.38722771406173706, + 0.4236871898174286, + 0.38722771406173706, + 0.10908228158950806, + 0.058192286640405655, + 0.05019022896885872, + 0.08140570670366287, + 0.1317952424287796 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5360", + "split": "test", + "style_correct": 0, + "style_probability": 0.5173951983451843, + "style_target": 0, + "support_probability": 0.175126314163208, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12382983380978539, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4809413552284241, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3246467113494873, + 0.531428873538971, + 0.827471911907196, + 0.3412798047065735, + 0.1459868997335434 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 1, + "style_probability": 0.5934387445449829, + "style_target": 1, + "support_probability": 0.36378365755081177, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001570790486606494, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7961180806159973, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5362:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3538930118083954, + 0.4769921898841858, + 0.8877654671669006, + 0.3149707317352295, + 0.1835429072380066 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5362", + "split": "test", + "style_correct": 1, + "style_probability": 0.1529550999403, + "style_target": 0, + "support_probability": 0.1608048677444458, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7287627978697842, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10385304689407349, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5037910342216492, + 0.19304002821445465, + 0.3746216893196106, + 0.9207195043563843, + 0.20899227261543274, + 0.05039723590016365 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 1, + "style_probability": 0.5729652047157288, + "style_target": 1, + "support_probability": 0.7659587860107422, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.819840361458926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06572885066270828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5367:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5767307877540588, + 0.20983247458934784, + 0.37987521290779114, + 0.9500582814216614, + 0.13270404934883118, + 0.10924176871776581 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5367", + "split": "test", + "style_correct": 0, + "style_probability": 0.7007699012756348, + "style_target": 0, + "support_probability": 0.8425870537757874, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13840655536237098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36039429903030396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43993526697158813, + 0.22091510891914368, + 0.48708376288414, + 0.8983186483383179, + 0.05721262842416763, + 0.10521852970123291, + 0.17023564875125885, + 0.24304142594337463, + 0.1251937597990036 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 1, + "style_probability": 0.5528655052185059, + "style_target": 1, + "support_probability": 0.2446327656507492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15201585439130308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21018384397029877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5371:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4380839765071869, + 0.2124636322259903, + 0.49674052000045776, + 0.9165691137313843, + 0.06867684423923492, + 0.08018092811107635 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5371", + "split": "test", + "style_correct": 1, + "style_probability": 0.3719402253627777, + "style_target": 0, + "support_probability": 0.41545459628105164, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09857265191567065, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5411166548728943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.397426575422287, + 0.6360753774642944, + 0.5900334715843201, + 0.3507135510444641, + 0.08186537027359009, + 0.05574948340654373, + 0.04606472700834274, + 0.06614934653043747, + 0.48062771558761597, + 0.22543208301067352, + 0.04023100063204765 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 1, + "style_probability": 0.541049063205719, + "style_target": 1, + "support_probability": 0.31679460406303406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00938463646608656, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5024300217628479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5377:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.47508499026298523, + 0.7870051860809326, + 0.7169382572174072, + 0.42677372694015503, + 0.057350531220436096, + 0.0416497141122818, + 0.07170228660106659, + 0.07251160591840744, + 0.054805684834718704, + 0.10925939679145813 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5377", + "split": "test", + "style_correct": 1, + "style_probability": 0.10674695670604706, + "style_target": 0, + "support_probability": 0.5162332057952881, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.638818676905931, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.19984541833400726, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6248897910118103, + 0.20043206214904785, + 0.5653220415115356, + 0.904086172580719, + 0.08792416751384735 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.6424830555915833, + "style_target": 1, + "support_probability": 0.7543529868125916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4333577946805889, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10737822949886322, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5379:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5132471919059753, + 0.2151201069355011, + 0.4934910833835602, + 0.8802977204322815, + 0.13779477775096893, + 0.05087954178452492, + 0.037625156342983246, + 0.07759809494018555, + 0.04625272750854492, + 0.08112482726573944, + 0.05589047074317932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5379", + "split": "test", + "style_correct": 1, + "style_probability": 0.243250772356987, + "style_target": 0, + "support_probability": 0.8075864315032959, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.852561296213814, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04433967173099518, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9323724508285522, + 0.23028317093849182, + 0.7011404037475586, + 0.23028317093849182, + 0.030680125579237938, + 0.07638142257928848 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.7821006774902344, + "style_target": 1, + "support_probability": 0.922991156578064, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0968074634382061, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.060274627059698105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5384:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9276505708694458, + 0.2487538605928421, + 0.6972364783287048, + 0.2487538605928421, + 0.05291275680065155, + 0.1372079998254776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5384", + "split": "test", + "style_correct": 1, + "style_probability": 0.13930478692054749, + "style_target": 0, + "support_probability": 0.8631757497787476, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9353267105098481, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03646649420261383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5393:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5227783918380737, + 0.9753536581993103, + 0.35312309861183167, + 0.34532350301742554, + 0.036981288343667984, + 0.03596938028931618, + 0.07755535840988159, + 0.03642136976122856, + 0.2407682090997696 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.75774085521698, + "style_target": 1, + "support_probability": 0.9474165439605713, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5723085363802483, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07721258699893951, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3281722664833069, + 0.9275640249252319, + 0.28295373916625977, + 0.3067542016506195, + 0.04607686772942543, + 0.051933787763118744, + 0.048725392669439316, + 0.04091313108801842, + 0.040386952459812164 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5393", + "split": "test", + "style_correct": 1, + "style_probability": 0.27895882725715637, + "style_target": 0, + "support_probability": 0.8884157538414001, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7276668245619929, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09657445549964905, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34845349192619324, + 0.9241182804107666, + 0.09389688819646835, + 0.34845349192619324, + 0.1155439093708992, + 0.06326987594366074, + 0.048300109803676605, + 0.03955547884106636, + 0.1267901062965393, + 0.03907028213143349, + 0.07605373114347458 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 1, + "style_probability": 0.8388447165489197, + "style_target": 1, + "support_probability": 0.6660789847373962, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3054136373606795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17869211733341217, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32831189036369324, + 0.8669259548187256, + 0.10774793475866318, + 0.32831189036369324, + 0.07710118591785431, + 0.0997905284166336, + 0.07956449687480927, + 0.03429386764764786, + 0.057700350880622864 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5401", + "split": "test", + "style_correct": 1, + "style_probability": 0.4495975971221924, + "style_target": 0, + "support_probability": 0.4334275424480438, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8793309642867797, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03849541023373604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4024859368801117, + 0.9862563014030457, + 0.4336550235748291, + 0.4024859368801117, + 0.11250482499599457 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.7294270396232605, + "style_target": 1, + "support_probability": 0.9422363638877869, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4179929547973078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.052904047071933746, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5417394638061523, + 0.9932689070701599, + 0.5259338021278381, + 0.5417394638061523, + 0.15415355563163757 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5409", + "split": "test", + "style_correct": 1, + "style_probability": 0.24208340048789978, + "style_target": 0, + "support_probability": 0.939582109451294, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9251395327544394, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18303783237934113, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9925530552864075, + 0.4861500859260559, + 0.45748642086982727, + 0.15999148786067963, + 0.08141651004552841, + 0.11713314056396484 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 1, + "style_probability": 0.9169766902923584, + "style_target": 1, + "support_probability": 0.8198699355125427, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8318080186221835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10303451865911484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5412:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9881142973899841, + 0.47633084654808044, + 0.3993293344974518, + 0.18216082453727722, + 0.05607803910970688, + 0.1409139335155487 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5412", + "split": "test", + "style_correct": 0, + "style_probability": 0.6682794094085693, + "style_target": 0, + "support_probability": 0.8973057270050049, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6351201416787866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3735309839248657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.185927152633667, + 0.185927152633667, + 0.8298464417457581, + 0.1464255154132843, + 0.08721642196178436, + 0.07623577117919922, + 0.04138820990920067, + 0.07785999029874802, + 0.083023801445961 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.7739840745925903, + "style_target": 1, + "support_probability": 0.8100317716598511, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3706801702307772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3493998646736145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5418:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17943352460861206, + 0.17943352460861206, + 0.8132356405258179, + 0.1698104739189148, + 0.09515346586704254, + 0.12655074894428253 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5418", + "split": "test", + "style_correct": 1, + "style_probability": 0.4857150614261627, + "style_target": 0, + "support_probability": 0.7200835943222046, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2569436625197561, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35959509015083313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3960894048213959, + 0.466928631067276, + 0.1863250881433487, + 0.777615487575531, + 0.07633950561285019, + 0.032678455114364624, + 0.042966313660144806, + 0.046747807413339615, + 0.021990686655044556, + 0.07472964376211166, + 0.06556164473295212 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.8288062810897827, + "style_target": 1, + "support_probability": 0.15736618638038635, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0523405733640847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33362191915512085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3987516164779663, + 0.5011729001998901, + 0.18191660940647125, + 0.8830562829971313, + 0.0913291946053505, + 0.0518747977912426, + 0.049399737268686295, + 0.06719156354665756, + 0.053722187876701355, + 0.1198858693242073 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5419", + "split": "test", + "style_correct": 1, + "style_probability": 0.2155342400074005, + "style_target": 0, + "support_probability": 0.3209502398967743, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07133162126391479, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5897068977355957, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2122984528541565, + 0.3313406705856323, + 0.2112278938293457, + 0.6653798818588257, + 0.2246031016111374 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 0, + "style_probability": 0.48974609375, + "style_target": 1, + "support_probability": 0.11907068639993668, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025526782582949714, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3826914131641388, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5427:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2068338394165039, + 0.4066565930843353, + 0.2367076575756073, + 0.745367705821991, + 0.13410812616348267 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5427", + "split": "test", + "style_correct": 1, + "style_probability": 0.2741306722164154, + "style_target": 0, + "support_probability": 0.38230541348457336, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.18913778859544195, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4612205922603607, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28018537163734436, + 0.7251464128494263, + 0.18820440769195557, + 0.20576409995555878, + 0.05901060625910759, + 0.10431227833032608 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 0, + "style_probability": 0.4376102089881897, + "style_target": 1, + "support_probability": 0.3208079934120178, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6635612291335562, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31132739782333374, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5429:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38030990958213806, + 0.7514320015907288, + 0.21893243491649628, + 0.22109782695770264, + 0.0595567561686039, + 0.04583411663770676 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5429", + "split": "test", + "style_correct": 0, + "style_probability": 0.8930459022521973, + "style_target": 0, + "support_probability": 0.5469566583633423, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.014241001395404742, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8877834677696228, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12762166559696198, + 0.4295702278614044, + 0.3170856833457947, + 0.4173122048377991, + 0.06700454652309418, + 0.06283528357744217, + 0.11526814848184586, + 0.05738973990082741, + 0.060746558010578156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 0, + "style_probability": 0.4463021755218506, + "style_target": 1, + "support_probability": 0.11010687053203583, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10256545378253358, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5150904655456543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5433:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11077365279197693, + 0.5186895728111267, + 0.3218955993652344, + 0.407390832901001, + 0.10105510801076889, + 0.054797522723674774 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5433", + "split": "test", + "style_correct": 1, + "style_probability": 0.3234677314758301, + "style_target": 0, + "support_probability": 0.4085671603679657, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06142449691774044, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5060874819755554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4439029395580292, + 0.8323376774787903, + 0.36630743741989136, + 0.20699013769626617, + 0.17456741631031036, + 0.10477882623672485, + 0.045263901352882385, + 0.06139688193798065, + 0.16111980378627777, + 0.1266959309577942, + 0.08700596541166306 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.5943689942359924, + "style_target": 1, + "support_probability": 0.05921473726630211, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020386820596147496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22165490686893463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4656856954097748, + 0.8961526155471802, + 0.4288296699523926, + 0.17030465602874756, + 0.11345108598470688, + 0.050717759877443314, + 0.0953613743185997, + 0.11608316749334335, + 0.05633135139942169, + 0.06575222313404083 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5441", + "split": "test", + "style_correct": 1, + "style_probability": 0.25630414485931396, + "style_target": 0, + "support_probability": 0.23677808046340942, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8921102273902797, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1897156834602356, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9853042960166931, + 0.3836694061756134, + 0.406818687915802, + 0.34491366147994995, + 0.042415961623191833 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.914478600025177, + "style_target": 1, + "support_probability": 0.6898564696311951, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6065852917581438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.102532297372818, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5448:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9811088442802429, + 0.40816807746887207, + 0.3848394751548767, + 0.3210529386997223, + 0.053394775837659836, + 0.05490976572036743, + 0.10104478895664215, + 0.06989841163158417, + 0.07704783231019974, + 0.1830323487520218, + 0.05412840098142624 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5448", + "split": "test", + "style_correct": 1, + "style_probability": 0.4815785884857178, + "style_target": 0, + "support_probability": 0.804591715335846, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8045084399719542, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17413857579231262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44788995385169983, + 0.4671270549297333, + 0.7572790384292603, + 0.8979362845420837, + 0.02575741708278656, + 0.06218089163303375 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.8238306641578674, + "style_target": 1, + "support_probability": 0.7994394898414612, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2629449658526877, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19035778939723969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5459:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4269064664840698, + 0.5729647874832153, + 0.7463973760604858, + 0.9334788918495178, + 0.0665130466222763, + 0.1513492316007614 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:5459", + "split": "test", + "style_correct": 1, + "style_probability": 0.3246510922908783, + "style_target": 0, + "support_probability": 0.789326548576355, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5442210921609139, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15275822579860687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46837952733039856, + 0.6885176301002502, + 0.9166486859321594, + 0.46837952733039856, + 0.10704322159290314, + 0.13791009783744812, + 0.0927576795220375, + 0.061540644615888596, + 0.14595946669578552 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 1, + "style_probability": 0.6997829675674438, + "style_target": 1, + "support_probability": 0.6106948256492615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3186211252699973, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2631489038467407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5490:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5131966471672058, + 0.5844463109970093, + 0.8830546736717224, + 0.5131966471672058, + 0.2978959083557129, + 0.1173834353685379, + 0.15209881961345673, + 0.07048054784536362, + 0.23299899697303772 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5490", + "split": "test", + "style_correct": 1, + "style_probability": 0.4629993736743927, + "style_target": 0, + "support_probability": 0.4899410605430603, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.717569089888638, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08536888659000397, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5296056270599365, + 0.4598384201526642, + 0.300508052110672, + 0.9709249138832092, + 0.06078029051423073, + 0.07892592251300812, + 0.17702682316303253, + 0.06663478910923004, + 0.07016248255968094, + 0.03997603803873062, + 0.10171705484390259 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.60585618019104, + "style_target": 1, + "support_probability": 0.7008507251739502, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3659943276194966, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0939880833029747, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5495:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49758604168891907, + 0.4951876103878021, + 0.3384925127029419, + 0.9444453716278076, + 0.10412708669900894, + 0.09676020592451096, + 0.21597500145435333, + 0.07850201427936554, + 0.3102301359176636 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5495", + "split": "test", + "style_correct": 1, + "style_probability": 0.20371980965137482, + "style_target": 0, + "support_probability": 0.5933681130409241, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8706424387490337, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21584922075271606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9615313410758972, + 0.2540995180606842, + 0.37404677271842957, + 0.5459278225898743, + 0.05699542909860611 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.8964110016822815, + "style_target": 1, + "support_probability": 0.7816577553749084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22124914036703824, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29544389247894287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5516:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9762557148933411, + 0.2792544364929199, + 0.3868325650691986, + 0.6053513288497925, + 0.09493755549192429 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5516", + "split": "test", + "style_correct": 1, + "style_probability": 0.421753853559494, + "style_target": 0, + "support_probability": 0.7203240990638733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09633651012141974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24007762968540192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7417219281196594, + 0.3307025134563446, + 0.34746402502059937, + 0.2881985902786255, + 0.05532664805650711, + 0.06988034397363663 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 1, + "style_probability": 0.9042809009552002, + "style_target": 1, + "support_probability": 0.14363375306129456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05409685101107929, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23081561923027039, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5524:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7466124892234802, + 0.3097623884677887, + 0.30992525815963745, + 0.28160420060157776, + 0.11546953022480011, + 0.1322619467973709 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5524", + "split": "test", + "style_correct": 0, + "style_probability": 0.7911548614501953, + "style_target": 0, + "support_probability": 0.1664622724056244, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5675524491421413, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18929460644721985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35735681653022766, + 0.28452757000923157, + 0.50935959815979, + 0.9127863645553589, + 0.03562384471297264, + 0.05465226620435715, + 0.035306040197610855, + 0.03095237910747528, + 0.04069563373923302 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 0, + "style_probability": 0.3370431065559387, + "style_target": 1, + "support_probability": 0.8918275237083435, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5056537625383761, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20297618210315704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5527:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39772069454193115, + 0.30363932251930237, + 0.5768900513648987, + 0.8951675295829773, + 0.08854009211063385, + 0.041690777987241745 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5527", + "split": "test", + "style_correct": 1, + "style_probability": 0.19523629546165466, + "style_target": 0, + "support_probability": 0.9020004868507385, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.445883568979486, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5565758943557739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16779503226280212, + 0.33402836322784424, + 0.6364879012107849, + 0.27441874146461487, + 0.09853031486272812, + 0.04746423289179802, + 0.07439679652452469, + 0.05586731433868408, + 0.09096015244722366, + 0.059213507920503616, + 0.08917246758937836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.8923381567001343, + "style_target": 1, + "support_probability": 0.506773054599762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03132954411561472, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.498935341835022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5530:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.21181881427764893, + 0.2856360375881195, + 0.8588141202926636, + 0.4433731734752655, + 0.0903361514210701, + 0.0609668530523777, + 0.04128419607877731, + 0.04845142364501953, + 0.04824794828891754, + 0.07996731996536255 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5530", + "split": "test", + "style_correct": 1, + "style_probability": 0.1212460920214653, + "style_target": 0, + "support_probability": 0.529241681098938, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3323666626015296, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.17746780812740326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21531954407691956, + 0.21666038036346436, + 0.4976294934749603, + 0.9088055491447449, + 0.16213378310203552 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.46698087453842163, + "style_target": 1, + "support_probability": 0.5805423855781555, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5249778862546393, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24075596034526825, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5535:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20659878849983215, + 0.2526232898235321, + 0.48372215032577515, + 0.8924071192741394, + 0.12926484644412994 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5535", + "split": "test", + "style_correct": 0, + "style_probability": 0.7021439671516418, + "style_target": 0, + "support_probability": 0.6101474761962891, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.3310606872157784, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3597300946712494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5553:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20757979154586792, + 0.9777687788009644, + 0.3523675203323364, + 0.41142377257347107, + 0.09315948933362961, + 0.08615847676992416 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 0, + "style_probability": 0.4878050684928894, + "style_target": 1, + "support_probability": 0.7258120775222778, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7280420676644184, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4852589964866638, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5553:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18866384029388428, + 0.9804861545562744, + 0.370295912027359, + 0.4422144889831543, + 0.03636738285422325, + 0.05532540753483772 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5553", + "split": "test", + "style_correct": 0, + "style_probability": 0.9010503888130188, + "style_target": 0, + "support_probability": 0.6030291318893433, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0601208181047781, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21390390396118164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.794136643409729, + 0.34930020570755005, + 0.2463013082742691, + 0.34930020570755005, + 0.16359016299247742, + 0.07657473534345627, + 0.10123929381370544, + 0.21879012882709503, + 0.13055174052715302 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 0, + "style_probability": 0.3129054009914398, + "style_target": 1, + "support_probability": 0.23980244994163513, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026143089750845228, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19459226727485657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5556:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8132121562957764, + 0.3853754699230194, + 0.2696703374385834, + 0.3853754699230194, + 0.045556556433439255, + 0.11675742268562317, + 0.15554259717464447, + 0.10692109167575836, + 0.14776340126991272 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5556", + "split": "test", + "style_correct": 1, + "style_probability": 0.11473675817251205, + "style_target": 0, + "support_probability": 0.5615720152854919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2103331472425775, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2772849500179291, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36365434527397156, + 0.6000385284423828, + 0.9003733396530151, + 0.36365434527397156, + 0.036719080060720444, + 0.05590173602104187, + 0.25992175936698914, + 0.04725809767842293, + 0.0593884103000164, + 0.09457212686538696, + 0.05291830003261566 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 1, + "style_probability": 0.5376474857330322, + "style_target": 1, + "support_probability": 0.4182913601398468, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32337789387857363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.44917139410972595, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5558:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3243013918399811, + 0.6341397762298584, + 0.8947439789772034, + 0.3243013918399811, + 0.0602693185210228, + 0.13527143001556396, + 0.05932023376226425, + 0.1345343291759491, + 0.12378855794668198, + 0.06505287438631058, + 0.04582161456346512 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5558", + "split": "test", + "style_correct": 0, + "style_probability": 0.7803564071655273, + "style_target": 0, + "support_probability": 0.33639398217201233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29897515070877745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42642685770988464, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.754699170589447, + 0.3913545608520508, + 0.5706080198287964, + 0.4735504984855652, + 0.09352938830852509 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.9090487360954285, + "style_target": 1, + "support_probability": 0.2053745687007904, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10451164853801485, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5516266822814941, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5560:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7306155562400818, + 0.3545542061328888, + 0.5249418616294861, + 0.40981370210647583, + 0.09382498264312744, + 0.17388807237148285, + 0.11553408205509186, + 0.18969124555587769, + 0.1602836698293686, + 0.12484979629516602 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5560", + "split": "test", + "style_correct": 1, + "style_probability": 0.45584186911582947, + "style_target": 0, + "support_probability": 0.11974573135375977, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7184971263761923, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17641882598400116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5580231547355652, + 0.9668259024620056, + 0.4500593841075897, + 0.33146247267723083, + 0.0699886828660965, + 0.053349971771240234 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.6639117002487183, + "style_target": 1, + "support_probability": 0.5599285960197449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.046490109087743654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2771330177783966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6211543083190918, + 0.9724562764167786, + 0.5012268424034119, + 0.3429606258869171, + 0.06071552261710167, + 0.06652592867612839 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5577", + "split": "test", + "style_correct": 1, + "style_probability": 0.0724453404545784, + "style_target": 0, + "support_probability": 0.535994291305542, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4029471787176941, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6359279751777649, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43332362174987793, + 0.8527860045433044, + 0.2528419494628906, + 0.3376484513282776, + 0.21571798622608185, + 0.09573324769735336, + 0.16414324939250946, + 0.12084528803825378, + 0.07801681756973267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 1, + "style_probability": 0.9005322456359863, + "style_target": 1, + "support_probability": 0.14187486469745636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05083673973441449, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7980715036392212, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5589:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39939939975738525, + 0.7705602645874023, + 0.2961837947368622, + 0.24801994860172272, + 0.09214036166667938, + 0.11178777366876602, + 0.09378132969141006, + 0.06508281081914902, + 0.2255346029996872 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5589", + "split": "test", + "style_correct": 0, + "style_probability": 0.5485177636146545, + "style_target": 0, + "support_probability": 0.08879026025533676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06744983606522982, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9248142242431641, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3486633896827698, + 0.27369654178619385, + 0.805831253528595, + 0.2621281147003174, + 0.07087530940771103, + 0.13300396502017975, + 0.18200089037418365, + 0.07987267524003983, + 0.16792815923690796, + 0.1168927326798439, + 0.08020485937595367 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.5807973146438599, + "style_target": 1, + "support_probability": 0.1396627128124237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026045342198369067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.874342679977417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5594:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2809690237045288, + 0.2731630206108093, + 0.8047800064086914, + 0.2785204350948334, + 0.0897405818104744, + 0.053477510809898376, + 0.06864754110574722, + 0.1185191348195076, + 0.08971822261810303 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5594", + "split": "test", + "style_correct": 1, + "style_probability": 0.15998516976833344, + "style_target": 0, + "support_probability": 0.15657132863998413, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00998095208745742, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6993897557258606, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4632384479045868, + 0.803256094455719, + 0.17025840282440186, + 0.3649771213531494, + 0.09307283908128738 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 1, + "style_probability": 0.7039618492126465, + "style_target": 1, + "support_probability": 0.04709247872233391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0018883813647013194, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6826133131980896, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5597:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.46991443634033203, + 0.8404005765914917, + 0.16718536615371704, + 0.36063867807388306, + 0.07327886670827866 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5597", + "split": "test", + "style_correct": 1, + "style_probability": 0.28136783838272095, + "style_target": 0, + "support_probability": 0.06811662018299103, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003239882491517379, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9508782625198364, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4367631673812866, + 0.10143379122018814, + 0.08296433836221695, + 0.30316391587257385, + 0.07434584945440292, + 0.07245977967977524 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 1, + "style_probability": 0.6808377504348755, + "style_target": 1, + "support_probability": 0.0012430890928953886, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00036533743312078606, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9444517493247986, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4522036910057068, + 0.13981039822101593, + 0.10527699440717697, + 0.29998350143432617, + 0.05506819114089012, + 0.08158532530069351 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5604", + "split": "test", + "style_correct": 0, + "style_probability": 0.5950407981872559, + "style_target": 0, + "support_probability": 0.0010323112364858389, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26702415940679103, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19989854097366333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22585083544254303, + 0.3281002640724182, + 0.7035370469093323, + 0.3152328431606293, + 0.04183412343263626, + 0.061512626707553864, + 0.048330266028642654, + 0.10898316651582718, + 0.039100151509046555 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.6861088275909424, + "style_target": 1, + "support_probability": 0.32518669962882996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11694618922897602, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12350114434957504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25575947761535645, + 0.2909489870071411, + 0.6684783697128296, + 0.33206698298454285, + 0.06238581985235214, + 0.06567805260419846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5610", + "split": "test", + "style_correct": 1, + "style_probability": 0.2298976331949234, + "style_target": 0, + "support_probability": 0.4596579074859619, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13163901929556232, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4761662781238556, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2617287337779999, + 0.3667084872722626, + 0.8862953186035156, + 0.11260508000850677, + 0.09515105187892914, + 0.02759775146842003, + 0.03284384682774544, + 0.1546929031610489, + 0.03859148547053337, + 0.04139203950762749, + 0.0889279842376709 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.7460336089134216, + "style_target": 1, + "support_probability": 0.1848916858434677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00865034467642345, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6354686617851257, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5614:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25449901819229126, + 0.4339134395122528, + 0.8713694214820862, + 0.13562768697738647, + 0.08443396538496017, + 0.038045551627874374, + 0.07178495824337006, + 0.07028122246265411, + 0.07189033925533295, + 0.09749075025320053 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5614", + "split": "test", + "style_correct": 1, + "style_probability": 0.1586088389158249, + "style_target": 0, + "support_probability": 0.17695441842079163, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.62971621315965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2671104073524475, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9412104487419128, + 0.40472519397735596, + 0.3982059061527252, + 0.5328201055526733, + 0.053650036454200745 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 0, + "style_probability": 0.3500240445137024, + "style_target": 1, + "support_probability": 0.8591683506965637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8073570899099992, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25107133388519287, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5628:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9568154811859131, + 0.3894646167755127, + 0.4080275893211365, + 0.5065472722053528, + 0.11434932798147202 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5628", + "split": "test", + "style_correct": 0, + "style_probability": 0.5609923005104065, + "style_target": 0, + "support_probability": 0.867389440536499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10868876699152709, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1978028565645218, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.683635413646698, + 0.24892295897006989, + 0.24538885056972504, + 0.2600127160549164, + 0.09442566335201263, + 0.12414190173149109 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.3606661856174469, + "style_target": 1, + "support_probability": 0.3199683427810669, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27981286380721393, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38021501898765564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5648:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7049174904823303, + 0.3058781623840332, + 0.21027396619319916, + 0.2348640412092209, + 0.05437454953789711, + 0.0782971903681755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5648", + "split": "test", + "style_correct": 0, + "style_probability": 0.9152768850326538, + "style_target": 0, + "support_probability": 0.20859098434448242, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3641285003123027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47895386815071106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7491164803504944, + 0.5703179836273193, + 0.17274650931358337, + 0.18409664928913116, + 0.08074484765529633, + 0.09983176738023758, + 0.15851084887981415, + 0.04520764574408531, + 0.1862654685974121 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 0, + "style_probability": 0.4622667133808136, + "style_target": 1, + "support_probability": 0.4236220419406891, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02687168607265367, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19899433851242065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8653199672698975, + 0.7085875272750854, + 0.25035804510116577, + 0.28532326221466064, + 0.05526673048734665, + 0.09518944472074509, + 0.07470958679914474, + 0.061825502663850784, + 0.11456816643476486 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:5652", + "split": "test", + "style_correct": 1, + "style_probability": 0.054605137556791306, + "style_target": 0, + "support_probability": 0.7941648960113525, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5678044053243916, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1572371870279312, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2525649070739746, + 0.8349515795707703, + 0.2295735627412796, + 0.13310901820659637, + 0.1788797527551651, + 0.14530713856220245, + 0.1073448657989502, + 0.11225303262472153, + 0.06484735012054443, + 0.07281208783388138 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 0, + "style_probability": 0.32736915349960327, + "style_target": 1, + "support_probability": 0.932007372379303, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6946663904356427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2539471685886383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24787266552448273, + 0.8044541478157043, + 0.27473652362823486, + 0.10208263993263245, + 0.17530842125415802, + 0.1232331395149231, + 0.08282095938920975, + 0.042262498289346695, + 0.13157910108566284, + 0.09275725483894348, + 0.164713054895401 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5665", + "split": "test", + "style_correct": 0, + "style_probability": 0.6407566070556641, + "style_target": 0, + "support_probability": 0.91899174451828, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5288315350684343, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23290427029132843, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.08423719555139542, + 0.30843469500541687, + 0.3714512884616852, + 0.7523183822631836, + 0.05766924470663071 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.8631956577301025, + "style_target": 1, + "support_probability": 0.6706781983375549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3163243637105211, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2392183244228363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5667:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.08978961408138275, + 0.2681328058242798, + 0.36011797189712524, + 0.7397118806838989, + 0.14358049631118774, + 0.19644112884998322, + 0.10967662930488586, + 0.15229730308055878, + 0.1017189472913742, + 0.1191454753279686, + 0.24832244217395782 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5667", + "split": "test", + "style_correct": 1, + "style_probability": 0.32055675983428955, + "style_target": 0, + "support_probability": 0.7182773947715759, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8899114617239653, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11513273417949677, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6401336193084717, + 0.9860007166862488, + 0.5812206864356995, + 0.38304048776626587, + 0.06939548999071121, + 0.02418862283229828 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.6545973420143127, + "style_target": 1, + "support_probability": 0.8944942951202393, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.600768541488911, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1551467329263687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6508883833885193, + 0.9907041192054749, + 0.5625705122947693, + 0.46901190280914307, + 0.05432264506816864, + 0.08521587401628494 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5669", + "split": "test", + "style_correct": 1, + "style_probability": 0.3471854329109192, + "style_target": 0, + "support_probability": 0.8826611042022705, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.024205633220283577, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8921770453453064, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5965719223022461, + 0.3192084729671478, + 0.33934420347213745, + 0.3192084729671478, + 0.11848984658718109, + 0.1319073587656021, + 0.1465366631746292, + 0.07956310361623764, + 0.1311679184436798 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 1, + "style_probability": 0.8087212443351746, + "style_target": 1, + "support_probability": 0.059949349611997604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01397770076484173, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9114856123924255, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5672:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6021496653556824, + 0.3128497004508972, + 0.31251680850982666, + 0.3128497004508972, + 0.13069632649421692, + 0.11816664040088654, + 0.0768180787563324, + 0.11594637483358383, + 0.22343683242797852 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5672", + "split": "test", + "style_correct": 0, + "style_probability": 0.5676822066307068, + "style_target": 0, + "support_probability": 0.06592082232236862, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.16419860295442845, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6033364534378052, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3381088078022003, + 0.9115440249443054, + 0.5076608657836914, + 0.3804449737071991, + 0.07051172107458115, + 0.07859847694635391, + 0.0958724245429039, + 0.08569138497114182, + 0.08289213478565216, + 0.06499861925840378 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.7763392329216003, + "style_target": 1, + "support_probability": 0.1793089061975479, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08972758376700617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5295574069023132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5681:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3303159177303314, + 0.8982073068618774, + 0.4938020706176758, + 0.36571669578552246, + 0.09100493788719177, + 0.06268314272165298, + 0.2829238474369049, + 0.6236660480499268, + 0.5064710974693298 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5681", + "split": "test", + "style_correct": 1, + "style_probability": 0.3598770201206207, + "style_target": 0, + "support_probability": 0.24748186767101288, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4900672925577308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10988409072160721, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9935458302497864, + 0.2881431579589844, + 0.6527335047721863, + 0.4226089119911194, + 0.1604703813791275 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 1, + "style_probability": 0.8985002040863037, + "style_target": 1, + "support_probability": 0.40419793128967285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22344425933419834, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1090899258852005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9981297850608826, + 0.2896641790866852, + 0.7520157694816589, + 0.4582081437110901, + 0.10148733109235764 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:5688", + "split": "test", + "style_correct": 0, + "style_probability": 0.6025935411453247, + "style_target": 0, + "support_probability": 0.5488642454147339, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1260920216039665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4136616289615631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30068522691726685, + 0.8716840744018555, + 0.2190120667219162, + 0.4019697904586792, + 0.2876734435558319, + 0.2069142907857895 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 1, + "style_probability": 0.7388197183609009, + "style_target": 1, + "support_probability": 0.1915307641029358, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17123672538787105, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4657660126686096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5705:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38188087940216064, + 0.8252847194671631, + 0.1809007227420807, + 0.3683951497077942, + 0.0724567174911499, + 0.0984494760632515 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5705", + "split": "test", + "style_correct": 0, + "style_probability": 0.7360644936561584, + "style_target": 0, + "support_probability": 0.2166689783334732, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8417100317946975, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02022060565650463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3501127362251282, + 0.9499231576919556, + 0.14457224309444427, + 0.19255582988262177, + 0.03745105117559433, + 0.0429837591946125, + 0.06280745565891266, + 0.0747186467051506, + 0.2685476243495941 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.8294790387153625, + "style_target": 1, + "support_probability": 0.8642821907997131, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5884119670657384, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009313391521573067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5709:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37782785296440125, + 0.943581759929657, + 0.1564980447292328, + 0.18743722140789032, + 0.04484415426850319, + 0.1452237367630005 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5709", + "split": "test", + "style_correct": 1, + "style_probability": 0.3363170027732849, + "style_target": 0, + "support_probability": 0.9045209884643555, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4224995161635831, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31277981400489807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7657256126403809, + 0.4559503495693207, + 0.22156542539596558, + 0.5999155044555664, + 0.051282960921525955, + 0.06968594342470169, + 0.02415834181010723, + 0.0356467068195343, + 0.11672510206699371, + 0.09126290678977966, + 0.09140311181545258 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.7640029788017273, + "style_target": 1, + "support_probability": 0.6357123851776123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04931832234052158, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27331456542015076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5716:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.854976236820221, + 0.6039035320281982, + 0.30533567070961, + 0.6923050880432129, + 0.09028361737728119, + 0.07732319831848145, + 0.13154996931552887, + 0.06830058246850967, + 0.04897225648164749, + 0.09047602117061615 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5716", + "split": "test", + "style_correct": 1, + "style_probability": 0.1724129021167755, + "style_target": 0, + "support_probability": 0.6954084038734436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5144045325882445, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.31586700677871704, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45242923498153687, + 0.7455190420150757, + 0.9058330059051514, + 0.6429622173309326, + 0.11709532886743546 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 1, + "style_probability": 0.7708954215049744, + "style_target": 1, + "support_probability": 0.6126237511634827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29618944651883794, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.35934939980506897, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5722:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5321829319000244, + 0.7234188914299011, + 0.8389260768890381, + 0.6056281924247742, + 0.12416977435350418 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:5722", + "split": "test", + "style_correct": 0, + "style_probability": 0.582231879234314, + "style_target": 0, + "support_probability": 0.4422414004802704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7721643940962508, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.040634676814079285, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41172298789024353, + 0.9771509170532227, + 0.3858146071434021, + 0.3307180404663086, + 0.1328691989183426, + 0.1109992191195488 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.8249865174293518, + "style_target": 1, + "support_probability": 0.5468103885650635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4654828671659939, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.026091894134879112, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5725:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41477179527282715, + 0.9741763472557068, + 0.3582969903945923, + 0.30402693152427673, + 0.16848358511924744 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5725", + "split": "test", + "style_correct": 1, + "style_probability": 0.3530539870262146, + "style_target": 0, + "support_probability": 0.6542885899543762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.000568016531049466, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9926401376724243, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35603949427604675, + 0.33655375242233276, + 0.40014463663101196, + 0.35603949427604675, + 0.07312790304422379, + 0.12032394111156464, + 0.1016007587313652, + 0.09894490987062454, + 0.09728622436523438 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.7947409152984619, + "style_target": 1, + "support_probability": 0.0031919353641569614, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004548959687578936, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.981618344783783, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5726:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4183250963687897, + 0.277843177318573, + 0.3705320358276367, + 0.4183250963687897, + 0.09043677151203156, + 0.277750164270401, + 0.19619639217853546, + 0.09773705154657364, + 0.14318965375423431 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5726", + "split": "test", + "style_correct": 1, + "style_probability": 0.4666888117790222, + "style_target": 0, + "support_probability": 0.009718411602079868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016230688685871443, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8370516896247864, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.45069900155067444, + 0.29730361700057983, + 0.21723993122577667, + 0.1126280426979065, + 0.10069489479064941, + 0.07783544808626175, + 0.058019936084747314, + 0.08428385108709335, + 0.08299677819013596, + 0.10225151479244232 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 1, + "style_probability": 0.7630677819252014, + "style_target": 1, + "support_probability": 0.005898140836507082, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017523285094081518, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.827904224395752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5732:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5046659708023071, + 0.26919063925743103, + 0.23097069561481476, + 0.1246715784072876, + 0.12211205810308456, + 0.10860921442508698, + 0.13260528445243835, + 0.12822435796260834, + 0.14205332100391388, + 0.17191492021083832, + 0.12610583007335663 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5732", + "split": "test", + "style_correct": 1, + "style_probability": 0.4926394820213318, + "style_target": 0, + "support_probability": 0.008749228902161121, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014699427189024507, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7657713890075684, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8518403172492981, + 0.3046033978462219, + 0.5140736103057861, + 0.21038506925106049, + 0.10283125936985016 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.2533396780490875, + "style_target": 1, + "support_probability": 0.09068606793880463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13525579911601415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7671677470207214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8656373620033264, + 0.3188212513923645, + 0.4934011399745941, + 0.21477675437927246, + 0.06201498210430145 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5744", + "split": "test", + "style_correct": 0, + "style_probability": 0.8748890161514282, + "style_target": 0, + "support_probability": 0.12378066778182983, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.06618147633840543, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.603934109210968, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3003026843070984, + 0.9043142795562744, + 0.37563684582710266, + 0.34334784746170044, + 0.18379969894886017, + 0.09811892360448837 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 0, + "style_probability": 0.3634832799434662, + "style_target": 1, + "support_probability": 0.2915252447128296, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0041816708463903165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8862257599830627, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5745:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4442363679409027, + 0.9257180690765381, + 0.3782765567302704, + 0.4034697413444519, + 0.03961476683616638, + 0.11001217365264893 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5745", + "split": "test", + "style_correct": 1, + "style_probability": 0.21979200839996338, + "style_target": 0, + "support_probability": 0.1931743025779724, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.28313696318441123, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24349196255207062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9353272914886475, + 0.24189460277557373, + 0.373134970664978, + 0.20187519490718842, + 0.024071235209703445, + 0.022234570235013962, + 0.08091665059328079, + 0.07277746498584747, + 0.057270798832178116 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 0, + "style_probability": 0.43943533301353455, + "style_target": 1, + "support_probability": 0.47828391194343567, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45621876013152374, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24160633981227875, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5756:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9261125326156616, + 0.2963980436325073, + 0.37820035219192505, + 0.1961171180009842, + 0.04940395802259445, + 0.05671559274196625, + 0.06384122371673584, + 0.05620961636304855, + 0.028302544727921486 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5756", + "split": "test", + "style_correct": 1, + "style_probability": 0.45892712473869324, + "style_target": 0, + "support_probability": 0.5763222575187683, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5706404814521875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12033874541521072, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.32982945442199707, + 0.37903183698654175, + 0.2713024318218231, + 0.941132664680481, + 0.043520327657461166, + 0.038932595402002335, + 0.03968137130141258, + 0.02666320838034153, + 0.09771134704351425, + 0.1541394740343094, + 0.052863288670778275 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.30684277415275574, + "style_target": 1, + "support_probability": 0.8344479203224182, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7149723939546035, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15114803612232208, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5767:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3407842218875885, + 0.3654303550720215, + 0.2507328689098358, + 0.9484890699386597, + 0.03054853528738022, + 0.028552265837788582, + 0.03214384242892265, + 0.024448653683066368, + 0.046094413846731186, + 0.05007816478610039, + 0.042622752487659454 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5767", + "split": "test", + "style_correct": 0, + "style_probability": 0.6100606322288513, + "style_target": 0, + "support_probability": 0.8106592893600464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008649974424004144, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.886469841003418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1737608164548874, + 0.2690456807613373, + 0.45126020908355713, + 0.3026329278945923, + 0.07619260251522064 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 0, + "style_probability": 0.4389362931251526, + "style_target": 1, + "support_probability": 0.003896137932315469, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001215613231816542, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.802867591381073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5770:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17925657331943512, + 0.2474822849035263, + 0.38278332352638245, + 0.30403053760528564, + 0.16875016689300537, + 0.3175508379936218, + 0.14242549240589142, + 0.15317361056804657, + 0.10446217656135559, + 0.11142800748348236 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5770", + "split": "test", + "style_correct": 1, + "style_probability": 0.26930028200149536, + "style_target": 0, + "support_probability": 0.007187745068222284, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7948551032960012, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.054905831813812256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8841408491134644, + 0.5607419610023499, + 0.11279920488595963, + 0.38107046484947205, + 0.12653809785842896, + 0.0678258016705513 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.6408641338348389, + "style_target": 1, + "support_probability": 0.8578745722770691, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6409699462980356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04744715988636017, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9360533952713013, + 0.6679057478904724, + 0.08483187854290009, + 0.37988027930259705, + 0.07918340712785721, + 0.15587130188941956 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5772", + "split": "test", + "style_correct": 1, + "style_probability": 0.4486536383628845, + "style_target": 0, + "support_probability": 0.8689254522323608, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007829427895853721, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.891066312789917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.285757452249527, + 0.4435770511627197, + 0.3048747181892395, + 0.5069806575775146, + 0.052157554775476456, + 0.06254765391349792, + 0.06076113134622574, + 0.05055651068687439, + 0.08700601011514664 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 1, + "style_probability": 0.634256899356842, + "style_target": 1, + "support_probability": 0.027173953130841255, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0795882981821876, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7003803849220276, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5773:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25735601782798767, + 0.4747467637062073, + 0.3358854353427887, + 0.5206478238105774, + 0.063698910176754, + 0.06278342753648758, + 0.06332851201295853, + 0.09034451842308044, + 0.04801955074071884 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5773", + "split": "test", + "style_correct": 0, + "style_probability": 0.8745519518852234, + "style_target": 0, + "support_probability": 0.11855097115039825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009855258622931956, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9380452036857605, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.35281479358673096, + 0.5119938254356384, + 0.1927243322134018, + 0.6544246077537537, + 0.09845077991485596, + 0.09092776477336884, + 0.10862003266811371, + 0.09296248853206635, + 0.19755025207996368, + 0.055672574788331985, + 0.07565092295408249 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 1, + "style_probability": 0.6808661222457886, + "style_target": 1, + "support_probability": 0.015862496569752693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00865525055992039, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.936410129070282, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5779:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42541539669036865, + 0.5006710290908813, + 0.1956097036600113, + 0.703988254070282, + 0.1496461182832718, + 0.050210677087306976, + 0.07488486915826797, + 0.10525034368038177, + 0.12651635706424713 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5779", + "split": "test", + "style_correct": 1, + "style_probability": 0.39214640855789185, + "style_target": 0, + "support_probability": 0.019163209944963455, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8795146399520385, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03471451997756958, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.938991129398346, + 0.3125295341014862, + 0.5437160730361938, + 0.16434712707996368, + 0.03698810189962387 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 1, + "style_probability": 0.5507992506027222, + "style_target": 1, + "support_probability": 0.9545033574104309, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19362821701526478, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0637788325548172, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5780:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9612390398979187, + 0.4347308874130249, + 0.6919715404510498, + 0.16633200645446777, + 0.23694384098052979 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5780", + "split": "test", + "style_correct": 1, + "style_probability": 0.11892609298229218, + "style_target": 0, + "support_probability": 0.9060747027397156, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20582578518430772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.537960946559906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4275052845478058, + 0.8689605593681335, + 0.36269912123680115, + 0.2725384533405304, + 0.14364191889762878, + 0.08517783880233765 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 1, + "style_probability": 0.6365708112716675, + "style_target": 1, + "support_probability": 0.2542647421360016, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49134787743762226, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3547428250312805, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4797036349773407, + 0.9160128235816956, + 0.3423631191253662, + 0.268428772687912, + 0.07530413568019867, + 0.04903395473957062 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5785", + "split": "test", + "style_correct": 0, + "style_probability": 0.8448014259338379, + "style_target": 0, + "support_probability": 0.44647738337516785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004924078348231387, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.968339741230011, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20862053334712982, + 0.610849142074585, + 0.27443939447402954, + 0.34402230381965637, + 0.05177932232618332, + 0.0352008119225502, + 0.1284346729516983, + 0.14032495021820068, + 0.036847688257694244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 0, + "style_probability": 0.3401012420654297, + "style_target": 1, + "support_probability": 0.03752967715263367, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003474436237579424, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9784290194511414, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5789:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22121889889240265, + 0.5949228405952454, + 0.2206048220396042, + 0.27823370695114136, + 0.04449784383177757, + 0.2523234486579895 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5789", + "split": "test", + "style_correct": 1, + "style_probability": 0.1633400171995163, + "style_target": 0, + "support_probability": 0.02936144545674324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3833805301911468, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1841345727443695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5995761156082153, + 0.26429715752601624, + 0.3994574546813965, + 0.8889646530151367, + 0.09023365378379822, + 0.05038432031869888, + 0.0558890663087368, + 0.061141472309827805, + 0.23759107291698456, + 0.11094692349433899, + 0.049719925969839096 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.7435261607170105, + "style_target": 1, + "support_probability": 0.39976415038108826, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21090381105653755, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10867255926132202, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5811:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5967515707015991, + 0.3866494596004486, + 0.406611829996109, + 0.9219503998756409, + 0.08113778382539749, + 0.07307523488998413, + 0.062010202556848526, + 0.07966476678848267, + 0.07856772094964981, + 0.09286272525787354 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5811", + "split": "test", + "style_correct": 1, + "style_probability": 0.3991600275039673, + "style_target": 0, + "support_probability": 0.5940226316452026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7987269134388839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17017978429794312, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.175559401512146, + 0.8564610481262207, + 0.4372471868991852, + 0.29503434896469116, + 0.036207832396030426 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.896294355392456, + "style_target": 1, + "support_probability": 0.8439419269561768, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42561375593643896, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20837253332138062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5817:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.176224023103714, + 0.8368324041366577, + 0.4137845039367676, + 0.2773921489715576, + 0.051120877265930176, + 0.08976180851459503, + 0.06874027103185654, + 0.05506742000579834, + 0.07185187190771103, + 0.058052174746990204, + 0.19472971558570862 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5817", + "split": "test", + "style_correct": 1, + "style_probability": 0.4574849605560303, + "style_target": 0, + "support_probability": 0.6284446120262146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22808902772451847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4506683051586151, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6331241726875305, + 0.31431302428245544, + 0.7103520631790161, + 0.3836085796356201, + 0.024838946759700775, + 0.03854289650917053 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.8677332401275635, + "style_target": 1, + "support_probability": 0.20711307227611542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.027147906521059584, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4379718601703644, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5818:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6897367238998413, + 0.36606112122535706, + 0.7472066283226013, + 0.40073302388191223, + 0.04228811338543892, + 0.09282661974430084 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5818", + "split": "test", + "style_correct": 1, + "style_probability": 0.4631856083869934, + "style_target": 0, + "support_probability": 0.24657443165779114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44251527370846944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10095403343439102, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31187260150909424, + 0.9394931197166443, + 0.3345987796783447, + 0.4755261242389679, + 0.050574228167533875, + 0.04040389508008957, + 0.022051392123103142, + 0.02859281562268734, + 0.035224106162786484 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 1, + "style_probability": 0.755058765411377, + "style_target": 1, + "support_probability": 0.4920005798339844, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7195572722357788, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03662852197885513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5821:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34706029295921326, + 0.9409179091453552, + 0.3378783166408539, + 0.4608656167984009, + 0.044419508427381516, + 0.04094167426228523, + 0.03967766836285591, + 0.08084005117416382, + 0.05453365668654442 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5821", + "split": "test", + "style_correct": 0, + "style_probability": 0.6665382385253906, + "style_target": 0, + "support_probability": 0.7644146680831909, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6059878180505684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23453256487846375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6329097747802734, + 0.2546139061450958, + 0.1159639060497284, + 0.8522160649299622, + 0.03039238043129444, + 0.02779124118387699, + 0.03593574836850166, + 0.06787507981061935, + 0.019761454313993454, + 0.03412284702062607, + 0.04835239425301552 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.8994709253311157, + "style_target": 1, + "support_probability": 0.520130455493927, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1664607084634895, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28663983941078186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5823:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7163202166557312, + 0.2342369556427002, + 0.13108861446380615, + 0.8065404891967773, + 0.06748475879430771, + 0.08142230659723282, + 0.08707919716835022, + 0.0548674538731575, + 0.06409653276205063 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:5823", + "split": "test", + "style_correct": 1, + "style_probability": 0.32746103405952454, + "style_target": 0, + "support_probability": 0.45797669887542725, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7504996280404796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2218194454908371, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9138980507850647, + 0.16921807825565338, + 0.5118927955627441, + 0.42176708579063416, + 0.08309116959571838 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.8423242568969727, + "style_target": 1, + "support_probability": 0.7111730575561523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011858109701051391, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5946762561798096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8935945630073547, + 0.1853971779346466, + 0.49857664108276367, + 0.4169861078262329, + 0.13665467500686646 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5828", + "split": "test", + "style_correct": 1, + "style_probability": 0.12392256408929825, + "style_target": 0, + "support_probability": 0.36282220482826233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5525103951971815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09915190190076828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4834117889404297, + 0.7395097017288208, + 0.2951645255088806, + 0.4834117889404297, + 0.05635279789566994, + 0.041257236152887344 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 1, + "style_probability": 0.7843060493469238, + "style_target": 1, + "support_probability": 0.772117555141449, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06545318901834474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4435165524482727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5829:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4774130880832672, + 0.5972250699996948, + 0.32639768719673157, + 0.4774130880832672, + 0.05316903442144394, + 0.1091800257563591 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5829", + "split": "test", + "style_correct": 0, + "style_probability": 0.5910469889640808, + "style_target": 0, + "support_probability": 0.12787507474422455, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5644397357663706, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3371599316596985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30542895197868347, + 0.1329670548439026, + 0.9699703454971313, + 0.6244412064552307, + 0.046252403408288956, + 0.027396565303206444, + 0.05027501657605171, + 0.03758611902594566, + 0.05998321995139122 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.6656787395477295, + "style_target": 1, + "support_probability": 0.6646904349327087, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21633134208906668, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.272062748670578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5830:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28267160058021545, + 0.1460084468126297, + 0.9528961777687073, + 0.5891904234886169, + 0.06382427364587784, + 0.1122112050652504 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5830", + "split": "test", + "style_correct": 1, + "style_probability": 0.20328469574451447, + "style_target": 0, + "support_probability": 0.6837245225906372, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5936685697649722, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3011093735694885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5145230889320374, + 0.2763839662075043, + 0.3332858085632324, + 0.6525579690933228, + 0.056142184883356094, + 0.045049652457237244, + 0.026528559625148773, + 0.07378850877285004, + 0.031343214213848114, + 0.04836392402648926 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.8993744254112244, + "style_target": 1, + "support_probability": 0.6277456879615784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06869723619520939, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2474694848060608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5891:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6315339803695679, + 0.286406546831131, + 0.405977338552475, + 0.7622294425964355, + 0.06632664054632187, + 0.04740476608276367, + 0.03061402030289173, + 0.06283188611268997, + 0.09075736254453659, + 0.11579647660255432 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5891", + "split": "test", + "style_correct": 1, + "style_probability": 0.15399377048015594, + "style_target": 0, + "support_probability": 0.7578341960906982, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005586948677759329, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9805013537406921, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.49574241042137146, + 0.369577556848526, + 0.19740335643291473, + 0.6754715442657471, + 0.21157151460647583 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.8509600758552551, + "style_target": 1, + "support_probability": 0.017725354060530663, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02639467003748975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9043517112731934, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.43536511063575745, + 0.3607339859008789, + 0.20849083364009857, + 0.6514703035354614, + 0.4528411030769348 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5899", + "split": "test", + "style_correct": 1, + "style_probability": 0.49177825450897217, + "style_target": 0, + "support_probability": 0.15121692419052124, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6469642698729743, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4133765697479248, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3637959063053131, + 0.2886960506439209, + 0.19957929849624634, + 0.8881970643997192, + 0.06250733137130737, + 0.11197251081466675 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.8210276961326599, + "style_target": 1, + "support_probability": 0.6334332823753357, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36462307319994386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3023604452610016, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5907:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3427310585975647, + 0.2801791727542877, + 0.18495376408100128, + 0.8616355657577515, + 0.11284960806369781 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5907", + "split": "test", + "style_correct": 1, + "style_probability": 0.46963199973106384, + "style_target": 0, + "support_probability": 0.631312906742096, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7393871255758812, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2552622854709625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19608935713768005, + 0.19350430369377136, + 0.1569288820028305, + 0.9220162630081177, + 0.042949240654706955, + 0.05543993040919304, + 0.02825716696679592, + 0.038633592426776886, + 0.09359301626682281 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.8756197690963745, + "style_target": 1, + "support_probability": 0.8170647025108337, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01195330525511196, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4217953085899353, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5911:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24531731009483337, + 0.24282322824001312, + 0.20316113531589508, + 0.9386352896690369, + 0.10718993842601776, + 0.057759806513786316, + 0.1208590492606163, + 0.1070256307721138, + 0.1640433818101883 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5911", + "split": "test", + "style_correct": 1, + "style_probability": 0.11559496074914932, + "style_target": 0, + "support_probability": 0.6688318252563477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18595653730564976, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7103082537651062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5203357934951782, + 0.3292462229728699, + 0.5203357934951782, + 0.514826774597168, + 0.044540293514728546, + 0.04601595923304558, + 0.08870351314544678, + 0.08242718130350113, + 0.02450535073876381, + 0.020832397043704987, + 0.0909566804766655 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 1, + "style_probability": 0.8834575414657593, + "style_target": 1, + "support_probability": 0.35720446705818176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09483146985347135, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7316281199455261, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5917:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4084424078464508, + 0.38771936297416687, + 0.4084424078464508, + 0.46966490149497986, + 0.06913865357637405, + 0.039460085332393646, + 0.042156364768743515, + 0.050535980612039566, + 0.07838679850101471, + 0.0701092928647995, + 0.08276540786027908 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5917", + "split": "test", + "style_correct": 0, + "style_probability": 0.5867806673049927, + "style_target": 0, + "support_probability": 0.42641907930374146, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0690556466385579, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.47262680530548096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37136512994766235, + 0.37880706787109375, + 0.34250015020370483, + 0.7725102305412292, + 0.0854833647608757 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.378682017326355, + "style_target": 1, + "support_probability": 0.4773099720478058, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24621256274987321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4709513485431671, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5919:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4048747420310974, + 0.42878761887550354, + 0.3500490188598633, + 0.7744703888893127, + 0.0498611181974411 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5919", + "split": "test", + "style_correct": 0, + "style_probability": 0.787022590637207, + "style_target": 0, + "support_probability": 0.5010929703712463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03948383536700106, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5071972608566284, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1970934122800827, + 0.32147279381752014, + 0.38435450196266174, + 0.3149760663509369, + 0.039540670812129974, + 0.08652559667825699 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 1, + "style_probability": 0.6063441038131714, + "style_target": 1, + "support_probability": 0.09921551495790482, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0040500388989805885, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4903196394443512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5929:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19915448129177094, + 0.31403565406799316, + 0.4665512442588806, + 0.32548606395721436, + 0.05225822329521179, + 0.11609983444213867 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5929", + "split": "test", + "style_correct": 1, + "style_probability": 0.3256821036338806, + "style_target": 0, + "support_probability": 0.13446645438671112, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06970520765612598, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5400677919387817, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.32979896664619446, + 0.24626819789409637, + 0.16558891534805298, + 0.857573390007019, + 0.20608246326446533, + 0.1058632880449295, + 0.10355541855096817, + 0.0797974020242691, + 0.08465699851512909 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.3490220010280609, + "style_target": 1, + "support_probability": 0.12111373990774155, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11943996182212759, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5071211457252502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5939:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.325387179851532, + 0.2598033547401428, + 0.1739123910665512, + 0.8648505806922913, + 0.13088156282901764, + 0.14110450446605682, + 0.09864002466201782, + 0.12334634363651276, + 0.12026352435350418 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5939", + "split": "test", + "style_correct": 0, + "style_probability": 0.5420264601707458, + "style_target": 0, + "support_probability": 0.1313064843416214, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6440224183103355, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06044397130608559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4565165340900421, + 0.42296743392944336, + 0.8786241412162781, + 0.4565165340900421, + 0.05940212309360504, + 0.06829961389303207, + 0.11113329976797104, + 0.04789569228887558, + 0.06142880767583847, + 0.12171609699726105 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.28872305154800415, + "style_target": 1, + "support_probability": 0.9086893200874329, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8750414040765265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03738250955939293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5958:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4295301139354706, + 0.42545250058174133, + 0.9200036525726318, + 0.4295301139354706, + 0.04716578498482704, + 0.09202475100755692, + 0.017463596537709236, + 0.023190902546048164, + 0.1421913206577301, + 0.07137414813041687, + 0.033460237085819244 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5958", + "split": "test", + "style_correct": 0, + "style_probability": 0.7737758159637451, + "style_target": 0, + "support_probability": 0.9506788849830627, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11863091060292422, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5692019462585449, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4222133755683899, + 0.925015926361084, + 0.2612946033477783, + 0.4730234444141388, + 0.07415610551834106 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 1, + "style_probability": 0.7518284320831299, + "style_target": 1, + "support_probability": 0.11972662806510925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.062205110659244944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.524889349937439, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:5969:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41920268535614014, + 0.924897313117981, + 0.2580401599407196, + 0.5411152243614197, + 0.0689697116613388, + 0.10318071395158768, + 0.11431936919689178, + 0.06232208013534546, + 0.06865542382001877, + 0.04770849272608757, + 0.14394548535346985 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5969", + "split": "test", + "style_correct": 0, + "style_probability": 0.5686509609222412, + "style_target": 0, + "support_probability": 0.12856800854206085, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8517099684192715, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05930592119693756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4479377865791321, + 0.8878483772277832, + 0.21423664689064026, + 0.3389284014701843, + 0.06574955582618713, + 0.07842062413692474 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 1, + "style_probability": 0.5503416061401367, + "style_target": 1, + "support_probability": 0.9831067323684692, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4682037492331937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07222133129835129, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5977:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47685086727142334, + 0.9479852914810181, + 0.3054813742637634, + 0.37803593277931213, + 0.055975448340177536, + 0.06815578788518906 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:5977", + "split": "test", + "style_correct": 1, + "style_probability": 0.20880714058876038, + "style_target": 0, + "support_probability": 0.9607616662979126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6784825762215405, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17758674919605255, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3013797700405121, + 0.46654805541038513, + 0.136581689119339, + 0.8294399976730347, + 0.06873446702957153, + 0.07731971144676208, + 0.04484479874372482, + 0.06505630910396576, + 0.0822351723909378 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 1, + "style_probability": 0.5922114849090576, + "style_target": 1, + "support_probability": 0.9005364775657654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7826355530912288, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16072788834571838, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.31008753180503845, + 0.4874444603919983, + 0.1571631133556366, + 0.8699689507484436, + 0.18501245975494385, + 0.045158158987760544, + 0.055304065346717834, + 0.028861822560429573, + 0.1119590774178505 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:5979", + "split": "test", + "style_correct": 0, + "style_probability": 0.7751978039741516, + "style_target": 0, + "support_probability": 0.8845627903938293, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26350486696707875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4644700586795807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4085114300251007, + 0.26591867208480835, + 0.8747352957725525, + 0.23818589746952057, + 0.08233936131000519, + 0.07974305748939514, + 0.10143603384494781, + 0.09298621863126755, + 0.07683825492858887, + 0.12501828372478485, + 0.2228667438030243 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.5845063924789429, + "style_target": 1, + "support_probability": 0.21800470352172852, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2821924111655427, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35198813676834106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5986:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3906005322933197, + 0.2658919095993042, + 0.8949154615402222, + 0.2345079928636551, + 0.06180867552757263, + 0.17710018157958984, + 0.09764846414327621, + 0.07600881159305573, + 0.09656771272420883 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:5986", + "split": "test", + "style_correct": 1, + "style_probability": 0.38349246978759766, + "style_target": 0, + "support_probability": 0.31319183111190796, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.871044620421216, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34815821051597595, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9483898878097534, + 0.25670483708381653, + 0.44130510091781616, + 0.27753564715385437, + 0.1354246735572815 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 1, + "style_probability": 0.8012613654136658, + "style_target": 1, + "support_probability": 0.7892225384712219, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7405092667589059, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20792846381664276, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5988:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9163024425506592, + 0.2791392505168915, + 0.45756766200065613, + 0.33458051085472107, + 0.15719158947467804 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:5988", + "split": "test", + "style_correct": 1, + "style_probability": 0.4238121211528778, + "style_target": 0, + "support_probability": 0.858790934085846, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5365525069808328, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29873037338256836, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4327247738838196, + 0.5818921327590942, + 0.628193736076355, + 0.4327247738838196, + 0.034622758626937866, + 0.04023277759552002 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.8703521490097046, + "style_target": 1, + "support_probability": 0.7640953660011292, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2812068512108503, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29383009672164917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:5990:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3966355323791504, + 0.5461141467094421, + 0.5995239615440369, + 0.3966355323791504, + 0.09973220527172089 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:5990", + "split": "test", + "style_correct": 1, + "style_probability": 0.37494853138923645, + "style_target": 0, + "support_probability": 0.7455592155456543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11927186594900263, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07414840906858444, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18669869005680084, + 0.7474626302719116, + 0.9722785949707031, + 0.23974262177944183, + 0.037413615733385086, + 0.0442352369427681, + 0.06746602058410645, + 0.0530916228890419, + 0.06349308043718338 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 0, + "style_probability": 0.48023220896720886, + "style_target": 1, + "support_probability": 0.2865835130214691, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04138472420736061, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08637448400259018, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6002:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21538293361663818, + 0.7843909859657288, + 0.9775027632713318, + 0.27211642265319824, + 0.03890576958656311, + 0.062255021184682846, + 0.04396424442529678, + 0.054551463574171066, + 0.08350012451410294 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6002", + "split": "test", + "style_correct": 1, + "style_probability": 0.21086981892585754, + "style_target": 0, + "support_probability": 0.3548630475997925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13456023669432324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2242300808429718, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3474572002887726, + 0.5528776049613953, + 0.5007157921791077, + 0.7102515697479248, + 0.04074055701494217, + 0.09887837618589401, + 0.053318846970796585, + 0.029207855463027954, + 0.031097061932086945, + 0.032173048704862595, + 0.058196548372507095 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8113268613815308, + "style_target": 1, + "support_probability": 0.14501287043094635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029192010750771047, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43926796317100525, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6011:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.37922918796539307, + 0.5837875604629517, + 0.4891197979450226, + 0.6301615238189697, + 0.15502507984638214, + 0.08205469697713852, + 0.1348246932029724, + 0.21922624111175537, + 0.08130000531673431, + 0.07453136891126633, + 0.0769437626004219 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6011", + "split": "test", + "style_correct": 0, + "style_probability": 0.5154633522033691, + "style_target": 0, + "support_probability": 0.05328739807009697, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016788131298033138, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8417810201644897, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29191434383392334, + 0.33197322487831116, + 0.39596062898635864, + 0.2471724897623062, + 0.14852647483348846 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.9238645434379578, + "style_target": 1, + "support_probability": 0.04349995777010918, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004618546228259469, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7931404709815979, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31943681836128235, + 0.3139185607433319, + 0.36516645550727844, + 0.21729648113250732, + 0.062362395226955414, + 0.08411421626806259, + 0.05679241567850113, + 0.13838955760002136, + 0.10754143446683884, + 0.08168657124042511, + 0.1711909919977188 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6013", + "split": "test", + "style_correct": 1, + "style_probability": 0.3791154623031616, + "style_target": 0, + "support_probability": 0.025423042476177216, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46066613136828494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3187883198261261, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6190723776817322, + 0.5353205800056458, + 0.4668411910533905, + 0.2310425043106079, + 0.03396710753440857, + 0.10837095230817795 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.9108002781867981, + "style_target": 1, + "support_probability": 0.4528619349002838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015198736157032051, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4173189401626587, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6016:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.765554666519165, + 0.6097140908241272, + 0.5219804644584656, + 0.2545027732849121, + 0.09312848001718521, + 0.2706933617591858 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6016", + "split": "test", + "style_correct": 1, + "style_probability": 0.1168074980378151, + "style_target": 0, + "support_probability": 0.4342549443244934, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3379626688535673, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15515953302383423, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3491610288619995, + 0.1819666177034378, + 0.3491610288619995, + 0.8047868013381958, + 0.04557677358388901, + 0.03384728729724884, + 0.029326187446713448, + 0.027332376688718796, + 0.13899610936641693 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 1, + "style_probability": 0.8018696904182434, + "style_target": 1, + "support_probability": 0.4351450502872467, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1765979602614998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17307013273239136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6027:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3620438575744629, + 0.1824813336133957, + 0.3620438575744629, + 0.7628682851791382, + 0.05428846925497055, + 0.055634934455156326, + 0.0676749125123024, + 0.105043925344944, + 0.03716132044792175 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6027", + "split": "test", + "style_correct": 0, + "style_probability": 0.5397152900695801, + "style_target": 0, + "support_probability": 0.5199431777000427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006056513635273317, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6518678069114685, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6003939509391785, + 0.31625720858573914, + 0.5430024862289429, + 0.7515236735343933, + 0.11335708945989609, + 0.0824328064918518, + 0.11586060374975204, + 0.09405330568552017, + 0.039368245750665665, + 0.0842626541852951, + 0.0768672451376915 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.7063645124435425, + "style_target": 1, + "support_probability": 0.023098301142454147, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003103555617309238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.702309250831604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6031:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.583347737789154, + 0.3166223168373108, + 0.5080094933509827, + 0.7080566883087158, + 0.22686511278152466, + 0.10510578751564026, + 0.21193262934684753, + 0.14445608854293823, + 0.12034584581851959 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6031", + "split": "test", + "style_correct": 1, + "style_probability": 0.37998393177986145, + "style_target": 0, + "support_probability": 0.014017201960086823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18602441296845296, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4542444050312042, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6685232520103455, + 0.629811704158783, + 0.8439754843711853, + 0.2630186676979065, + 0.11405026912689209 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 0, + "style_probability": 0.37384334206581116, + "style_target": 1, + "support_probability": 0.48087507486343384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024172985139731644, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5627988576889038, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7513529658317566, + 0.7426174283027649, + 0.9045085906982422, + 0.3138871490955353, + 0.2063857913017273 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6032", + "split": "test", + "style_correct": 1, + "style_probability": 0.1613488346338272, + "style_target": 0, + "support_probability": 0.56706303358078, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02234340226938336, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7943471670150757, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2036251425743103, + 0.15880273282527924, + 0.16753654181957245, + 0.5837348699569702, + 0.07814259827136993, + 0.07827470451593399 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.37887629866600037, + "style_target": 1, + "support_probability": 0.08828230947256088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01701948456064439, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8284115195274353, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6035:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20417340099811554, + 0.14169932901859283, + 0.17373345792293549, + 0.5765054225921631, + 0.1289495974779129, + 0.12573562562465668 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6035", + "split": "test", + "style_correct": 0, + "style_probability": 0.5636640191078186, + "style_target": 0, + "support_probability": 0.048782624304294586, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.12275775426566152, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6092433929443359, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4580201506614685, + 0.4116151034832001, + 0.6673577427864075, + 0.68302983045578, + 0.029330112040042877, + 0.1222783550620079, + 0.19710929691791534, + 0.12373148649930954, + 0.10486889630556107 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.3453387916088104, + "style_target": 1, + "support_probability": 0.38434451818466187, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5637529636939007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5224888324737549, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6038:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5018444061279297, + 0.37363001704216003, + 0.7019497156143188, + 0.667043149471283, + 0.05373036861419678, + 0.07261783629655838, + 0.02908269874751568, + 0.07226572930812836, + 0.06084423139691353 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6038", + "split": "test", + "style_correct": 0, + "style_probability": 0.904329776763916, + "style_target": 0, + "support_probability": 0.594082772731781, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14713163369513357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.317960649728775, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6715561151504517, + 0.3796249032020569, + 0.39190173149108887, + 0.8121241927146912, + 0.0780041441321373, + 0.027843287214636803, + 0.04292343184351921, + 0.07416507601737976, + 0.20944614708423615, + 0.14323574304580688, + 0.10831072926521301 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 0, + "style_probability": 0.47652333974838257, + "style_target": 1, + "support_probability": 0.2488614022731781, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0280809403146769, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42796584963798523, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6039:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7276828289031982, + 0.40601783990859985, + 0.4691886007785797, + 0.8399215936660767, + 0.17028233408927917, + 0.13252229988574982, + 0.1060580462217331, + 0.14615552127361298, + 0.10837814211845398, + 0.14716199040412903 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:6039", + "split": "test", + "style_correct": 1, + "style_probability": 0.2831405997276306, + "style_target": 0, + "support_probability": 0.30416619777679443, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01140977164184209, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8414673209190369, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3544413447380066, + 0.08870936185121536, + 0.4577709138393402, + 0.19231121242046356, + 0.08097974210977554 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 0, + "style_probability": 0.4569898545742035, + "style_target": 1, + "support_probability": 0.07178972661495209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0004927740687094008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.96462082862854, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6048:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3318510949611664, + 0.197639599442482, + 0.5253375768661499, + 0.209299698472023, + 0.16684873402118683 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6048", + "split": "test", + "style_correct": 1, + "style_probability": 0.18637153506278992, + "style_target": 0, + "support_probability": 0.028458021581172943, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10156085468967556, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5096482634544373, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7264721989631653, + 0.19631102681159973, + 0.10108038783073425, + 0.19631102681159973, + 0.049925796687603, + 0.07615573704242706 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.4860225319862366, + "style_target": 1, + "support_probability": 0.37382376194000244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.687073556456788, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30452796816825867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6060:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7744244337081909, + 0.18083800375461578, + 0.07871348410844803, + 0.18083800375461578, + 0.04618922248482704, + 0.08999758213758469 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6060", + "split": "test", + "style_correct": 0, + "style_probability": 0.935584545135498, + "style_target": 0, + "support_probability": 0.6435355544090271, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35067913605915635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33368220925331116, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4360688626766205, + 0.1368919461965561, + 0.8091505765914917, + 0.4360688626766205, + 0.03365381062030792, + 0.06832410395145416, + 0.062375739216804504, + 0.07604527473449707, + 0.0566757470369339 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 0, + "style_probability": 0.4664095938205719, + "style_target": 1, + "support_probability": 0.4480668306350708, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2120882308687655, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.30782756209373474, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6066:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4148746728897095, + 0.11840644478797913, + 0.8048301935195923, + 0.4148746728897095, + 0.056082166731357574, + 0.09385306388139725 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6066", + "split": "test", + "style_correct": 1, + "style_probability": 0.26146048307418823, + "style_target": 0, + "support_probability": 0.46998330950737, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03343468349905842, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9092145562171936, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27992773056030273, + 0.28035470843315125, + 0.5028662085533142, + 0.3682482838630676, + 0.06210596486926079, + 0.1249481663107872, + 0.1015147715806961, + 0.05006071925163269, + 0.10475385189056396, + 0.07375220954418182, + 0.13733728229999542 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 0, + "style_probability": 0.4577307403087616, + "style_target": 1, + "support_probability": 0.12054254859685898, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0027506644031932695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7408901453018188, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6070:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.35741424560546875, + 0.4627927839756012, + 0.7473829388618469, + 0.3498499393463135, + 0.06598442047834396, + 0.07823915779590607, + 0.0969642698764801, + 0.10553580522537231, + 0.08224966377019882, + 0.14775767922401428 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6070", + "split": "test", + "style_correct": 1, + "style_probability": 0.06973472237586975, + "style_target": 0, + "support_probability": 0.1872473657131195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7188926420276687, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4221963882446289, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26915058493614197, + 0.6762111186981201, + 0.93619704246521, + 0.5396642684936523, + 0.04994940757751465 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.7106412053108215, + "style_target": 1, + "support_probability": 0.6031038165092468, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13746981431658423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6659007668495178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6074:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27323615550994873, + 0.6335386633872986, + 0.907692015171051, + 0.47688421607017517, + 0.15208426117897034, + 0.22791612148284912, + 0.07451736181974411, + 0.07097292691469193, + 0.11011194437742233, + 0.1120319813489914, + 0.23642079532146454 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6074", + "split": "test", + "style_correct": 1, + "style_probability": 0.2896081507205963, + "style_target": 0, + "support_probability": 0.270051509141922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04406579689545143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.40600910782814026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5610383152961731, + 0.3969578146934509, + 0.9488971829414368, + 0.36621251702308655, + 0.04148562625050545, + 0.10568693280220032 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.7669440507888794, + "style_target": 1, + "support_probability": 0.06186347454786301, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009572414912200953, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.42400288581848145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6078:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6542072892189026, + 0.4202186167240143, + 0.9668614864349365, + 0.38532403111457825, + 0.0797569677233696, + 0.10798933357000351 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6078", + "split": "test", + "style_correct": 1, + "style_probability": 0.29530349373817444, + "style_target": 0, + "support_probability": 0.10249023139476776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006358833437932976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7457570433616638, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46031007170677185, + 0.2417612373828888, + 0.3536224067211151, + 0.7820560932159424, + 0.06519701331853867, + 0.11092378944158554, + 0.09441883116960526, + 0.12215318530797958, + 0.14630264043807983 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 1, + "style_probability": 0.7716050744056702, + "style_target": 1, + "support_probability": 0.022030938416719437, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011595086888673053, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5882899165153503, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.453718364238739, + 0.2833060920238495, + 0.38827821612358093, + 0.7476283311843872, + 0.10030809789896011, + 0.06300096213817596, + 0.049049556255340576, + 0.07944604754447937, + 0.07607020437717438 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6079", + "split": "test", + "style_correct": 0, + "style_probability": 0.6399056911468506, + "style_target": 0, + "support_probability": 0.038879115134477615, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.040345538393821816, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7375898957252502, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13727779686450958, + 0.862092137336731, + 0.13746093213558197, + 0.22759170830249786, + 0.22402648627758026, + 0.21184243261814117, + 0.09878290444612503, + 0.19844526052474976, + 0.1676325649023056, + 0.10489054769277573, + 0.06709899008274078 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.7657284736633301, + "style_target": 1, + "support_probability": 0.039183661341667175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05985270357363959, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4129096567630768, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6085:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1324533373117447, + 0.8541572093963623, + 0.14136679470539093, + 0.2561044692993164, + 0.15437419712543488, + 0.10617305338382721, + 0.06375285238027573, + 0.1631438136100769, + 0.09714508801698685 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6085", + "split": "test", + "style_correct": 1, + "style_probability": 0.4213561713695526, + "style_target": 0, + "support_probability": 0.14024896919727325, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7404502895516956, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.11790915578603745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.26196327805519104, + 0.2702546715736389, + 0.3358519971370697, + 0.9953771829605103, + 0.061113983392715454 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.6872188448905945, + "style_target": 1, + "support_probability": 0.6729782819747925, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32925212041081764, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16517841815948486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6119:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28814107179641724, + 0.3060108423233032, + 0.36755862832069397, + 0.9961749315261841, + 0.08183635771274567 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6119", + "split": "test", + "style_correct": 1, + "style_probability": 0.3026188313961029, + "style_target": 0, + "support_probability": 0.66053706407547, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.16809782825552236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5193132758140564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1966456025838852, + 0.922049880027771, + 0.20562896132469177, + 0.5098109841346741, + 0.13200819492340088, + 0.045822326093912125 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 0, + "style_probability": 0.40271317958831787, + "style_target": 1, + "support_probability": 0.5091395974159241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26228751322619814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34971317648887634, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6121:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20581020414829254, + 0.8803796172142029, + 0.2447902113199234, + 0.5300473570823669, + 0.05398358032107353, + 0.10723870992660522 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6121", + "split": "test", + "style_correct": 1, + "style_probability": 0.42846256494522095, + "style_target": 0, + "support_probability": 0.588895320892334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11809478430133068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.282967746257782, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6437045931816101, + 0.34015145897865295, + 0.15931302309036255, + 0.9337645173072815, + 0.027714084833860397, + 0.027458567172288895, + 0.03054204210639, + 0.2055339813232422, + 0.08859981596469879 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.8208980560302734, + "style_target": 1, + "support_probability": 0.0831560343503952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06012570462169185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1202264204621315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6135:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6235300302505493, + 0.30429908633232117, + 0.15267780423164368, + 0.9301024675369263, + 0.2098568081855774, + 0.16385342180728912 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6135", + "split": "test", + "style_correct": 1, + "style_probability": 0.2835152745246887, + "style_target": 0, + "support_probability": 0.15022410452365875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0443783887245086, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7870941162109375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1267024129629135, + 0.5935887694358826, + 0.45514044165611267, + 0.30026015639305115, + 0.0486702136695385, + 0.03539101779460907, + 0.044323619455099106, + 0.03504027798771858, + 0.05559885874390602, + 0.23993292450904846, + 0.11617700755596161 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.8147274851799011, + "style_target": 1, + "support_probability": 0.1332990974187851, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0023765694054355977, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.768354594707489, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6158:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18339942395687103, + 0.7572714686393738, + 0.422273725271225, + 0.3728179335594177, + 0.10274161398410797, + 0.07550043612718582, + 0.16030757129192352, + 0.12992161512374878, + 0.09220833331346512, + 0.16843871772289276 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6158", + "split": "test", + "style_correct": 1, + "style_probability": 0.12644752860069275, + "style_target": 0, + "support_probability": 0.1738254874944687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.273510284187104, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2258070707321167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30884668231010437, + 0.3834569752216339, + 0.2770789563655853, + 0.8588050007820129, + 0.09226170927286148 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 0, + "style_probability": 0.48408007621765137, + "style_target": 1, + "support_probability": 0.4156767427921295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019389617281154844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.252971887588501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6159:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2878825068473816, + 0.5331659913063049, + 0.2749497592449188, + 0.9175205826759338, + 0.1589176207780838 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6159", + "split": "test", + "style_correct": 1, + "style_probability": 0.14207172393798828, + "style_target": 0, + "support_probability": 0.3296872675418854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4024538992642981, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16371318697929382, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.852739155292511, + 0.3202621340751648, + 0.33871519565582275, + 0.3389795124530792, + 0.06660845875740051, + 0.0622691810131073 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.4801342487335205, + "style_target": 1, + "support_probability": 0.6563709378242493, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.596227857561356, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1524558961391449, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6160:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8947944045066833, + 0.4169549345970154, + 0.3622957766056061, + 0.4012407958507538, + 0.09192971140146255, + 0.04381614178419113 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6160", + "split": "test", + "style_correct": 0, + "style_probability": 0.7395253777503967, + "style_target": 0, + "support_probability": 0.661881685256958, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007061520800129896, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8988838791847229, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5459503531455994, + 0.35418686270713806, + 0.4872492551803589, + 0.35418686270713806, + 0.06973984092473984, + 0.1385519951581955, + 0.1106582060456276, + 0.10016503930091858, + 0.13883653283119202 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 1, + "style_probability": 0.5706291198730469, + "style_target": 1, + "support_probability": 0.010447442531585693, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015391540608208902, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7847179770469666, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6164:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5989689230918884, + 0.3213924765586853, + 0.5225402116775513, + 0.3213924765586853, + 0.11127462983131409 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6164", + "split": "test", + "style_correct": 1, + "style_probability": 0.46752163767814636, + "style_target": 0, + "support_probability": 0.024423925206065178, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7286383432282761, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16465117037296295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3140566945075989, + 0.2187955677509308, + 0.3032982647418976, + 0.9818431735038757, + 0.057017840445041656, + 0.2436458021402359, + 0.24742406606674194, + 0.17939051985740662, + 0.137132927775383, + 0.15746118128299713 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 1, + "style_probability": 0.5962682366371155, + "style_target": 1, + "support_probability": 0.6587689518928528, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.258337327724357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1354207843542099, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6167:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25817033648490906, + 0.22887460887432098, + 0.3124389052391052, + 0.9955783486366272, + 0.12069956213235855, + 0.10502398759126663, + 0.09458906948566437, + 0.16781306266784668, + 0.07094766944646835, + 0.08474425971508026 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6167", + "split": "test", + "style_correct": 1, + "style_probability": 0.2023455649614334, + "style_target": 0, + "support_probability": 0.7564163208007812, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06453935671566312, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5583130717277527, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8644007444381714, + 0.34456753730773926, + 0.1963759809732437, + 0.48489701747894287, + 0.14481987059116364 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.3734082877635956, + "style_target": 1, + "support_probability": 0.20219235122203827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13855494102155497, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6307575702667236, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6168:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.888447642326355, + 0.3859362304210663, + 0.2109835296869278, + 0.532318651676178, + 0.0782870426774025 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6168", + "split": "test", + "style_correct": 0, + "style_probability": 0.8499558568000793, + "style_target": 0, + "support_probability": 0.14592576026916504, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5737667546956366, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0979822650551796, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9366311430931091, + 0.38346806168556213, + 0.3324429392814636, + 0.38346806168556213, + 0.06762578338384628, + 0.08162437379360199 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 1, + "style_probability": 0.6644784212112427, + "style_target": 1, + "support_probability": 0.8413272500038147, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31104231444676245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1342913657426834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6176:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9742956757545471, + 0.403734415769577, + 0.38743335008621216, + 0.403734415769577, + 0.07762079685926437, + 0.09104928374290466 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6176", + "split": "test", + "style_correct": 0, + "style_probability": 0.5231623649597168, + "style_target": 0, + "support_probability": 0.8290510773658752, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.004031317905119792, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9825014472007751, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44044193625450134, + 0.2704382836818695, + 0.27303797006607056, + 0.19180405139923096, + 0.11727827042341232, + 0.10787604749202728, + 0.11607532948255539, + 0.1132536381483078, + 0.1284182071685791 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 0, + "style_probability": 0.4626160264015198, + "style_target": 1, + "support_probability": 0.01297875214368105, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007271746944554511, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9675222039222717, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6184:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5117916464805603, + 0.3154459297657013, + 0.2734910845756531, + 0.19300933182239532, + 0.0754227414727211, + 0.12269821763038635, + 0.13238069415092468, + 0.05777982249855995, + 0.12179844826459885 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6184", + "split": "test", + "style_correct": 0, + "style_probability": 0.5918958187103271, + "style_target": 0, + "support_probability": 0.014478771016001701, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5852724725938856, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04873286560177803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9828581809997559, + 0.35084474086761475, + 0.4781171679496765, + 0.3424507677555084, + 0.05500474199652672, + 0.09114473313093185, + 0.17765161395072937, + 0.13111071288585663, + 0.0885903462767601, + 0.13235515356063843 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.45885738730430603, + "style_target": 1, + "support_probability": 0.8502506613731384, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8755336889315402, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04427949711680412, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6196:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9896719455718994, + 0.363875150680542, + 0.5422917008399963, + 0.36031582951545715, + 0.08436768501996994, + 0.07349059730768204, + 0.10243555903434753, + 0.0646410882472992, + 0.11387410759925842, + 0.11468593031167984, + 0.09724550694227219 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6196", + "split": "test", + "style_correct": 0, + "style_probability": 0.8658743500709534, + "style_target": 0, + "support_probability": 0.8786919713020325, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6530980186223303, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08206785470247269, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4532341957092285, + 0.4532341957092285, + 0.8590134382247925, + 0.2869740426540375, + 0.13502612709999084 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 0, + "style_probability": 0.37347912788391113, + "style_target": 1, + "support_probability": 0.9366969466209412, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48561575246603544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10532200336456299, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6218:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4869045615196228, + 0.4869045615196228, + 0.9674662947654724, + 0.2927102744579315, + 0.16688749194145203 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6218", + "split": "test", + "style_correct": 1, + "style_probability": 0.2959270179271698, + "style_target": 0, + "support_probability": 0.958189845085144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15725842357035089, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29078155755996704, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29622170329093933, + 0.2795393764972687, + 0.5720868706703186, + 0.60481196641922, + 0.2612168490886688, + 0.12171945720911026 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 0, + "style_probability": 0.42017605900764465, + "style_target": 1, + "support_probability": 0.4771096706390381, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3547764803031476, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19724397361278534, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23915976285934448, + 0.20724648237228394, + 0.6160367131233215, + 0.6425474286079407, + 0.035668060183525085, + 0.10579528659582138 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6219", + "split": "test", + "style_correct": 0, + "style_probability": 0.545898973941803, + "style_target": 0, + "support_probability": 0.8116300702095032, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06262408163034161, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6706579923629761, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5755195021629333, + 0.6464744210243225, + 0.24622158706188202, + 0.44363343715667725, + 0.04128577560186386, + 0.08126349002122879, + 0.1622227281332016, + 0.1209356039762497, + 0.10674946010112762 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 1, + "style_probability": 0.5209013819694519, + "style_target": 1, + "support_probability": 0.12292927503585815, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.27074031563971346, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5727108716964722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6221:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6360398530960083, + 0.6561959981918335, + 0.22769136726856232, + 0.4401346743106842, + 0.09405422955751419, + 0.15651851892471313, + 0.07545618712902069, + 0.04206570237874985, + 0.1364600658416748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6221", + "split": "test", + "style_correct": 0, + "style_probability": 0.8568767309188843, + "style_target": 0, + "support_probability": 0.2217341661453247, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015679587347763757, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2844238579273224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3417185842990875, + 0.4388718903064728, + 0.49558866024017334, + 0.21962954103946686, + 0.11793461441993713, + 0.07468871772289276, + 0.08841574937105179, + 0.08918432146310806, + 0.3910883069038391, + 0.1552908569574356 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 0, + "style_probability": 0.48260360956192017, + "style_target": 1, + "support_probability": 0.05405138060450554, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011359714818668096, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6336632966995239, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6238:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4463910758495331, + 0.5464928150177002, + 0.5539905428886414, + 0.2488866150379181, + 0.08215606957674026, + 0.080644391477108, + 0.1461755484342575, + 0.1148366928100586, + 0.08458729833364487, + 0.11760180443525314 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6238", + "split": "test", + "style_correct": 1, + "style_probability": 0.2024865448474884, + "style_target": 0, + "support_probability": 0.06755173206329346, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.893081636465876, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05303172767162323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2883061468601227, + 0.34032461047172546, + 0.9714381098747253, + 0.47988617420196533, + 0.050592318177223206 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.7298330068588257, + "style_target": 1, + "support_probability": 0.9140933156013489, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6980763714458327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08753804862499237, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6246:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2392154335975647, + 0.38053590059280396, + 0.9406045079231262, + 0.5093007683753967, + 0.09273725748062134 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6246", + "split": "test", + "style_correct": 1, + "style_probability": 0.39750775694847107, + "style_target": 0, + "support_probability": 0.858990490436554, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10655431808468839, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7974412441253662, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2391936182975769, + 0.5205227136611938, + 0.2169380635023117, + 0.8778271079063416, + 0.1590389907360077, + 0.059076469391584396 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.7453578114509583, + "style_target": 1, + "support_probability": 0.20640254020690918, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09492491288029292, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6058971285820007, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6251:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24531272053718567, + 0.5266258120536804, + 0.2307523638010025, + 0.8721536993980408, + 0.08676357567310333, + 0.12817727029323578, + 0.12737102806568146, + 0.08108021318912506, + 0.25445714592933655, + 0.17012375593185425 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6251", + "split": "test", + "style_correct": 1, + "style_probability": 0.32818278670310974, + "style_target": 0, + "support_probability": 0.2967909276485443, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03406173538706536, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8355630040168762, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21096909046173096, + 0.5935567617416382, + 0.19831012189388275, + 0.5744068622589111, + 0.07629990577697754, + 0.06508256494998932, + 0.0513729490339756, + 0.10238783061504364, + 0.07006356120109558 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.7702282071113586, + "style_target": 1, + "support_probability": 0.0784943550825119, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005826636615258284, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7582671642303467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6253:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22319822013378143, + 0.6807357668876648, + 0.21761596202850342, + 0.6510313749313354, + 0.10225380957126617, + 0.10686755180358887, + 0.07696627080440521, + 0.06339021772146225, + 0.09135214984416962 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6253", + "split": "test", + "style_correct": 1, + "style_probability": 0.22937723994255066, + "style_target": 0, + "support_probability": 0.28387928009033203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8826370797475392, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06154054030776024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6260:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9776901602745056, + 0.4100448787212372, + 0.13266758620738983, + 0.32425960898399353, + 0.07662612199783325, + 0.053199950605630875, + 0.044058818370103836, + 0.03997395187616348, + 0.08615008741617203, + 0.07245665788650513 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 1, + "style_probability": 0.8926352858543396, + "style_target": 1, + "support_probability": 0.7930862903594971, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7163677159757107, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05349693447351456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6260:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9696257710456848, + 0.35969629883766174, + 0.13282041251659393, + 0.33804765343666077, + 0.06106676906347275, + 0.07441046833992004, + 0.04199281334877014, + 0.027622254565358162, + 0.04876072704792023, + 0.04194095730781555, + 0.08682151138782501 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6260", + "split": "test", + "style_correct": 0, + "style_probability": 0.6520791053771973, + "style_target": 0, + "support_probability": 0.8109698295593262, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7433361984534949, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1348523646593094, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4458134174346924, + 0.5498926043510437, + 0.952541172504425, + 0.4458134174346924, + 0.02672179415822029 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.7497590184211731, + "style_target": 1, + "support_probability": 0.845859706401825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3814523708988986, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0785699263215065, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3806520700454712, + 0.5165771842002869, + 0.9385547637939453, + 0.3806520700454712, + 0.044537413865327835, + 0.03674940764904022, + 0.05783946067094803, + 0.11354203522205353, + 0.06320544332265854 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6266", + "split": "test", + "style_correct": 1, + "style_probability": 0.27578628063201904, + "style_target": 0, + "support_probability": 0.8093798756599426, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7091401417969578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.08382081985473633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.09525186568498611, + 0.3293893337249756, + 0.3603523075580597, + 0.9562705755233765, + 0.030190521851181984, + 0.03884520009160042 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.7683467268943787, + "style_target": 1, + "support_probability": 0.5563669204711914, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31131635375336586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09983120858669281, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6271:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.08669811487197876, + 0.42325928807258606, + 0.5181872248649597, + 0.9868956208229065, + 0.09646721929311752, + 0.07611741870641708 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6271", + "split": "test", + "style_correct": 1, + "style_probability": 0.2656647562980652, + "style_target": 0, + "support_probability": 0.6449592709541321, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20469562796392887, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5180445909500122, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.282649427652359, + 0.13594657182693481, + 0.42084503173828125, + 0.18457931280136108, + 0.04007474705576897, + 0.06726347655057907, + 0.05780702829360962, + 0.04183461144566536, + 0.04181963950395584 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.8763664960861206, + "style_target": 1, + "support_probability": 0.15850339829921722, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06543647257189678, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5030907988548279, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6286:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2866571843624115, + 0.14390169084072113, + 0.4629725515842438, + 0.21901002526283264, + 0.08683174103498459, + 0.1105799600481987, + 0.0878172367811203, + 0.22022125124931335, + 0.059421081095933914 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6286", + "split": "test", + "style_correct": 1, + "style_probability": 0.45515501499176025, + "style_target": 0, + "support_probability": 0.1601593792438507, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.61291816868534, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1778678148984909, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.42497652769088745, + 0.1487838476896286, + 0.627203643321991, + 0.8925057649612427, + 0.024417825043201447, + 0.04443870112299919, + 0.05002502351999283, + 0.04051661491394043, + 0.04522048309445381, + 0.020355038344860077 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 0, + "style_probability": 0.45149871706962585, + "style_target": 1, + "support_probability": 0.8849880695343018, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47964265652218785, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15815746784210205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6291:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39613187313079834, + 0.15643370151519775, + 0.6241645216941833, + 0.8601018786430359, + 0.018135759979486465, + 0.039770595729351044 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6291", + "split": "test", + "style_correct": 1, + "style_probability": 0.23833858966827393, + "style_target": 0, + "support_probability": 0.8964245319366455, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09511696742460171, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.796920120716095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2528494894504547, + 0.5755536556243896, + 0.2981211245059967, + 0.3136221766471863, + 0.13072654604911804 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.8842301368713379, + "style_target": 1, + "support_probability": 0.13579176366329193, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0019705642258283497, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8662391304969788, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6293:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2970849275588989, + 0.6033044457435608, + 0.44052910804748535, + 0.3237534165382385, + 0.1074535921216011 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6293", + "split": "test", + "style_correct": 1, + "style_probability": 0.24341373145580292, + "style_target": 0, + "support_probability": 0.07664696127176285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08694597965952512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3298673927783966, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5499765276908875, + 0.32305198907852173, + 0.38233789801597595, + 0.9627997875213623, + 0.2118532657623291, + 0.23889818787574768 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 1, + "style_probability": 0.8380183577537537, + "style_target": 1, + "support_probability": 0.0746181309223175, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05751616880587119, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3003852069377899, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6299:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.47069671750068665, + 0.3118226230144501, + 0.2899053692817688, + 0.9469115734100342, + 0.09710226207971573, + 0.2245098203420639 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6299", + "split": "test", + "style_correct": 0, + "style_probability": 0.615373432636261, + "style_target": 0, + "support_probability": 0.09767033904790878, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5462409326174367, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16901245713233948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34786033630371094, + 0.1471187174320221, + 0.9832561016082764, + 0.16061703860759735, + 0.07871787250041962, + 0.0704779252409935, + 0.15093496441841125, + 0.3410504162311554, + 0.40802496671676636 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.74629807472229, + "style_target": 1, + "support_probability": 0.5665115118026733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2832285346811769, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1296241730451584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6302:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3530988097190857, + 0.15587428212165833, + 0.9751009345054626, + 0.17691579461097717, + 0.11115096509456635 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6302", + "split": "test", + "style_correct": 1, + "style_probability": 0.3017498254776001, + "style_target": 0, + "support_probability": 0.6393064260482788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5811303551267102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2699615955352783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3288639783859253, + 0.846283495426178, + 0.5174439549446106, + 0.4739624261856079, + 0.061381954699754715, + 0.033082861453294754, + 0.0282941535115242, + 0.047246020287275314, + 0.1063702404499054, + 0.0686325803399086, + 0.05489350110292435 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.8206184506416321, + "style_target": 1, + "support_probability": 0.5871537327766418, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11947362354931546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28926777839660645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6309:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.33501997590065, + 0.8718084692955017, + 0.5537253022193909, + 0.5332021117210388, + 0.05928045138716698, + 0.041669443249702454, + 0.05081452429294586, + 0.08258354663848877, + 0.08032657206058502, + 0.10753367096185684 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6309", + "split": "test", + "style_correct": 1, + "style_probability": 0.2861533761024475, + "style_target": 0, + "support_probability": 0.7003055214881897, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0072201814303370915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4903121888637543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9305158853530884, + 0.26277297735214233, + 0.25241780281066895, + 0.38279494643211365, + 0.11879460513591766 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 1, + "style_probability": 0.5757995247840881, + "style_target": 1, + "support_probability": 0.02784404158592224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013526295911544745, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2898560166358948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6324:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9383751153945923, + 0.32782262563705444, + 0.2714889347553253, + 0.37427404522895813, + 0.15852800011634827 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6324", + "split": "test", + "style_correct": 1, + "style_probability": 0.3722888231277466, + "style_target": 0, + "support_probability": 0.06465975940227509, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7019382654248146, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09983737766742706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9740443229675293, + 0.2845669388771057, + 0.061042528599500656, + 0.2410839945077896, + 0.05225665122270584, + 0.041892725974321365 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 0, + "style_probability": 0.31097283959388733, + "style_target": 1, + "support_probability": 0.9037858843803406, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7761967162177826, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05293085798621178, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6364:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9570159912109375, + 0.2850458025932312, + 0.06936147809028625, + 0.22150146961212158, + 0.10330494493246078, + 0.10965284705162048, + 0.20592597126960754, + 0.08042044937610626, + 0.06601512432098389, + 0.12055820971727371, + 0.058287329971790314 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6364", + "split": "test", + "style_correct": 1, + "style_probability": 0.17831112444400787, + "style_target": 0, + "support_probability": 0.9506904482841492, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6607039584860246, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.040109455585479736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.11846378445625305, + 0.3235761821269989, + 0.24031515419483185, + 0.9543930292129517, + 0.1287364512681961, + 0.06226189434528351, + 0.040452100336551666, + 0.0644434466958046, + 0.054844461381435394 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.7042160034179688, + "style_target": 1, + "support_probability": 0.5197747945785522, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08148279996877504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05678620934486389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6387:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15015828609466553, + 0.4092623293399811, + 0.287150502204895, + 0.9656447172164917, + 0.079310841858387, + 0.07661109417676926, + 0.07082785665988922, + 0.06660176068544388, + 0.09179368615150452 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6387", + "split": "test", + "style_correct": 1, + "style_probability": 0.20643243193626404, + "style_target": 0, + "support_probability": 0.5263555645942688, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003255732213625349, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8459277749061584, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42324116826057434, + 0.5995815992355347, + 0.49274152517318726, + 0.30946019291877747, + 0.03859437629580498, + 0.11034900695085526, + 0.10780681669712067, + 0.03966284170746803, + 0.030599769204854965, + 0.06438063085079193, + 0.049913402646780014 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 1, + "style_probability": 0.5407041907310486, + "style_target": 1, + "support_probability": 0.01507427729666233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0041894196950467985, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7646165490150452, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6389:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.47171151638031006, + 0.5660760998725891, + 0.47840017080307007, + 0.3031420409679413, + 0.041429340839385986, + 0.10354851931333542, + 0.13308939337730408, + 0.054751958698034286, + 0.18392117321491241, + 0.17390938103199005 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6389", + "split": "test", + "style_correct": 1, + "style_probability": 0.302115797996521, + "style_target": 0, + "support_probability": 0.025865184143185616, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6223058724115127, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.25384509563446045, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5198308229446411, + 0.9307745099067688, + 0.612888753414154, + 0.47153618931770325, + 0.06615597009658813 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.8100292682647705, + "style_target": 1, + "support_probability": 0.791009247303009, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4859856661786139, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15208519995212555, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6393:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4710560142993927, + 0.9189969897270203, + 0.55511075258255, + 0.4759732186794281, + 0.0648462325334549, + 0.08948318660259247, + 0.20848803222179413, + 0.05514271557331085, + 0.0966658964753151 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6393", + "split": "test", + "style_correct": 1, + "style_probability": 0.4660273492336273, + "style_target": 0, + "support_probability": 0.843055009841919, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03871642440372348, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6006741523742676, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38676121830940247, + 0.3335673213005066, + 0.27357515692710876, + 0.37285107374191284, + 0.2086864560842514, + 0.1358143389225006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.7523044347763062, + "style_target": 1, + "support_probability": 0.25938087701797485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0024500918814083774, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9135167002677917, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6396:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41366860270500183, + 0.29435014724731445, + 0.3032294511795044, + 0.4630175828933716, + 0.1533181369304657, + 0.1982789784669876 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6396", + "split": "test", + "style_correct": 1, + "style_probability": 0.21473537385463715, + "style_target": 0, + "support_probability": 0.14395731687545776, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5448309178888415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.323963463306427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5377123951911926, + 0.9323028922080994, + 0.2322939932346344, + 0.3427767753601074, + 0.0626562088727951, + 0.18176285922527313, + 0.06914746761322021, + 0.16834236681461334, + 0.0876365602016449 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 1, + "style_probability": 0.8244457244873047, + "style_target": 1, + "support_probability": 0.40278708934783936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2605595410994699, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3112460970878601, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6401:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5378362536430359, + 0.9162741303443909, + 0.25255268812179565, + 0.36819490790367126, + 0.046495433896780014, + 0.11158428341150284, + 0.104067362844944, + 0.11976709216833115, + 0.09163583070039749 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6401", + "split": "test", + "style_correct": 0, + "style_probability": 0.5469114780426025, + "style_target": 0, + "support_probability": 0.39245808124542236, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022347430145321145, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8207049369812012, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.27949535846710205, + 0.2202649712562561, + 0.2390747219324112, + 0.37626686692237854, + 0.049833156168460846, + 0.0995555892586708, + 0.07006219029426575, + 0.08403316140174866, + 0.22410054504871368, + 0.10730002820491791 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.9174546599388123, + "style_target": 1, + "support_probability": 0.09730062633752823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00584634706278048, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9006884694099426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6405:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24441823363304138, + 0.2733120918273926, + 0.22690154612064362, + 0.2975952923297882, + 0.17798921465873718, + 0.24391143023967743 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6405", + "split": "test", + "style_correct": 1, + "style_probability": 0.3509286940097809, + "style_target": 0, + "support_probability": 0.0659538209438324, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8284519379359807, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2466159462928772, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8932327032089233, + 0.5604848265647888, + 0.46537140011787415, + 0.42263466119766235, + 0.09040574729442596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.908332109451294, + "style_target": 1, + "support_probability": 0.672167956829071, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3576205703509654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18702039122581482, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6408:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.965427815914154, + 0.6921826601028442, + 0.5290703773498535, + 0.5064820051193237, + 0.1995532512664795 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6408", + "split": "test", + "style_correct": 1, + "style_probability": 0.2543577253818512, + "style_target": 0, + "support_probability": 0.7711833119392395, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03340344802972028, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5260935425758362, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.10120602697134018, + 0.35039958357810974, + 0.6868873834609985, + 0.40764012932777405, + 0.05279037728905678, + 0.0829186961054802 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 1, + "style_probability": 0.7970229983329773, + "style_target": 1, + "support_probability": 0.06841354817152023, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031172691115810565, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5030748844146729, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6410:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1178169697523117, + 0.35287266969680786, + 0.6659667491912842, + 0.47478049993515015, + 0.039007194340229034, + 0.1100936308503151 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6410", + "split": "test", + "style_correct": 0, + "style_probability": 0.6498286128044128, + "style_target": 0, + "support_probability": 0.07892091572284698, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06697792469886554, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5242831707000732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5951433181762695, + 0.7444196939468384, + 0.40482738614082336, + 0.19547933340072632, + 0.10083183646202087, + 0.19704526662826538, + 0.1918959766626358, + 0.0713706910610199, + 0.10500805824995041 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.6740393042564392, + "style_target": 1, + "support_probability": 0.0796721950173378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0656707462836774, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.271882027387619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6413:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5917667150497437, + 0.7389729022979736, + 0.397306352853775, + 0.2236110121011734, + 0.08510904759168625 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6413", + "split": "test", + "style_correct": 1, + "style_probability": 0.2572758197784424, + "style_target": 0, + "support_probability": 0.1644562929868698, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9426440754559557, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05324097350239754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6419:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22094766795635223, + 0.43897759914398193, + 0.07090797275304794, + 0.9662622809410095, + 0.03880317136645317, + 0.03606029227375984, + 0.07151190936565399, + 0.044548340141773224, + 0.09250185638666153, + 0.06535307317972183 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.8417326807975769, + "style_target": 1, + "support_probability": 0.9456543922424316, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5204281474811125, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03646865859627724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6419:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.28652623295783997, + 0.5931566953659058, + 0.07826375216245651, + 0.9889131188392639, + 0.06632616370916367, + 0.04388044402003288, + 0.052487537264823914, + 0.07133889943361282, + 0.0935087502002716, + 0.13396614789962769 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6419", + "split": "test", + "style_correct": 1, + "style_probability": 0.13416838645935059, + "style_target": 0, + "support_probability": 0.9627686142921448, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.013237568211887934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16207802295684814, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25059401988983154, + 0.7678091526031494, + 0.2425062358379364, + 0.3242303133010864, + 0.1350225806236267 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 0, + "style_probability": 0.405534565448761, + "style_target": 1, + "support_probability": 0.06741812825202942, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004154435018079993, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22073641419410706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6425:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29051488637924194, + 0.8913345336914062, + 0.2215024083852768, + 0.36101922392845154, + 0.1844913214445114 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6425", + "split": "test", + "style_correct": 1, + "style_probability": 0.2343083918094635, + "style_target": 0, + "support_probability": 0.12158169597387314, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.601393745185831, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.091937355697155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6430:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29267174005508423, + 0.2578953206539154, + 0.9776819348335266, + 0.27080607414245605, + 0.03478783741593361, + 0.08179471641778946 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 1, + "style_probability": 0.6895975470542908, + "style_target": 1, + "support_probability": 0.6142817139625549, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9195867064731473, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10363654047250748, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6430:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2911083996295929, + 0.2222311645746231, + 0.9893869161605835, + 0.2960031032562256, + 0.10677119344472885, + 0.16746604442596436 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6430", + "split": "test", + "style_correct": 0, + "style_probability": 0.9394766688346863, + "style_target": 0, + "support_probability": 0.7035748958587646, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02403827104052736, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9062806367874146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5291711688041687, + 0.4214622974395752, + 0.3528994619846344, + 0.539199709892273, + 0.06999804079532623, + 0.10164865106344223, + 0.0583319328725338, + 0.06147352233529091, + 0.0687706470489502 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.65220707654953, + "style_target": 1, + "support_probability": 0.06311191618442535, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007415936448490031, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8996206521987915, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6436:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4942612051963806, + 0.39161649346351624, + 0.31283336877822876, + 0.5468966960906982, + 0.13075968623161316 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6436", + "split": "test", + "style_correct": 1, + "style_probability": 0.32379353046417236, + "style_target": 0, + "support_probability": 0.0656629130244255, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1250143747216832, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6161802411079407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48523417115211487, + 0.8745495080947876, + 0.5171250700950623, + 0.23498259484767914, + 0.11700358986854553, + 0.2156830132007599, + 0.06313563138246536, + 0.07303641736507416, + 0.12714532017707825, + 0.08754178136587143, + 0.12252164632081985 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 1, + "style_probability": 0.7673134803771973, + "style_target": 1, + "support_probability": 0.06593061238527298, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06123786701958523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3974888324737549, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6441:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5750197768211365, + 0.9288675785064697, + 0.5395062565803528, + 0.24187248945236206, + 0.06310789287090302, + 0.05112161487340927, + 0.08349249511957169, + 0.11857137829065323, + 0.11776584386825562, + 0.12612973153591156 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6441", + "split": "test", + "style_correct": 1, + "style_probability": 0.3095114231109619, + "style_target": 0, + "support_probability": 0.24666310846805573, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6815964196837072, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0814935564994812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9623802900314331, + 0.26631471514701843, + 0.46076107025146484, + 0.4445320665836334, + 0.19339057803153992 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.3765907287597656, + "style_target": 1, + "support_probability": 0.750617265701294, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.894880526370141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0983496904373169, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6444:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9811796545982361, + 0.21227191388607025, + 0.4574565589427948, + 0.4366344213485718, + 0.05096954107284546 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6444", + "split": "test", + "style_correct": 0, + "style_probability": 0.7733376026153564, + "style_target": 0, + "support_probability": 0.7795659303665161, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8509599116654343, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.046709094196558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25575172901153564, + 0.17044681310653687, + 0.9951350092887878, + 0.30811044573783875, + 0.0780089795589447, + 0.094178207218647 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 0, + "style_probability": 0.44670408964157104, + "style_target": 1, + "support_probability": 0.9535588622093201, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4646511636003083, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15187199413776398, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6450:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2663401663303375, + 0.23552793264389038, + 0.9990783929824829, + 0.28961774706840515, + 0.06056773290038109, + 0.0929020568728447 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6450", + "split": "test", + "style_correct": 1, + "style_probability": 0.22801411151885986, + "style_target": 0, + "support_probability": 0.9323884844779968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0027586883261715434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7615231275558472, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2064426839351654, + 0.35925614833831787, + 0.2503679394721985, + 0.2653293013572693, + 0.12749747931957245, + 0.17393244802951813, + 0.12110849469900131, + 0.18010282516479492, + 0.23717890679836273 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.26326054334640503, + "style_target": 1, + "support_probability": 0.021715756505727768, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002104773353952746, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8232924342155457, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6453:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.269710510969162, + 0.36770787835121155, + 0.2525709569454193, + 0.24989061057567596, + 0.12702886760234833, + 0.08233592659235, + 0.08341328054666519, + 0.13201624155044556, + 0.07005874812602997 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6453", + "split": "test", + "style_correct": 0, + "style_probability": 0.593356728553772, + "style_target": 0, + "support_probability": 0.005177565850317478, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.23424326603502624, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5212149024009705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16249755024909973, + 0.21735554933547974, + 0.8505182862281799, + 0.3227574825286865, + 0.09025298058986664, + 0.12087272852659225, + 0.09747718274593353, + 0.07506460696458817, + 0.07219690084457397, + 0.08188612014055252, + 0.14048239588737488 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 1, + "style_probability": 0.5244585871696472, + "style_target": 1, + "support_probability": 0.4107882082462311, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2572141312769549, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7080223560333252, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6454:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1742335557937622, + 0.2156527191400528, + 0.8462403416633606, + 0.3337766230106354, + 0.06684164702892303, + 0.059139594435691833, + 0.11034435033798218, + 0.051935020834207535, + 0.12433858215808868, + 0.06714187562465668 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6454", + "split": "test", + "style_correct": 0, + "style_probability": 0.8993393778800964, + "style_target": 0, + "support_probability": 0.16913481056690216, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3352320107213984, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3152294456958771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.604653537273407, + 0.09806270897388458, + 0.3344728648662567, + 0.2355659008026123, + 0.06304491311311722 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 1, + "style_probability": 0.6495856046676636, + "style_target": 1, + "support_probability": 0.5251870155334473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2414350044911284, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24110369384288788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6464:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6254283785820007, + 0.08569089323282242, + 0.36240407824516296, + 0.2677987813949585, + 0.0534074641764164, + 0.13122858107089996, + 0.04853641614317894, + 0.1641569882631302, + 0.15807583928108215 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6464", + "split": "test", + "style_correct": 0, + "style_probability": 0.5187628269195557, + "style_target": 0, + "support_probability": 0.49757057428359985, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10574094585112803, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.788379967212677, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16224907338619232, + 0.48603346943855286, + 0.16224907338619232, + 0.47218695282936096, + 0.05464650318026543, + 0.06935343146324158 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 0, + "style_probability": 0.49676868319511414, + "style_target": 1, + "support_probability": 0.2692314088344574, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00055430732599588, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.94126296043396, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6475:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21064631640911102, + 0.44797319173812866, + 0.21064631640911102, + 0.4868999421596527, + 0.08144677430391312, + 0.16090862452983856 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6475", + "split": "test", + "style_correct": 1, + "style_probability": 0.15868723392486572, + "style_target": 0, + "support_probability": 0.03938467800617218, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13285349315569572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6603963375091553, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7350373864173889, + 0.4972321093082428, + 0.18265286087989807, + 0.13238881528377533, + 0.14048510789871216, + 0.07096734642982483, + 0.05876458063721657, + 0.11905453354120255, + 0.13597151637077332 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 1, + "style_probability": 0.6887880563735962, + "style_target": 1, + "support_probability": 0.1722124069929123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1337360011075397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6178366541862488, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6480:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.812102198600769, + 0.48741504549980164, + 0.19821327924728394, + 0.134953573346138, + 0.14725980162620544, + 0.13532304763793945, + 0.06391710042953491, + 0.06129904091358185, + 0.12725159525871277 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6480", + "split": "test", + "style_correct": 0, + "style_probability": 0.7966444492340088, + "style_target": 0, + "support_probability": 0.2539483606815338, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0045259509871037355, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8380359411239624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17566120624542236, + 0.3881150186061859, + 0.4361853301525116, + 0.6296378374099731, + 0.08607632666826248, + 0.0951404720544815, + 0.11188416182994843, + 0.12034594267606735, + 0.051372040063142776, + 0.0615864172577858, + 0.04959398880600929 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 0, + "style_probability": 0.43830206990242004, + "style_target": 1, + "support_probability": 0.015208241529762745, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026908215316028264, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7807271480560303, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6499:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17774032056331635, + 0.34375274181365967, + 0.4410693943500519, + 0.593633770942688, + 0.15547305345535278, + 0.16041988134384155 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6499", + "split": "test", + "style_correct": 1, + "style_probability": 0.2881165146827698, + "style_target": 0, + "support_probability": 0.00989786721765995, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05994121288681243, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7290546894073486, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.539948582649231, + 0.35435545444488525, + 0.10847809910774231, + 0.35435545444488525, + 0.16839498281478882 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 1, + "style_probability": 0.872514545917511, + "style_target": 1, + "support_probability": 0.14068301022052765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08566690047282677, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6591681241989136, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6500:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5853720903396606, + 0.36786696314811707, + 0.12349895387887955, + 0.36786696314811707, + 0.10841616243124008 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6500", + "split": "test", + "style_correct": 0, + "style_probability": 0.6354475617408752, + "style_target": 0, + "support_probability": 0.1946551650762558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4038794713582377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16671745479106903, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43739020824432373, + 0.43739020824432373, + 0.16339686512947083, + 0.9537591934204102, + 0.0509784072637558, + 0.08585785329341888 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.7129566669464111, + "style_target": 1, + "support_probability": 0.5334945321083069, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1657939331064231, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18104304373264313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6511:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4360709488391876, + 0.4360709488391876, + 0.16448438167572021, + 0.9344135522842407, + 0.0909309834241867, + 0.13537627458572388, + 0.03192431107163429, + 0.07944028824567795, + 0.15857858955860138, + 0.1296943873167038, + 0.053123634308576584 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6511", + "split": "test", + "style_correct": 1, + "style_probability": 0.3305899500846863, + "style_target": 0, + "support_probability": 0.4380694627761841, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7419106245556294, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1152416542172432, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21541766822338104, + 0.9751535058021545, + 0.39966538548469543, + 0.14830662310123444, + 0.03637022152543068, + 0.030152570456266403, + 0.05257703736424446, + 0.04573190584778786, + 0.10797357559204102 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.7520384192466736, + "style_target": 1, + "support_probability": 0.7836040258407593, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13424722572174463, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11654791235923767, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6513:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26936933398246765, + 0.9815024733543396, + 0.427590548992157, + 0.15911665558815002, + 0.03624492511153221, + 0.04650364816188812, + 0.09765592217445374, + 0.08397826552391052, + 0.134502112865448 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6513", + "split": "test", + "style_correct": 1, + "style_probability": 0.1749851405620575, + "style_target": 0, + "support_probability": 0.7710418701171875, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12752201125256057, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24780108034610748, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4427177906036377, + 0.8179363012313843, + 0.7178373336791992, + 0.21953178942203522, + 0.11419336497783661, + 0.12175306677818298, + 0.11916299909353256, + 0.05015850439667702, + 0.08544602245092392, + 0.1996363252401352 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 1, + "style_probability": 0.7564084529876709, + "style_target": 1, + "support_probability": 0.2128448337316513, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14153108571561113, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25119927525520325, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6517:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45324426889419556, + 0.8153261542320251, + 0.7277057766914368, + 0.19754748046398163, + 0.09286804497241974, + 0.19162923097610474, + 0.06357843428850174, + 0.05935520678758621, + 0.07410624623298645, + 0.16162650287151337, + 0.1043834462761879 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:6517", + "split": "test", + "style_correct": 0, + "style_probability": 0.5783580541610718, + "style_target": 0, + "support_probability": 0.2762819230556488, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.25345902146612076, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.041361913084983826, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22841787338256836, + 0.37149158120155334, + 0.9058638215065002, + 0.43652230501174927, + 0.09863309562206268 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.376751184463501, + "style_target": 1, + "support_probability": 0.5838537812232971, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4354034406599982, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03444045037031174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6518:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23559625446796417, + 0.3570724129676819, + 0.9373809695243835, + 0.4321184754371643, + 0.08250485360622406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6518", + "split": "test", + "style_correct": 0, + "style_probability": 0.6002342700958252, + "style_target": 0, + "support_probability": 0.7102388739585876, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8398510246941129, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11145485937595367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9696431756019592, + 0.2134227305650711, + 0.10757429152727127, + 0.29603710770606995, + 0.14451156556606293, + 0.11577606946229935 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.7080168128013611, + "style_target": 1, + "support_probability": 0.8955813646316528, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7847315872216392, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06289668381214142, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6520:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.980215847492218, + 0.2249753624200821, + 0.11013317853212357, + 0.24702729284763336, + 0.07607730478048325, + 0.11378093808889389, + 0.06050894781947136, + 0.11775587499141693, + 0.07476252317428589, + 0.16395732760429382, + 0.1297365128993988 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6520", + "split": "test", + "style_correct": 1, + "style_probability": 0.3796355724334717, + "style_target": 0, + "support_probability": 0.943161129951477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007439816343227987, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7936407923698425, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46463388204574585, + 0.3566211462020874, + 0.2859741449356079, + 0.25531500577926636, + 0.09555844962596893, + 0.1863008737564087, + 0.20043271780014038, + 0.16647960245609283, + 0.14373043179512024 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 0, + "style_probability": 0.48176246881484985, + "style_target": 1, + "support_probability": 0.03856322169303894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031044839883814786, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6248316168785095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6523:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5745330452919006, + 0.4513095021247864, + 0.2644695043563843, + 0.22988027334213257, + 0.09034204483032227, + 0.05175864323973656, + 0.07385735213756561, + 0.08493596315383911, + 0.13544796407222748 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6523", + "split": "test", + "style_correct": 1, + "style_probability": 0.14198853075504303, + "style_target": 0, + "support_probability": 0.17628055810928345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46928955709995895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17910213768482208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8653083443641663, + 0.3961644470691681, + 0.22652165591716766, + 0.30963006615638733, + 0.07537075132131577, + 0.06855709850788116, + 0.18579475581645966, + 0.08020074665546417, + 0.1469704806804657, + 0.14980581402778625 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 1, + "style_probability": 0.5219111442565918, + "style_target": 1, + "support_probability": 0.6565033197402954, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6574314241204887, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10335666686296463, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6528:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9181995987892151, + 0.47113481163978577, + 0.2103242576122284, + 0.36746811866760254, + 0.09249213337898254, + 0.06131769344210625, + 0.059360746294260025, + 0.02914467826485634, + 0.031198730692267418, + 0.17990747094154358, + 0.1153445690870285 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6528", + "split": "test", + "style_correct": 0, + "style_probability": 0.6075201630592346, + "style_target": 0, + "support_probability": 0.8309097290039062, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006451371236123769, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.623003363609314, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22447989881038666, + 0.3860307037830353, + 0.3581582307815552, + 0.22447989881038666, + 0.06778782606124878 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 0, + "style_probability": 0.33980923891067505, + "style_target": 1, + "support_probability": 0.12116469442844391, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004537627574025678, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6895115971565247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6529:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17762985825538635, + 0.3946729004383087, + 0.2977832555770874, + 0.17762985825538635, + 0.10266460478305817, + 0.08512242138385773, + 0.09308277815580368, + 0.07347854226827621, + 0.08055903017520905 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6529", + "split": "test", + "style_correct": 1, + "style_probability": 0.20994789898395538, + "style_target": 0, + "support_probability": 0.04751696065068245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12833516882749407, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18688547611236572, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1947091668844223, + 0.9751031994819641, + 0.29166215658187866, + 0.3291073143482208, + 0.06362269818782806, + 0.04918823763728142 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.5308020114898682, + "style_target": 1, + "support_probability": 0.2721349895000458, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01901567432118112, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2955304980278015, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6533:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26192668080329895, + 0.9935357570648193, + 0.25085529685020447, + 0.3938959240913391, + 0.07168816030025482, + 0.10663702338933945 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6533", + "split": "test", + "style_correct": 1, + "style_probability": 0.12924394011497498, + "style_target": 0, + "support_probability": 0.46133944392204285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01172852017741199, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9314870238304138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.470936119556427, + 0.18089021742343903, + 0.2804930806159973, + 0.5614244937896729, + 0.18441781401634216, + 0.1294633150100708, + 0.10835554450750351, + 0.21133100986480713, + 0.12067658454179764 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.4713858962059021, + "style_target": 1, + "support_probability": 0.041454918682575226, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07905662516006283, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8592363595962524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6552:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5463024377822876, + 0.21453075110912323, + 0.2654007077217102, + 0.5642282366752625, + 0.0656648725271225, + 0.16240935027599335, + 0.05405065044760704, + 0.0683991014957428, + 0.15745317935943604 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6552", + "split": "test", + "style_correct": 0, + "style_probability": 0.8548365831375122, + "style_target": 0, + "support_probability": 0.06944994628429413, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8010226826280586, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2762911021709442, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8824625611305237, + 0.22029851377010345, + 0.22358664870262146, + 0.2088165581226349, + 0.08806319534778595, + 0.13614970445632935, + 0.09143158793449402, + 0.0684565007686615, + 0.09091222286224365, + 0.10410687327384949 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.6786779761314392, + "style_target": 1, + "support_probability": 0.9649102091789246, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.702685082109916, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19483894109725952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6556:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.893338143825531, + 0.2676193118095398, + 0.19539080560207367, + 0.24367204308509827, + 0.08879895508289337, + 0.08601852506399155 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6556", + "split": "test", + "style_correct": 1, + "style_probability": 0.39151668548583984, + "style_target": 0, + "support_probability": 0.971617579460144, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.25707154262400067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12331603467464447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29925158619880676, + 0.2637123763561249, + 0.20146170258522034, + 0.9075540900230408, + 0.09217630326747894 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 1, + "style_probability": 0.7821598649024963, + "style_target": 1, + "support_probability": 0.37779107689857483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22338634353715744, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10250930488109589, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6563:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3085024058818817, + 0.2793373465538025, + 0.20049601793289185, + 0.8654165267944336, + 0.05451137572526932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6563", + "split": "test", + "style_correct": 0, + "style_probability": 0.6246620416641235, + "style_target": 0, + "support_probability": 0.40942302346229553, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10943458398047624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19995494186878204, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9478135704994202, + 0.20830315351486206, + 0.22826452553272247, + 0.40572816133499146, + 0.11685546487569809, + 0.13277733325958252 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 1, + "style_probability": 0.8035576939582825, + "style_target": 1, + "support_probability": 0.12731771171092987, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.080845057633665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15905925631523132, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6564:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9300974011421204, + 0.22276319563388824, + 0.22220592200756073, + 0.396904319524765, + 0.09827809035778046, + 0.06707451492547989, + 0.2731230854988098, + 0.07210515439510345, + 0.1573105752468109, + 0.1614033728837967 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6564", + "split": "test", + "style_correct": 1, + "style_probability": 0.4460085928440094, + "style_target": 0, + "support_probability": 0.19599351286888123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4176300840423224, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2359100878238678, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2513347566127777, + 0.9361382722854614, + 0.18494689464569092, + 0.5936257243156433, + 0.044100672006607056, + 0.09723037481307983, + 0.11015582084655762, + 0.0863552987575531, + 0.10471020638942719 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.6994359493255615, + "style_target": 1, + "support_probability": 0.5762768387794495, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0918690002778213, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23439118266105652, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6573:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2702349126338959, + 0.9501585960388184, + 0.25383979082107544, + 0.6449881792068481, + 0.062351979315280914, + 0.09254965931177139, + 0.1024191752076149, + 0.05711621791124344, + 0.1580529808998108 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6573", + "split": "test", + "style_correct": 1, + "style_probability": 0.23437470197677612, + "style_target": 0, + "support_probability": 0.6570817828178406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3153079601462423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19041378796100616, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14963214099407196, + 0.3523837924003601, + 0.2116444706916809, + 0.924588680267334, + 0.04415431246161461, + 0.0974876657128334, + 0.0362936332821846, + 0.08423997461795807, + 0.09667564183473587, + 0.15653526782989502, + 0.1379808634519577 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 1, + "style_probability": 0.7838760018348694, + "style_target": 1, + "support_probability": 0.2641059160232544, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2030638997020979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2738633453845978, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6575:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.16731658577919006, + 0.36638757586479187, + 0.20993512868881226, + 0.901286780834198, + 0.11879126727581024, + 0.08389227092266083, + 0.11167167127132416, + 0.06322809308767319, + 0.17238029837608337, + 0.17202134430408478, + 0.09341668337583542 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6575", + "split": "test", + "style_correct": 0, + "style_probability": 0.5961244106292725, + "style_target": 0, + "support_probability": 0.23478631675243378, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12866789166199943, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5868493914604187, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29900068044662476, + 0.39348140358924866, + 0.6816532015800476, + 0.29900068044662476, + 0.10874442756175995 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.9142950773239136, + "style_target": 1, + "support_probability": 0.08409781754016876, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04854207347411066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5066337585449219, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6578:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2967500388622284, + 0.39511197805404663, + 0.7106314301490784, + 0.2967500388622284, + 0.07336828857660294, + 0.060499031096696854, + 0.09813366830348969, + 0.041286490857601166, + 0.10986130684614182 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6578", + "split": "test", + "style_correct": 1, + "style_probability": 0.4585953652858734, + "style_target": 0, + "support_probability": 0.10619040578603745, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16512005987214987, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26055222749710083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2646123766899109, + 0.17435133457183838, + 0.31749486923217773, + 0.6367869973182678, + 0.043643247336149216, + 0.057505834847688675 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.5153726935386658, + "style_target": 1, + "support_probability": 0.5618167519569397, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006161589267642545, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30390214920043945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6587:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3179582953453064, + 0.17758142948150635, + 0.41056787967681885, + 0.6903019547462463, + 0.05917132645845413, + 0.09909120202064514 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6587", + "split": "test", + "style_correct": 1, + "style_probability": 0.09221669286489487, + "style_target": 0, + "support_probability": 0.39334678649902344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03914909451878668, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9450197815895081, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.15487220883369446, + 0.21221014857292175, + 0.6586208939552307, + 0.4581010341644287, + 0.0625729188323021, + 0.09580666571855545, + 0.1188686192035675, + 0.053342051804065704, + 0.06156997010111809 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 1, + "style_probability": 0.942112386226654, + "style_target": 1, + "support_probability": 0.028438260778784752, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013825731295254995, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9830998182296753, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16083984076976776, + 0.21691398322582245, + 0.5167860984802246, + 0.3274553120136261, + 0.0895000547170639, + 0.1226118728518486, + 0.08834878355264664, + 0.14089086651802063, + 0.12681512534618378 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6590", + "split": "test", + "style_correct": 0, + "style_probability": 0.5480577945709229, + "style_target": 0, + "support_probability": 0.011420577764511108, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01963414636805449, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9415270686149597, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.44203048944473267, + 0.3655197024345398, + 0.25079476833343506, + 0.3878686726093292, + 0.0993306040763855, + 0.05444536358118057, + 0.0890740305185318, + 0.20424920320510864, + 0.06010982766747475, + 0.08390020579099655 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.8875921964645386, + "style_target": 1, + "support_probability": 0.039615899324417114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005122200063217622, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.845500111579895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44357573986053467, + 0.36715224385261536, + 0.2236400544643402, + 0.3592923879623413, + 0.07888275384902954, + 0.2736184597015381 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6604", + "split": "test", + "style_correct": 1, + "style_probability": 0.4776427149772644, + "style_target": 0, + "support_probability": 0.060268599539995193, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6670138292579146, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07635516673326492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9851352572441101, + 0.3078289330005646, + 0.25339168310165405, + 0.20230025053024292, + 0.08182578533887863 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 1, + "style_probability": 0.574604332447052, + "style_target": 1, + "support_probability": 0.6750763654708862, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14724725764273305, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09845321625471115, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9914706349372864, + 0.2984803318977356, + 0.3027378022670746, + 0.22182026505470276, + 0.10259108245372772 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6612", + "split": "test", + "style_correct": 1, + "style_probability": 0.2119937688112259, + "style_target": 0, + "support_probability": 0.6004127264022827, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4138260740271349, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47250062227249146, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7195815443992615, + 0.19423934817314148, + 0.32460010051727295, + 0.12416939437389374, + 0.04697595164179802, + 0.0849165990948677 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 1, + "style_probability": 0.7692987322807312, + "style_target": 1, + "support_probability": 0.5149362087249756, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1744065034740991, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4036393463611603, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6616:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7475850582122803, + 0.2440863996744156, + 0.296329140663147, + 0.12914423644542694, + 0.12036512792110443, + 0.12602104246616364 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6616", + "split": "test", + "style_correct": 1, + "style_probability": 0.3248424530029297, + "style_target": 0, + "support_probability": 0.5416996479034424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20838293736554017, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7188571095466614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38265153765678406, + 0.257422536611557, + 0.23044314980506897, + 0.5726765990257263, + 0.18349061906337738, + 0.15249796211719513, + 0.05409286543726921, + 0.17021535336971283, + 0.12074078619480133 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.8277146220207214, + "style_target": 1, + "support_probability": 0.15241801738739014, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05649866491112344, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6029597520828247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6639:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32121923565864563, + 0.24536697566509247, + 0.22973856329917908, + 0.49222391843795776, + 0.3852996230125427 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6639", + "split": "test", + "style_correct": 1, + "style_probability": 0.2560618817806244, + "style_target": 0, + "support_probability": 0.12762543559074402, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5928410310273122, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10848928987979889, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.26813217997550964, + 0.816365659236908, + 0.19039951264858246, + 0.2852264642715454, + 0.11472789198160172, + 0.12802979350090027, + 0.08350873738527298, + 0.14475291967391968, + 0.05430141091346741, + 0.06312152743339539, + 0.07363934069871902 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.5839980244636536, + "style_target": 1, + "support_probability": 0.8455044627189636, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.116513679078432, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12741170823574066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6645:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2762708365917206, + 0.8489557504653931, + 0.20515768229961395, + 0.29558154940605164, + 0.13756051659584045, + 0.14781148731708527, + 0.08170799911022186, + 0.07258692383766174, + 0.10502886772155762, + 0.12535424530506134 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6645", + "split": "test", + "style_correct": 1, + "style_probability": 0.15510763227939606, + "style_target": 0, + "support_probability": 0.8240402936935425, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0671474438488562, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9047936797142029, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28308963775634766, + 0.15825940668582916, + 0.5818267464637756, + 0.6603773236274719, + 0.12003853917121887 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 1, + "style_probability": 0.8499612212181091, + "style_target": 1, + "support_probability": 0.10584770143032074, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04816840185273107, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8738842606544495, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6647:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27891528606414795, + 0.176513209939003, + 0.5792714953422546, + 0.5816128253936768, + 0.1672055721282959 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6647", + "split": "test", + "style_correct": 0, + "style_probability": 0.5172532200813293, + "style_target": 0, + "support_probability": 0.1373540461063385, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11979408209220566, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5440560579299927, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6186744570732117, + 0.3554733991622925, + 0.8367056846618652, + 0.5661925673484802, + 0.08471149951219559, + 0.11772776395082474 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.6755594611167908, + "style_target": 1, + "support_probability": 0.27796539664268494, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045189155337395086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5074180364608765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6650:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5644215941429138, + 0.363863080739975, + 0.8103718757629395, + 0.5235510468482971, + 0.0619242899119854, + 0.05591757595539093, + 0.05805278569459915, + 0.11330348253250122, + 0.10042527318000793, + 0.08327630162239075, + 0.09523294121026993 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6650", + "split": "test", + "style_correct": 1, + "style_probability": 0.32127824425697327, + "style_target": 0, + "support_probability": 0.21025298535823822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20685220858409178, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6619817614555359, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7904861569404602, + 0.19982537627220154, + 0.33301734924316406, + 0.6049535870552063, + 0.04398879408836365, + 0.03554170951247215, + 0.10809528827667236, + 0.044965099543333054, + 0.1044098362326622 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.8365911841392517, + "style_target": 1, + "support_probability": 0.17366288602352142, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04166247511060761, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6367025971412659, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6652:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8530524373054504, + 0.21091115474700928, + 0.360850065946579, + 0.6481673121452332, + 0.029813140630722046, + 0.09738234430551529, + 0.04856620728969574, + 0.036776427179574966, + 0.07831314206123352 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6652", + "split": "test", + "style_correct": 1, + "style_probability": 0.3941621482372284, + "style_target": 0, + "support_probability": 0.3089456856250763, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7905246667231579, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03355823829770088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5477004647254944, + 0.2888069450855255, + 0.6953879594802856, + 0.9596243500709534, + 0.039840955287218094, + 0.05225479602813721, + 0.06358679383993149, + 0.020067058503627777, + 0.0994773656129837, + 0.04116738960146904, + 0.09038573503494263 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 1, + "style_probability": 0.5000863075256348, + "style_target": 1, + "support_probability": 0.8569393754005432, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5495672196412613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05351603776216507, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6655:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5051704049110413, + 0.28515979647636414, + 0.6156588792800903, + 0.9185690879821777, + 0.029232321307063103, + 0.02683565579354763, + 0.05205817520618439, + 0.11974550783634186, + 0.06994304805994034, + 0.0407179594039917, + 0.054362304508686066 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6655", + "split": "test", + "style_correct": 1, + "style_probability": 0.4671666622161865, + "style_target": 0, + "support_probability": 0.7296940684318542, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22157767748741897, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.35630562901496887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41305530071258545, + 0.8347746729850769, + 0.2566602826118469, + 0.4766603410243988, + 0.20853447914123535 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 1, + "style_probability": 0.5100998878479004, + "style_target": 1, + "support_probability": 0.41425821185112, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.49572004406600806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.29831576347351074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6657:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4196985960006714, + 0.8676074147224426, + 0.26002517342567444, + 0.5047179460525513, + 0.08574439585208893 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6657", + "split": "test", + "style_correct": 0, + "style_probability": 0.843163013458252, + "style_target": 0, + "support_probability": 0.5009894967079163, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3712127987823166, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.061482179909944534, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29347681999206543, + 0.8065640926361084, + 0.3742864429950714, + 0.0852484479546547, + 0.06941651552915573, + 0.11968211084604263 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 0, + "style_probability": 0.25321274995803833, + "style_target": 1, + "support_probability": 0.824873149394989, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05763554809487426, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13971319794654846, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6669:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2667487859725952, + 0.9210476279258728, + 0.3272984027862549, + 0.09732364863157272, + 0.04919803887605667, + 0.09891846776008606 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6669", + "split": "test", + "style_correct": 1, + "style_probability": 0.1688523292541504, + "style_target": 0, + "support_probability": 0.7707683444023132, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006220393183630433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9204983115196228, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43204495310783386, + 0.2696724534034729, + 0.37588897347450256, + 0.7099871635437012, + 0.06219475716352463, + 0.06846415251493454, + 0.07070093601942062, + 0.18336725234985352, + 0.22243772447109222 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 0, + "style_probability": 0.27007144689559937, + "style_target": 1, + "support_probability": 0.03656813129782677, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00596114663393943, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8753763437271118, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6671:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46118250489234924, + 0.25557780265808105, + 0.395751953125, + 0.6760829091072083, + 0.07006046921014786, + 0.050463683903217316, + 0.11555497348308563, + 0.07491627335548401, + 0.19386281073093414 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6671", + "split": "test", + "style_correct": 0, + "style_probability": 0.6062041521072388, + "style_target": 0, + "support_probability": 0.026635130867362022, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00815601067572713, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47274652123451233, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1829742044210434, + 0.6021831631660461, + 0.2067512720823288, + 0.22276388108730316, + 0.08134526759386063, + 0.0794813483953476, + 0.12453195452690125, + 0.11593478918075562, + 0.313414067029953, + 0.0920405462384224, + 0.10060907900333405 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.41201043128967285, + "style_target": 1, + "support_probability": 0.02317676693201065, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00939623543342061, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.49780046939849854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18997499346733093, + 0.6173537969589233, + 0.17793583869934082, + 0.22726596891880035, + 0.0818694680929184, + 0.052501130849123, + 0.04298419877886772, + 0.09215568006038666, + 0.24392743408679962, + 0.11626577377319336, + 0.12292124330997467 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6673", + "split": "test", + "style_correct": 0, + "style_probability": 0.7469942569732666, + "style_target": 0, + "support_probability": 0.011616343632340431, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5192777966602087, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0852070301771164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7840129733085632, + 0.37147530913352966, + 0.9798169732093811, + 0.33633148670196533, + 0.03709936887025833 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 1, + "style_probability": 0.5821741819381714, + "style_target": 1, + "support_probability": 0.8192565441131592, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12031631480949213, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10339294373989105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6678:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8120038509368896, + 0.34694379568099976, + 0.9835341572761536, + 0.3776977062225342, + 0.08451008796691895 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6678", + "split": "test", + "style_correct": 1, + "style_probability": 0.2600184977054596, + "style_target": 0, + "support_probability": 0.7923547029495239, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.3711143502896021, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8202716112136841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20938917994499207, + 0.7149655818939209, + 0.35664132237434387, + 0.3630964756011963, + 0.0789102166891098, + 0.0932910218834877 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 1, + "style_probability": 0.8805034160614014, + "style_target": 1, + "support_probability": 0.3002481460571289, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.31890128854802846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9073355197906494, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20186278223991394, + 0.6498979926109314, + 0.3310474753379822, + 0.27342528104782104, + 0.10843520611524582, + 0.185990110039711 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6681", + "split": "test", + "style_correct": 0, + "style_probability": 0.5897499918937683, + "style_target": 0, + "support_probability": 0.4458371698856354, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.019760257676539442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8670800924301147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18508031964302063, + 0.742591142654419, + 0.48388671875, + 0.42547616362571716, + 0.152213454246521, + 0.15385784208774567, + 0.1289042979478836, + 0.09374719858169556, + 0.1592792123556137 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.8083184957504272, + "style_target": 1, + "support_probability": 0.019690796732902527, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005092720428763546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8570181727409363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.17143210768699646, + 0.7425547242164612, + 0.45302990078926086, + 0.41210344433784485, + 0.23539943993091583 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6685", + "split": "test", + "style_correct": 1, + "style_probability": 0.2894665598869324, + "style_target": 0, + "support_probability": 0.04164579138159752, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9568814751841792, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11990945041179657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6704:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9933397173881531, + 0.4171049892902374, + 0.2334754317998886, + 0.32509154081344604, + 0.11701731383800507, + 0.17332807183265686, + 0.16929559409618378, + 0.14700445532798767, + 0.06435255706310272, + 0.10447429120540619, + 0.08458877354860306 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.8319841027259827, + "style_target": 1, + "support_probability": 0.9549662470817566, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.27281383370803525, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.167481929063797, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6704:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9986580610275269, + 0.49066901206970215, + 0.2252984195947647, + 0.4423762857913971, + 0.07139866799116135, + 0.07817284762859344, + 0.2204669713973999, + 0.09953580051660538, + 0.10587198287248611, + 0.1405845284461975 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6704", + "split": "test", + "style_correct": 1, + "style_probability": 0.11458786576986313, + "style_target": 0, + "support_probability": 0.9319130778312683, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6232540886847993, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22082364559173584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6763115525245667, + 0.5124638080596924, + 0.44171351194381714, + 0.4231759309768677, + 0.02989501692354679 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 1, + "style_probability": 0.8714202046394348, + "style_target": 1, + "support_probability": 0.8484323024749756, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4330373804959464, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38471168279647827, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6712:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5985976457595825, + 0.5466675162315369, + 0.4019986093044281, + 0.3707386255264282, + 0.06618036329746246 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6712", + "split": "test", + "style_correct": 0, + "style_probability": 0.5339147448539734, + "style_target": 0, + "support_probability": 0.814541757106781, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13074565482139633, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9541127681732178, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2367345094680786, + 0.6070890426635742, + 0.3694728910923004, + 0.5339750647544861, + 0.10989043116569519, + 0.08020778000354767 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.8417578935623169, + "style_target": 1, + "support_probability": 0.14093101024627686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0038453199992655165, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9514961838722229, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6713:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21230831742286682, + 0.5423728227615356, + 0.3593801259994507, + 0.5326083898544312, + 0.13723309338092804, + 0.24944695830345154, + 0.09376247227191925, + 0.11626475304365158, + 0.15532028675079346, + 0.08658774197101593, + 0.17979300022125244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6713", + "split": "test", + "style_correct": 1, + "style_probability": 0.2971644401550293, + "style_target": 0, + "support_probability": 0.038659460842609406, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22662183087313895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10350148379802704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.46349793672561646, + 0.872089684009552, + 0.516808271408081, + 0.2681255638599396, + 0.09076043218374252, + 0.054567739367485046, + 0.05931699648499489, + 0.04925976321101189, + 0.07311069220304489 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 1, + "style_probability": 0.5597239136695862, + "style_target": 1, + "support_probability": 0.4845427870750427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011564351977564008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24970810115337372, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6721:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5357257723808289, + 0.8851245641708374, + 0.5702621936798096, + 0.2651399075984955, + 0.06217123195528984, + 0.1130034551024437, + 0.1285506784915924, + 0.08280188590288162, + 0.26492413878440857 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6721", + "split": "test", + "style_correct": 1, + "style_probability": 0.10090424865484238, + "style_target": 0, + "support_probability": 0.38566187024116516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06473146787970308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6731878519058228, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3047943115234375, + 0.2943812608718872, + 0.7372347116470337, + 0.4675693213939667, + 0.06816567480564117, + 0.05988844484090805, + 0.0881836786866188, + 0.07385444641113281, + 0.07100168615579605, + 0.1113959550857544, + 0.06468802690505981 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 1, + "style_probability": 0.7115654945373535, + "style_target": 1, + "support_probability": 0.10332164913415909, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07562862088365208, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7165115475654602, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6722:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28057387471199036, + 0.3105836808681488, + 0.7015013694763184, + 0.5378591418266296, + 0.0767720490694046, + 0.07867543399333954, + 0.05664670094847679, + 0.09732530266046524, + 0.09593872725963593, + 0.34472912549972534, + 0.06989334523677826 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6722", + "split": "test", + "style_correct": 0, + "style_probability": 0.6488434672355652, + "style_target": 0, + "support_probability": 0.11282248049974442, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7288497425409872, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14136838912963867, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25755688548088074, + 0.9076070189476013, + 0.07892554253339767, + 0.25755688548088074, + 0.12437169253826141 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 1, + "style_probability": 0.8427695631980896, + "style_target": 1, + "support_probability": 0.6145093441009521, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30919902835433355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08500853925943375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6729:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23580341041088104, + 0.895699679851532, + 0.08288402110338211, + 0.23580341041088104, + 0.04822506010532379, + 0.04615209624171257, + 0.05806628242135048, + 0.11034601181745529, + 0.10331623256206512 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6729", + "split": "test", + "style_correct": 1, + "style_probability": 0.2707239091396332, + "style_target": 0, + "support_probability": 0.6468253135681152, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.057331947684658546, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7143805027008057, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43100255727767944, + 0.19320222735404968, + 0.26298826932907104, + 0.4848918616771698, + 0.0734981819987297, + 0.09004342555999756 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.7983720898628235, + "style_target": 1, + "support_probability": 0.12635506689548492, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00257679283622042, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7357625961303711, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6756:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4458535611629486, + 0.23427274823188782, + 0.28901806473731995, + 0.5145506262779236, + 0.08833353966474533, + 0.11598870903253555 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6756", + "split": "test", + "style_correct": 1, + "style_probability": 0.2516942322254181, + "style_target": 0, + "support_probability": 0.12144733220338821, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.21074894924100998, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5831680297851562, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6174265742301941, + 0.2553003132343292, + 0.30225494503974915, + 0.4903407394886017, + 0.07531966269016266, + 0.19831715524196625, + 0.09903319925069809, + 0.08166992664337158, + 0.078303761780262 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 1, + "style_probability": 0.93890380859375, + "style_target": 1, + "support_probability": 0.17595817148685455, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05882509168304749, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5086321830749512, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6772:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6446762084960938, + 0.25086355209350586, + 0.3031795918941498, + 0.45810380578041077, + 0.062639519572258, + 0.10941271483898163, + 0.08802863955497742, + 0.06333785504102707, + 0.15105503797531128 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6772", + "split": "test", + "style_correct": 0, + "style_probability": 0.607485294342041, + "style_target": 0, + "support_probability": 0.2735533118247986, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7082303566090324, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4422467052936554, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3440205156803131, + 0.6664084196090698, + 0.9452510476112366, + 0.5095692873001099, + 0.043025873601436615, + 0.06125382333993912, + 0.06105126440525055, + 0.09389416873455048, + 0.09043362736701965, + 0.0595637783408165, + 0.09007110446691513 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 1, + "style_probability": 0.8572961091995239, + "style_target": 1, + "support_probability": 0.567092776298523, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.41438527727460084, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.33440962433815, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6778:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33063971996307373, + 0.5872962474822998, + 0.9339884519577026, + 0.45212051272392273, + 0.13152965903282166, + 0.0835564136505127 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6778", + "split": "test", + "style_correct": 1, + "style_probability": 0.48898327350616455, + "style_target": 0, + "support_probability": 0.6373486518859863, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7546806199752609, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04055498540401459, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9217467308044434, + 0.08389759063720703, + 0.99456787109375, + 0.9217467308044434, + 0.07140422612428665 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.6330492496490479, + "style_target": 1, + "support_probability": 0.8406022787094116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1746736245586753, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06060093268752098, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6790:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.948590874671936, + 0.12023425102233887, + 0.9977929592132568, + 0.948590874671936, + 0.10108682513237 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6790", + "split": "test", + "style_correct": 1, + "style_probability": 0.10336221009492874, + "style_target": 0, + "support_probability": 0.853701651096344, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.417498923632742, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05976869538426399, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46524956822395325, + 0.9294755458831787, + 0.6560183167457581, + 0.1629503071308136, + 0.05873994901776314, + 0.050348103046417236 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 1, + "style_probability": 0.7327836155891418, + "style_target": 1, + "support_probability": 0.4839465320110321, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2715982319782011, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08162485808134079, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.43179628252983093, + 0.8896254897117615, + 0.5703073143959045, + 0.18330448865890503, + 0.10259126871824265, + 0.08410273492336273 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6793", + "split": "test", + "style_correct": 0, + "style_probability": 0.5943571329116821, + "style_target": 0, + "support_probability": 0.3805304765701294, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7539579824366456, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03982551768422127, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.262645959854126, + 0.9958341121673584, + 0.5078924894332886, + 0.41700002551078796, + 0.07116850465536118, + 0.06593465059995651, + 0.16692310571670532, + 0.04744305834174156, + 0.04370453953742981 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.6553179025650024, + "style_target": 1, + "support_probability": 0.7874694466590881, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.534184606666404, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04017442464828491, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6805:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25155016779899597, + 0.992299497127533, + 0.5010089874267578, + 0.4025176465511322, + 0.18791836500167847 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6805", + "split": "test", + "style_correct": 1, + "style_probability": 0.354400098323822, + "style_target": 0, + "support_probability": 0.8006197810173035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32201384846581865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4587155282497406, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8124500513076782, + 0.5223328471183777, + 0.4645996689796448, + 0.20394264161586761, + 0.08471343666315079, + 0.026934465393424034, + 0.06701802462339401, + 0.12023225426673889, + 0.14609819650650024, + 0.08825104683637619, + 0.048242516815662384 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.8163585066795349, + "style_target": 1, + "support_probability": 0.3961099088191986, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008970985097690731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6320429444313049, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6816:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8389036059379578, + 0.5643128156661987, + 0.4288886487483978, + 0.27022719383239746, + 0.08565634489059448, + 0.05337492749094963, + 0.1144832894206047, + 0.16669100522994995, + 0.10498707741498947, + 0.09990300983190536 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6816", + "split": "test", + "style_correct": 1, + "style_probability": 0.18204258382320404, + "style_target": 0, + "support_probability": 0.264117956161499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44021528049431596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09654105454683304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1829575151205063, + 0.943716824054718, + 0.16864296793937683, + 0.11778520792722702, + 0.07047433406114578 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 1, + "style_probability": 0.5272373557090759, + "style_target": 1, + "support_probability": 0.5784723162651062, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08362783852185185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09523230791091919, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6817:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21391870081424713, + 0.9775848984718323, + 0.19533208012580872, + 0.15594744682312012, + 0.0777561143040657 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6817", + "split": "test", + "style_correct": 1, + "style_probability": 0.14130648970603943, + "style_target": 0, + "support_probability": 0.6938574314117432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05415755597212701, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6718199849128723, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37649816274642944, + 0.4235621690750122, + 0.8349583148956299, + 0.3478846549987793, + 0.1277066022157669, + 0.07996362447738647 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 1, + "style_probability": 0.561185896396637, + "style_target": 1, + "support_probability": 0.1306811273097992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044369574815385164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7759121060371399, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6822:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35375240445137024, + 0.46510934829711914, + 0.8515651226043701, + 0.34479570388793945, + 0.04256749898195267, + 0.14084382355213165 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6822", + "split": "test", + "style_correct": 0, + "style_probability": 0.755415678024292, + "style_target": 0, + "support_probability": 0.11861097067594528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22816316807427486, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48751363158226013, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2626006305217743, + 0.21206706762313843, + 0.12041248381137848, + 0.7161369323730469, + 0.15418195724487305, + 0.08123570680618286, + 0.07343044131994247, + 0.09756620973348618, + 0.12687274813652039 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 1, + "style_probability": 0.6224969625473022, + "style_target": 1, + "support_probability": 0.34508058428764343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09995928701585656, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.45939695835113525, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6827:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2771683633327484, + 0.2052435725927353, + 0.12454482167959213, + 0.7391493320465088, + 0.11828335374593735 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6827", + "split": "test", + "style_correct": 1, + "style_probability": 0.3673841059207916, + "style_target": 0, + "support_probability": 0.2958885431289673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010355847858614093, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8503066301345825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20890021324157715, + 0.2200373262166977, + 0.3606100082397461, + 0.16141380369663239, + 0.0700879842042923, + 0.08925671875476837, + 0.17740730941295624, + 0.06318828463554382, + 0.041485030204057693, + 0.07007808983325958, + 0.14171256124973297 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 1, + "style_probability": 0.5165044069290161, + "style_target": 1, + "support_probability": 0.05880502238869667, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005776942190335587, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8669186234474182, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.24298134446144104, + 0.2953338623046875, + 0.4464591145515442, + 0.21624207496643066, + 0.0846526026725769, + 0.03568797558546066, + 0.04164468124508858, + 0.08262363821268082, + 0.04010165482759476, + 0.07736755162477493 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6836", + "split": "test", + "style_correct": 1, + "style_probability": 0.08701816946268082, + "style_target": 0, + "support_probability": 0.11764653027057648, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9291971962786363, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.047586921602487564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6841:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3890267312526703, + 0.9880461692810059, + 0.2518026828765869, + 0.4123269021511078, + 0.03479660674929619 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 1, + "style_probability": 0.8191899061203003, + "style_target": 1, + "support_probability": 0.8940714001655579, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8404841945332181, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.023426445201039314, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6841:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.38102778792381287, + 0.981581449508667, + 0.2198074460029602, + 0.367289662361145, + 0.06952414661645889, + 0.054956864565610886, + 0.04476356878876686, + 0.07521239668130875, + 0.06388077884912491 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6841", + "split": "test", + "style_correct": 0, + "style_probability": 0.5152205228805542, + "style_target": 0, + "support_probability": 0.9146444201469421, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05816211908384217, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9310060143470764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3157234191894531, + 0.39197981357574463, + 0.5180347561836243, + 0.3161507844924927, + 0.059723321348428726, + 0.04849337413907051 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.8189000487327576, + "style_target": 1, + "support_probability": 0.07550554722547531, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013372562683459652, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9687173366546631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6842:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28642287850379944, + 0.3380107879638672, + 0.5826159119606018, + 0.33776232600212097, + 0.0415881872177124, + 0.1095222756266594 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6842", + "split": "test", + "style_correct": 1, + "style_probability": 0.19054736196994781, + "style_target": 0, + "support_probability": 0.08992395550012589, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13893682739083957, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8352681994438171, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8921874761581421, + 0.16257037222385406, + 0.1433076113462448, + 0.29635345935821533, + 0.16679538786411285, + 0.037272822111845016, + 0.13002678751945496, + 0.03947753831744194, + 0.05100448429584503 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 1, + "style_probability": 0.6762052774429321, + "style_target": 1, + "support_probability": 0.19417570531368256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23435476704233513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6506772637367249, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6844:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8706610798835754, + 0.20409409701824188, + 0.12166035175323486, + 0.3564263880252838, + 0.10545670241117477, + 0.098895363509655, + 0.07015658915042877, + 0.027544323354959488, + 0.10357557237148285 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6844", + "split": "test", + "style_correct": 0, + "style_probability": 0.5754994750022888, + "style_target": 0, + "support_probability": 0.28332817554473877, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19416500681482066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5056556463241577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:positive", + "evidence_budget": 14, + "evidence_probabilities": [ + 0.4051820933818817, + 0.979276180267334, + 0.1697756052017212, + 0.3084808588027954, + 0.056099023669958115, + 0.09769251197576523, + 0.13272970914840698, + 0.07400033622980118, + 0.13161595165729523, + 0.284074991941452, + 0.08839116990566254, + 0.17121215164661407, + 0.06971073895692825, + 0.07348790764808655 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.7022174596786499, + "style_target": 1, + "support_probability": 0.10407660901546478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08609102915916457, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4620242416858673, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6845:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4393904209136963, + 0.9685148000717163, + 0.17276285588741302, + 0.34255683422088623, + 0.08079133182764053, + 0.1152322068810463 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6845", + "split": "test", + "style_correct": 1, + "style_probability": 0.3273444175720215, + "style_target": 0, + "support_probability": 0.10640603303909302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11295369532144141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.35750478506088257, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.12084469944238663, + 0.19965074956417084, + 0.21966469287872314, + 0.9693594574928284, + 0.10918112099170685 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.6045199036598206, + "style_target": 1, + "support_probability": 0.34222498536109924, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02116361466522276, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.35886090993881226, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6847:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1397029161453247, + 0.253019243478775, + 0.27924761176109314, + 0.9775036573410034, + 0.08152319490909576 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6847", + "split": "test", + "style_correct": 1, + "style_probability": 0.17189405858516693, + "style_target": 0, + "support_probability": 0.41148728132247925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4702901243608757, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15794719755649567, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9785188436508179, + 0.1325126588344574, + 0.2633696496486664, + 0.18753713369369507, + 0.20154346525669098, + 0.09608381986618042 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 1, + "style_probability": 0.7489507794380188, + "style_target": 1, + "support_probability": 0.5113679766654968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24604221646357868, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3225557506084442, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6856:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9349085092544556, + 0.18860885500907898, + 0.30428650975227356, + 0.18907159566879272, + 0.13195781409740448, + 0.09860426187515259 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:6856", + "split": "test", + "style_correct": 0, + "style_probability": 0.6264719367027283, + "style_target": 0, + "support_probability": 0.35306528210639954, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058134053698606365, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4458204209804535, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4153832495212555, + 0.3091653883457184, + 0.7691219449043274, + 0.8900527358055115, + 0.048429980874061584, + 0.08626170456409454, + 0.0870233029127121, + 0.13939927518367767, + 0.09913395345211029 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.6061474084854126, + "style_target": 1, + "support_probability": 0.18923692405223846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058260683172948184, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.36094143986701965, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6863:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3928333818912506, + 0.3073108494281769, + 0.7679677605628967, + 0.8668996095657349, + 0.07959995418787003 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6863", + "split": "test", + "style_correct": 1, + "style_probability": 0.3799857199192047, + "style_target": 0, + "support_probability": 0.27430012822151184, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013026223549065682, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33569982647895813, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5939713716506958, + 0.5679771900177002, + 0.2644144594669342, + 0.5496978759765625, + 0.08449284732341766, + 0.0329788513481617, + 0.04419485107064247, + 0.03913398087024689, + 0.07531298696994781, + 0.07121207565069199 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.7313385605812073, + "style_target": 1, + "support_probability": 0.04008276388049126, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002716084306736449, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3501870632171631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6301093697547913, + 0.6235144138336182, + 0.28074297308921814, + 0.5924485921859741, + 0.05531077831983566, + 0.04420657455921173, + 0.024351857602596283, + 0.04684751480817795, + 0.051094118505716324, + 0.08067581802606583 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6864", + "split": "test", + "style_correct": 1, + "style_probability": 0.22516365349292755, + "style_target": 0, + "support_probability": 0.0649368166923523, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2221141373642368, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4152136445045471, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23959439992904663, + 0.34080010652542114, + 0.9181742668151855, + 0.36909225583076477, + 0.19997426867485046 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.7240418195724487, + "style_target": 1, + "support_probability": 0.3258988559246063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05913688386875738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7283550500869751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6894:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21155330538749695, + 0.32296547293663025, + 0.8445950746536255, + 0.35962972044944763, + 0.12318199127912521 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6894", + "split": "test", + "style_correct": 1, + "style_probability": 0.4628729820251465, + "style_target": 0, + "support_probability": 0.11091339588165283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3958445499279577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4234498143196106, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36575713753700256, + 0.20151007175445557, + 0.7439364194869995, + 0.36575713753700256, + 0.08885060995817184, + 0.0923641175031662 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.9204932451248169, + "style_target": 1, + "support_probability": 0.25707224011421204, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04666903121418377, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6020029783248901, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6906:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30387529730796814, + 0.16906653344631195, + 0.6840932965278625, + 0.30387529730796814, + 0.07198485732078552, + 0.07609935849905014, + 0.07732921838760376, + 0.045318424701690674, + 0.21382425725460052, + 0.06684313714504242, + 0.07808388024568558 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6906", + "split": "test", + "style_correct": 1, + "style_probability": 0.43749189376831055, + "style_target": 0, + "support_probability": 0.07926823198795319, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13859629514055613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21820485591888428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2160979062318802, + 0.21254423260688782, + 0.8958912491798401, + 0.2882443368434906, + 0.07587257027626038, + 0.052373554557561874, + 0.11033430695533752, + 0.29702234268188477, + 0.35868313908576965 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.7505609393119812, + "style_target": 1, + "support_probability": 0.12361148744821548, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.045128657465263844, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24188224971294403, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6910:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25922536849975586, + 0.2508372962474823, + 0.9241722226142883, + 0.3358587324619293, + 0.07376500964164734, + 0.07724519073963165, + 0.09283492714166641, + 0.11806067824363708, + 0.12615081667900085 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6910", + "split": "test", + "style_correct": 1, + "style_probability": 0.4696222245693207, + "style_target": 0, + "support_probability": 0.10012202709913254, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11144467964610529, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7664808630943298, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5383124351501465, + 0.43343937397003174, + 0.5383124351501465, + 0.7749677896499634, + 0.07110366225242615, + 0.1780959665775299, + 0.07579176872968674, + 0.10732369124889374, + 0.10050852596759796, + 0.20731380581855774 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 1, + "style_probability": 0.9395147562026978, + "style_target": 1, + "support_probability": 0.06755702197551727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020192130183142473, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7769007682800293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6915:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5111535787582397, + 0.48818886280059814, + 0.5111535787582397, + 0.7709980607032776, + 0.06366787850856781, + 0.3291914463043213, + 0.10167712718248367, + 0.12806689739227295, + 0.16456802189350128, + 0.0768488422036171, + 0.09433915466070175 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6915", + "split": "test", + "style_correct": 0, + "style_probability": 0.5354796051979065, + "style_target": 0, + "support_probability": 0.06709564477205276, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.157489151995442, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.41336891055107117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3515404164791107, + 0.09527599066495895, + 0.25979626178741455, + 0.6885550618171692, + 0.06128741800785065 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 1, + "style_probability": 0.7963785529136658, + "style_target": 1, + "support_probability": 0.3319782018661499, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11322029503098463, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39207443594932556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6919:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37668928503990173, + 0.09941548854112625, + 0.2382679432630539, + 0.6553506851196289, + 0.0560225285589695, + 0.06439564377069473, + 0.14575864374637604, + 0.05940501019358635, + 0.1271236091852188 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6919", + "split": "test", + "style_correct": 0, + "style_probability": 0.5433813333511353, + "style_target": 0, + "support_probability": 0.33336782455444336, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5237832523986441, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3412165939807892, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2924376428127289, + 0.9348118305206299, + 0.21489381790161133, + 0.4939194321632385, + 0.0668712705373764, + 0.051923178136348724 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.8894513845443726, + "style_target": 1, + "support_probability": 0.3492097556591034, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.065873921996193, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3248137831687927, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6925:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3041174113750458, + 0.9756265878677368, + 0.19101335108280182, + 0.5336691737174988, + 0.05661981552839279, + 0.092232845723629 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:6925", + "split": "test", + "style_correct": 1, + "style_probability": 0.2864430248737335, + "style_target": 0, + "support_probability": 0.5035407543182373, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17164570431247658, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6087989807128906, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4678824245929718, + 0.26541101932525635, + 0.8805049657821655, + 0.2423117458820343, + 0.06367528438568115, + 0.05439365282654762, + 0.08274651318788528, + 0.08653804659843445, + 0.050798449665308 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 1, + "style_probability": 0.8069061636924744, + "style_target": 1, + "support_probability": 0.16755612194538116, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09019149110836411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5490739941596985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6926:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4904481768608093, + 0.26536378264427185, + 0.8345673680305481, + 0.2801051139831543, + 0.05678222328424454, + 0.0553983673453331, + 0.06136909872293472, + 0.0445362888276577, + 0.09097537398338318 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6926", + "split": "test", + "style_correct": 0, + "style_probability": 0.5563421845436096, + "style_target": 0, + "support_probability": 0.2267877161502838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3707090775085575, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27310657501220703, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.305698037147522, + 0.3947334289550781, + 0.6851646900177002, + 0.6941484808921814, + 0.07791963964700699, + 0.0704280287027359, + 0.11536398530006409, + 0.0514332577586174, + 0.04593559354543686, + 0.04786205291748047, + 0.037528377026319504 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 1, + "style_probability": 0.917373776435852, + "style_target": 1, + "support_probability": 0.12568511068820953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16659887562455292, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2328973412513733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6938:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3007986843585968, + 0.39198949933052063, + 0.6868239641189575, + 0.6819778084754944, + 0.07593324780464172, + 0.13654693961143494 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6938", + "split": "test", + "style_correct": 0, + "style_probability": 0.6528999209403992, + "style_target": 0, + "support_probability": 0.20523716509342194, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.892581495716616, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16381266713142395, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6943:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38968855142593384, + 0.2612515091896057, + 0.38968855142593384, + 0.9412956833839417, + 0.16196046769618988 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.8727294206619263, + "style_target": 1, + "support_probability": 0.8510230183601379, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08952236976822237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16876918077468872, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6943:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5877680778503418, + 0.25938647985458374, + 0.5877680778503418, + 0.9791660904884338, + 0.11663959920406342 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6943", + "split": "test", + "style_correct": 1, + "style_probability": 0.08806043118238449, + "style_target": 0, + "support_probability": 0.8573939800262451, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.046331097730014634, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7574955224990845, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7050327062606812, + 0.326608806848526, + 0.8441752195358276, + 0.7050327062606812, + 0.08593696355819702, + 0.19533300399780273 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 1, + "style_probability": 0.638641357421875, + "style_target": 1, + "support_probability": 0.26634180545806885, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07662803936945295, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8756638169288635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6962:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6728725433349609, + 0.34874027967453003, + 0.7908880710601807, + 0.6728725433349609, + 0.1009882241487503, + 0.12284805625677109 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:6962", + "split": "test", + "style_correct": 0, + "style_probability": 0.6324930787086487, + "style_target": 0, + "support_probability": 0.17969974875450134, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5814614431722731, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4057394862174988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.417766273021698, + 0.6396848559379578, + 0.4000588357448578, + 0.6649678349494934, + 0.06330554932355881, + 0.07190949469804764, + 0.05800095945596695, + 0.07060429453849792, + 0.06929413229227066 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.8577067852020264, + "style_target": 1, + "support_probability": 0.6466080546379089, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1189178350434617, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5210564732551575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6965:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34867534041404724, + 0.6020059585571289, + 0.37814247608184814, + 0.5854223966598511, + 0.07223331928253174 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6965", + "split": "test", + "style_correct": 1, + "style_probability": 0.2424197942018509, + "style_target": 0, + "support_probability": 0.3851691484451294, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.827110038117368, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1700725555419922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.583976149559021, + 0.16466864943504333, + 0.505649983882904, + 0.9427516460418701, + 0.07215261459350586, + 0.41777393221855164, + 0.09117908030748367, + 0.04538402333855629, + 0.09673958271741867, + 0.09706874191761017 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.7594975829124451, + "style_target": 1, + "support_probability": 0.8297263383865356, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.17947208371709425, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10711881518363953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6967:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6494693756103516, + 0.20513035356998444, + 0.5750860571861267, + 0.9655841588973999, + 0.10829827189445496, + 0.11321059614419937, + 0.07899884879589081, + 0.10555925965309143, + 0.04146281257271767, + 0.13187971711158752 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6967", + "split": "test", + "style_correct": 1, + "style_probability": 0.09213721007108688, + "style_target": 0, + "support_probability": 0.8801277875900269, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6537716391285642, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.09269747883081436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.48463162779808044, + 0.31718239188194275, + 0.2165253609418869, + 0.9113478660583496, + 0.042247191071510315 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 0, + "style_probability": 0.356052428483963, + "style_target": 1, + "support_probability": 0.8514537811279297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10889049987155944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.12297680228948593, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6971:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5430748462677002, + 0.23890958726406097, + 0.19814422726631165, + 0.9865624308586121, + 0.11811238527297974 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6971", + "split": "test", + "style_correct": 1, + "style_probability": 0.11612964421510696, + "style_target": 0, + "support_probability": 0.7436610460281372, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6509725583946988, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.27414363622665405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3175175189971924, + 0.269397497177124, + 0.9177083969116211, + 0.8088529109954834, + 0.06245985999703407, + 0.07427126169204712 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 1, + "style_probability": 0.5413841009140015, + "style_target": 1, + "support_probability": 0.8145489692687988, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8009278483910975, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.23088909685611725, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6974:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27912813425064087, + 0.1969047337770462, + 0.9702253341674805, + 0.8531609773635864, + 0.09881973266601562, + 0.0927525982260704 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:6974", + "split": "test", + "style_correct": 0, + "style_probability": 0.7126602530479431, + "style_target": 0, + "support_probability": 0.8469315767288208, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1589180070973697, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6244211792945862, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6020367741584778, + 0.3344680368900299, + 0.26554837822914124, + 0.3100343942642212, + 0.03965376317501068, + 0.06931741535663605, + 0.03221441060304642, + 0.061195652931928635, + 0.054190900176763535 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.5932217836380005, + "style_target": 1, + "support_probability": 0.4092126488685608, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11962711936465721, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4733554422855377, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6975:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6503892540931702, + 0.32713353633880615, + 0.2840125858783722, + 0.3173801898956299, + 0.07489608973264694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6975", + "split": "test", + "style_correct": 1, + "style_probability": 0.4567546546459198, + "style_target": 0, + "support_probability": 0.5120700597763062, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008814127390609218, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7508080005645752, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1670486181974411, + 0.1672673523426056, + 0.3698714077472687, + 0.18219859898090363, + 0.10330624878406525, + 0.11115651577711105, + 0.06218781694769859, + 0.07175678759813309, + 0.06638921052217484, + 0.04873073101043701, + 0.04861264303326607 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 0, + "style_probability": 0.3894155025482178, + "style_target": 1, + "support_probability": 0.045914240181446075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011505949521584127, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6770807504653931, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:6998:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20814044773578644, + 0.25470659136772156, + 0.41312772035598755, + 0.2494499832391739, + 0.13360270857810974, + 0.0710555836558342, + 0.0610407255589962, + 0.12304119020700455, + 0.12046964466571808, + 0.11781305074691772 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6998", + "split": "test", + "style_correct": 1, + "style_probability": 0.08830293267965317, + "style_target": 0, + "support_probability": 0.11935482174158096, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.09697951406344087, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6481791734695435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30405062437057495, + 0.27877649664878845, + 0.2092406451702118, + 0.719624936580658, + 0.06777150928974152 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.2585357427597046, + "style_target": 1, + "support_probability": 0.29959407448768616, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1312064203459995, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.833736002445221, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:6999:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28822410106658936, + 0.29291778802871704, + 0.16657201945781708, + 0.6961001753807068, + 0.22511547803878784 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:6999", + "split": "test", + "style_correct": 0, + "style_probability": 0.7708002328872681, + "style_target": 0, + "support_probability": 0.2142970860004425, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13087582559682964, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2579381465911865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46118324995040894, + 0.8852235674858093, + 0.3794460892677307, + 0.45684415102005005, + 0.031230945140123367, + 0.04708107188344002 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 0, + "style_probability": 0.26380297541618347, + "style_target": 1, + "support_probability": 0.4427151679992676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08570322219276072, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25586262345314026, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7001:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4680856466293335, + 0.9558118581771851, + 0.4714299738407135, + 0.49137449264526367, + 0.04117622599005699, + 0.09460058808326721 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7001", + "split": "test", + "style_correct": 1, + "style_probability": 0.24691833555698395, + "style_target": 0, + "support_probability": 0.5397059917449951, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15696453969970928, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4053295850753784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3735421895980835, + 0.311911404132843, + 0.05609573796391487, + 0.529040515422821, + 0.06343042850494385, + 0.11940530687570572, + 0.051467668265104294, + 0.08281870931386948, + 0.0833020806312561 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 0, + "style_probability": 0.22547878324985504, + "style_target": 1, + "support_probability": 0.7080244421958923, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2836273271508567, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4951583445072174, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7012:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39343681931495667, + 0.2902567386627197, + 0.08012314140796661, + 0.5259614586830139, + 0.06395919620990753, + 0.06284324824810028, + 0.05895672366023064, + 0.10551121085882187, + 0.06016820669174194 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7012", + "split": "test", + "style_correct": 1, + "style_probability": 0.4625348448753357, + "style_target": 0, + "support_probability": 0.8142131567001343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004894146607770685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9491111636161804, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31306809186935425, + 0.7056803107261658, + 0.16577334702014923, + 0.18521252274513245, + 0.12953200936317444, + 0.08602399379014969, + 0.08459848910570145, + 0.13183583319187164, + 0.06281396001577377, + 0.14131593704223633, + 0.10844592750072479 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.15756095945835114, + "style_target": 1, + "support_probability": 0.0370526984333992, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004291606026466112, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9740784764289856, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7013:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2766496539115906, + 0.7000621557235718, + 0.17195211350917816, + 0.1424880176782608, + 0.11124908179044724, + 0.11294157058000565, + 0.10522900521755219, + 0.241684228181839, + 0.0898781567811966, + 0.07924015074968338, + 0.1527082920074463 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7013", + "split": "test", + "style_correct": 0, + "style_probability": 0.6432437300682068, + "style_target": 0, + "support_probability": 0.02179374173283577, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5414446124945798, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6153848767280579, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6625362038612366, + 0.6882738471031189, + 0.27574509382247925, + 0.5435851216316223, + 0.07119534909725189 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.8627022504806519, + "style_target": 1, + "support_probability": 0.6858917474746704, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011619281019249248, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8001328706741333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7032:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6190931797027588, + 0.7739858627319336, + 0.3310675621032715, + 0.4894793927669525, + 0.19922372698783875 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7032", + "split": "test", + "style_correct": 1, + "style_probability": 0.12317588925361633, + "style_target": 0, + "support_probability": 0.4097866415977478, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5327201095701071, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22733429074287415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4794102907180786, + 0.32739949226379395, + 0.9827799797058105, + 0.7154279351234436, + 0.03734797611832619, + 0.04156574606895447 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 1, + "style_probability": 0.6632277369499207, + "style_target": 1, + "support_probability": 0.6210107803344727, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5996746717016777, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21605618298053741, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7042:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4683256447315216, + 0.2986615002155304, + 0.9526618719100952, + 0.6969808340072632, + 0.03298318758606911, + 0.0580110140144825 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7042", + "split": "test", + "style_correct": 0, + "style_probability": 0.6373733878135681, + "style_target": 0, + "support_probability": 0.6632899641990662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3471735886829351, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7100093364715576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26675155758857727, + 0.31358158588409424, + 0.26675155758857727, + 0.9089829325675964, + 0.09768717736005783, + 0.11612803488969803, + 0.107661172747612, + 0.04110949486494064, + 0.08420781046152115 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.8331704139709473, + "style_target": 1, + "support_probability": 0.3061257004737854, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04830068150007971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7093794941902161, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7055:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3053939938545227, + 0.3139263093471527, + 0.3053939938545227, + 0.8674785494804382, + 0.140994131565094 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7055", + "split": "test", + "style_correct": 1, + "style_probability": 0.24354852735996246, + "style_target": 0, + "support_probability": 0.27727851271629333, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04513759541125184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8638527393341064, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5205918550491333, + 0.2515588700771332, + 0.5636460185050964, + 0.6519758105278015, + 0.058592818677425385, + 0.040089935064315796, + 0.042121730744838715, + 0.07896777242422104, + 0.061272576451301575, + 0.07147395610809326, + 0.09070305526256561 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.7395718097686768, + "style_target": 1, + "support_probability": 0.1254122406244278, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005518547828906151, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8572070002555847, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7059:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5686986446380615, + 0.25853443145751953, + 0.4911205470561981, + 0.7474981546401978, + 0.0797603651881218, + 0.08405138552188873, + 0.08235359936952591, + 0.06933878362178802, + 0.04242420196533203, + 0.1539505273103714 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7059", + "split": "test", + "style_correct": 1, + "style_probability": 0.08655130118131638, + "style_target": 0, + "support_probability": 0.4451413154602051, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004961835889272235, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5584513545036316, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6518661975860596, + 0.8106803894042969, + 0.25140371918678284, + 0.6518661975860596, + 0.11843525618314743 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 1, + "style_probability": 0.813123345375061, + "style_target": 1, + "support_probability": 0.018061600625514984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00980458174227938, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4457406997680664, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7065:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6878637075424194, + 0.836066484451294, + 0.27390095591545105, + 0.6878637075424194, + 0.07594430446624756 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7065", + "split": "test", + "style_correct": 0, + "style_probability": 0.6650731563568115, + "style_target": 0, + "support_probability": 0.03970487415790558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.036508607510510334, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8399907350540161, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.0953243225812912, + 0.562953531742096, + 0.32107824087142944, + 0.0558135025203228, + 0.13109643757343292, + 0.08069852739572525 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.8304541110992432, + "style_target": 1, + "support_probability": 0.12620413303375244, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010086130061460974, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8265476226806641, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7066:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13131040334701538, + 0.5158287882804871, + 0.29522764682769775, + 0.07268571853637695, + 0.05198579281568527, + 0.10645588487386703, + 0.3445529341697693, + 0.10751795768737793, + 0.08694059401750565, + 0.16184818744659424, + 0.19737648963928223 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7066", + "split": "test", + "style_correct": 1, + "style_probability": 0.2983402907848358, + "style_target": 0, + "support_probability": 0.07791386544704437, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6875389509150942, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.26565614342689514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.48407262563705444, + 0.19062787294387817, + 0.9191253781318665, + 0.38795003294944763, + 0.10499189794063568, + 0.11269068717956543, + 0.1931389719247818, + 0.07747502624988556, + 0.14546848833560944 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.9104287624359131, + "style_target": 1, + "support_probability": 0.48180413246154785, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1225573979806769, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23167850077152252, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7074:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5568140149116516, + 0.15931636095046997, + 0.9921253323554993, + 0.4907485246658325, + 0.08160710334777832, + 0.11099415272474289, + 0.10636574774980545, + 0.1117192879319191, + 0.14081402122974396 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7074", + "split": "test", + "style_correct": 1, + "style_probability": 0.2559203505516052, + "style_target": 0, + "support_probability": 0.546323835849762, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15845079302478382, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5765531063079834, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36232349276542664, + 0.25989776849746704, + 0.36232349276542664, + 0.9336625337600708, + 0.11247651278972626, + 0.10904242843389511, + 0.1093069538474083, + 0.0548238679766655, + 0.13616733253002167, + 0.061542145907878876, + 0.07473177462816238 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 1, + "style_probability": 0.7117446064949036, + "style_target": 1, + "support_probability": 0.27027255296707153, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05238862436078695, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8010323643684387, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7075:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39700260758399963, + 0.3081185817718506, + 0.39700260758399963, + 0.8914251923561096, + 0.128641277551651, + 0.09674666076898575, + 0.16425396502017975, + 0.15286077558994293, + 0.10100416094064713, + 0.11648564040660858, + 0.12466919422149658 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7075", + "split": "test", + "style_correct": 0, + "style_probability": 0.5938397645950317, + "style_target": 0, + "support_probability": 0.09363993257284164, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6415352717640701, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33885830640792847, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7300295829772949, + 0.5748926997184753, + 0.42229291796684265, + 0.28440922498703003, + 0.05116601288318634 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 1, + "style_probability": 0.830405592918396, + "style_target": 1, + "support_probability": 0.7571172118186951, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4413414374644233, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4627172350883484, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7092:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6772474646568298, + 0.5174673795700073, + 0.43267929553985596, + 0.26308250427246094, + 0.05072873458266258, + 0.04526998847723007, + 0.0723487138748169, + 0.09167836606502533, + 0.2166629135608673 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7092", + "split": "test", + "style_correct": 1, + "style_probability": 0.4807497262954712, + "style_target": 0, + "support_probability": 0.6789942979812622, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7398309574964657, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.055013544857501984, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4123924970626831, + 0.7064425349235535, + 0.9858397245407104, + 0.48925819993019104, + 0.023111989721655846, + 0.04345967248082161 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.5463494062423706, + "style_target": 1, + "support_probability": 0.7730997800827026, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20304262838611467, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.06557587534189224, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7110:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4718680679798126, + 0.7320879101753235, + 0.9907535314559937, + 0.5150063037872314, + 0.042807966470718384, + 0.07800266891717911 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7110", + "split": "test", + "style_correct": 1, + "style_probability": 0.17214615643024445, + "style_target": 0, + "support_probability": 0.7692910432815552, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8184162116547604, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1432880461215973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3252390921115875, + 0.7647689580917358, + 0.8536843657493591, + 0.5735782384872437, + 0.036748774349689484, + 0.044178057461977005, + 0.07496708631515503, + 0.04232171177864075, + 0.033206015825271606 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 1, + "style_probability": 0.7448165416717529, + "style_target": 1, + "support_probability": 0.9130850434303284, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6428544438408075, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.263272762298584, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7118:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33162274956703186, + 0.7551907896995544, + 0.7357415556907654, + 0.5446537733078003, + 0.07315423339605331, + 0.033683471381664276, + 0.07995899766683578, + 0.06983254849910736, + 0.10137762874364853 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7118", + "split": "test", + "style_correct": 0, + "style_probability": 0.5289662480354309, + "style_target": 0, + "support_probability": 0.8789833784103394, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3351138156371008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.748850405216217, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6978650093078613, + 0.43386563658714294, + 0.39324772357940674, + 0.6734211444854736, + 0.05029657110571861, + 0.06008908897638321, + 0.033657725900411606, + 0.06217828020453453, + 0.09980539232492447, + 0.05051384121179581, + 0.041772447526454926 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.869573175907135, + "style_target": 1, + "support_probability": 0.3244720995426178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14508277324516072, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7387405037879944, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7119:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.689327597618103, + 0.3816753327846527, + 0.4056988060474396, + 0.6437805891036987, + 0.1116524338722229, + 0.11260323971509933 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7119", + "split": "test", + "style_correct": 1, + "style_probability": 0.3895178735256195, + "style_target": 0, + "support_probability": 0.29112327098846436, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.462323156534552, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1342582106590271, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.984235405921936, + 0.1475554257631302, + 0.35387271642684937, + 0.23776915669441223, + 0.07279025763273239 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.7936753034591675, + "style_target": 1, + "support_probability": 0.4049850404262543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1420075009643167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1382933259010315, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7124:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.986101508140564, + 0.18979232013225555, + 0.37862908840179443, + 0.26501724123954773, + 0.14388427138328552 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7124", + "split": "test", + "style_correct": 1, + "style_probability": 0.41729000210762024, + "style_target": 0, + "support_probability": 0.4746851921081543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5885141588150589, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17479051649570465, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8489137291908264, + 0.5313649773597717, + 0.9813287258148193, + 0.8489137291908264, + 0.07613806426525116, + 0.06430158764123917 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 1, + "style_probability": 0.7158886194229126, + "style_target": 1, + "support_probability": 0.6068513989448547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7171825709013859, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1625794917345047, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7126:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8465745449066162, + 0.4957111179828644, + 0.9612987041473389, + 0.8465745449066162, + 0.13738222420215607, + 0.09140997380018234 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:7126", + "split": "test", + "style_correct": 0, + "style_probability": 0.6604333519935608, + "style_target": 0, + "support_probability": 0.7343957424163818, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0032862787716039774, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9788475632667542, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24493367969989777, + 0.3762260377407074, + 0.8228014707565308, + 0.3183015286922455, + 0.12543649971485138, + 0.11281265318393707, + 0.13875272870063782, + 0.11184853315353394, + 0.11327709257602692 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.826352059841156, + "style_target": 1, + "support_probability": 0.011308196932077408, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001857949115388183, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9569151401519775, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7134:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22951152920722961, + 0.36039474606513977, + 0.8140186667442322, + 0.29733923077583313, + 0.18416166305541992 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7134", + "split": "test", + "style_correct": 1, + "style_probability": 0.28679755330085754, + "style_target": 0, + "support_probability": 0.011238046921789646, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7765064827911486, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20488467812538147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9683923721313477, + 0.4238361418247223, + 0.48120036721229553, + 0.6462409496307373, + 0.08447093516588211, + 0.056961916387081146, + 0.08416887372732162, + 0.09709538519382477, + 0.1522345095872879, + 0.061372581869363785, + 0.08713630586862564 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.8624750971794128, + "style_target": 1, + "support_probability": 0.8259358406066895, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.09141457361449934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19708819687366486, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7140:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9854100942611694, + 0.47000762820243835, + 0.5371412634849548, + 0.6076179146766663, + 0.17348553240299225, + 0.1651400327682495, + 0.08641485869884491, + 0.05316377803683281, + 0.07060211896896362, + 0.21280673146247864 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7140", + "split": "test", + "style_correct": 1, + "style_probability": 0.21895144879817963, + "style_target": 0, + "support_probability": 0.8491680026054382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.625130351314624, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17757898569107056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4727709889411926, + 0.4124314486980438, + 0.7045347690582275, + 0.33754897117614746, + 0.08402565866708755 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 1, + "style_probability": 0.8077772259712219, + "style_target": 1, + "support_probability": 0.7889178991317749, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46396117935656633, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26864928007125854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7150:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38411945104599, + 0.3525887429714203, + 0.6706401109695435, + 0.3085678815841675, + 0.11541126668453217 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7150", + "split": "test", + "style_correct": 0, + "style_probability": 0.5840372443199158, + "style_target": 0, + "support_probability": 0.7083165645599365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41047078090457134, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6255566477775574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.44763249158859253, + 0.5816320776939392, + 0.4081208109855652, + 0.18487031757831573, + 0.05515721067786217, + 0.0948244109749794 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.9188308715820312, + "style_target": 1, + "support_probability": 0.3080129325389862, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19143870531673102, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.32020869851112366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7169:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4387781023979187, + 0.655640721321106, + 0.4098469614982605, + 0.2355993688106537, + 0.04985805228352547, + 0.25840067863464355, + 0.18996001780033112, + 0.052922897040843964, + 0.056947361677885056, + 0.1684391051530838, + 0.09957391023635864 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7169", + "split": "test", + "style_correct": 1, + "style_probability": 0.21490560472011566, + "style_target": 0, + "support_probability": 0.5303970575332642, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7515218713469807, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08102521300315857, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9662282466888428, + 0.3137078285217285, + 0.48710373044013977, + 0.5191565752029419, + 0.051818493753671646, + 0.05035866051912308, + 0.05279265344142914, + 0.06860093027353287, + 0.08730565756559372 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.6120912432670593, + "style_target": 1, + "support_probability": 0.8265582919120789, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10535396281763586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14110462367534637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7176:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.97889643907547, + 0.3107938766479492, + 0.541763186454773, + 0.6038234829902649, + 0.10807971656322479, + 0.0308353491127491, + 0.07914218306541443, + 0.06664717197418213, + 0.08657307177782059 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7176", + "split": "test", + "style_correct": 1, + "style_probability": 0.09803072363138199, + "style_target": 0, + "support_probability": 0.8260871767997742, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7766875123201409, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12654589116573334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36062824726104736, + 0.9833834767341614, + 0.29721158742904663, + 0.4948830306529999, + 0.05183543637394905, + 0.051418550312519073, + 0.041543371975421906, + 0.04999594762921333, + 0.04342379420995712, + 0.06661512702703476, + 0.09817206859588623 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 1, + "style_probability": 0.6852417588233948, + "style_target": 1, + "support_probability": 0.6889997720718384, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6932619284494876, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18736325204372406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7181:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3522448241710663, + 0.9652180671691895, + 0.2723636031150818, + 0.4858141839504242, + 0.058305900543928146, + 0.255527138710022, + 0.17558133602142334, + 0.07257235050201416, + 0.1048550233244896, + 0.16507476568222046 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7181", + "split": "test", + "style_correct": 1, + "style_probability": 0.4998319745063782, + "style_target": 0, + "support_probability": 0.7065992951393127, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12109675435463617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33876216411590576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7150393128395081, + 0.1871047168970108, + 0.1595638543367386, + 0.6751217246055603, + 0.05920768901705742 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 0, + "style_probability": 0.4772152006626129, + "style_target": 1, + "support_probability": 0.4625992774963379, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03180837200545738, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6720603704452515, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7188:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6882773637771606, + 0.19612738490104675, + 0.18817657232284546, + 0.7171638011932373, + 0.06832502782344818 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7188", + "split": "test", + "style_correct": 0, + "style_probability": 0.638545036315918, + "style_target": 0, + "support_probability": 0.22479252517223358, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.26954434311637243, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.0936306044459343, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2697623670101166, + 0.18247197568416595, + 0.18083737790584564, + 0.9382598400115967, + 0.06065208092331886, + 0.04824329540133476 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 0, + "style_probability": 0.2766772508621216, + "style_target": 1, + "support_probability": 0.5724606513977051, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4038457744426758, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03903292492032051, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7199:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2670615315437317, + 0.15917359292507172, + 0.18927019834518433, + 0.9547661542892456, + 0.05376088246703148, + 0.11427322030067444, + 0.04629375413060188, + 0.09119342267513275, + 0.0453222393989563, + 0.11091337352991104, + 0.1301688402891159 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7199", + "split": "test", + "style_correct": 1, + "style_probability": 0.18689098954200745, + "style_target": 0, + "support_probability": 0.7345197796821594, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8415517612610728, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03291326016187668, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3028482496738434, + 0.44642528891563416, + 0.9790322780609131, + 0.13315804302692413, + 0.0597054623067379, + 0.06441819667816162, + 0.08017951995134354, + 0.1109272837638855, + 0.08195467293262482 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.7239184379577637, + "style_target": 1, + "support_probability": 0.9001024961471558, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8036197786801865, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.010248500853776932, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7202:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2508488893508911, + 0.5025050640106201, + 0.9981535077095032, + 0.10613244026899338, + 0.08953581005334854, + 0.05177078768610954, + 0.08322515338659286, + 0.05421145260334015, + 0.073258176445961 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7202", + "split": "test", + "style_correct": 1, + "style_probability": 0.21335560083389282, + "style_target": 0, + "support_probability": 0.984904408454895, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10882169052452184, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5135144591331482, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6754084229469299, + 0.18490618467330933, + 0.44988226890563965, + 0.7433240413665771, + 0.07718227058649063, + 0.07594414800405502, + 0.07176424562931061, + 0.1341669261455536, + 0.05371196195483208, + 0.11664775758981705, + 0.07163672894239426 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 1, + "style_probability": 0.6585612297058105, + "style_target": 1, + "support_probability": 0.15510176122188568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09927530307214916, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4551064670085907, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7206:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6850959658622742, + 0.1581738144159317, + 0.5106061697006226, + 0.7379244565963745, + 0.03252503275871277, + 0.07544833421707153, + 0.0722997635602951, + 0.05779843032360077, + 0.0491560660302639, + 0.03572012484073639, + 0.06683769077062607 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7206", + "split": "test", + "style_correct": 0, + "style_probability": 0.6342536211013794, + "style_target": 0, + "support_probability": 0.2137242555618286, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07403618113254938, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5027405619621277, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4141826331615448, + 0.22389070689678192, + 0.4141826331615448, + 0.7441670298576355, + 0.07240690290927887 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 1, + "style_probability": 0.5240177512168884, + "style_target": 1, + "support_probability": 0.33900943398475647, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02168948163460028, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7974967956542969, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7211:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4329119622707367, + 0.2710157334804535, + 0.4329119622707367, + 0.7159643769264221, + 0.08331012725830078, + 0.07061942666769028, + 0.10848478972911835, + 0.04878528043627739, + 0.08752387762069702 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7211", + "split": "test", + "style_correct": 1, + "style_probability": 0.3837326467037201, + "style_target": 0, + "support_probability": 0.04516487568616867, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.31514691032092834, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4151328206062317, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3260943293571472, + 0.26142072677612305, + 0.31245800852775574, + 0.8682126402854919, + 0.052125658839941025, + 0.09363050758838654 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.5642609596252441, + "style_target": 1, + "support_probability": 0.5557191371917725, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.016014651431894285, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2545804977416992, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7219:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28551551699638367, + 0.29509589076042175, + 0.31164658069610596, + 0.9386059641838074, + 0.0614580437541008, + 0.12683595716953278 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7219", + "split": "test", + "style_correct": 1, + "style_probability": 0.12858699262142181, + "style_target": 0, + "support_probability": 0.6008606553077698, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.011851193707191902, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9032419919967651, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2652435004711151, + 0.4383203089237213, + 0.15948732197284698, + 0.22890974581241608, + 0.12049625813961029, + 0.15106463432312012, + 0.10636793076992035, + 0.06726448982954025, + 0.12159370630979538 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.27963095903396606, + "style_target": 1, + "support_probability": 0.08138751983642578, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09301468669624491, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7269700765609741, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7229:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2603139281272888, + 0.5019654035568237, + 0.1861167699098587, + 0.28050661087036133, + 0.1365109086036682, + 0.046327345073223114, + 0.04158743470907211, + 0.07650464028120041, + 0.037187330424785614 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7229", + "split": "test", + "style_correct": 0, + "style_probability": 0.73581862449646, + "style_target": 0, + "support_probability": 0.22711625695228577, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5221942972477933, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3010551929473877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.45194193720817566, + 0.41885948181152344, + 0.8366328477859497, + 0.28012147545814514, + 0.03648468106985092, + 0.06112751364707947, + 0.11081057786941528, + 0.12436419725418091, + 0.0834478884935379, + 0.10595578700304031 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 1, + "style_probability": 0.5053967833518982, + "style_target": 1, + "support_probability": 0.6674291491508484, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4471780479300662, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15948784351348877, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7234:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45844778418540955, + 0.3845129907131195, + 0.854769229888916, + 0.307583212852478, + 0.0561952106654644, + 0.06235066056251526 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7234", + "split": "test", + "style_correct": 1, + "style_probability": 0.3674808740615845, + "style_target": 0, + "support_probability": 0.758630096912384, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005158514294126282, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2535950541496277, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47691214084625244, + 0.7490813732147217, + 0.45752719044685364, + 0.15743903815746307, + 0.10185945779085159 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 0, + "style_probability": 0.16467873752117157, + "style_target": 1, + "support_probability": 0.08494007587432861, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009480342843201647, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2607141435146332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7237:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5001362562179565, + 0.7637354731559753, + 0.5084722638130188, + 0.13018789887428284, + 0.07122300565242767 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7237", + "split": "test", + "style_correct": 1, + "style_probability": 0.20821569859981537, + "style_target": 0, + "support_probability": 0.09703682363033295, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17820892897201546, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.36041855812072754, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11627888679504395, + 0.4660069942474365, + 0.7204358577728271, + 0.18991751968860626, + 0.1013711616396904, + 0.12547999620437622 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.30612829327583313, + "style_target": 1, + "support_probability": 0.5194187760353088, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5466722169989247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.35667645931243896, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7238:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13544519245624542, + 0.4862435758113861, + 0.7403088212013245, + 0.1855180710554123, + 0.09000933915376663, + 0.05475682020187378 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7238", + "split": "test", + "style_correct": 0, + "style_probability": 0.8807134628295898, + "style_target": 0, + "support_probability": 0.47557270526885986, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.057985754757015055, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6779558062553406, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13904120028018951, + 0.19230179488658905, + 0.09157240390777588, + 0.5721737742424011, + 0.11087505519390106, + 0.11722537130117416, + 0.10699833929538727, + 0.11956971883773804, + 0.14593660831451416 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.594375729560852, + "style_target": 1, + "support_probability": 0.0911613404750824, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028639189317366746, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4522123634815216, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7239:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.12489277124404907, + 0.16859793663024902, + 0.11812632530927658, + 0.7160202264785767, + 0.061016764491796494, + 0.07853087037801743, + 0.11102613061666489, + 0.04089561477303505, + 0.07102985680103302 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7239", + "split": "test", + "style_correct": 1, + "style_probability": 0.37491849064826965, + "style_target": 0, + "support_probability": 0.22447261214256287, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.37986162896316067, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12111517786979675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17704831063747406, + 0.5202764272689819, + 0.17519396543502808, + 0.914497435092926, + 0.09075098484754562, + 0.045445192605257034, + 0.12257609516382217, + 0.10621828585863113, + 0.1295127272605896, + 0.092823825776577, + 0.12045792490243912 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.3776310384273529, + "style_target": 1, + "support_probability": 0.574407160282135, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5700508973165058, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13891148567199707, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7247:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1994544267654419, + 0.4730414152145386, + 0.18983639776706696, + 0.8919710516929626, + 0.08595097064971924, + 0.24633808434009552, + 0.26754534244537354, + 0.07054336369037628, + 0.028315570205450058, + 0.028524303808808327, + 0.07255356013774872 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7247", + "split": "test", + "style_correct": 0, + "style_probability": 0.6258236169815063, + "style_target": 0, + "support_probability": 0.5451332926750183, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6191950621633211, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16420483589172363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5516206622123718, + 0.7043635249137878, + 0.9487172365188599, + 0.21760010719299316, + 0.0516919381916523 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.7692391276359558, + "style_target": 1, + "support_probability": 0.6850503087043762, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.521552514802277, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0855470597743988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7250:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.546626091003418, + 0.6720391511917114, + 0.9318765997886658, + 0.20555582642555237, + 0.05207324028015137, + 0.05659450218081474, + 0.1503901332616806, + 0.05889197066426277, + 0.10325147211551666 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7250", + "split": "test", + "style_correct": 1, + "style_probability": 0.4614521861076355, + "style_target": 0, + "support_probability": 0.7899935245513916, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.823089532779541, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06731023639440536, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7259:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7020782828330994, + 0.42402729392051697, + 0.22121784090995789, + 0.986091673374176, + 0.07703735679388046, + 0.0804092288017273 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.6602411270141602, + "style_target": 1, + "support_probability": 0.9026492238044739, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.032032751167889735, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13122588396072388, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7259:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7506905198097229, + 0.4674317538738251, + 0.25671347975730896, + 0.9914407730102539, + 0.11903878301382065, + 0.1729033887386322 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:7259", + "split": "test", + "style_correct": 1, + "style_probability": 0.04761483892798424, + "style_target": 0, + "support_probability": 0.81219083070755, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6704105344137902, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16594542562961578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6156541109085083, + 0.8795519471168518, + 0.4890437126159668, + 0.2627560496330261, + 0.11286979913711548, + 0.2942974269390106, + 0.11094047874212265, + 0.13102345168590546, + 0.06627438962459564 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 1, + "style_probability": 0.847454845905304, + "style_target": 1, + "support_probability": 0.3638436198234558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2651469765958847, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3095126450061798, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7261:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5534114241600037, + 0.8282622694969177, + 0.4444139301776886, + 0.2614923119544983, + 0.0951187014579773, + 0.20087766647338867, + 0.08495355397462845, + 0.06173000484704971, + 0.13013002276420593 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7261", + "split": "test", + "style_correct": 0, + "style_probability": 0.7594835162162781, + "style_target": 0, + "support_probability": 0.13350717723369598, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.44879509130491085, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22240738570690155, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42921650409698486, + 0.4316745102405548, + 0.47512587904930115, + 0.3963174819946289, + 0.03827299550175667, + 0.03306164592504501, + 0.08046390116214752, + 0.1390940248966217, + 0.0470278225839138, + 0.06524499505758286, + 0.04269714653491974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.9100868105888367, + "style_target": 1, + "support_probability": 0.7373033165931702, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20526230610817645, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37490731477737427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3880139887332916, + 0.385415643453598, + 0.4286125600337982, + 0.35934171080589294, + 0.07008357346057892, + 0.06371574848890305 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7265", + "split": "test", + "style_correct": 1, + "style_probability": 0.3955034017562866, + "style_target": 0, + "support_probability": 0.648556113243103, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006571250749813905, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7665316462516785, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2531333565711975, + 0.33674636483192444, + 0.54637610912323, + 0.3057357370853424, + 0.10076960176229477 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 0, + "style_probability": 0.4591708183288574, + "style_target": 1, + "support_probability": 0.0596650093793869, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0028423057723328546, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8980023264884949, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7266:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.205436572432518, + 0.28697511553764343, + 0.4633762538433075, + 0.28148946166038513, + 0.09957908093929291, + 0.10794313251972198, + 0.09224126487970352, + 0.06846824288368225, + 0.20242995023727417 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7266", + "split": "test", + "style_correct": 1, + "style_probability": 0.13713222742080688, + "style_target": 0, + "support_probability": 0.029588283970952034, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4090815858909345, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23498918116092682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5159069895744324, + 0.8897081613540649, + 0.40241530537605286, + 0.41167041659355164, + 0.20077019929885864, + 0.07239465415477753 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 0, + "style_probability": 0.325357586145401, + "style_target": 1, + "support_probability": 0.5531741976737976, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044761044288104575, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37048444151878357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7269:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5520068407058716, + 0.9587817788124084, + 0.4567928612232208, + 0.29957592487335205, + 0.0873744860291481, + 0.09380736947059631 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7269", + "split": "test", + "style_correct": 1, + "style_probability": 0.12509141862392426, + "style_target": 0, + "support_probability": 0.4854656755924225, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7938112776214652, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15064848959445953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7276:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9762366414070129, + 0.18978159129619598, + 0.6705750823020935, + 0.4368341863155365, + 0.08869429677724838, + 0.06399457156658173, + 0.10904309898614883, + 0.236839160323143, + 0.09966286271810532 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 1, + "style_probability": 0.6129062175750732, + "style_target": 1, + "support_probability": 0.9083998799324036, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9034598813547065, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1494094282388687, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7276:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9941463470458984, + 0.16334380209445953, + 0.6435129046440125, + 0.4108676016330719, + 0.0969337746500969, + 0.04205997660756111, + 0.06182578206062317, + 0.09306913614273071, + 0.10201514512300491 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7276", + "split": "test", + "style_correct": 0, + "style_probability": 0.84602952003479, + "style_target": 0, + "support_probability": 0.9042305946350098, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10814270266020998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3757777214050293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3893822729587555, + 0.8517887592315674, + 0.12322387099266052, + 0.47901651263237, + 0.0582905113697052, + 0.04031446948647499, + 0.09168951213359833, + 0.058982301503419876, + 0.07825583219528198, + 0.1531921923160553, + 0.06325241178274155 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 1, + "style_probability": 0.6815673112869263, + "style_target": 1, + "support_probability": 0.3448929488658905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04669831824556515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.36998775601387024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7277:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3801189661026001, + 0.8741589188575745, + 0.1094556674361229, + 0.46920278668403625, + 0.06887777149677277, + 0.1472403109073639 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7277", + "split": "test", + "style_correct": 1, + "style_probability": 0.4549153447151184, + "style_target": 0, + "support_probability": 0.32913991808891296, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10619331151181298, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.21565556526184082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36491432785987854, + 0.9722278118133545, + 0.2699047327041626, + 0.36491432785987854, + 0.05450470373034477 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 1, + "style_probability": 0.5504133105278015, + "style_target": 1, + "support_probability": 0.13764601945877075, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04252989296720621, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.22139909863471985, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7281:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3997768759727478, + 0.9942031502723694, + 0.2542411983013153, + 0.3997768759727478, + 0.10624384135007858 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7281", + "split": "test", + "style_correct": 1, + "style_probability": 0.25678905844688416, + "style_target": 0, + "support_probability": 0.2290934920310974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2747961689094644, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24636210501194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6772077679634094, + 0.27864620089530945, + 0.6390582323074341, + 0.6772077679634094, + 0.08894457668066025, + 0.04742632433772087 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 1, + "style_probability": 0.6432929039001465, + "style_target": 1, + "support_probability": 0.3802894949913025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3195698251221941, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.43314310908317566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7282:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6932381391525269, + 0.30931010842323303, + 0.6801406741142273, + 0.6932381391525269, + 0.09715811163187027, + 0.18116313219070435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7282", + "split": "test", + "style_correct": 0, + "style_probability": 0.8907976150512695, + "style_target": 0, + "support_probability": 0.11134051531553268, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4866662446295855, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17841540277004242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37348946928977966, + 0.9226009249687195, + 0.4590819478034973, + 0.5034630298614502, + 0.13027721643447876, + 0.056657690554857254, + 0.1600877344608307, + 0.08047347515821457, + 0.03206732124090195 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 0, + "style_probability": 0.498276948928833, + "style_target": 1, + "support_probability": 0.7088314294815063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4620841736819869, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13738006353378296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7285:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39287152886390686, + 0.9525418281555176, + 0.48168572783470154, + 0.5449482202529907, + 0.02756189927458763 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7285", + "split": "test", + "style_correct": 1, + "style_probability": 0.33648258447647095, + "style_target": 0, + "support_probability": 0.8478423953056335, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31895954602663856, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4395449459552765, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8581610321998596, + 0.638813316822052, + 0.19040335714817047, + 0.20611543953418732, + 0.05255407467484474, + 0.05144328624010086, + 0.29776233434677124, + 0.12437859922647476, + 0.19188076257705688, + 0.1112629845738411, + 0.0536976158618927 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.6152232885360718, + "style_target": 1, + "support_probability": 0.2831089496612549, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22926653899104646, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22030849754810333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7288:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9391799569129944, + 0.7251123785972595, + 0.17416715621948242, + 0.22396960854530334, + 0.06470021605491638, + 0.08247457444667816, + 0.04466523230075836, + 0.08101644366979599, + 0.08106772601604462, + 0.11987695842981339 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:7288", + "split": "test", + "style_correct": 1, + "style_probability": 0.21828801929950714, + "style_target": 0, + "support_probability": 0.6199193000793457, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39517162811012696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.31818634271621704, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2654779255390167, + 0.2087765634059906, + 0.14621365070343018, + 0.9928194880485535, + 0.12023898959159851 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.8192766308784485, + "style_target": 1, + "support_probability": 0.2757648229598999, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16995467062969635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2324857860803604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7291:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26526063680648804, + 0.22295381128787994, + 0.16471245884895325, + 0.9862486720085144, + 0.1370616853237152, + 0.13564956188201904, + 0.11049922555685043, + 0.09042850881814957, + 0.07432229071855545 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7291", + "split": "test", + "style_correct": 1, + "style_probability": 0.4293211102485657, + "style_target": 0, + "support_probability": 0.29741933941841125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21697354103827138, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17670251429080963, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3607075810432434, + 0.18433202803134918, + 0.5244762301445007, + 0.5359334945678711, + 0.035820651799440384, + 0.06926469504833221 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.8621124625205994, + "style_target": 1, + "support_probability": 0.17369334399700165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03456792833049426, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.20581279695034027, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7303:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4253414571285248, + 0.2443428337574005, + 0.5465448498725891, + 0.6486440300941467, + 0.04640410095453262, + 0.0694730207324028 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7303", + "split": "test", + "style_correct": 1, + "style_probability": 0.1909550279378891, + "style_target": 0, + "support_probability": 0.44876164197921753, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7929530940584292, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06627020239830017, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2935634255409241, + 0.8948753476142883, + 0.2236190289258957, + 0.30668914318084717, + 0.042114563286304474, + 0.04020819440484047, + 0.057024996727705, + 0.08197140693664551, + 0.0376497246325016 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 1, + "style_probability": 0.7306216955184937, + "style_target": 1, + "support_probability": 0.9149516820907593, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.649484511381047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11417118459939957, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2891346216201782, + 0.8073493242263794, + 0.3083972930908203, + 0.3295832574367523, + 0.03839823603630066, + 0.07019031792879105, + 0.06023528799414635, + 0.08744402229785919, + 0.10932155698537827 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7305", + "split": "test", + "style_correct": 0, + "style_probability": 0.5064844489097595, + "style_target": 0, + "support_probability": 0.9191283583641052, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9322025068677461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09987423568964005, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.44530466198921204, + 0.46909376978874207, + 0.5357232689857483, + 0.9808134436607361, + 0.07307910919189453, + 0.17394126951694489, + 0.2125377655029297, + 0.055038273334503174, + 0.07730190455913544, + 0.09717068076133728, + 0.0868603065609932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.880025327205658, + "style_target": 1, + "support_probability": 0.8814946413040161, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7812262624765154, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.04661525413393974, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7315:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46646982431411743, + 0.442436158657074, + 0.48247379064559937, + 0.974845290184021, + 0.28948792815208435, + 0.10836663842201233 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7315", + "split": "test", + "style_correct": 1, + "style_probability": 0.3982454836368561, + "style_target": 0, + "support_probability": 0.9398386478424072, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9636998967017049, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.051826342940330505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7316:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9930340051651001, + 0.5843867063522339, + 0.4597024619579315, + 0.47019901871681213, + 0.0939946249127388 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.8665250539779663, + "style_target": 1, + "support_probability": 0.936360239982605, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7635866207186126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.038488611578941345, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7316:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9979817867279053, + 0.596491277217865, + 0.5615366697311401, + 0.4734242558479309, + 0.15018230676651 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7316", + "split": "test", + "style_correct": 1, + "style_probability": 0.18588297069072723, + "style_target": 0, + "support_probability": 0.979009747505188, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.668235630095495, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16137707233428955, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5007901191711426, + 0.9110280871391296, + 0.2794419527053833, + 0.5007901191711426, + 0.04821883141994476, + 0.17203737795352936 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 1, + "style_probability": 0.8937399387359619, + "style_target": 1, + "support_probability": 0.6116073131561279, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5075754837705944, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17749951779842377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7321:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4484288990497589, + 0.9027870893478394, + 0.2953733205795288, + 0.4484288990497589, + 0.07115931808948517, + 0.09662327170372009 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7321", + "split": "test", + "style_correct": 0, + "style_probability": 0.5766074657440186, + "style_target": 0, + "support_probability": 0.6566200256347656, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06294405810049053, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5141113996505737, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9553815126419067, + 0.1171250194311142, + 0.3735438883304596, + 0.3036268353462219, + 0.08259525150060654, + 0.0442592017352581, + 0.23688453435897827, + 0.07083812355995178, + 0.10645978897809982 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 1, + "style_probability": 0.7699253559112549, + "style_target": 1, + "support_probability": 0.06507907807826996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020524173177743237, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5069682598114014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7332:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.935828447341919, + 0.13376225531101227, + 0.34628286957740784, + 0.30202028155326843, + 0.22967219352722168 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7332", + "split": "test", + "style_correct": 1, + "style_probability": 0.2732168138027191, + "style_target": 0, + "support_probability": 0.10458839684724808, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4242925628639149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02548930048942566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2525171637535095, + 0.1516249179840088, + 0.9953609108924866, + 0.2525171637535095, + 0.08599567413330078, + 0.03009050153195858, + 0.10986153781414032, + 0.02608848735690117, + 0.30342379212379456, + 0.12852461636066437, + 0.06841669231653214 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.6887611150741577, + "style_target": 1, + "support_probability": 0.4986378848552704, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14665718099426517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.031206712126731873, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7334:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2880497872829437, + 0.17667551338672638, + 0.9975042939186096, + 0.2880497872829437, + 0.028792304918169975, + 0.03262781724333763, + 0.0807667076587677, + 0.08513220399618149, + 0.07316968590021133, + 0.10776516050100327 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7334", + "split": "test", + "style_correct": 1, + "style_probability": 0.27168571949005127, + "style_target": 0, + "support_probability": 0.5550772547721863, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6224725810810625, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13984648883342743, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9848736524581909, + 0.4217654764652252, + 0.13480161130428314, + 0.39914315938949585, + 0.09775974601507187 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 0, + "style_probability": 0.24111314117908478, + "style_target": 1, + "support_probability": 0.9024013876914978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7613669938760382, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1952032446861267, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7340:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9787344932556152, + 0.4047241508960724, + 0.1512705683708191, + 0.4282061457633972, + 0.08459869027137756 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7340", + "split": "test", + "style_correct": 1, + "style_probability": 0.44604265689849854, + "style_target": 0, + "support_probability": 0.8848996758460999, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1505823348550539, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6014795303344727, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3314686715602875, + 0.9176966547966003, + 0.43336543440818787, + 0.24486789107322693, + 0.05373586714267731, + 0.09450006484985352 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.2124016135931015, + "style_target": 1, + "support_probability": 0.596775472164154, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2894847945090895, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7527124881744385, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7341:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3577432930469513, + 0.9464973211288452, + 0.4012749493122101, + 0.23196059465408325, + 0.24677108228206635, + 0.07510245591402054 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7341", + "split": "test", + "style_correct": 0, + "style_probability": 0.630429744720459, + "style_target": 0, + "support_probability": 0.5150147676467896, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2100721689944933, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4236718714237213, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2487349510192871, + 0.9254295229911804, + 0.4609224498271942, + 0.3493095636367798, + 0.16059015691280365, + 0.2109704315662384, + 0.1057620421051979, + 0.06170382350683212, + 0.14242768287658691 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.5651391744613647, + "style_target": 1, + "support_probability": 0.3160921037197113, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2818346587895153, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16735003888607025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2956071197986603, + 0.963572084903717, + 0.5812702775001526, + 0.4395049214363098, + 0.13121114671230316, + 0.07826713472604752, + 0.09040072560310364, + 0.1064702644944191, + 0.09839581698179245 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7349", + "split": "test", + "style_correct": 1, + "style_probability": 0.48405683040618896, + "style_target": 0, + "support_probability": 0.6673884391784668, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7012846548872887, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09537801891565323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27396148443222046, + 0.25739210844039917, + 0.9555497765541077, + 0.2783811688423157, + 0.14939072728157043, + 0.12695464491844177, + 0.12805727124214172, + 0.06080496311187744, + 0.08730261772871017, + 0.0497041791677475, + 0.06890436261892319 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 0, + "style_probability": 0.37102270126342773, + "style_target": 1, + "support_probability": 0.9209944605827332, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7608269781178479, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09910229593515396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7354:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21120324730873108, + 0.21079552173614502, + 0.9375877380371094, + 0.23577307164669037, + 0.0482875220477581, + 0.057298753410577774, + 0.047292426228523254, + 0.0618319995701313, + 0.0550694465637207, + 0.0982821062207222, + 0.049850862473249435 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7354", + "split": "test", + "style_correct": 0, + "style_probability": 0.5732684135437012, + "style_target": 0, + "support_probability": 0.9119041562080383, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5802220846416688, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4785599112510681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3079565465450287, + 0.33408182859420776, + 0.3988806903362274, + 0.9206233620643616, + 0.0554954819381237 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.8734445571899414, + "style_target": 1, + "support_probability": 0.6201224327087402, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.43653597020504264, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2482420653104782, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7355:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2910007834434509, + 0.3197983503341675, + 0.40602296590805054, + 0.9222279191017151, + 0.0679418221116066, + 0.09132793545722961, + 0.05323091894388199, + 0.049857139587402344, + 0.08967960625886917 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7355", + "split": "test", + "style_correct": 1, + "style_probability": 0.4581531286239624, + "style_target": 0, + "support_probability": 0.7745199799537659, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16092667092424406, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5968183279037476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5733354687690735, + 0.33205264806747437, + 0.566496729850769, + 0.8635580539703369, + 0.034654583781957626, + 0.053554777055978775 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.8184815049171448, + "style_target": 1, + "support_probability": 0.116083525121212, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006147265310921133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7354849576950073, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7363:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6333185434341431, + 0.3851083219051361, + 0.6439763903617859, + 0.9075411558151245, + 0.037533409893512726, + 0.056699980050325394 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7363", + "split": "test", + "style_correct": 1, + "style_probability": 0.1827401965856552, + "style_target": 0, + "support_probability": 0.13302457332611084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10129945986586009, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2622225880622864, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8623015880584717, + 0.1554451286792755, + 0.31292328238487244, + 0.19031625986099243, + 0.0710170716047287, + 0.06545925885438919, + 0.07168054580688477, + 0.05413008853793144, + 0.04722414165735245 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 1, + "style_probability": 0.5718958377838135, + "style_target": 1, + "support_probability": 0.21729300916194916, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08008804490395958, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5192750692367554, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7368:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8311483860015869, + 0.22870010137557983, + 0.38418298959732056, + 0.21784496307373047, + 0.13720543682575226, + 0.18557818233966827, + 0.07816684991121292, + 0.20472712814807892, + 0.10282409191131592 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7368", + "split": "test", + "style_correct": 0, + "style_probability": 0.5105516314506531, + "style_target": 0, + "support_probability": 0.16655118763446808, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9549125205790361, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0604373961687088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3892661929130554, + 0.48072394728660583, + 0.9890305399894714, + 0.652895450592041, + 0.05942409113049507, + 0.03480881080031395, + 0.046083126217126846, + 0.06578462570905685, + 0.06633162498474121, + 0.05353610962629318, + 0.04723312333226204 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.8732326626777649, + "style_target": 1, + "support_probability": 0.9278513789176941, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8004750543645045, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04407969117164612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7374:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4105021357536316, + 0.5024597644805908, + 0.9837919473648071, + 0.657953143119812, + 0.059122733771800995, + 0.04388657584786415 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7374", + "split": "test", + "style_correct": 1, + "style_probability": 0.4238005578517914, + "style_target": 0, + "support_probability": 0.9428380131721497, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6937951650129222, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4575643539428711, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28486257791519165, + 0.5416494011878967, + 0.2783261835575104, + 0.8949957489967346, + 0.04839807376265526 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.8719382882118225, + "style_target": 1, + "support_probability": 0.5922505855560303, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02043877189126198, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7528181076049805, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7395:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34087878465652466, + 0.6516637206077576, + 0.2874795198440552, + 0.9485414028167725, + 0.11267407238483429 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7395", + "split": "test", + "style_correct": 1, + "style_probability": 0.1666087806224823, + "style_target": 0, + "support_probability": 0.36668911576271057, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06559685456163278, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7639988660812378, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2761686146259308, + 0.31301748752593994, + 0.32108721137046814, + 0.2761686146259308, + 0.1548493206501007, + 0.07122056186199188 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 1, + "style_probability": 0.9021512269973755, + "style_target": 1, + "support_probability": 0.19520340859889984, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022392309964786247, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5202761888504028, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7406:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26303982734680176, + 0.29544302821159363, + 0.3159709572792053, + 0.26303982734680176, + 0.10410615801811218, + 0.17690935730934143 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7406", + "split": "test", + "style_correct": 0, + "style_probability": 0.6275225281715393, + "style_target": 0, + "support_probability": 0.1898607611656189, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.028271787408752354, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7929110527038574, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4218751788139343, + 0.7063751816749573, + 0.4480225741863251, + 0.43479761481285095, + 0.1378694325685501, + 0.07621587812900543, + 0.06692247092723846, + 0.2612829804420471, + 0.15467888116836548 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.778207004070282, + "style_target": 1, + "support_probability": 0.03198489174246788, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03441887571649938, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.634225606918335, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4068652093410492, + 0.6781684756278992, + 0.4062964618206024, + 0.3872782588005066, + 0.28920385241508484 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7409", + "split": "test", + "style_correct": 1, + "style_probability": 0.4729151129722595, + "style_target": 0, + "support_probability": 0.13580405712127686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10286756243764653, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.442878782749176, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.47677889466285706, + 0.7420452237129211, + 0.7822040319442749, + 0.5204849243164062, + 0.02991662360727787, + 0.05512712523341179, + 0.07653583586215973, + 0.02939840592443943, + 0.03513326495885849, + 0.07997377216815948, + 0.10749977082014084 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 1, + "style_probability": 0.689599871635437, + "style_target": 1, + "support_probability": 0.2203553169965744, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07816904679560864, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3893834948539734, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7410:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5046257972717285, + 0.7483488917350769, + 0.8085640072822571, + 0.5252620577812195, + 0.0766032487154007, + 0.052117787301540375, + 0.12293989211320877, + 0.11182985454797745, + 0.08122950047254562, + 0.13061346113681793 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7410", + "split": "test", + "style_correct": 0, + "style_probability": 0.5163844227790833, + "style_target": 0, + "support_probability": 0.302464097738266, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7604073914961873, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10735104233026505, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23393763601779938, + 0.43663886189460754, + 0.6766313314437866, + 0.9672756195068359, + 0.12876872718334198 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.7006755471229553, + "style_target": 1, + "support_probability": 0.8258422017097473, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30851682037295447, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.06814783811569214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7411:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18457190692424774, + 0.3981751799583435, + 0.7703081965446472, + 0.9896103143692017, + 0.1373056173324585 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7411", + "split": "test", + "style_correct": 1, + "style_probability": 0.313861608505249, + "style_target": 0, + "support_probability": 0.8311961889266968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021101425619274753, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7197800874710083, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3418996036052704, + 0.5399661064147949, + 0.268136203289032, + 0.4666721224784851, + 0.04765521362423897, + 0.18215368688106537 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 0, + "style_probability": 0.4968470633029938, + "style_target": 1, + "support_probability": 0.07277660071849823, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02236556299589454, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8192878365516663, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7415:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.31531691551208496, + 0.5219380259513855, + 0.22345295548439026, + 0.4950772523880005, + 0.06802317500114441, + 0.06886761635541916 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7415", + "split": "test", + "style_correct": 0, + "style_probability": 0.7688692212104797, + "style_target": 0, + "support_probability": 0.05468731001019478, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.458003350800542, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21142826974391937, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.288109689950943, + 0.9258249998092651, + 0.0840255618095398, + 0.502592921257019, + 0.054434146732091904, + 0.0826074406504631, + 0.04503430426120758, + 0.04910877346992493, + 0.1162201389670372 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 1, + "style_probability": 0.5304443836212158, + "style_target": 1, + "support_probability": 0.5864626169204712, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17042643375071886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17902344465255737, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25198379158973694, + 0.9321312308311462, + 0.0767684206366539, + 0.45085015892982483, + 0.08611949533224106 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7417", + "split": "test", + "style_correct": 1, + "style_probability": 0.31508883833885193, + "style_target": 0, + "support_probability": 0.5358027815818787, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41984269797683466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13309067487716675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9821493029594421, + 0.5065321326255798, + 0.40118589997291565, + 0.34117329120635986, + 0.10080484300851822, + 0.09656578302383423, + 0.10236953943967819, + 0.04685753211379051, + 0.05654682219028473, + 0.13125476241111755 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 1, + "style_probability": 0.5401442050933838, + "style_target": 1, + "support_probability": 0.5787236094474792, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1336553631878985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20469385385513306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7420:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9983835220336914, + 0.47864773869514465, + 0.4102984666824341, + 0.4053848385810852, + 0.07119326293468475, + 0.0525142140686512, + 0.06010900437831879, + 0.06660899519920349, + 0.060021232813596725, + 0.1323486715555191 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7420", + "split": "test", + "style_correct": 1, + "style_probability": 0.2693444490432739, + "style_target": 0, + "support_probability": 0.6983598470687866, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009355453783250312, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7945424914360046, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25280728936195374, + 0.5981197357177734, + 0.2546643912792206, + 0.43086907267570496, + 0.06746326386928558 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.8106604218482971, + "style_target": 1, + "support_probability": 0.019612984731793404, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0037768708580444834, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8026787638664246, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7437:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.277618944644928, + 0.5578646063804626, + 0.23093363642692566, + 0.39511898159980774, + 0.10228224843740463, + 0.21961164474487305, + 0.15482811629772186, + 0.08312314003705978, + 0.13057519495487213 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7437", + "split": "test", + "style_correct": 1, + "style_probability": 0.3233121633529663, + "style_target": 0, + "support_probability": 0.009999952279031277, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5960930736301506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12236537039279938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28655296564102173, + 0.9740463495254517, + 0.24012809991836548, + 0.24523799121379852, + 0.06716687977313995, + 0.06491228193044662 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.6195828914642334, + "style_target": 1, + "support_probability": 0.8050870895385742, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.326638813894192, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14321458339691162, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7473:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3333036005496979, + 0.9755576848983765, + 0.27193430066108704, + 0.281899631023407, + 0.040538836270570755, + 0.08862155675888062 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7473", + "split": "test", + "style_correct": 1, + "style_probability": 0.3899822533130646, + "style_target": 0, + "support_probability": 0.8270025849342346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16523934582046174, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3600168228149414, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13668376207351685, + 0.4120779037475586, + 0.6635127663612366, + 0.20116852223873138, + 0.04878762736916542, + 0.03992104530334473, + 0.05206175521016121, + 0.08406411111354828, + 0.04673721641302109 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 1, + "style_probability": 0.7686178684234619, + "style_target": 1, + "support_probability": 0.21475762128829956, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12470561678566039, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3996865153312683, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7478:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16248957812786102, + 0.4064108729362488, + 0.6639875769615173, + 0.21293343603610992, + 0.0351925790309906, + 0.14693064987659454, + 0.06657963246107101, + 0.04084574431180954, + 0.09115812182426453 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7478", + "split": "test", + "style_correct": 0, + "style_probability": 0.5143250226974487, + "style_target": 0, + "support_probability": 0.24617742002010345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2758254273639693, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6876521706581116, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1253681182861328, + 0.17615503072738647, + 0.711491584777832, + 0.7706457376480103, + 0.06571070104837418, + 0.04668306186795235, + 0.03703700378537178, + 0.03229374811053276, + 0.04586845636367798, + 0.047727275639772415, + 0.08153880387544632 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 1, + "style_probability": 0.8642286658287048, + "style_target": 1, + "support_probability": 0.18999764323234558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26624840166761216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3004491329193115, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7479:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12927933037281036, + 0.2026187628507614, + 0.6950134038925171, + 0.7537519335746765, + 0.09428704530000687, + 0.06911321729421616 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7479", + "split": "test", + "style_correct": 1, + "style_probability": 0.43666085600852966, + "style_target": 0, + "support_probability": 0.5980134010314941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.586644474959769, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3942442238330841, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33091306686401367, + 0.16627821326255798, + 0.9019819498062134, + 0.42185676097869873, + 0.1271364986896515 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.6184021830558777, + "style_target": 1, + "support_probability": 0.5517932176589966, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10390256667906739, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.45297735929489136, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7482:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3696051239967346, + 0.18702755868434906, + 0.913966953754425, + 0.40785109996795654, + 0.14051572978496552 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7482", + "split": "test", + "style_correct": 1, + "style_probability": 0.20268763601779938, + "style_target": 0, + "support_probability": 0.5427911281585693, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.05320748071832025, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8714877963066101, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6453114151954651, + 0.41661790013313293, + 0.2995755672454834, + 0.30582714080810547, + 0.19047005474567413, + 0.11775195598602295 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 1, + "style_probability": 0.8832127451896667, + "style_target": 1, + "support_probability": 0.07159950584173203, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12755712779481954, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7900774478912354, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.612201452255249, + 0.37657344341278076, + 0.3157801032066345, + 0.32626286149024963, + 0.07810159772634506, + 0.058961670845746994 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7488", + "split": "test", + "style_correct": 0, + "style_probability": 0.708480179309845, + "style_target": 0, + "support_probability": 0.17066720128059387, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.005377404187921763, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7043091654777527, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.27747586369514465, + 0.48840296268463135, + 0.30100885033607483, + 0.27747586369514465, + 0.053617071360349655, + 0.10559795796871185, + 0.09300520271062851, + 0.3486972451210022, + 0.14510105550289154 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.63644939661026, + "style_target": 1, + "support_probability": 0.06650533527135849, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0021273484140660872, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8549728393554688, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7493:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2918006181716919, + 0.45642244815826416, + 0.2885674238204956, + 0.2918006181716919, + 0.12305597960948944 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7493", + "split": "test", + "style_correct": 1, + "style_probability": 0.20001055300235748, + "style_target": 0, + "support_probability": 0.04051485285162926, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10150487429996247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8122996687889099, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31365537643432617, + 0.16516199707984924, + 0.31365537643432617, + 0.7032394409179688, + 0.06749171018600464, + 0.06411339342594147, + 0.07569989562034607, + 0.0927094966173172, + 0.04311547055840492, + 0.20549553632736206, + 0.08675897866487503 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.8052656650543213, + "style_target": 1, + "support_probability": 0.13421832025051117, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03673288380823103, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.568769097328186, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7499:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.43555498123168945, + 0.19420433044433594, + 0.43555498123168945, + 0.7912521958351135, + 0.07754208892583847, + 0.06886377930641174, + 0.11818049103021622, + 0.05981390178203583, + 0.04980828985571861, + 0.08520322293043137 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7499", + "split": "test", + "style_correct": 1, + "style_probability": 0.15887974202632904, + "style_target": 0, + "support_probability": 0.5497748255729675, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1724882113930164, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05988427251577377, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3198685944080353, + 0.4191058576107025, + 0.3198685944080353, + 0.880110502243042, + 0.14751940965652466 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 1, + "style_probability": 0.8399091958999634, + "style_target": 1, + "support_probability": 0.2899056077003479, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1328908406324603, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.05874277278780937, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7507:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3894675076007843, + 0.43241041898727417, + 0.3894675076007843, + 0.8482996821403503, + 0.08518389612436295 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7507", + "split": "test", + "style_correct": 0, + "style_probability": 0.6182590126991272, + "style_target": 0, + "support_probability": 0.34385979175567627, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6885128135307355, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2236129194498062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9825719594955444, + 0.3551696836948395, + 0.43486812710762024, + 0.32744377851486206, + 0.07367970049381256, + 0.0838632583618164 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 1, + "style_probability": 0.7025988101959229, + "style_target": 1, + "support_probability": 0.7873995304107666, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4879289956607628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18685610592365265, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7524:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9735768437385559, + 0.3382302224636078, + 0.40406355261802673, + 0.3397141695022583, + 0.04321630671620369, + 0.07391681522130966, + 0.05358515679836273, + 0.19876845180988312, + 0.07898658514022827, + 0.06640022993087769, + 0.05902528762817383 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7524", + "split": "test", + "style_correct": 1, + "style_probability": 0.43759456276893616, + "style_target": 0, + "support_probability": 0.7825696468353271, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10531610359731314, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27891480922698975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8101219534873962, + 0.17679761350154877, + 0.21020786464214325, + 0.3619033396244049, + 0.06187952309846878, + 0.051445674151182175, + 0.04487691819667816, + 0.12215004116296768, + 0.1293896734714508 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.7268936038017273, + "style_target": 1, + "support_probability": 0.23204508423805237, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0095475611133411, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.302337646484375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7540:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8461031317710876, + 0.19347965717315674, + 0.2773771286010742, + 0.43553799390792847, + 0.079274982213974, + 0.10557343810796738, + 0.09482614696025848, + 0.07870572060346603, + 0.1403847634792328 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7540", + "split": "test", + "style_correct": 1, + "style_probability": 0.1880972683429718, + "style_target": 0, + "support_probability": 0.2517453730106354, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6468705297850619, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11171650141477585, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20973089337348938, + 0.15029747784137726, + 0.956506609916687, + 0.24831272661685944, + 0.08031290024518967, + 0.1664281189441681, + 0.03068903274834156, + 0.053142961114645004, + 0.027123557403683662, + 0.09039971232414246, + 0.11169246584177017 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 1, + "style_probability": 0.5798081755638123, + "style_target": 1, + "support_probability": 0.6456096768379211, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.32738869209305577, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21159727871418, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7542:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20004770159721375, + 0.18352122604846954, + 0.9435027837753296, + 0.2487671673297882, + 0.09495209902524948, + 0.14859895408153534, + 0.10454893112182617, + 0.3003728985786438, + 0.10519565641880035, + 0.06093433126807213, + 0.06886763125658035 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7542", + "split": "test", + "style_correct": 1, + "style_probability": 0.342224657535553, + "style_target": 0, + "support_probability": 0.46319955587387085, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.020409711500863903, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6934282183647156, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42348629236221313, + 0.4420357048511505, + 0.6921287178993225, + 0.44262343645095825, + 0.05699656903743744 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.7650342583656311, + "style_target": 1, + "support_probability": 0.07269264757633209, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010947547858563663, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7291427254676819, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7543:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3747195303440094, + 0.39550724625587463, + 0.6138343811035156, + 0.3994903862476349, + 0.07934897392988205, + 0.06313289701938629, + 0.07709431648254395, + 0.0771200880408287, + 0.11106077581644058 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7543", + "split": "test", + "style_correct": 1, + "style_probability": 0.34768497943878174, + "style_target": 0, + "support_probability": 0.03290242329239845, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0011503262399070737, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9921950101852417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6603851318359375, + 0.31215226650238037, + 0.31938081979751587, + 0.2428673803806305, + 0.10441739857196808, + 0.053410355001688004 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.8708829283714294, + "style_target": 1, + "support_probability": 0.004187611397355795, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005046769019650363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9853765368461609, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7552:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7129140496253967, + 0.3840749263763428, + 0.4130922853946686, + 0.3142322897911072, + 0.10794944316148758, + 0.1386726349592209 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7552", + "split": "test", + "style_correct": 1, + "style_probability": 0.284614622592926, + "style_target": 0, + "support_probability": 0.01254216767847538, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8995110713870779, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13384631276130676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21056364476680756, + 0.9315279126167297, + 0.22935329377651215, + 0.21056364476680756, + 0.0781242623925209, + 0.10935016721487045, + 0.07001394778490067, + 0.05289028212428093, + 0.09409723430871964 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 1, + "style_probability": 0.9172177910804749, + "style_target": 1, + "support_probability": 0.8752455711364746, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8542023281263234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.037698760628700256, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7553:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19720111787319183, + 0.9321995377540588, + 0.2190960794687271, + 0.19720111787319183, + 0.06164130941033363, + 0.0804021954536438, + 0.05930132046341896, + 0.15862149000167847, + 0.09703061729669571 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7553", + "split": "test", + "style_correct": 0, + "style_probability": 0.5200592279434204, + "style_target": 0, + "support_probability": 0.9488821625709534, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.696665349128118, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31084826588630676, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4094926714897156, + 0.7321113348007202, + 0.7928233742713928, + 0.6735955476760864, + 0.03292490541934967, + 0.033739157021045685, + 0.04491797462105751, + 0.12806564569473267, + 0.04049500823020935, + 0.02899879217147827, + 0.0693984255194664 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 1, + "style_probability": 0.8469207882881165, + "style_target": 1, + "support_probability": 0.6930211186408997, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6304332051113235, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2984848618507385, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7565:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39334094524383545, + 0.6379380822181702, + 0.7841973900794983, + 0.5892647504806519, + 0.07481082528829575, + 0.04895241931080818 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7565", + "split": "test", + "style_correct": 0, + "style_probability": 0.5002256035804749, + "style_target": 0, + "support_probability": 0.8178150653839111, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.19867852309140588, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.42779210209846497, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.563583254814148, + 0.3007599115371704, + 0.12058962881565094, + 0.7017992734909058, + 0.1536117047071457 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 0, + "style_probability": 0.4529339075088501, + "style_target": 1, + "support_probability": 0.41516751050949097, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2179393504093596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24432586133480072, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7571:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.610720157623291, + 0.2571001946926117, + 0.10865584015846252, + 0.7296520471572876, + 0.09429119527339935, + 0.10583287477493286, + 0.08733151108026505, + 0.10451958328485489, + 0.12740646302700043 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7571", + "split": "test", + "style_correct": 1, + "style_probability": 0.27604666352272034, + "style_target": 0, + "support_probability": 0.668929934501648, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.28027524779742835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09702862799167633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30196109414100647, + 0.18988093733787537, + 0.29097628593444824, + 0.7983062863349915, + 0.0645497739315033, + 0.07811126112937927 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.5715983510017395, + "style_target": 1, + "support_probability": 0.40037626028060913, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04537002790578271, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1788448840379715, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7579:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4015048146247864, + 0.165084108710289, + 0.33459237217903137, + 0.8510788083076477, + 0.06336323171854019, + 0.13581688702106476 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7579", + "split": "test", + "style_correct": 1, + "style_probability": 0.2321755290031433, + "style_target": 0, + "support_probability": 0.42447441816329956, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06722969039440585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6124340295791626, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.49053388833999634, + 0.33018821477890015, + 0.7503342628479004, + 0.3624957203865051, + 0.05517285689711571, + 0.10922826826572418, + 0.10081619024276733, + 0.09935465455055237, + 0.14778895676136017 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 1, + "style_probability": 0.5910935997962952, + "style_target": 1, + "support_probability": 0.12500588595867157, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04153899806833517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7105503082275391, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7602:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4454941749572754, + 0.3441447913646698, + 0.7792559862136841, + 0.36628538370132446, + 0.08347903937101364, + 0.1431494802236557, + 0.09540025144815445, + 0.07974895089864731, + 0.0896541103720665 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7602", + "split": "test", + "style_correct": 0, + "style_probability": 0.8484929203987122, + "style_target": 0, + "support_probability": 0.03022800199687481, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.20256070376653756, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3281513452529907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.27880576252937317, + 0.38930830359458923, + 0.8939964175224304, + 0.2709420323371887, + 0.07830625772476196, + 0.07172795385122299, + 0.07084331661462784, + 0.06810121983289719, + 0.08707492798566818, + 0.06305626779794693, + 0.09303338080644608 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 1, + "style_probability": 0.5733422636985779, + "style_target": 1, + "support_probability": 0.29298824071884155, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1892604936425997, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.16633190214633942, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7610:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3027012348175049, + 0.3752104341983795, + 0.9303911924362183, + 0.30276691913604736, + 0.09912341088056564, + 0.1325853168964386 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7610", + "split": "test", + "style_correct": 1, + "style_probability": 0.25713256001472473, + "style_target": 0, + "support_probability": 0.49554407596588135, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18563671935419726, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3004811406135559, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31347256898880005, + 0.10463207960128784, + 0.1295001208782196, + 0.8405727744102478, + 0.14025422930717468 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 1, + "style_probability": 0.7387001514434814, + "style_target": 1, + "support_probability": 0.3370605409145355, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23552587212056864, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24488475918769836, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7619:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2980688810348511, + 0.12950171530246735, + 0.16216175258159637, + 0.7962744235992432, + 0.044772010296583176 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7619", + "split": "test", + "style_correct": 0, + "style_probability": 0.554643452167511, + "style_target": 0, + "support_probability": 0.3700779974460602, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015150972843985146, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7979341149330139, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4765733480453491, + 0.7649039030075073, + 0.15761040151119232, + 0.5202615857124329, + 0.10591660439968109, + 0.14004112780094147 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.6927088499069214, + "style_target": 1, + "support_probability": 0.049334727227687836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.026500597900898537, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7579424381256104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7625:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5041642189025879, + 0.6711788773536682, + 0.17450439929962158, + 0.5113255977630615, + 0.09333108365535736, + 0.0811290591955185, + 0.15052542090415955, + 0.09044508635997772, + 0.14125555753707886, + 0.1503426730632782 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7625", + "split": "test", + "style_correct": 1, + "style_probability": 0.4307858347892761, + "style_target": 0, + "support_probability": 0.06099274381995201, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1429604726794089, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6582609415054321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5656560659408569, + 0.5047619342803955, + 0.2690280079841614, + 0.40921884775161743, + 0.05273152515292168, + 0.04166901856660843, + 0.09667928516864777, + 0.06993390619754791, + 0.09396322071552277 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 1, + "style_probability": 0.9124792814254761, + "style_target": 1, + "support_probability": 0.04617388918995857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23501490844096648, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4065505266189575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7637:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5515140891075134, + 0.5679399967193604, + 0.3300979733467102, + 0.44408032298088074, + 0.06581775099039078, + 0.11653843522071838, + 0.05122213065624237, + 0.06431273370981216, + 0.0778278335928917 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7637", + "split": "test", + "style_correct": 0, + "style_probability": 0.807996392250061, + "style_target": 0, + "support_probability": 0.3506579101085663, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8155711697060964, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08166772127151489, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17310956120491028, + 0.19442805647850037, + 0.17310956120491028, + 0.8983473181724548, + 0.04073246940970421, + 0.04977146163582802, + 0.02776975929737091, + 0.029907438904047012, + 0.10422229021787643, + 0.0739847868680954 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 1, + "style_probability": 0.8235256671905518, + "style_target": 1, + "support_probability": 0.928777813911438, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7358789007713717, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08631016314029694, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7643:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1863068789243698, + 0.2227807641029358, + 0.1863068789243698, + 0.8497986197471619, + 0.17981000244617462, + 0.1000293716788292, + 0.08141420781612396, + 0.11420560628175735, + 0.06548471003770828, + 0.04722591117024422 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7643", + "split": "test", + "style_correct": 0, + "style_probability": 0.7061948180198669, + "style_target": 0, + "support_probability": 0.9282928109169006, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3653860746484731, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3903270959854126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7749334573745728, + 0.5895081162452698, + 0.16170503199100494, + 0.4288563132286072, + 0.10051329433917999 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 1, + "style_probability": 0.5191749334335327, + "style_target": 1, + "support_probability": 0.41105854511260986, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4712726790764386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4192448556423187, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8164016604423523, + 0.533189594745636, + 0.1344556361436844, + 0.3352874219417572, + 0.10894198715686798 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7656", + "split": "test", + "style_correct": 0, + "style_probability": 0.8129655122756958, + "style_target": 0, + "support_probability": 0.3629660904407501, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011575462664895068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8587487936019897, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8371070027351379, + 0.5612232685089111, + 0.33551040291786194, + 0.18805475533008575, + 0.1951659619808197, + 0.15999552607536316 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.6176869869232178, + "style_target": 1, + "support_probability": 0.024757931008934975, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002153602521829001, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9465458989143372, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7660:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8042921423912048, + 0.45800361037254333, + 0.323138564825058, + 0.1563378870487213, + 0.12046538293361664, + 0.16080963611602783, + 0.1466083973646164, + 0.12833191454410553, + 0.12420139461755753, + 0.12540268898010254, + 0.15607470273971558 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7660", + "split": "test", + "style_correct": 1, + "style_probability": 0.35126766562461853, + "style_target": 0, + "support_probability": 0.0076275416649878025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.38066306338380684, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07334379851818085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18780691921710968, + 0.2584535777568817, + 0.18780691921710968, + 0.9831122756004333, + 0.04560934007167816, + 0.055733826011419296, + 0.08128196746110916, + 0.07927002757787704, + 0.08329392969608307 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.5518889427185059, + "style_target": 1, + "support_probability": 0.49611300230026245, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10450794868923197, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09579242020845413, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7667:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23930895328521729, + 0.36034369468688965, + 0.23930895328521729, + 0.9945316314697266, + 0.043376028537750244, + 0.06477607786655426, + 0.05139248073101044, + 0.038065843284130096, + 0.04918430373072624 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7667", + "split": "test", + "style_correct": 1, + "style_probability": 0.1962001472711563, + "style_target": 0, + "support_probability": 0.579725444316864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.32157508620161934, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20286501944065094, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3162734806537628, + 0.41474223136901855, + 0.9553565979003906, + 0.8381603360176086, + 0.0873955562710762, + 0.066811703145504, + 0.07227428257465363, + 0.06484369188547134, + 0.14339718222618103, + 0.0835300162434578 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 1, + "style_probability": 0.5998372435569763, + "style_target": 1, + "support_probability": 0.5314366817474365, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5914271199724226, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23050129413604736, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7674:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30290094017982483, + 0.40652117133140564, + 0.9691956043243408, + 0.8671042919158936, + 0.05916275084018707, + 0.04749877005815506, + 0.05363839492201805, + 0.060574986040592194, + 0.10029736161231995, + 0.058793481439352036, + 0.03612571954727173 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7674", + "split": "test", + "style_correct": 0, + "style_probability": 0.8445543050765991, + "style_target": 0, + "support_probability": 0.5843647718429565, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7454464724936365, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08596550673246384, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3987228572368622, + 0.5240064263343811, + 0.6104564666748047, + 0.9975280165672302, + 0.07532304525375366 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.7276027202606201, + "style_target": 1, + "support_probability": 0.7904303669929504, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3314134168198475, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.14243578910827637, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7675:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.46865060925483704, + 0.5633957982063293, + 0.6812305450439453, + 0.9991810917854309, + 0.10559455305337906 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7675", + "split": "test", + "style_correct": 1, + "style_probability": 0.27774596214294434, + "style_target": 0, + "support_probability": 0.7780526876449585, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7052982080117971, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07144899666309357, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32407355308532715, + 0.18514230847358704, + 0.34983640909194946, + 0.988353967666626, + 0.03546172007918358, + 0.05588771775364876 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 1, + "style_probability": 0.786474347114563, + "style_target": 1, + "support_probability": 0.6418933868408203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7049644540419919, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05729469656944275, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7679:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34924453496932983, + 0.16745616495609283, + 0.3572450280189514, + 0.9783600568771362, + 0.07112389802932739, + 0.07549446821212769 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7679", + "split": "test", + "style_correct": 0, + "style_probability": 0.599884569644928, + "style_target": 0, + "support_probability": 0.7609906792640686, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10957450948456682, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37631312012672424, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3884579837322235, + 0.7002812027931213, + 0.682319700717926, + 0.8523076772689819, + 0.07186263054609299, + 0.058738093823194504, + 0.09776964038610458, + 0.06593754887580872, + 0.04115426540374756 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.7543090581893921, + "style_target": 1, + "support_probability": 0.24578902125358582, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05006109619999367, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.30176034569740295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7683:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4334886968135834, + 0.6779260635375977, + 0.6431329250335693, + 0.8114859461784363, + 0.10581926256418228 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7683", + "split": "test", + "style_correct": 1, + "style_probability": 0.3331228792667389, + "style_target": 0, + "support_probability": 0.25233569741249084, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.79497794101691, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07374823838472366, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9852834939956665, + 0.46476951241493225, + 0.25702163577079773, + 0.5216293334960938, + 0.06836257129907608, + 0.11012428253889084, + 0.02482454478740692, + 0.035788919776678085, + 0.030667835846543312, + 0.037204544991254807 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.7394676804542542, + "style_target": 1, + "support_probability": 0.7678429484367371, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.45996430361050505, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05894405022263527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7684:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9922045469284058, + 0.49464935064315796, + 0.30368292331695557, + 0.5757975578308105, + 0.051000434905290604, + 0.05798693001270294, + 0.03737347573041916, + 0.11003798246383667, + 0.06462285667657852, + 0.10483875125646591 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7684", + "split": "test", + "style_correct": 1, + "style_probability": 0.2672031819820404, + "style_target": 0, + "support_probability": 0.8735666275024414, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.043203472930466454, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5007028579711914, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6970745325088501, + 0.3319863975048065, + 0.7147309184074402, + 0.7002712488174438, + 0.045799512416124344 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 0, + "style_probability": 0.35290369391441345, + "style_target": 1, + "support_probability": 0.3464527130126953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004518296979299786, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5922399759292603, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7706:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8010604977607727, + 0.37730029225349426, + 0.86626797914505, + 0.8158566355705261, + 0.14624421298503876 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7706", + "split": "test", + "style_correct": 1, + "style_probability": 0.12965816259384155, + "style_target": 0, + "support_probability": 0.36250194907188416, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.42458443350205494, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.15930376946926117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6305577754974365, + 0.775028645992279, + 0.16831250488758087, + 0.8285350203514099, + 0.04444032907485962, + 0.07448940724134445 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 0, + "style_probability": 0.3982182443141937, + "style_target": 1, + "support_probability": 0.7129769921302795, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.771359901265768, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09764432907104492, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7707:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7351940274238586, + 0.8064531683921814, + 0.13659021258354187, + 0.8934586048126221, + 0.042321477085351944, + 0.05038904771208763 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:7707", + "split": "test", + "style_correct": 0, + "style_probability": 0.7806970477104187, + "style_target": 0, + "support_probability": 0.8317323327064514, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6030610796390761, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2546447813510895, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24940833449363708, + 0.24312731623649597, + 0.921377420425415, + 0.39520514011383057, + 0.12833787500858307, + 0.11793944239616394, + 0.1656482070684433, + 0.11600789427757263, + 0.15985845029354095 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.5672146081924438, + "style_target": 1, + "support_probability": 0.7498621344566345, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.46337421621887737, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.22575481235980988, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7714:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27911487221717834, + 0.18900775909423828, + 0.9466265439987183, + 0.34026989340782166, + 0.15944358706474304 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7714", + "split": "test", + "style_correct": 1, + "style_probability": 0.33717042207717896, + "style_target": 0, + "support_probability": 0.7269806861877441, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28725684717284317, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2592654824256897, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9123048186302185, + 0.27890923619270325, + 0.27755632996559143, + 0.30657437443733215, + 0.23974108695983887, + 0.08042663335800171, + 0.14701077342033386, + 0.1436779946088791, + 0.24712608754634857, + 0.12460587173700333, + 0.1480720490217209 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 1, + "style_probability": 0.5662696957588196, + "style_target": 1, + "support_probability": 0.29244622588157654, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15433952894353453, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24827046692371368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7732:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9544836282730103, + 0.3572516143321991, + 0.2867708206176758, + 0.2607404291629791, + 0.0863717794418335, + 0.11639045178890228, + 0.11003192514181137, + 0.11296435445547104, + 0.08976565301418304, + 0.19464850425720215 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7732", + "split": "test", + "style_correct": 0, + "style_probability": 0.5387618541717529, + "style_target": 0, + "support_probability": 0.28307291865348816, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5420978536619784, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3114994764328003, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.525578498840332, + 0.17249614000320435, + 0.9009667634963989, + 0.4302233159542084, + 0.3274053633213043 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 1, + "style_probability": 0.7637997269630432, + "style_target": 1, + "support_probability": 0.4184337854385376, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12032395531003104, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6046167016029358, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7742:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47708648443222046, + 0.15815311670303345, + 0.9311029314994812, + 0.4115179777145386, + 0.19905170798301697 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7742", + "split": "test", + "style_correct": 0, + "style_probability": 0.7256725430488586, + "style_target": 0, + "support_probability": 0.15352347493171692, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6024056806965383, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12543365359306335, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38093140721321106, + 0.8638396859169006, + 0.3180016279220581, + 0.3493017852306366, + 0.07704748958349228, + 0.24745331704616547 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.662114679813385, + "style_target": 1, + "support_probability": 0.858827531337738, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5740455633319819, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10083373636007309, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7757:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.34893447160720825, + 0.9015783667564392, + 0.28412216901779175, + 0.2789602279663086, + 0.07630392163991928, + 0.07692775130271912, + 0.04745055362582207, + 0.0820755809545517, + 0.170941561460495, + 0.04626224562525749, + 0.04587017744779587 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7757", + "split": "test", + "style_correct": 1, + "style_probability": 0.398095965385437, + "style_target": 0, + "support_probability": 0.920866847038269, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0029112046010243415, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9238778948783875, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35543641448020935, + 0.30878233909606934, + 0.2886272072792053, + 0.6399345993995667, + 0.12401983141899109, + 0.08542841672897339, + 0.22930710017681122, + 0.08505318313837051, + 0.40358009934425354 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 1, + "style_probability": 0.5050327777862549, + "style_target": 1, + "support_probability": 0.01848953403532505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007830856819083523, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9633021950721741, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7771:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3583190143108368, + 0.2841540277004242, + 0.3889588713645935, + 0.7286406755447388, + 0.09672167897224426, + 0.21710637211799622, + 0.17800703644752502, + 0.14246472716331482, + 0.186310276389122 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7771", + "split": "test", + "style_correct": 1, + "style_probability": 0.37686887383461, + "style_target": 0, + "support_probability": 0.02307990752160549, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017831525797095548, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9079564809799194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2358238250017166, + 0.698566734790802, + 0.2358238250017166, + 0.16347381472587585, + 0.0761963427066803, + 0.051278892904520035, + 0.06015785038471222, + 0.06280363351106644, + 0.16102339327335358, + 0.10322026163339615, + 0.26918888092041016 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 0, + "style_probability": 0.49190467596054077, + "style_target": 1, + "support_probability": 0.005127361975610256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005210348441914758, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8475587964057922, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7777:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25932997465133667, + 0.7488571405410767, + 0.25932997465133667, + 0.17458780109882355, + 0.05290515720844269, + 0.03974640369415283, + 0.05560843646526337, + 0.07167845219373703, + 0.06421928107738495, + 0.12517209351062775 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7777", + "split": "test", + "style_correct": 0, + "style_probability": 0.8188923597335815, + "style_target": 0, + "support_probability": 0.008097575046122074, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.29064717901594683, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.37288886308670044, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2550410330295563, + 0.2532437741756439, + 0.3554479479789734, + 0.8683627247810364, + 0.06287983059883118 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.5033663511276245, + "style_target": 1, + "support_probability": 0.5080236792564392, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0440526449412153, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3526861071586609, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7779:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31634798645973206, + 0.3086823523044586, + 0.416560560464859, + 0.945455014705658, + 0.14136214554309845 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7779", + "split": "test", + "style_correct": 1, + "style_probability": 0.28551197052001953, + "style_target": 0, + "support_probability": 0.3618888556957245, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15827291896347795, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5813157558441162, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8511065244674683, + 0.451582670211792, + 0.14748887717723846, + 0.451582670211792, + 0.12480802834033966, + 0.07871123403310776 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 0, + "style_probability": 0.37261736392974854, + "style_target": 1, + "support_probability": 0.33373886346817017, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17649734552201934, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5811128616333008, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7788:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7704258561134338, + 0.4430014491081238, + 0.18266037106513977, + 0.4430014491081238, + 0.08384695649147034, + 0.08722031861543655 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7788", + "split": "test", + "style_correct": 0, + "style_probability": 0.525446355342865, + "style_target": 0, + "support_probability": 0.29173868894577026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.40731487489127716, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08015972375869751, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5448986291885376, + 0.25327709317207336, + 0.5739225149154663, + 0.9776608943939209, + 0.06441951543092728, + 0.0358450673520565, + 0.03599749132990837, + 0.08155892789363861, + 0.062087226659059525 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 0, + "style_probability": 0.44660305976867676, + "style_target": 1, + "support_probability": 0.596542477607727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6294091195234159, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1252904236316681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7795:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5435225367546082, + 0.27810660004615784, + 0.5815467834472656, + 0.9828577041625977, + 0.022279761731624603, + 0.07334397733211517, + 0.1297776997089386, + 0.031266894191503525, + 0.04692745581269264 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7795", + "split": "test", + "style_correct": 0, + "style_probability": 0.8533845543861389, + "style_target": 0, + "support_probability": 0.48455771803855896, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020755562195688085, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4710286557674408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5593419671058655, + 0.17655783891677856, + 0.76394122838974, + 0.5163601636886597, + 0.15992240607738495, + 0.15195389091968536, + 0.1392146348953247, + 0.08363693952560425, + 0.06564847379922867, + 0.11096672713756561, + 0.24395278096199036 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 0, + "style_probability": 0.3684147596359253, + "style_target": 1, + "support_probability": 0.0343325138092041, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03574355222650239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4363015592098236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7798:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5521788597106934, + 0.18416981399059296, + 0.8695381283760071, + 0.5435808897018433, + 0.11051017045974731, + 0.07797403633594513, + 0.11658161878585815, + 0.1303449422121048, + 0.14605915546417236, + 0.18587492406368256 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:7798", + "split": "test", + "style_correct": 1, + "style_probability": 0.4939749836921692, + "style_target": 0, + "support_probability": 0.13815928995609283, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11530389075858326, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3227144479751587, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22487019002437592, + 0.3863847553730011, + 0.28146687150001526, + 0.7805408835411072, + 0.126213937997818 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 0, + "style_probability": 0.4449741542339325, + "style_target": 1, + "support_probability": 0.331529438495636, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28567403637772776, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.24965360760688782, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7803:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22784501314163208, + 0.3609994351863861, + 0.2270665317773819, + 0.7499812245368958, + 0.06121024116873741 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7803", + "split": "test", + "style_correct": 0, + "style_probability": 0.5681942105293274, + "style_target": 0, + "support_probability": 0.5516906380653381, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4998781381724626, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02765791118144989, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9518439173698425, + 0.2164149284362793, + 0.28108060359954834, + 0.268912136554718, + 0.10003255307674408, + 0.08355779945850372 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 1, + "style_probability": 0.5253660678863525, + "style_target": 1, + "support_probability": 0.8697342872619629, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6919561119046165, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05057132989168167, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7812:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9602407217025757, + 0.20509149134159088, + 0.2830091118812561, + 0.24709218740463257, + 0.032219287008047104, + 0.07997569441795349 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7812", + "split": "test", + "style_correct": 0, + "style_probability": 0.8548414707183838, + "style_target": 0, + "support_probability": 0.7880269885063171, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5302552077685618, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.058146972209215164, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24201273918151855, + 0.45082300901412964, + 0.13820986449718475, + 0.9109294414520264, + 0.10161285847425461, + 0.059598829597234726, + 0.04535470902919769, + 0.20501385629177094, + 0.12217973172664642 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 0, + "style_probability": 0.334656298160553, + "style_target": 1, + "support_probability": 0.7293081879615784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.28559708415566654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0972789078950882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7815:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22299817204475403, + 0.5005941390991211, + 0.1739335060119629, + 0.9575813412666321, + 0.08702847361564636, + 0.03620433807373047, + 0.06794694811105728, + 0.07936844974756241, + 0.08639200776815414 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7815", + "split": "test", + "style_correct": 1, + "style_probability": 0.18770046532154083, + "style_target": 0, + "support_probability": 0.7216907143592834, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4362427479001596, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.025754526257514954, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9973575472831726, + 0.3065285384654999, + 0.47123950719833374, + 0.26523712277412415, + 0.06353375315666199, + 0.14371627569198608, + 0.12974244356155396, + 0.0899050384759903, + 0.05367409437894821, + 0.1180693507194519, + 0.09481356292963028 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.4338027238845825, + "style_target": 1, + "support_probability": 0.5913660526275635, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4566966014029106, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029660586267709732, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7836:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9920874238014221, + 0.3452838063240051, + 0.468289315700531, + 0.2971959114074707, + 0.07360207289457321, + 0.10634557902812958, + 0.0876367837190628, + 0.07679194211959839, + 0.08348841220140457, + 0.050180546939373016 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7836", + "split": "test", + "style_correct": 0, + "style_probability": 0.6693428158760071, + "style_target": 0, + "support_probability": 0.4431374669075012, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01220740298382772, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.88458651304245, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2691190540790558, + 0.4175794720649719, + 0.34836921095848083, + 0.11201675981283188, + 0.043390464037656784 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.763311505317688, + "style_target": 1, + "support_probability": 0.0762808695435524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017020459040023581, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9585691094398499, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7848:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.268877238035202, + 0.4081299304962158, + 0.33752167224884033, + 0.11355230212211609, + 0.07153277844190598, + 0.08153343200683594, + 0.06250199675559998, + 0.05748379975557327, + 0.12315188348293304 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7848", + "split": "test", + "style_correct": 1, + "style_probability": 0.22732065618038177, + "style_target": 0, + "support_probability": 0.016113968566060066, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12696675712487604, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7502520084381104, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2920101284980774, + 0.20348072052001953, + 0.09573729336261749, + 0.39629173278808594, + 0.13827930390834808, + 0.06924878805875778 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.9219434857368469, + "style_target": 1, + "support_probability": 0.17384091019630432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0041710956176142224, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.57429039478302, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36637914180755615, + 0.23575526475906372, + 0.15280646085739136, + 0.5544077754020691, + 0.2279273271560669, + 0.1414208561182022 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7851", + "split": "test", + "style_correct": 1, + "style_probability": 0.2835896611213684, + "style_target": 0, + "support_probability": 0.1320306807756424, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.038611071358940485, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1842503845691681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23759721219539642, + 0.15073798596858978, + 0.20758821070194244, + 0.9151995182037354, + 0.2280924767255783, + 0.04591246694326401, + 0.06627725064754486, + 0.06699781119823456, + 0.03969047963619232 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.6641598343849182, + "style_target": 1, + "support_probability": 0.050824977457523346, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04999117432257806, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13592441380023956, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7859:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2237069010734558, + 0.14358754456043243, + 0.18751917779445648, + 0.9032732844352722, + 0.18771369755268097, + 0.16264189779758453, + 0.06207145005464554, + 0.10627627372741699, + 0.18255111575126648 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7859", + "split": "test", + "style_correct": 1, + "style_probability": 0.37198469042778015, + "style_target": 0, + "support_probability": 0.09281453490257263, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7773939216897077, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08560191094875336, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9658958911895752, + 0.24152013659477234, + 0.1755601018667221, + 0.4245425760746002, + 0.13030849397182465, + 0.08238210529088974, + 0.26594164967536926, + 0.03724249079823494, + 0.0531749501824379, + 0.10285954177379608, + 0.0392586886882782 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.9095824956893921, + "style_target": 1, + "support_probability": 0.48817023634910583, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.44027116867725974, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0460033193230629, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7860:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9579247236251831, + 0.23298873007297516, + 0.1710260957479477, + 0.4662938117980957, + 0.08349364995956421, + 0.15232686698436737 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7860", + "split": "test", + "style_correct": 1, + "style_probability": 0.44557473063468933, + "style_target": 0, + "support_probability": 0.6711921095848083, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3827177685580434, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09240701794624329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47769299149513245, + 0.7714706063270569, + 0.14719343185424805, + 0.4851309657096863, + 0.1213328018784523 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 1, + "style_probability": 0.5236948728561401, + "style_target": 1, + "support_probability": 0.6269091963768005, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25487875692192397, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21578645706176758, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7861:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4663415253162384, + 0.785729706287384, + 0.13242262601852417, + 0.5453236103057861, + 0.1505354642868042, + 0.1765783578157425, + 0.04131565988063812, + 0.05621970444917679, + 0.10371508449316025 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7861", + "split": "test", + "style_correct": 1, + "style_probability": 0.43467020988464355, + "style_target": 0, + "support_probability": 0.45943281054496765, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.41122045867366097, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3893856406211853, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8006101250648499, + 0.29358288645744324, + 0.14222452044487, + 0.29358288645744324, + 0.05492079257965088, + 0.1042931005358696 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 0, + "style_probability": 0.29767802357673645, + "style_target": 1, + "support_probability": 0.7999510765075684, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018637830385773265, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41775086522102356, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7880:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8738397359848022, + 0.33340710401535034, + 0.19630080461502075, + 0.33340710401535034, + 0.06790583580732346, + 0.11319298297166824 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7880", + "split": "test", + "style_correct": 1, + "style_probability": 0.0633816123008728, + "style_target": 0, + "support_probability": 0.6880038380622864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024881800138703888, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.930285632610321, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.274017870426178, + 0.33668267726898193, + 0.5326176881790161, + 0.417213499546051, + 0.058872658759355545, + 0.0637441873550415, + 0.07732033729553223, + 0.1131720170378685, + 0.09050741046667099 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 1, + "style_probability": 0.5638043880462646, + "style_target": 1, + "support_probability": 0.1151069924235344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016010159962648204, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.904421329498291, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7887:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.30786392092704773, + 0.2609708905220032, + 0.5519096255302429, + 0.4039446711540222, + 0.056860655546188354, + 0.09855663776397705, + 0.06466792523860931, + 0.12055496871471405, + 0.06494651734828949 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7887", + "split": "test", + "style_correct": 0, + "style_probability": 0.6621139049530029, + "style_target": 0, + "support_probability": 0.1146770492196083, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023385091705186326, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7559150457382202, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.30085960030555725, + 0.16623999178409576, + 0.17710164189338684, + 0.576737105846405, + 0.05897877737879753, + 0.036230653524398804, + 0.05817965418100357, + 0.044870346784591675, + 0.05347879230976105, + 0.08942990750074387 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.567868173122406, + "style_target": 1, + "support_probability": 0.04394328594207764, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004178052401610233, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7768940329551697, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7896:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.26889851689338684, + 0.13629290461540222, + 0.21844668686389923, + 0.5351251363754272, + 0.04661605879664421, + 0.05381481721997261 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7896", + "split": "test", + "style_correct": 1, + "style_probability": 0.3945452868938446, + "style_target": 0, + "support_probability": 0.03215990215539932, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.36854297381534096, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3037135899066925, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4520736336708069, + 0.4471661150455475, + 0.4520736336708069, + 0.9642776250839233, + 0.1875407099723816 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.8048245310783386, + "style_target": 1, + "support_probability": 0.3812490701675415, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23906658396587058, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.2895769774913788, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7899:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4327845871448517, + 0.43719521164894104, + 0.4327845871448517, + 0.9505224823951721, + 0.07981007546186447 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7899", + "split": "test", + "style_correct": 1, + "style_probability": 0.49386662244796753, + "style_target": 0, + "support_probability": 0.49627232551574707, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010981087363235444, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.909143328666687, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.13817447423934937, + 0.2989104688167572, + 0.4744187593460083, + 0.20977868139743805, + 0.09545031934976578, + 0.3016049265861511 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 1, + "style_probability": 0.8512528538703918, + "style_target": 1, + "support_probability": 0.010128073394298553, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002121524909290406, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.851242184638977, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7926:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.14693917334079742, + 0.3115925192832947, + 0.44074103236198425, + 0.1852700412273407, + 0.18462111055850983, + 0.08977235108613968, + 0.13701081275939941, + 0.0543888621032238, + 0.08264578878879547, + 0.07288946211338043 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7926", + "split": "test", + "style_correct": 0, + "style_probability": 0.5303702354431152, + "style_target": 0, + "support_probability": 0.005626342259347439, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.29547576486237404, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8024724721908569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18097823858261108, + 0.6179612874984741, + 0.5107103586196899, + 0.13452884554862976, + 0.025429347530007362, + 0.08714764565229416, + 0.0767877846956253, + 0.0197284072637558, + 0.06473518162965775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.9009496569633484, + "style_target": 1, + "support_probability": 0.26564931869506836, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006950471127899813, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.667683482170105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7929:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1943860501050949, + 0.6604082584381104, + 0.6082102060317993, + 0.15268205106258392, + 0.05060375854372978, + 0.15581655502319336, + 0.041631437838077545, + 0.04236598312854767, + 0.056192200630903244 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7929", + "split": "test", + "style_correct": 1, + "style_probability": 0.17067348957061768, + "style_target": 0, + "support_probability": 0.2891347408294678, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8592463931286716, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08963007479906082, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24381178617477417, + 0.3825494945049286, + 0.2721077799797058, + 0.9643522500991821, + 0.08051932603120804, + 0.03362707793712616, + 0.05279109999537468, + 0.07820948213338852, + 0.10146943479776382, + 0.03440536558628082, + 0.13077746331691742 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 1, + "style_probability": 0.8858848214149475, + "style_target": 1, + "support_probability": 0.7634869813919067, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6124435674433926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06767787039279938, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7971:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.24558468163013458, + 0.3893183171749115, + 0.19676348567008972, + 0.9555292129516602, + 0.0882788747549057, + 0.08879073709249496, + 0.13340766727924347, + 0.08677069842815399, + 0.08994971960783005, + 0.02440514601767063, + 0.04741664230823517 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:7971", + "split": "test", + "style_correct": 0, + "style_probability": 0.5391721725463867, + "style_target": 0, + "support_probability": 0.8080379366874695, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03336461126489709, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.656214714050293, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6422969102859497, + 0.2052355855703354, + 0.57684326171875, + 0.565129816532135, + 0.15864422917366028 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.5953207612037659, + "style_target": 1, + "support_probability": 0.10391567647457123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.031112093237723563, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6876145601272583, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:7976:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6494626998901367, + 0.19460853934288025, + 0.5763540267944336, + 0.5479278564453125, + 0.1327672004699707, + 0.1613181084394455, + 0.05283442139625549, + 0.06788833439350128, + 0.16408097743988037 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:7976", + "split": "test", + "style_correct": 1, + "style_probability": 0.30306217074394226, + "style_target": 0, + "support_probability": 0.12640710175037384, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09668279371283008, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37324684858322144, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8640327453613281, + 0.19236353039741516, + 0.36124488711357117, + 0.5014916658401489, + 0.06963028758764267, + 0.16138708591461182 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.8562365174293518, + "style_target": 1, + "support_probability": 0.05733495205640793, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.008200285217405678, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5572506189346313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7984:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9312019348144531, + 0.2821398377418518, + 0.4244159162044525, + 0.5607854723930359, + 0.07703211903572083, + 0.08858726173639297 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7984", + "split": "test", + "style_correct": 1, + "style_probability": 0.33526042103767395, + "style_target": 0, + "support_probability": 0.062257979065179825, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8181515846748937, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06733155250549316, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8972885608673096, + 0.15182636678218842, + 0.31829577684402466, + 0.22766059637069702, + 0.2042320966720581, + 0.09413396567106247, + 0.11829457432031631, + 0.02655795030295849, + 0.06224091351032257 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 1, + "style_probability": 0.8076842427253723, + "style_target": 1, + "support_probability": 0.856709897518158, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4359380776877515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15370872616767883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7988:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8392467498779297, + 0.15776963531970978, + 0.3232516646385193, + 0.2347448766231537, + 0.05860671401023865, + 0.08031468838453293, + 0.03143731877207756, + 0.06835892796516418, + 0.07500756531953812 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:7988", + "split": "test", + "style_correct": 0, + "style_probability": 0.5057220458984375, + "style_target": 0, + "support_probability": 0.7143141031265259, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6268982034666557, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17053942382335663, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15353234112262726, + 0.3264315128326416, + 0.7607144117355347, + 0.9772096276283264, + 0.08457371592521667, + 0.05039357393980026, + 0.08465485274791718, + 0.07085239887237549, + 0.10793661326169968, + 0.05629682540893555, + 0.12433532625436783 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 1, + "style_probability": 0.8567891120910645, + "style_target": 1, + "support_probability": 0.5024431943893433, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3316380243857555, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1339852213859558, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7991:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18078193068504333, + 0.35062748193740845, + 0.7364770174026489, + 0.9705596566200256, + 0.06143109127879143, + 0.08748184889554977 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:7991", + "split": "test", + "style_correct": 1, + "style_probability": 0.43417030572891235, + "style_target": 0, + "support_probability": 0.5423150062561035, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5506025695441394, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09312479197978973, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5348423719406128, + 0.8785800933837891, + 0.29019564390182495, + 0.36804673075675964, + 0.19964656233787537 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 1, + "style_probability": 0.5495120882987976, + "style_target": 1, + "support_probability": 0.7260379791259766, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.20292549817584063, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.18357984721660614, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:7997:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.585436224937439, + 0.919403612613678, + 0.28461164236068726, + 0.46890169382095337, + 0.09509947896003723 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:7997", + "split": "test", + "style_correct": 1, + "style_probability": 0.4711751937866211, + "style_target": 0, + "support_probability": 0.6379405856132507, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010055871132130686, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7823382616043091, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.233234703540802, + 0.4950304627418518, + 0.4546535909175873, + 0.48794278502464294, + 0.15925152599811554, + 0.115901879966259 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 0, + "style_probability": 0.32942894101142883, + "style_target": 1, + "support_probability": 0.04805968329310417, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006436244673583643, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.827582597732544, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8005:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2322617769241333, + 0.4421145021915436, + 0.4600617587566376, + 0.45067569613456726, + 0.2515237033367157, + 0.164561927318573 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8005", + "split": "test", + "style_correct": 0, + "style_probability": 0.5569932460784912, + "style_target": 0, + "support_probability": 0.027952702715992928, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2939262783928367, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05918882042169571, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7308035492897034, + 0.16392654180526733, + 0.8520442843437195, + 0.7224811911582947, + 0.10189257562160492, + 0.15856397151947021, + 0.1178475171327591, + 0.05160076171159744, + 0.18748368322849274 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.4255930781364441, + "style_target": 1, + "support_probability": 0.5075682997703552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5052795400972698, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07343128323554993, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8006:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7394206523895264, + 0.14203234016895294, + 0.8810311555862427, + 0.7243291735649109, + 0.04784567281603813, + 0.07985692471265793, + 0.1060858815908432, + 0.03597293794155121, + 0.11159182339906693 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:8006", + "split": "test", + "style_correct": 0, + "style_probability": 0.7727007269859314, + "style_target": 0, + "support_probability": 0.40171924233436584, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009930929502281085, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6991101503372192, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.43556728959083557, + 0.2400270700454712, + 0.4063620865345001, + 0.24646969139575958, + 0.06141488626599312, + 0.04582465812563896, + 0.12114232778549194, + 0.17843876779079437, + 0.12955354154109955, + 0.07679375261068344, + 0.4147292673587799 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 0, + "style_probability": 0.20826345682144165, + "style_target": 1, + "support_probability": 0.04714014381170273, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003513502912773671, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9549329876899719, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8018:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3675983250141144, + 0.20897413790225983, + 0.37848660349845886, + 0.3332582712173462, + 0.12763634324073792, + 0.08947666734457016, + 0.13622725009918213, + 0.1251259446144104, + 0.06836046278476715, + 0.11969838291406631 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8018", + "split": "test", + "style_correct": 1, + "style_probability": 0.09880071133375168, + "style_target": 0, + "support_probability": 0.014452483505010605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09481234293243766, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9245010018348694, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23278404772281647, + 0.4306880831718445, + 0.44509246945381165, + 0.24786493182182312, + 0.08341065049171448 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 1, + "style_probability": 0.8610849976539612, + "style_target": 1, + "support_probability": 0.18163008987903595, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07133795368303854, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8937925100326538, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8021:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.22781655192375183, + 0.44892534613609314, + 0.46192529797554016, + 0.29093316197395325, + 0.06997060030698776 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8021", + "split": "test", + "style_correct": 0, + "style_probability": 0.6390966773033142, + "style_target": 0, + "support_probability": 0.24814513325691223, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7946391108052495, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13817624747753143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2591090798377991, + 0.2301260381937027, + 0.9201744794845581, + 0.3504549264907837, + 0.031120292842388153, + 0.05246995761990547 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.7478214502334595, + "style_target": 1, + "support_probability": 0.8599011898040771, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.19012106736450815, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4487263858318329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8026:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2520595192909241, + 0.21048714220523834, + 0.8742010593414307, + 0.3191853165626526, + 0.06380409002304077, + 0.054338764399290085, + 0.1990509331226349, + 0.18202780187129974, + 0.11067298799753189, + 0.08826898783445358, + 0.12997065484523773 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8026", + "split": "test", + "style_correct": 1, + "style_probability": 0.3981236219406128, + "style_target": 0, + "support_probability": 0.44341006875038147, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8896943797996109, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18032030761241913, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9338294863700867, + 0.057181358337402344, + 0.13616591691970825, + 0.3395450711250305, + 0.054214607924222946, + 0.04902389273047447, + 0.0277290940284729, + 0.026859352365136147, + 0.020392021164298058 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.8377485871315002, + "style_target": 1, + "support_probability": 0.9234107732772827, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4391819241184969, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1664770394563675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8036:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9742499589920044, + 0.07888326048851013, + 0.19710437953472137, + 0.45400869846343994, + 0.07527289539575577, + 0.044279374182224274, + 0.08270511776208878, + 0.047692496329545975, + 0.0774419978260994 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8036", + "split": "test", + "style_correct": 1, + "style_probability": 0.1940477341413498, + "style_target": 0, + "support_probability": 0.941013753414154, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1296655663740731, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2625634968280792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.39404499530792236, + 0.2801585793495178, + 0.8376690745353699, + 0.7401008605957031, + 0.04007289186120033, + 0.049887340515851974, + 0.0434722900390625, + 0.035450149327516556, + 0.04140513390302658, + 0.05587078630924225 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 1, + "style_probability": 0.7813794612884521, + "style_target": 1, + "support_probability": 0.2922481596469879, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00809865539267496, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5240598320960999, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8047:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4240814745426178, + 0.268951952457428, + 0.7170073986053467, + 0.6418182253837585, + 0.08929769694805145, + 0.11075998842716217, + 0.07035151869058609, + 0.17542684078216553, + 0.3128860592842102, + 0.0897592231631279, + 0.07214006036520004 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8047", + "split": "test", + "style_correct": 0, + "style_probability": 0.500877857208252, + "style_target": 0, + "support_probability": 0.034496672451496124, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19472690853897046, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.42534491419792175, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.423324316740036, + 0.26365378499031067, + 0.423324316740036, + 0.8342738747596741, + 0.08346473425626755 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.8067765235900879, + "style_target": 1, + "support_probability": 0.33591219782829285, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017134244988564976, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7170231342315674, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8051:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39983463287353516, + 0.25321927666664124, + 0.39983463287353516, + 0.7973005771636963, + 0.15034843981266022, + 0.0878547877073288, + 0.10433518141508102, + 0.04075517877936363, + 0.11045130342245102 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8051", + "split": "test", + "style_correct": 1, + "style_probability": 0.19468346238136292, + "style_target": 0, + "support_probability": 0.07992859184741974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48351272626716835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.26501286029815674, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3334379196166992, + 0.37162303924560547, + 0.94990473985672, + 0.46832942962646484, + 0.08535308390855789, + 0.03961740806698799 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.7702571749687195, + "style_target": 1, + "support_probability": 0.5961182117462158, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0930869379178101, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.24961833655834198, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8054:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3493553698062897, + 0.4064949154853821, + 0.9684370160102844, + 0.45620250701904297, + 0.05798497796058655, + 0.09917288273572922 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8054", + "split": "test", + "style_correct": 1, + "style_probability": 0.2761974334716797, + "style_target": 0, + "support_probability": 0.6688135862350464, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5370349391891516, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21706731617450714, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9364446997642517, + 0.46696048974990845, + 0.4336162507534027, + 0.21692846715450287, + 0.204244464635849, + 0.12538263201713562, + 0.06988456845283508, + 0.0934198722243309, + 0.09672267735004425 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 1, + "style_probability": 0.8640474081039429, + "style_target": 1, + "support_probability": 0.48125335574150085, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.18886101810980802, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2721068263053894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8068:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9310992956161499, + 0.47957396507263184, + 0.4662882685661316, + 0.223066508769989, + 0.12444032728672028, + 0.2516148090362549, + 0.07689377665519714, + 0.14836722612380981, + 0.0885092243552208 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8068", + "split": "test", + "style_correct": 0, + "style_probability": 0.6018959879875183, + "style_target": 0, + "support_probability": 0.4157329499721527, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.009080638437700682, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9054712653160095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4569128453731537, + 0.3448643088340759, + 0.47436392307281494, + 0.6318021416664124, + 0.05330713465809822, + 0.18622343242168427, + 0.21779845654964447, + 0.25768762826919556, + 0.09769059717655182, + 0.09038145840167999, + 0.0779297798871994 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.7800895571708679, + "style_target": 1, + "support_probability": 0.026060152798891068, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017395042965191738, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8556625247001648, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8071:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4461789131164551, + 0.3230559527873993, + 0.47042399644851685, + 0.5964287519454956, + 0.07635164260864258, + 0.07570468634366989 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8071", + "split": "test", + "style_correct": 1, + "style_probability": 0.39529839158058167, + "style_target": 0, + "support_probability": 0.02087056078016758, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.04635805828875284, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.31478047370910645, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3150826096534729, + 0.796400249004364, + 0.37117379903793335, + 0.5260125398635864, + 0.06307002902030945 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.5914844274520874, + "style_target": 1, + "support_probability": 0.19375355541706085, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.027319793280426323, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35275012254714966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8073:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.26411551237106323, + 0.7874565720558167, + 0.39216575026512146, + 0.4774651825428009, + 0.04980391636490822, + 0.06739528477191925, + 0.10687559098005295, + 0.13981497287750244, + 0.1311611384153366 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8073", + "split": "test", + "style_correct": 1, + "style_probability": 0.34595587849617004, + "style_target": 0, + "support_probability": 0.15288089215755463, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6329290426876284, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06835503876209259, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3345399498939514, + 0.20431238412857056, + 0.11464741080999374, + 0.9335868954658508, + 0.044096995145082474, + 0.05117550119757652 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 0, + "style_probability": 0.4305838942527771, + "style_target": 1, + "support_probability": 0.705311119556427, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07435212539022552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.09731531143188477, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8075:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29278334975242615, + 0.20400072634220123, + 0.15656855702400208, + 0.9828213453292847, + 0.10150790214538574, + 0.10864829272031784 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8075", + "split": "test", + "style_correct": 1, + "style_probability": 0.09070592373609543, + "style_target": 0, + "support_probability": 0.7255926728248596, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4891233003788855, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17953622341156006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8055838346481323, + 0.26734670996665955, + 0.366455078125, + 0.1659374088048935, + 0.11635751277208328, + 0.0905221700668335, + 0.2113730013370514, + 0.09816060960292816, + 0.07441931962966919 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 1, + "style_probability": 0.6970840692520142, + "style_target": 1, + "support_probability": 0.6814197897911072, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6931926073139323, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2609599828720093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8080:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8296526670455933, + 0.29685625433921814, + 0.38292500376701355, + 0.18301911652088165, + 0.07586982846260071, + 0.09240211546421051, + 0.16383837163448334, + 0.1003895029425621, + 0.10536240041255951 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8080", + "split": "test", + "style_correct": 0, + "style_probability": 0.9020869135856628, + "style_target": 0, + "support_probability": 0.5601381063461304, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5620247581321074, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13112957775592804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7326835989952087, + 0.2320558726787567, + 0.6410849690437317, + 0.4121895432472229, + 0.07044602930545807, + 0.05908231437206268, + 0.10564516484737396, + 0.16049213707447052, + 0.09608297795057297, + 0.0837763175368309, + 0.05255555734038353 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 1, + "style_probability": 0.5574401617050171, + "style_target": 1, + "support_probability": 0.7674004435539246, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3965948614522929, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07173991948366165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8082:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7513085603713989, + 0.21444940567016602, + 0.6360600590705872, + 0.3703864514827728, + 0.07693995535373688, + 0.11323228478431702 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8082", + "split": "test", + "style_correct": 1, + "style_probability": 0.3087260127067566, + "style_target": 0, + "support_probability": 0.7727827429771423, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.43724209919046686, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30275940895080566, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6543788313865662, + 0.45829886198043823, + 0.32689645886421204, + 0.45829886198043823, + 0.17038285732269287 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 1, + "style_probability": 0.5876741409301758, + "style_target": 1, + "support_probability": 0.5500973463058472, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10070112409671594, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23426850140094757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8096:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7808835506439209, + 0.5278344750404358, + 0.3345164358615875, + 0.5278344750404358, + 0.10904911905527115 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8096", + "split": "test", + "style_correct": 1, + "style_probability": 0.246530681848526, + "style_target": 0, + "support_probability": 0.7091581225395203, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.014892839672752352, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5136398673057556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3984249532222748, + 0.3203931450843811, + 0.3984249532222748, + 0.6600428819656372, + 0.04146532341837883, + 0.04986497759819031 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.4806879162788391, + "style_target": 1, + "support_probability": 0.10658340901136398, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009256285249722997, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5059324502944946, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8099:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3523779511451721, + 0.3468403220176697, + 0.3523779511451721, + 0.6613407731056213, + 0.09299842268228531, + 0.09859969466924667 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8099", + "split": "test", + "style_correct": 0, + "style_probability": 0.7815989255905151, + "style_target": 0, + "support_probability": 0.028036560863256454, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5542804077726977, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20025262236595154, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37139439582824707, + 0.5660589933395386, + 0.6515740752220154, + 0.714056670665741, + 0.058372464030981064, + 0.05898967757821083, + 0.0336311049759388, + 0.029270654544234276, + 0.053273268043994904 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 0, + "style_probability": 0.44150206446647644, + "style_target": 1, + "support_probability": 0.9169641137123108, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6042689801244698, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.07745394855737686, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8115:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34932267665863037, + 0.6016806960105896, + 0.7638899683952332, + 0.7155917882919312, + 0.07176768034696579 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:8115", + "split": "test", + "style_correct": 1, + "style_probability": 0.2197938710451126, + "style_target": 0, + "support_probability": 0.96295565366745, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4399593045285712, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25199174880981445, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38230952620506287, + 0.3996160626411438, + 0.9645248055458069, + 0.2692950665950775, + 0.22278061509132385, + 0.09373127669095993, + 0.09655879437923431, + 0.06890123337507248, + 0.04924174025654793, + 0.12012046575546265 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.520171046257019, + "style_target": 1, + "support_probability": 0.5381510257720947, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2248853004967941, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12609632313251495, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8126:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36511775851249695, + 0.4059273898601532, + 0.9948215484619141, + 0.21783101558685303, + 0.08576947450637817, + 0.0720188245177269, + 0.1302674412727356, + 0.12342936545610428, + 0.11799027025699615, + 0.16427741944789886 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8126", + "split": "test", + "style_correct": 1, + "style_probability": 0.18150894343852997, + "style_target": 0, + "support_probability": 0.6564657092094421, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.012184614434902045, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9125580787658691, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2906242609024048, + 0.5968648791313171, + 0.17061245441436768, + 0.3845953643321991, + 0.06110313534736633 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.7949005961418152, + "style_target": 1, + "support_probability": 0.022809892892837524, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02756464289640004, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6891885995864868, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8131:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2988945245742798, + 0.5113022923469543, + 0.1631557196378708, + 0.35658374428749084, + 0.06890951842069626, + 0.2507525086402893, + 0.11901522427797318, + 0.10428989678621292, + 0.12975072860717773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8131", + "split": "test", + "style_correct": 1, + "style_probability": 0.31224706768989563, + "style_target": 0, + "support_probability": 0.1045997217297554, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8388155963101553, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1258208155632019, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18435156345367432, + 0.9095401763916016, + 0.27535614371299744, + 0.3216785788536072, + 0.058848850429058075, + 0.03619297593832016 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.872926652431488, + "style_target": 1, + "support_probability": 0.8649469017982483, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0253734565033078, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23745661973953247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8133:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29617443680763245, + 0.9563845992088318, + 0.3842164874076843, + 0.402599960565567, + 0.053691472858190536, + 0.11324360966682434 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8133", + "split": "test", + "style_correct": 1, + "style_probability": 0.08183056861162186, + "style_target": 0, + "support_probability": 0.8346923589706421, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9032629961396879, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1035323217511177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9560493230819702, + 0.13984233140945435, + 0.1458544135093689, + 0.607499897480011, + 0.021415991708636284, + 0.04705780744552612, + 0.034374259412288666, + 0.03859741985797882, + 0.028807416558265686 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 1, + "style_probability": 0.8513074517250061, + "style_target": 1, + "support_probability": 0.8917643427848816, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8380050584889888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14209328591823578, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8136:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9270305633544922, + 0.1394972801208496, + 0.15174491703510284, + 0.5259267687797546, + 0.06291107088327408, + 0.043698858469724655, + 0.0530083142220974, + 0.15564899146556854, + 0.07158105820417404 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8136", + "split": "test", + "style_correct": 0, + "style_probability": 0.6686533093452454, + "style_target": 0, + "support_probability": 0.8994297385215759, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8074282158873914, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0423065610229969, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28841477632522583, + 0.45352593064308167, + 0.3198554813861847, + 0.8393058180809021, + 0.020339269191026688, + 0.018687419593334198, + 0.017785362899303436, + 0.05945226550102234, + 0.015665613114833832, + 0.022992532700300217, + 0.019382577389478683 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.8569053411483765, + "style_target": 1, + "support_probability": 0.8955045342445374, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7089892081939553, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.036288779228925705, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8139:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30730047821998596, + 0.4458431303501129, + 0.3408564329147339, + 0.8220064640045166, + 0.06777480244636536, + 0.13010480999946594 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8139", + "split": "test", + "style_correct": 1, + "style_probability": 0.4916517734527588, + "style_target": 0, + "support_probability": 0.906410813331604, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07663978282312378, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4251149594783783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3077108860015869, + 0.6325919032096863, + 0.1750926524400711, + 0.23016680777072906, + 0.09455003589391708 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.794445276260376, + "style_target": 1, + "support_probability": 0.3624362051486969, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01755343207093852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4945423901081085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8141:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3406599462032318, + 0.7009609937667847, + 0.21981573104858398, + 0.28108853101730347, + 0.09243667125701904 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8141", + "split": "test", + "style_correct": 1, + "style_probability": 0.464184045791626, + "style_target": 0, + "support_probability": 0.3532600402832031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8589700421436888, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08418747782707214, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2684549391269684, + 0.1348997950553894, + 0.9957979321479797, + 0.42692258954048157, + 0.2834373116493225, + 0.09534276276826859 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 1, + "style_probability": 0.7405246496200562, + "style_target": 1, + "support_probability": 0.7999629974365234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7680107828134588, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09243813902139664, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8146:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27285030484199524, + 0.17383387684822083, + 0.9823824763298035, + 0.4397576153278351, + 0.09655055403709412, + 0.064577616751194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8146", + "split": "test", + "style_correct": 0, + "style_probability": 0.6849801540374756, + "style_target": 0, + "support_probability": 0.77008056640625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07093583156399495, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7588293552398682, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3827746510505676, + 0.30006495118141174, + 0.39310377836227417, + 0.30006495118141174, + 0.05856170877814293, + 0.17005029320716858, + 0.15500853955745697, + 0.21989241242408752, + 0.17620053887367249 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.9242416024208069, + "style_target": 1, + "support_probability": 0.09209798276424408, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0036725355450607246, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7640040516853333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8176:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3717695474624634, + 0.21872423589229584, + 0.4166683852672577, + 0.21872423589229584, + 0.10893801599740982 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8176", + "split": "test", + "style_correct": 1, + "style_probability": 0.29982417821884155, + "style_target": 0, + "support_probability": 0.04432576894760132, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8259862893148124, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1169239804148674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.425807386636734, + 0.8037182092666626, + 0.851315975189209, + 0.6236714124679565, + 0.10985331237316132, + 0.059476181864738464, + 0.030298519879579544, + 0.08120764791965485, + 0.13156992197036743, + 0.23054206371307373, + 0.06356418132781982 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.8945797085762024, + "style_target": 1, + "support_probability": 0.8845685124397278, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2364661384399689, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.047648657113313675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8182:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5512969493865967, + 0.8055135011672974, + 0.929496169090271, + 0.7585176825523376, + 0.0712551698088646, + 0.03567301481962204, + 0.05708276480436325, + 0.06418895721435547, + 0.07912729680538177, + 0.1695450246334076 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:8182", + "split": "test", + "style_correct": 1, + "style_probability": 0.11848005652427673, + "style_target": 0, + "support_probability": 0.8747143149375916, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3586449573624506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8301575779914856, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4246363639831543, + 0.9356787800788879, + 0.6867345571517944, + 0.08724550902843475, + 0.03607054427266121 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 1, + "style_probability": 0.7805307507514954, + "style_target": 1, + "support_probability": 0.3572220206260681, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44897729582404006, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5380062460899353, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8185:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5039638876914978, + 0.8976009488105774, + 0.6128506660461426, + 0.1212918758392334, + 0.2764430642127991 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8185", + "split": "test", + "style_correct": 0, + "style_probability": 0.5785032510757446, + "style_target": 0, + "support_probability": 0.5248743295669556, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9141335685924085, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05255391076207161, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7130768299102783, + 0.2679772675037384, + 0.2582674026489258, + 0.993610680103302, + 0.11404544115066528, + 0.08528970927000046 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 1, + "style_probability": 0.7604236602783203, + "style_target": 1, + "support_probability": 0.8854329586029053, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7175898688235449, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.044421110302209854, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8191:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6656338572502136, + 0.23823866248130798, + 0.24184927344322205, + 0.987775444984436, + 0.10646103322505951, + 0.06847914308309555, + 0.13907845318317413, + 0.14967849850654602, + 0.07258659601211548, + 0.10239662230014801 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8191", + "split": "test", + "style_correct": 1, + "style_probability": 0.32944783568382263, + "style_target": 0, + "support_probability": 0.8837368488311768, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6913591411192463, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18769966065883636, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18215353786945343, + 0.1265985369682312, + 0.32519111037254333, + 0.9564562439918518, + 0.05847069248557091, + 0.060061126947402954, + 0.058693189173936844, + 0.09027130156755447, + 0.037868913263082504 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.8687037229537964, + "style_target": 1, + "support_probability": 0.6267634034156799, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05152791606545958, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21861547231674194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8199:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20907476544380188, + 0.14424653351306915, + 0.3657093048095703, + 0.9752435088157654, + 0.10780158638954163, + 0.07750055938959122, + 0.06015612557530403, + 0.06720855832099915, + 0.14303234219551086 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8199", + "split": "test", + "style_correct": 1, + "style_probability": 0.1864350587129593, + "style_target": 0, + "support_probability": 0.6257676482200623, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12595758385057734, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7607222199440002, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1530141681432724, + 0.6202012896537781, + 0.18331541121006012, + 0.5664787888526917, + 0.05359714850783348, + 0.05820569023489952, + 0.11304409801959991, + 0.239991694688797, + 0.06216606870293617, + 0.17627131938934326, + 0.03069261461496353 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.854540228843689, + "style_target": 1, + "support_probability": 0.065361887216568, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014860437272091083, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8461610674858093, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8201:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15974055230617523, + 0.617919921875, + 0.23268812894821167, + 0.4954456388950348, + 0.060084227472543716, + 0.06325611472129822, + 0.2860637307167053, + 0.06549647450447083, + 0.05648351460695267, + 0.08461056649684906, + 0.10033570975065231 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8201", + "split": "test", + "style_correct": 1, + "style_probability": 0.4719121754169464, + "style_target": 0, + "support_probability": 0.053989849984645844, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8777444097590673, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15747950971126556, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.923059344291687, + 0.16754376888275146, + 0.3387458622455597, + 0.727707028388977, + 0.030057094991207123 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.8549871444702148, + "style_target": 1, + "support_probability": 0.9075469374656677, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6904715789008335, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08086473494768143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8204:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8916679620742798, + 0.20464983582496643, + 0.31144919991493225, + 0.7287936806678772, + 0.045886121690273285, + 0.07874985039234161, + 0.14022642374038696, + 0.08328171074390411, + 0.06776928901672363 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8204", + "split": "test", + "style_correct": 1, + "style_probability": 0.4090501368045807, + "style_target": 0, + "support_probability": 0.91373211145401, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.5913235329940818, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.05826190486550331, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33035939931869507, + 0.15965934097766876, + 0.8732853531837463, + 0.24483633041381836, + 0.10196153819561005, + 0.04050924628973007 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.8932739496231079, + "style_target": 1, + "support_probability": 0.5618220567703247, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029845889578796306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0935843214392662, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8210:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3495927155017853, + 0.22336389124393463, + 0.9381424188613892, + 0.2815031111240387, + 0.09128526598215103, + 0.11566801369190216 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8210", + "split": "test", + "style_correct": 1, + "style_probability": 0.19097602367401123, + "style_target": 0, + "support_probability": 0.6564596891403198, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9242936999093309, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16678817570209503, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8213:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25525957345962524, + 0.9897540211677551, + 0.26721298694610596, + 0.2517564296722412, + 0.08645513653755188, + 0.08318480104207993, + 0.058035384863615036, + 0.14417825639247894, + 0.06365688890218735 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 1, + "style_probability": 0.8849045038223267, + "style_target": 1, + "support_probability": 0.7459629774093628, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8117405637563307, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1603558361530304, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8213:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2850053012371063, + 0.9874321222305298, + 0.27409428358078003, + 0.2736801505088806, + 0.13075138628482819, + 0.0887204110622406, + 0.15193034708499908, + 0.1444530040025711, + 0.07168513536453247 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8213", + "split": "test", + "style_correct": 0, + "style_probability": 0.5957217216491699, + "style_target": 0, + "support_probability": 0.731471061706543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.015199798091648615, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9148929715156555, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.23509155213832855, + 0.3884528875350952, + 0.4388047158718109, + 0.45943501591682434, + 0.05480159446597099, + 0.08803302049636841, + 0.09633639454841614, + 0.03904850035905838, + 0.027978476136922836, + 0.15554121136665344, + 0.0212213434278965 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.7704233527183533, + "style_target": 1, + "support_probability": 0.06067030131816864, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01294083046307698, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8622189164161682, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8223:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2358841896057129, + 0.3871142268180847, + 0.4294413626194, + 0.4655846059322357, + 0.11730611324310303, + 0.08560027182102203 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8223", + "split": "test", + "style_correct": 1, + "style_probability": 0.379579097032547, + "style_target": 0, + "support_probability": 0.11246439814567566, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016185933822858153, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7518307566642761, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36283931136131287, + 0.6513187289237976, + 0.25513920187950134, + 0.30048859119415283, + 0.06285731494426727 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.777096688747406, + "style_target": 1, + "support_probability": 0.09850234538316727, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00042252437242127505, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8376151919364929, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8225:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3740044832229614, + 0.6525740027427673, + 0.26536548137664795, + 0.29010915756225586, + 0.05085853859782219 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8225", + "split": "test", + "style_correct": 1, + "style_probability": 0.10542631149291992, + "style_target": 0, + "support_probability": 0.11852864921092987, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9434698060816871, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.06661687791347504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8231:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4278903007507324, + 0.24815697968006134, + 0.27661532163619995, + 0.9977840781211853, + 0.07257745414972305, + 0.11322355270385742 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 1, + "style_probability": 0.8448898792266846, + "style_target": 1, + "support_probability": 0.9379574060440063, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9076270034036504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.03888319432735443, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8231:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4257325530052185, + 0.23966865241527557, + 0.261918842792511, + 0.9928804039955139, + 0.06603524833917618, + 0.0695992037653923 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8231", + "split": "test", + "style_correct": 0, + "style_probability": 0.6173861026763916, + "style_target": 0, + "support_probability": 0.963693380355835, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.2662742084742149, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08548009395599365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9706274271011353, + 0.3826771378517151, + 0.26239633560180664, + 0.35607922077178955, + 0.1664922684431076, + 0.06872561573982239, + 0.23367221653461456, + 0.106612928211689, + 0.1729317158460617 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.6884220242500305, + "style_target": 1, + "support_probability": 0.3412087857723236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15318222402798387, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06960974633693695, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8240:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9607183337211609, + 0.377159982919693, + 0.26210832595825195, + 0.3473038077354431, + 0.0854736790060997 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8240", + "split": "test", + "style_correct": 1, + "style_probability": 0.3502344489097595, + "style_target": 0, + "support_probability": 0.4346098303794861, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7175999737514722, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3609101474285126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2825719118118286, + 0.1579783409833908, + 0.9254127144813538, + 0.37879255414009094, + 0.05878162384033203, + 0.03940718248486519, + 0.07433411478996277, + 0.15492160618305206, + 0.09415077418088913, + 0.05194413661956787 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.8154446482658386, + "style_target": 1, + "support_probability": 0.671267032623291, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3125210122301496, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14860105514526367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8246:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.34977638721466064, + 0.15473227202892303, + 0.9669380187988281, + 0.3915466070175171, + 0.08132205158472061, + 0.04396655037999153, + 0.07453658431768417, + 0.0915624275803566, + 0.05665163695812225, + 0.08774661272764206 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8246", + "split": "test", + "style_correct": 1, + "style_probability": 0.19754476845264435, + "style_target": 0, + "support_probability": 0.8568309545516968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.002814467744775562, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6581336855888367, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18531538546085358, + 0.2174641489982605, + 0.5283114910125732, + 0.41855084896087646, + 0.10292096436023712 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 1, + "style_probability": 0.8423610925674438, + "style_target": 1, + "support_probability": 0.008522001095116138, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.013005440479836091, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.46130210161209106, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8258:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20836977660655975, + 0.22611680626869202, + 0.5932115316390991, + 0.40524303913116455, + 0.10470022261142731 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8258", + "split": "test", + "style_correct": 0, + "style_probability": 0.5263880491256714, + "style_target": 0, + "support_probability": 0.05205979943275452, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022661101141211223, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6496469378471375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38872191309928894, + 0.274177610874176, + 0.546006441116333, + 0.2834322452545166, + 0.08863773941993713, + 0.08468569815158844 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.8401982188224792, + "style_target": 1, + "support_probability": 0.029155852273106575, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006040698232478769, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5023428797721863, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8262:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.33786407113075256, + 0.29721105098724365, + 0.5186269879341125, + 0.29389914870262146, + 0.10449501127004623, + 0.049123238772153854, + 0.14142382144927979, + 0.15435118973255157, + 0.058228399604558945, + 0.11333592236042023, + 0.07611668109893799 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8262", + "split": "test", + "style_correct": 1, + "style_probability": 0.3106442093849182, + "style_target": 0, + "support_probability": 0.04861114174127579, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006225940018383614, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9787295460700989, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23345406353473663, + 0.6370032429695129, + 0.15502823889255524, + 0.3524244427680969, + 0.11792653799057007, + 0.14332325756549835, + 0.09002982079982758, + 0.08914150297641754, + 0.07088306546211243 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.7782588005065918, + "style_target": 1, + "support_probability": 0.029195483773946762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009304872148296405, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9655957221984863, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8269:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.254041850566864, + 0.6823491454124451, + 0.17947955429553986, + 0.40929925441741943, + 0.10556677728891373, + 0.1797478348016739, + 0.10129399597644806, + 0.07793746888637543, + 0.16106174886226654 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8269", + "split": "test", + "style_correct": 1, + "style_probability": 0.12002364546060562, + "style_target": 0, + "support_probability": 0.07480737566947937, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005302323101790268, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6927098035812378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2064298689365387, + 0.34347018599510193, + 0.21544340252876282, + 0.48941725492477417, + 0.12668220698833466, + 0.04952535033226013, + 0.062355317175388336, + 0.10004635155200958, + 0.11092964559793472, + 0.03840916231274605, + 0.04148517921566963 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.681526780128479, + "style_target": 1, + "support_probability": 0.0021622488275170326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006525574906003328, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7565442323684692, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8287:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22422967851161957, + 0.36762675642967224, + 0.22517727315425873, + 0.4775443971157074, + 0.094845712184906, + 0.07476964592933655, + 0.06488306820392609, + 0.0665007159113884, + 0.1382942646741867, + 0.16043497622013092, + 0.09958581626415253 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8287", + "split": "test", + "style_correct": 1, + "style_probability": 0.48972463607788086, + "style_target": 0, + "support_probability": 0.004775795619934797, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3559928961791119, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37074652314186096, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8830686211585999, + 0.1857379525899887, + 0.14267097413539886, + 0.1811504364013672, + 0.03462176397442818 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.7494729161262512, + "style_target": 1, + "support_probability": 0.6413739323616028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3278472630470617, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16750989854335785, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8288:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8591945767402649, + 0.2099866271018982, + 0.14942526817321777, + 0.1754298359155655, + 0.07726315408945084, + 0.10204731673002243, + 0.08074240386486053, + 0.1326139122247696, + 0.09715422242879868 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8288", + "split": "test", + "style_correct": 1, + "style_probability": 0.38826435804367065, + "style_target": 0, + "support_probability": 0.7870957255363464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003126735552890835, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9484292268753052, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23450185358524323, + 0.6096540689468384, + 0.208614781498909, + 0.23450185358524323, + 0.07378175854682922, + 0.0538889542222023 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.7175138592720032, + "style_target": 1, + "support_probability": 0.009189586155116558, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003796555332298278, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9495702385902405, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8293:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2661038041114807, + 0.6584663391113281, + 0.21208268404006958, + 0.2661038041114807, + 0.06768541038036346, + 0.1408509612083435 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8293", + "split": "test", + "style_correct": 1, + "style_probability": 0.1429223269224167, + "style_target": 0, + "support_probability": 0.023452214896678925, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.772924505988648, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21912619471549988, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9625982046127319, + 0.28907284140586853, + 0.41126707196235657, + 0.28907284140586853, + 0.0398026667535305, + 0.04596142843365669, + 0.035009048879146576, + 0.05536163970828056, + 0.030420513823628426 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 1, + "style_probability": 0.680314302444458, + "style_target": 1, + "support_probability": 0.8473017811775208, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7779035712394773, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3216301500797272, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8300:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9303097724914551, + 0.3267921209335327, + 0.3860304355621338, + 0.3267921209335327, + 0.1564706265926361, + 0.08683446794748306, + 0.07586730271577835, + 0.05690787732601166, + 0.0988548994064331 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8300", + "split": "test", + "style_correct": 0, + "style_probability": 0.6415367722511292, + "style_target": 0, + "support_probability": 0.8390053510665894, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.029432492533825735, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9159880876541138, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2637439966201782, + 0.2208377569913864, + 0.4823925793170929, + 0.2637439966201782, + 0.07864618301391602, + 0.06767109036445618, + 0.08393418788909912, + 0.08668208867311478, + 0.06833270937204361, + 0.049087509512901306, + 0.03969556465744972 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.897072434425354, + "style_target": 1, + "support_probability": 0.05016424134373665, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006751504119726303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8819093704223633, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8310:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2689228951931, + 0.20236258208751678, + 0.4772337079048157, + 0.2689228951931, + 0.0451180674135685, + 0.08860519528388977 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8310", + "split": "test", + "style_correct": 1, + "style_probability": 0.31174641847610474, + "style_target": 0, + "support_probability": 0.08176057040691376, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6668390571502414, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.34490084648132324, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28906917572021484, + 0.4137541949748993, + 0.130097895860672, + 0.8811726570129395, + 0.10833513736724854 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.7485737204551697, + "style_target": 1, + "support_probability": 0.6944403052330017, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.031234092045554492, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5296226143836975, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8315:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3303876221179962, + 0.46176543831825256, + 0.11387139558792114, + 0.9012481570243835, + 0.11665846407413483 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8315", + "split": "test", + "style_correct": 1, + "style_probability": 0.1383441537618637, + "style_target": 0, + "support_probability": 0.5047224760055542, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5242944782647498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.19533176720142365, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25636446475982666, + 0.3095577359199524, + 0.32195428013801575, + 0.918076753616333, + 0.15895478427410126, + 0.13139601051807404 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 1, + "style_probability": 0.8641608357429504, + "style_target": 1, + "support_probability": 0.6364091038703918, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18411746099990098, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2670440673828125, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8319:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2310357391834259, + 0.3797968626022339, + 0.3575237989425659, + 0.8737425208091736, + 0.19346697628498077, + 0.1010555624961853 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8319", + "split": "test", + "style_correct": 0, + "style_probability": 0.6586958765983582, + "style_target": 0, + "support_probability": 0.33167964220046997, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13316417666349345, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8371349573135376, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19617384672164917, + 0.14976869523525238, + 0.16937056183815002, + 0.33650535345077515, + 0.09513397514820099, + 0.11543837934732437, + 0.06880181282758713, + 0.22648373246192932, + 0.15013611316680908 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.8604598045349121, + "style_target": 1, + "support_probability": 0.3720373809337616, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024499125043608846, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8045076131820679, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8323:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.19132088124752045, + 0.218556210398674, + 0.14366590976715088, + 0.39714515209198, + 0.1429262012243271 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8323", + "split": "test", + "style_correct": 1, + "style_probability": 0.28404027223587036, + "style_target": 0, + "support_probability": 0.22093982994556427, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13527257755419075, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7333220839500427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.20949460566043854, + 0.302702933549881, + 0.36503273248672485, + 0.7293241620063782, + 0.10874863713979721, + 0.10862758755683899, + 0.2975821793079376, + 0.13209408521652222, + 0.09066949039697647, + 0.17548435926437378, + 0.20205660164356232 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.8663926124572754, + "style_target": 1, + "support_probability": 0.08674481511116028, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0250881447818545, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5956558585166931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8347:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2295415997505188, + 0.36968475580215454, + 0.42706939578056335, + 0.7889775633811951, + 0.22533974051475525, + 0.07904402911663055, + 0.13882257044315338, + 0.1345200538635254, + 0.08598385006189346, + 0.15617048740386963 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8347", + "split": "test", + "style_correct": 1, + "style_probability": 0.26879891753196716, + "style_target": 0, + "support_probability": 0.18924885988235474, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11492288985924137, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4914669692516327, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6885203719139099, + 0.2604729235172272, + 0.278847873210907, + 0.22045636177062988, + 0.10228390991687775 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 1, + "style_probability": 0.8079293966293335, + "style_target": 1, + "support_probability": 0.13375243544578552, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09967036083111933, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3440231382846832, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8348:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6645906567573547, + 0.2641568183898926, + 0.26396676898002625, + 0.23639243841171265, + 0.06074078753590584 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8348", + "split": "test", + "style_correct": 0, + "style_probability": 0.5229315757751465, + "style_target": 0, + "support_probability": 0.2483045607805252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4135128617250068, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5426610112190247, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.25881484150886536, + 0.40747812390327454, + 0.2515193819999695, + 0.7520117163658142, + 0.06496432423591614, + 0.10517608374357224 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.7731902003288269, + "style_target": 1, + "support_probability": 0.44734203815460205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06354221472717292, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6971976161003113, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8351:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2666969895362854, + 0.46020442247390747, + 0.26085522770881653, + 0.7185661196708679, + 0.13550163805484772, + 0.12588846683502197, + 0.10094574093818665, + 0.04759462922811508, + 0.05329512804746628, + 0.09944065660238266, + 0.16276578605175018 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8351", + "split": "test", + "style_correct": 1, + "style_probability": 0.366605281829834, + "style_target": 0, + "support_probability": 0.12581683695316315, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0020202335526493267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.940693199634552, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2702880799770355, + 0.3509623408317566, + 0.18847431242465973, + 0.23354168236255646, + 0.05578283220529556, + 0.07720032334327698, + 0.0747346505522728, + 0.06622384488582611, + 0.07973021268844604 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.7162949442863464, + "style_target": 1, + "support_probability": 0.014414952136576176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006819723933592926, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9100328087806702, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8358:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2706245183944702, + 0.39338552951812744, + 0.21459682285785675, + 0.2721100151538849, + 0.05635117366909981, + 0.16570697724819183, + 0.07132326811552048, + 0.07731163501739502, + 0.09094861894845963 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8358", + "split": "test", + "style_correct": 1, + "style_probability": 0.28076112270355225, + "style_target": 0, + "support_probability": 0.0525907427072525, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7150627569598811, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18679159879684448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.6168916821479797, + 0.7311588525772095, + 0.5388784408569336, + 0.44968506693840027, + 0.13431699573993683, + 0.06983841955661774, + 0.043436840176582336, + 0.06576205044984818, + 0.04285212978720665, + 0.04418295994400978, + 0.054634321480989456 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 1, + "style_probability": 0.8325526118278503, + "style_target": 1, + "support_probability": 0.9480229020118713, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5654632827434725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4068562686443329, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8365:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5396183729171753, + 0.6002143025398254, + 0.5836012363433838, + 0.46439385414123535, + 0.06217221915721893, + 0.03876689821481705, + 0.0505039244890213, + 0.08849021792411804, + 0.031522978097200394, + 0.05140813812613487, + 0.04845280572772026 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8365", + "split": "test", + "style_correct": 0, + "style_probability": 0.5838146805763245, + "style_target": 0, + "support_probability": 0.9569160342216492, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.742848202415356, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07111397385597229, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8174561858177185, + 0.32679399847984314, + 0.4108646810054779, + 0.373354434967041, + 0.08236540108919144 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 1, + "style_probability": 0.7285011410713196, + "style_target": 1, + "support_probability": 0.8967283964157104, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8291581826502216, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15451167523860931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8376:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8863930106163025, + 0.3203844130039215, + 0.40479639172554016, + 0.3643616735935211, + 0.046852707862854004 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8376", + "split": "test", + "style_correct": 0, + "style_probability": 0.8823229670524597, + "style_target": 0, + "support_probability": 0.8268740773200989, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007563945739702927, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9616854190826416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6850332617759705, + 0.3475136458873749, + 0.19362163543701172, + 0.3309820294380188, + 0.31499040126800537, + 0.09470298886299133 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 1, + "style_probability": 0.5442594885826111, + "style_target": 1, + "support_probability": 0.03376017510890961, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0009688702631806266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9866334199905396, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8380:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7268660068511963, + 0.26334133744239807, + 0.16861288249492645, + 0.322000116109848, + 0.11214806884527206, + 0.22014012932777405, + 0.10657183080911636, + 0.14541393518447876, + 0.08587422966957092, + 0.1386122703552246, + 0.10215195268392563 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8380", + "split": "test", + "style_correct": 1, + "style_probability": 0.306993305683136, + "style_target": 0, + "support_probability": 0.005038352683186531, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8493455429572805, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1739831566810608, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8384:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9334921836853027, + 0.4529060423374176, + 0.3372765779495239, + 0.34632429480552673, + 0.06753810495138168, + 0.07911153882741928, + 0.05408569425344467, + 0.03552935644984245, + 0.05208496004343033 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.5028019547462463, + "style_target": 1, + "support_probability": 0.8925000429153442, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3127777462226007, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.22219157218933105, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8384:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9841554164886475, + 0.5543840527534485, + 0.31462788581848145, + 0.32564279437065125, + 0.06639967113733292, + 0.06550467759370804, + 0.07464578747749329, + 0.06156367436051369, + 0.05654841288924217 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8384", + "split": "test", + "style_correct": 1, + "style_probability": 0.1802544742822647, + "style_target": 0, + "support_probability": 0.8266237378120422, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0033754333827452627, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9482700228691101, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4140954315662384, + 0.3418017625808716, + 0.18106210231781006, + 0.5223443508148193, + 0.07371087372303009, + 0.04991840198636055, + 0.13678766787052155, + 0.084046371281147, + 0.10572493821382523, + 0.11966419219970703, + 0.06366681307554245 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 1, + "style_probability": 0.5664544701576233, + "style_target": 1, + "support_probability": 0.01224130392074585, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001726438108899942, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9588767886161804, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8387:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3858160972595215, + 0.3089033365249634, + 0.18744869530200958, + 0.5865421295166016, + 0.09065084159374237, + 0.13692606985569, + 0.07137633115053177, + 0.07712492346763611, + 0.07831422239542007, + 0.05541759729385376, + 0.12571722269058228 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8387", + "split": "test", + "style_correct": 0, + "style_probability": 0.5459842085838318, + "style_target": 0, + "support_probability": 0.010561619885265827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006655351354592661, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9740049839019775, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29971379041671753, + 0.4391319751739502, + 0.469074010848999, + 0.5472385287284851, + 0.1027316078543663 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.7467911839485168, + "style_target": 1, + "support_probability": 0.04078945890069008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006641480955373729, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.9822613000869751, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3727858066558838, + 0.4962858259677887, + 0.5735974907875061, + 0.6581646203994751, + 0.0790424719452858 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8399", + "split": "test", + "style_correct": 1, + "style_probability": 0.1719091534614563, + "style_target": 0, + "support_probability": 0.04119078069925308, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04067989109376669, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2772718071937561, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.36044031381607056, + 0.40747350454330444, + 0.41380414366722107, + 0.7483577728271484, + 0.08486685901880264, + 0.08712981641292572 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 1, + "style_probability": 0.5449951887130737, + "style_target": 1, + "support_probability": 0.3443998396396637, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08104152720226308, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24272435903549194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8403:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3984113037586212, + 0.40930020809173584, + 0.4399394094944, + 0.7474557757377625, + 0.03891189396381378, + 0.100234255194664 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8403", + "split": "test", + "style_correct": 0, + "style_probability": 0.5161852240562439, + "style_target": 0, + "support_probability": 0.32227444648742676, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5187343445323833, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5088515877723694, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1604674607515335, + 0.7310596108436584, + 0.619311511516571, + 0.42512232065200806, + 0.14573028683662415, + 0.08730242401361465, + 0.021820249035954475, + 0.06697000563144684, + 0.10554498434066772 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 1, + "style_probability": 0.8735501766204834, + "style_target": 1, + "support_probability": 0.5582752227783203, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39269776005005497, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.48489752411842346, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8406:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20705090463161469, + 0.7132039070129395, + 0.5735072493553162, + 0.44912534952163696, + 0.0593409538269043 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8406", + "split": "test", + "style_correct": 0, + "style_probability": 0.5629610419273376, + "style_target": 0, + "support_probability": 0.6109051704406738, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4418495738886819, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1558239459991455, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3733474612236023, + 0.42009633779525757, + 0.4844770133495331, + 0.9482759237289429, + 0.12795089185237885, + 0.14811691641807556, + 0.1214531660079956, + 0.06000233069062233, + 0.08962634950876236, + 0.12733444571495056 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.774676501750946, + "style_target": 1, + "support_probability": 0.4916042983531952, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14891894366472513, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19155313074588776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8415:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.39214223623275757, + 0.46411678194999695, + 0.5577284097671509, + 0.9643215537071228, + 0.2463301569223404, + 0.09788599610328674, + 0.15278968214988708, + 0.1891285628080368, + 0.11916680634021759, + 0.15331335365772247 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8415", + "split": "test", + "style_correct": 1, + "style_probability": 0.38929271697998047, + "style_target": 0, + "support_probability": 0.6082606911659241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11362622040394396, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5421355366706848, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4874630868434906, + 0.28174641728401184, + 0.48217153549194336, + 0.3177361786365509, + 0.05401753634214401 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 0, + "style_probability": 0.4233815670013428, + "style_target": 1, + "support_probability": 0.4271298944950104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004811659984093218, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.743912935256958, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8417:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6064258217811584, + 0.328093945980072, + 0.5276961922645569, + 0.3384284973144531, + 0.07742727547883987 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8417", + "split": "test", + "style_correct": 1, + "style_probability": 0.13935217261314392, + "style_target": 0, + "support_probability": 0.26190727949142456, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02208852191534527, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5246508121490479, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2511657178401947, + 0.8377083539962769, + 0.6573252081871033, + 0.31406351923942566, + 0.09808714687824249, + 0.19708111882209778 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 1, + "style_probability": 0.5904943943023682, + "style_target": 1, + "support_probability": 0.07396481186151505, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05199113235883956, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4771568477153778, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8421:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2451624572277069, + 0.8541784286499023, + 0.6610855460166931, + 0.31060588359832764, + 0.05374981835484505, + 0.055548787117004395 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8421", + "split": "test", + "style_correct": 0, + "style_probability": 0.8079640865325928, + "style_target": 0, + "support_probability": 0.0700829029083252, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.303038455293283, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12064238637685776, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7498633861541748, + 0.6108641028404236, + 0.2331973910331726, + 0.49491047859191895, + 0.04774593934416771, + 0.02214369736611843, + 0.06548971682786942, + 0.0612325593829155, + 0.03804456815123558 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 1, + "style_probability": 0.5733539462089539, + "style_target": 1, + "support_probability": 0.5268020033836365, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15284506505552817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08952881395816803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8433:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7584170699119568, + 0.6055551171302795, + 0.20104478299617767, + 0.5746638178825378, + 0.05553571879863739 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8433", + "split": "test", + "style_correct": 1, + "style_probability": 0.332057923078537, + "style_target": 0, + "support_probability": 0.5558619499206543, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08129031658079588, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6366511583328247, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8279420733451843, + 0.5110086798667908, + 0.550753116607666, + 0.3119947910308838, + 0.07364654541015625, + 0.11232912540435791, + 0.08432286232709885, + 0.0649922713637352, + 0.20699246227741241, + 0.23192580044269562, + 0.06329937279224396 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.6761806011199951, + "style_target": 1, + "support_probability": 0.07853569090366364, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02522472989173852, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.45862650871276855, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8436:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8959432244300842, + 0.5747042298316956, + 0.6730995178222656, + 0.25127097964286804, + 0.10810669511556625, + 0.0923740491271019, + 0.1543797105550766, + 0.0975986048579216, + 0.0704297125339508, + 0.12887440621852875 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8436", + "split": "test", + "style_correct": 1, + "style_probability": 0.2697683870792389, + "style_target": 0, + "support_probability": 0.3977106213569641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13596401801997612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.8695584535598755, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24651283025741577, + 0.25276410579681396, + 0.10973874479532242, + 0.7422666549682617, + 0.15357555449008942 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.7881477475166321, + "style_target": 1, + "support_probability": 0.22181761264801025, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015871081167335713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8553390502929688, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8451:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23849083483219147, + 0.2206900715827942, + 0.10156912356615067, + 0.7208726406097412, + 0.22914189100265503, + 0.15620402991771698, + 0.22841832041740417, + 0.12432773411273956, + 0.34892749786376953 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8451", + "split": "test", + "style_correct": 1, + "style_probability": 0.17934410274028778, + "style_target": 0, + "support_probability": 0.0839957669377327, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.044300859993184886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8483408689498901, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6455562114715576, + 0.4540681838989258, + 0.8290693163871765, + 0.27386143803596497, + 0.20910122990608215, + 0.0357794351875782 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.7162388563156128, + "style_target": 1, + "support_probability": 0.11889860033988953, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005225537488686564, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9243280291557312, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8454:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6177507638931274, + 0.5300886631011963, + 0.8480831980705261, + 0.31982555985450745, + 0.08654236793518066, + 0.2031199336051941 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8454", + "split": "test", + "style_correct": 1, + "style_probability": 0.25296923518180847, + "style_target": 0, + "support_probability": 0.09358306974172592, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.035357679373635424, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6494743227958679, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6995583772659302, + 0.29495635628700256, + 0.24374797940254211, + 0.40108877420425415, + 0.24441823363304138, + 0.10679555684328079, + 0.13562211394309998, + 0.18897990882396698, + 0.216993510723114 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.8479085564613342, + "style_target": 1, + "support_probability": 0.09373974055051804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017449506063026095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5621514916419983, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8460:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6386644244194031, + 0.2931152880191803, + 0.24157609045505524, + 0.3881186246871948, + 0.11562734097242355, + 0.08088816702365875, + 0.10462573915719986, + 0.061552636325359344, + 0.08576077967882156 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8460", + "split": "test", + "style_correct": 1, + "style_probability": 0.4257219433784485, + "style_target": 0, + "support_probability": 0.1084497943520546, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.43965451152133994, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06744087487459183, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9361492991447449, + 0.19591939449310303, + 0.15640783309936523, + 0.16310574114322662, + 0.019324472174048424, + 0.01891886070370674, + 0.15554335713386536, + 0.05527491122484207, + 0.1296743005514145, + 0.13210754096508026, + 0.1444743275642395 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.6929461359977722, + "style_target": 1, + "support_probability": 0.5217610001564026, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15654374464932985, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04682784155011177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8464:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9134752154350281, + 0.19562382996082306, + 0.183558851480484, + 0.17450344562530518, + 0.028768574818968773, + 0.034815508872270584 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8464", + "split": "test", + "style_correct": 1, + "style_probability": 0.2527705430984497, + "style_target": 0, + "support_probability": 0.5848731994628906, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.042065958731515966, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9137042164802551, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4066237509250641, + 0.5018482804298401, + 0.4681156575679779, + 0.4066237509250641, + 0.03117361292243004 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.86258465051651, + "style_target": 1, + "support_probability": 0.09431823343038559, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0016238544673252418, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9418827891349792, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8465:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.49369874596595764, + 0.5803314447402954, + 0.5713885426521301, + 0.49369874596595764, + 0.07133854180574417 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8465", + "split": "test", + "style_correct": 1, + "style_probability": 0.22005325555801392, + "style_target": 0, + "support_probability": 0.049690429121255875, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0013124811074189805, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9657235741615295, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22721411287784576, + 0.4523152709007263, + 0.46045106649398804, + 0.33671408891677856, + 0.08852001279592514, + 0.05813024938106537 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 1, + "style_probability": 0.6573424339294434, + "style_target": 1, + "support_probability": 0.012242693454027176, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00727853392088529, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8664122819900513, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8471:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22963781654834747, + 0.33913013339042664, + 0.5063614249229431, + 0.3459520637989044, + 0.16694416105747223, + 0.05379877984523773 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8471", + "split": "test", + "style_correct": 0, + "style_probability": 0.5837777256965637, + "style_target": 0, + "support_probability": 0.03223409131169319, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.48059296398781015, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30481868982315063, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36302250623703003, + 0.26369255781173706, + 0.36302250623703003, + 0.7852087616920471, + 0.02350984700024128, + 0.07013404369354248, + 0.05142274498939514, + 0.03681477904319763, + 0.03622190281748772 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.7488608360290527, + "style_target": 1, + "support_probability": 0.5493295192718506, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3062501343816848, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.23116996884346008, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8483:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38103199005126953, + 0.2298916131258011, + 0.38103199005126953, + 0.7759833335876465, + 0.13397186994552612 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8483", + "split": "test", + "style_correct": 1, + "style_probability": 0.27022674679756165, + "style_target": 0, + "support_probability": 0.7598763704299927, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3670768286889903, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7359800338745117, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2126743495464325, + 0.8904563188552856, + 0.2126743495464325, + 0.3678210973739624, + 0.08426067233085632, + 0.04530585557222366, + 0.063017837703228, + 0.04573207348585129, + 0.08912356197834015, + 0.06850866973400116, + 0.11412815004587173 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.5706387162208557, + "style_target": 1, + "support_probability": 0.42075780034065247, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03982057479435363, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7783039212226868, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8485:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2475435733795166, + 0.9410446882247925, + 0.2475435733795166, + 0.43038153648376465, + 0.23983639478683472, + 0.08620838820934296, + 0.35324448347091675, + 0.08724091202020645, + 0.05477466806769371, + 0.11146216839551926 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8485", + "split": "test", + "style_correct": 1, + "style_probability": 0.11072079837322235, + "style_target": 0, + "support_probability": 0.32954323291778564, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06299316388370979, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7322172522544861, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2572219669818878, + 0.3592982590198517, + 0.7906352281570435, + 0.2572219669818878, + 0.0758393257856369 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.5575363636016846, + "style_target": 1, + "support_probability": 0.12462641298770905, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005056814858752645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8588981032371521, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8486:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31665679812431335, + 0.3937790095806122, + 0.8900411128997803, + 0.31665679812431335, + 0.09334209561347961 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8486", + "split": "test", + "style_correct": 1, + "style_probability": 0.2998220920562744, + "style_target": 0, + "support_probability": 0.05773172900080681, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19036441016556438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.27114468812942505, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3080698251724243, + 0.23816117644309998, + 0.24671372771263123, + 0.9684786796569824, + 0.06682457774877548, + 0.18783427774906158 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 1, + "style_probability": 0.5278837084770203, + "style_target": 1, + "support_probability": 0.2727331519126892, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.23819392725110866, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.26091864705085754, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8487:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.30627378821372986, + 0.20792752504348755, + 0.23640616238117218, + 0.9860941171646118, + 0.03488290309906006, + 0.12298538535833359 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8487", + "split": "test", + "style_correct": 0, + "style_probability": 0.6117368340492249, + "style_target": 0, + "support_probability": 0.31240788102149963, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.02739414516034389, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3473890721797943, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3167736232280731, + 0.09430039674043655, + 0.4620223939418793, + 0.8809669613838196, + 0.10470889508724213, + 0.14314298331737518, + 0.07862182706594467, + 0.07070491462945938, + 0.14627471566200256 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 0, + "style_probability": 0.39437827467918396, + "style_target": 1, + "support_probability": 0.07321155071258545, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.016007456076662423, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.43978551030158997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8491:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28564462065696716, + 0.09566780179738998, + 0.4522499144077301, + 0.8872948288917542, + 0.19035765528678894 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8491", + "split": "test", + "style_correct": 1, + "style_probability": 0.2828727662563324, + "style_target": 0, + "support_probability": 0.06020192801952362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.026666415099119156, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7848091721534729, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6189935207366943, + 0.31088483333587646, + 0.43664103746414185, + 0.31088483333587646, + 0.0619836263358593, + 0.08765200525522232, + 0.10543067753314972, + 0.06670834124088287, + 0.0406542532145977, + 0.15769419074058533 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 1, + "style_probability": 0.5957809090614319, + "style_target": 1, + "support_probability": 0.082171730697155, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009470572580827506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6886492967605591, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8492:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8185148239135742, + 0.32259324193000793, + 0.5084301233291626, + 0.32259324193000793, + 0.10441217571496964, + 0.11201417446136475, + 0.07514713704586029, + 0.049581192433834076, + 0.05954471603035927, + 0.12081921845674515 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8492", + "split": "test", + "style_correct": 1, + "style_probability": 0.2210201621055603, + "style_target": 0, + "support_probability": 0.15871474146842957, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2831900316286884, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7938494086265564, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3049113154411316, + 0.9133592247962952, + 0.33862411975860596, + 0.2658262550830841, + 0.1077481061220169 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.8301469087600708, + "style_target": 1, + "support_probability": 0.2673574984073639, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04804712386832444, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7983594536781311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8497:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2803783416748047, + 0.8938814401626587, + 0.29364922642707825, + 0.2395303100347519, + 0.10130680352449417, + 0.0944041758775711, + 0.1810958981513977, + 0.11611845344305038, + 0.11057189106941223 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8497", + "split": "test", + "style_correct": 1, + "style_probability": 0.23688073456287384, + "style_target": 0, + "support_probability": 0.20126844942569733, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5046284650698816, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3055744469165802, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9659122824668884, + 0.23868964612483978, + 0.4100341796875, + 0.23868964612483978, + 0.09821408241987228, + 0.11920391023159027 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.7118378281593323, + "style_target": 1, + "support_probability": 0.661362886428833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04379144128194845, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4081428050994873, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8504:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9787842631340027, + 0.27603814005851746, + 0.47982749342918396, + 0.27603814005851746, + 0.04668906703591347, + 0.11531724035739899 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8504", + "split": "test", + "style_correct": 1, + "style_probability": 0.2090584635734558, + "style_target": 0, + "support_probability": 0.6300739645957947, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.863592482922389, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.221848726272583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9088299870491028, + 0.5241140127182007, + 0.48831743001937866, + 0.44462355971336365, + 0.03523886948823929, + 0.028450611978769302, + 0.09496207535266876, + 0.08300399780273438, + 0.14454729855060577 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 1, + "style_probability": 0.8744195699691772, + "style_target": 1, + "support_probability": 0.9080992937088013, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7372901290138428, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38123196363449097, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8516:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8358372449874878, + 0.3998332917690277, + 0.382314532995224, + 0.31216028332710266, + 0.1478898525238037, + 0.10447968542575836, + 0.07765663415193558, + 0.08921056985855103, + 0.15696647763252258 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8516", + "split": "test", + "style_correct": 0, + "style_probability": 0.6884641647338867, + "style_target": 0, + "support_probability": 0.8773109912872314, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7887739812267434, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1932830661535263, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9220151305198669, + 0.16894114017486572, + 0.16894114017486572, + 0.19933298230171204, + 0.03896869346499443, + 0.06158009544014931, + 0.04109877347946167, + 0.029773589223623276, + 0.058112382888793945, + 0.04426391050219536, + 0.07873917371034622 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 1, + "style_probability": 0.880918025970459, + "style_target": 1, + "support_probability": 0.4680752158164978, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36552019383459466, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12840183079242706, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8540:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9203696846961975, + 0.1640874445438385, + 0.1640874445438385, + 0.18624967336654663, + 0.08163458108901978, + 0.05088438093662262 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8540", + "split": "test", + "style_correct": 1, + "style_probability": 0.32144859433174133, + "style_target": 0, + "support_probability": 0.46305346488952637, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8034797147135606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1583864986896515, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.27036675810813904, + 0.9612916707992554, + 0.23928247392177582, + 0.4613693952560425, + 0.04338282719254494 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.8581605553627014, + "style_target": 1, + "support_probability": 0.5243149995803833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5455717402950029, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08073752373456955, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8541:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31996068358421326, + 0.9847207069396973, + 0.2667097747325897, + 0.5481051802635193, + 0.15678934752941132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8541", + "split": "test", + "style_correct": 1, + "style_probability": 0.4652247428894043, + "style_target": 0, + "support_probability": 0.5787244439125061, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0003843790751267564, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9931514263153076, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38728561997413635, + 0.31016072630882263, + 0.5082678198814392, + 0.31016072630882263, + 0.12360117584466934, + 0.04076055809855461 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 1, + "style_probability": 0.5326067209243774, + "style_target": 1, + "support_probability": 0.004862031899392605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0014622742483991896, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9744802713394165, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8548:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46161016821861267, + 0.3484143316745758, + 0.5262096524238586, + 0.3484143316745758, + 0.13459865748882294, + 0.06859547644853592 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8548", + "split": "test", + "style_correct": 1, + "style_probability": 0.4398556351661682, + "style_target": 0, + "support_probability": 0.015534989535808563, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.31851356996964597, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21192209422588348, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.39580437541007996, + 0.5305415987968445, + 0.33436089754104614, + 0.43767523765563965, + 0.06798533350229263, + 0.06925797462463379, + 0.0997951552271843, + 0.1479140967130661, + 0.07388028502464294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.8420990705490112, + "style_target": 1, + "support_probability": 0.43482694029808044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.12425458954986368, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15313468873500824, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8553:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37498217821121216, + 0.47878608107566833, + 0.29994603991508484, + 0.4069591462612152, + 0.24080190062522888 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8553", + "split": "test", + "style_correct": 1, + "style_probability": 0.39532676339149475, + "style_target": 0, + "support_probability": 0.5127368569374084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4625887606733876, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12090485543012619, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15642209351062775, + 0.9025557637214661, + 0.4691464602947235, + 0.16321542859077454, + 0.02269730716943741, + 0.11115524172782898, + 0.05391651764512062, + 0.042286984622478485, + 0.045123856514692307, + 0.0683799609541893, + 0.029381314292550087 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.6395565271377563, + "style_target": 1, + "support_probability": 0.5495461225509644, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04191911968624296, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1978788524866104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8557:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1667126715183258, + 0.9322919249534607, + 0.5213524699211121, + 0.16971127688884735, + 0.0537966825067997, + 0.0521819107234478, + 0.07323488593101501, + 0.04091064631938934, + 0.03181371092796326, + 0.05373875051736832 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8557", + "split": "test", + "style_correct": 1, + "style_probability": 0.12877696752548218, + "style_target": 0, + "support_probability": 0.596523642539978, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7452305377843871, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1432371586561203, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24341769516468048, + 0.4409794211387634, + 0.8539081811904907, + 0.1745631992816925, + 0.055301111191511154 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 1, + "style_probability": 0.8205697536468506, + "style_target": 1, + "support_probability": 0.6486548185348511, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4027216369408926, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.302047461271286, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8558:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24268685281276703, + 0.39089614152908325, + 0.7867928147315979, + 0.18002000451087952, + 0.07851572334766388 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8558", + "split": "test", + "style_correct": 0, + "style_probability": 0.59555983543396, + "style_target": 0, + "support_probability": 0.4142630100250244, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10761498348004218, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7654585242271423, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7896751165390015, + 0.42046183347702026, + 0.2980232834815979, + 0.30702266097068787, + 0.05288771912455559, + 0.13665767014026642 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.8577950596809387, + "style_target": 1, + "support_probability": 0.07984796166419983, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022479146993628185, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6758783459663391, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8560:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.772326648235321, + 0.38395556807518005, + 0.28300178050994873, + 0.29374784231185913, + 0.1253313571214676, + 0.2181767076253891, + 0.04103336110711098, + 0.06436672806739807, + 0.05738838389515877, + 0.06919893622398376, + 0.16011080145835876 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8560", + "split": "test", + "style_correct": 1, + "style_probability": 0.30750325322151184, + "style_target": 0, + "support_probability": 0.062493011355400085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6059234424313047, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27130234241485596, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9447791576385498, + 0.2648151218891144, + 0.2105056494474411, + 0.34508016705513, + 0.06210397928953171, + 0.04602805897593498, + 0.07156828790903091, + 0.0742250308394432, + 0.052636630833148956 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.7502521276473999, + "style_target": 1, + "support_probability": 0.6449381113052368, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11950004707091688, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2192966192960739, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8572:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9721748232841492, + 0.28295600414276123, + 0.246305450797081, + 0.3953518271446228, + 0.05857464671134949, + 0.035661712288856506, + 0.13012398779392242, + 0.08747756481170654, + 0.1952458769083023 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8572", + "split": "test", + "style_correct": 1, + "style_probability": 0.1949460506439209, + "style_target": 0, + "support_probability": 0.70075523853302, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24972884040625587, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5667181015014648, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.23681554198265076, + 0.8993557095527649, + 0.11427176743745804, + 0.2475965917110443, + 0.17384691536426544, + 0.07449999451637268, + 0.10883188247680664, + 0.0909237414598465, + 0.22295881807804108, + 0.15414516627788544 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 1, + "style_probability": 0.8651726245880127, + "style_target": 1, + "support_probability": 0.19724887609481812, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1797700053548512, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.42900222539901733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8585:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.22799769043922424, + 0.8988305926322937, + 0.1409369260072708, + 0.23575261235237122, + 0.1184549629688263, + 0.129192054271698, + 0.13864058256149292, + 0.10686437040567398, + 0.11604437977075577, + 0.31723350286483765 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8585", + "split": "test", + "style_correct": 0, + "style_probability": 0.5787734985351562, + "style_target": 0, + "support_probability": 0.28509244322776794, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7230808895331435, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.38964974880218506, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18057896196842194, + 0.16925042867660522, + 0.9410409331321716, + 0.3830188512802124, + 0.055788472294807434 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 1, + "style_probability": 0.8514408469200134, + "style_target": 1, + "support_probability": 0.712730884552002, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.35091095102312236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3760952055454254, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8590:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23610955476760864, + 0.16967681050300598, + 0.9262152314186096, + 0.4012922942638397, + 0.11825928837060928, + 0.08804935216903687, + 0.16582317650318146, + 0.17692776024341583, + 0.14798817038536072 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8590", + "split": "test", + "style_correct": 1, + "style_probability": 0.3216172754764557, + "style_target": 0, + "support_probability": 0.7163724303245544, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.923792810025315, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.03810698539018631, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.27071455121040344, + 0.2278342843055725, + 0.9988743662834167, + 0.22705593705177307, + 0.053871672600507736, + 0.04742199555039406 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.7281712293624878, + "style_target": 1, + "support_probability": 0.8649042844772339, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7700419288044102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.029162440448999405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8606:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.28298893570899963, + 0.21492436528205872, + 0.9995829463005066, + 0.2635362148284912, + 0.05211881175637245, + 0.10795529931783676 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8606", + "split": "test", + "style_correct": 1, + "style_probability": 0.4254339933395386, + "style_target": 0, + "support_probability": 0.9102715253829956, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8858506268350119, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03490590676665306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22319549322128296, + 0.2666528522968292, + 0.31799426674842834, + 0.9986931681632996, + 0.02688026800751686, + 0.0981372520327568, + 0.04239692911505699, + 0.05366145074367523, + 0.04877668619155884 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 1, + "style_probability": 0.7148300409317017, + "style_target": 1, + "support_probability": 0.8224591612815857, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7791080154989488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06590358912944794, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8607:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2307819128036499, + 0.2616131603717804, + 0.3148588240146637, + 0.9953115582466125, + 0.12013885378837585, + 0.0862252414226532, + 0.06973753869533539, + 0.05445687845349312, + 0.14238959550857544 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8607", + "split": "test", + "style_correct": 0, + "style_probability": 0.6891168355941772, + "style_target": 0, + "support_probability": 0.6255387663841248, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.017771179521017817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8744103908538818, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30257269740104675, + 0.3027633726596832, + 0.364108681678772, + 0.36010146141052246, + 0.03885776549577713, + 0.17263492941856384, + 0.1306178867816925, + 0.08079905062913895, + 0.05546445772051811, + 0.06594760715961456, + 0.060218434780836105 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.7755691409111023, + "style_target": 1, + "support_probability": 0.07935676723718643, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001984892142002704, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9155016541481018, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8613:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2634609639644623, + 0.2794463634490967, + 0.33598828315734863, + 0.2795412242412567, + 0.19692568480968475, + 0.18621540069580078 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8613", + "split": "test", + "style_correct": 1, + "style_probability": 0.2946675717830658, + "style_target": 0, + "support_probability": 0.022500772029161453, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.424543062101474, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4136197566986084, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37590664625167847, + 0.3446328639984131, + 0.2697599232196808, + 0.8987110257148743, + 0.0765329897403717 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.8414477705955505, + "style_target": 1, + "support_probability": 0.30266234278678894, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0030064410869090467, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5621756315231323, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8627:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38529065251350403, + 0.4102954864501953, + 0.2726806104183197, + 0.9055781960487366, + 0.16860252618789673 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8627", + "split": "test", + "style_correct": 1, + "style_probability": 0.08579882979393005, + "style_target": 0, + "support_probability": 0.16903336346149445, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07335090585917213, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.656456470489502, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3878348171710968, + 0.5481252670288086, + 0.19089394807815552, + 0.3516022562980652, + 0.08800923079252243, + 0.07033921778202057 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.868919312953949, + "style_target": 1, + "support_probability": 0.11750620603561401, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004729330297994494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.699661135673523, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8640:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4176807403564453, + 0.41654863953590393, + 0.19318422675132751, + 0.28154119849205017, + 0.13382700085639954, + 0.09871481359004974 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8640", + "split": "test", + "style_correct": 1, + "style_probability": 0.44914671778678894, + "style_target": 0, + "support_probability": 0.07023811340332031, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04967197477549301, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6279234886169434, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3005199134349823, + 0.8744277358055115, + 0.30307814478874207, + 0.43003541231155396, + 0.07287774980068207, + 0.13851632177829742, + 0.03900734707713127, + 0.06967847794294357, + 0.16221000254154205 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 1, + "style_probability": 0.7515634894371033, + "style_target": 1, + "support_probability": 0.0764928087592125, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023514961907613596, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.634902834892273, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8653:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31753379106521606, + 0.8385109901428223, + 0.3053511083126068, + 0.39912211894989014, + 0.09001621603965759 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8653", + "split": "test", + "style_correct": 0, + "style_probability": 0.5245260000228882, + "style_target": 0, + "support_probability": 0.07626973092556, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5256293989991718, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4344480633735657, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.6191657185554504, + 0.3156827390193939, + 0.6191657185554504, + 0.7007532119750977, + 0.05051914602518082, + 0.037481214851140976, + 0.05321969836950302, + 0.018354080617427826, + 0.0366988368332386, + 0.04039068520069122 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.7854979038238525, + "style_target": 1, + "support_probability": 0.7678521275520325, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00900401726596589, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4873187839984894, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8655:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5432541370391846, + 0.36461740732192993, + 0.5432541370391846, + 0.7448891997337341, + 0.08405894786119461, + 0.08099696040153503, + 0.10479246824979782, + 0.08373735100030899, + 0.07271663099527359, + 0.12984693050384521 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8655", + "split": "test", + "style_correct": 1, + "style_probability": 0.10541421920061111, + "style_target": 0, + "support_probability": 0.5624849200248718, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007240845512089141, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4543881118297577, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8647904992103577, + 0.6729925870895386, + 0.29494524002075195, + 0.2770385146141052, + 0.0807608962059021 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.6055482625961304, + "style_target": 1, + "support_probability": 0.018515603616833687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012656405126415917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46775105595588684, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8656:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8386778235435486, + 0.6696704626083374, + 0.3045525550842285, + 0.29010164737701416, + 0.05089860409498215 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8656", + "split": "test", + "style_correct": 1, + "style_probability": 0.4532979130744934, + "style_target": 0, + "support_probability": 0.03555275872349739, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006330715602911654, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.900709867477417, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.23313122987747192, + 0.5768335461616516, + 0.7492388486862183, + 0.22628647089004517, + 0.14403335750102997, + 0.1251833736896515 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.7797269225120544, + "style_target": 1, + "support_probability": 0.015184897929430008, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0023794005236132394, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9482783675193787, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8658:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2742920219898224, + 0.5685805678367615, + 0.7065762877464294, + 0.20304512977600098, + 0.18267236649990082, + 0.10993122309446335, + 0.11787924915552139, + 0.06472215056419373, + 0.2049553245306015, + 0.09298289567232132, + 0.14961320161819458 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8658", + "split": "test", + "style_correct": 1, + "style_probability": 0.3721502125263214, + "style_target": 0, + "support_probability": 0.006794267799705267, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4809491849772413, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28351330757141113, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6579234600067139, + 0.22315490245819092, + 0.3881358504295349, + 0.6852642297744751, + 0.07152890413999557, + 0.05678696185350418, + 0.03397759050130844, + 0.10880190879106522, + 0.058065176010131836 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.8712996244430542, + "style_target": 1, + "support_probability": 0.5300034880638123, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.023671332656299844, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3046916425228119, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8663:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7677354216575623, + 0.3657548427581787, + 0.49546313285827637, + 0.8053938150405884, + 0.11736238747835159, + 0.15374323725700378, + 0.11545486748218536, + 0.07496456801891327, + 0.10095853358507156 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8663", + "split": "test", + "style_correct": 1, + "style_probability": 0.12359896302223206, + "style_target": 0, + "support_probability": 0.492922306060791, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4179644216796614, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03706271946430206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5801467895507812, + 0.972969114780426, + 0.608044445514679, + 0.24066153168678284, + 0.08601449429988861, + 0.10389601439237595, + 0.04457086697220802, + 0.024077530950307846, + 0.05361645296216011, + 0.06599799543619156, + 0.07108909636735916 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 1, + "style_probability": 0.6754119992256165, + "style_target": 1, + "support_probability": 0.4355412423610687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2721717568259532, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04143459349870682, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8665:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5347257256507874, + 0.9574525952339172, + 0.6519158482551575, + 0.2618747055530548, + 0.053654514253139496, + 0.05326005071401596, + 0.03718619421124458, + 0.09454558789730072, + 0.285724401473999, + 0.21999868750572205, + 0.03858073055744171 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8665", + "split": "test", + "style_correct": 0, + "style_probability": 0.6588163375854492, + "style_target": 0, + "support_probability": 0.2863779664039612, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.09944964957147252, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6944329142570496, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.47052815556526184, + 0.4234737753868103, + 0.3426837623119354, + 0.26451408863067627, + 0.1191367357969284 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 1, + "style_probability": 0.5283187031745911, + "style_target": 1, + "support_probability": 0.44397416710853577, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46094114073787656, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5339380502700806, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8681:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5507459044456482, + 0.4773232042789459, + 0.4470679759979248, + 0.2974982261657715, + 0.0814017802476883 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8681", + "split": "test", + "style_correct": 0, + "style_probability": 0.9454756379127502, + "style_target": 0, + "support_probability": 0.5765981078147888, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.006428143825376488, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.950416624546051, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3658016324043274, + 0.22912602126598358, + 0.15398289263248444, + 0.2009328454732895, + 0.06913673877716064, + 0.12549391388893127 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 0, + "style_probability": 0.4459660053253174, + "style_target": 1, + "support_probability": 0.05700966343283653, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0026897959749589565, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9609331488609314, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8685:negative", + "evidence_budget": 12, + "evidence_probabilities": [ + 0.39322808384895325, + 0.2610584497451782, + 0.19361567497253418, + 0.1672820746898651, + 0.08676093071699142, + 0.08622820675373077, + 0.09266207367181778, + 0.07126323133707047, + 0.0703018382191658, + 0.1098984107375145, + 0.09053674340248108, + 0.07448606193065643 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8685", + "split": "test", + "style_correct": 1, + "style_probability": 0.41531887650489807, + "style_target": 0, + "support_probability": 0.015064341016113758, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.30314496773652877, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12499015033245087, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22287015616893768, + 0.2755314111709595, + 0.15793289244174957, + 0.9780880212783813, + 0.08672551810741425, + 0.13254331052303314, + 0.07383838295936584, + 0.04727580025792122, + 0.1264481097459793 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 0, + "style_probability": 0.4011194109916687, + "style_target": 1, + "support_probability": 0.46015307307243347, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.02972593388126121, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.31570398807525635, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8687:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2301410436630249, + 0.30135756731033325, + 0.18230347335338593, + 0.992915153503418, + 0.04618927463889122, + 0.1766485571861267, + 0.10372927784919739, + 0.145238995552063, + 0.2803615927696228 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8687", + "split": "test", + "style_correct": 1, + "style_probability": 0.06374869495630264, + "style_target": 0, + "support_probability": 0.5074602365493774, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6640087149064584, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08046610653400421, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9685096740722656, + 0.30501267313957214, + 0.30501267313957214, + 0.3838188052177429, + 0.08899784088134766, + 0.08478772640228271, + 0.10604109615087509, + 0.0443069152534008, + 0.04833109304308891, + 0.15943831205368042 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 1, + "style_probability": 0.5734878778457642, + "style_target": 1, + "support_probability": 0.7215222120285034, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7979742117005628, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05488433688879013, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8708:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9779115915298462, + 0.3388640582561493, + 0.3388640582561493, + 0.404029905796051, + 0.06834175437688828, + 0.07067795842885971, + 0.13403870165348053, + 0.25619184970855713, + 0.050087831914424896, + 0.06921062618494034 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8708", + "split": "test", + "style_correct": 0, + "style_probability": 0.815946102142334, + "style_target": 0, + "support_probability": 0.6771335601806641, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07048140494145817, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5632696151733398, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4696069061756134, + 0.5824798345565796, + 0.4030981957912445, + 0.35788750648498535, + 0.3131897747516632 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.6777801513671875, + "style_target": 1, + "support_probability": 0.10023219138383865, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008971734169159267, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8504272103309631, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8709:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5306792855262756, + 0.48290619254112244, + 0.43376606702804565, + 0.3474178910255432, + 0.16881698369979858 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8709", + "split": "test", + "style_correct": 1, + "style_probability": 0.11305750161409378, + "style_target": 0, + "support_probability": 0.02315630204975605, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15625363067618195, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10354577749967575, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9833332896232605, + 0.7580927610397339, + 0.39120712876319885, + 0.572144627571106, + 0.12656384706497192, + 0.24220937490463257 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 0, + "style_probability": 0.4489664137363434, + "style_target": 1, + "support_probability": 0.5344567894935608, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.20574571476152492, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12280181795358658, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8712:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9664245843887329, + 0.7082038521766663, + 0.4308479130268097, + 0.4634903073310852, + 0.0995076596736908, + 0.07856809347867966 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:8712", + "split": "test", + "style_correct": 1, + "style_probability": 0.49776288866996765, + "style_target": 0, + "support_probability": 0.44231727719306946, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3854210739491126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5934935212135315, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33027371764183044, + 0.12434787303209305, + 0.427964448928833, + 0.9628767371177673, + 0.12709051370620728, + 0.18651124835014343, + 0.16721276938915253, + 0.0744679793715477, + 0.17001520097255707 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 1, + "style_probability": 0.7588633894920349, + "style_target": 1, + "support_probability": 0.30585435032844543, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15712805245712502, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.5388520359992981, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8723:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30912545323371887, + 0.13923120498657227, + 0.42369288206100464, + 0.9547159671783447, + 0.15826907753944397 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8723", + "split": "test", + "style_correct": 1, + "style_probability": 0.35506540536880493, + "style_target": 0, + "support_probability": 0.34323766827583313, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2584047351029266, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24695253372192383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2346445471048355, + 0.9738032221794128, + 0.4810153841972351, + 0.6184399127960205, + 0.16771939396858215, + 0.09612765163183212, + 0.09169000387191772, + 0.0699668601155281, + 0.093494713306427, + 0.059417665004730225, + 0.07725158333778381 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 1, + "style_probability": 0.7529093027114868, + "style_target": 1, + "support_probability": 0.2163495570421219, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09539903196471756, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17331744730472565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8729:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2524228096008301, + 0.9810061454772949, + 0.4738474190235138, + 0.6151891946792603, + 0.15085168182849884, + 0.052938129752874374, + 0.1143651157617569, + 0.11294756829738617, + 0.1002902239561081, + 0.18784834444522858 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8729", + "split": "test", + "style_correct": 1, + "style_probability": 0.36501461267471313, + "style_target": 0, + "support_probability": 0.3908796012401581, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03290551917467499, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.762626051902771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10761627554893494, + 0.7462934851646423, + 0.14881685376167297, + 0.1353970170021057, + 0.10140487551689148 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 1, + "style_probability": 0.6170211434364319, + "style_target": 1, + "support_probability": 0.0778840109705925, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002325381055550025, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8422725200653076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8733:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.10348719358444214, + 0.8464585542678833, + 0.14026060700416565, + 0.13340842723846436, + 0.10445187985897064 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8733", + "split": "test", + "style_correct": 1, + "style_probability": 0.3372853994369507, + "style_target": 0, + "support_probability": 0.059430044144392014, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.46816873035363576, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4428129494190216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.39242079854011536, + 0.3203648328781128, + 0.2565951645374298, + 0.7973305583000183, + 0.06559548527002335, + 0.030932364985346794 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 1, + "style_probability": 0.5831235647201538, + "style_target": 1, + "support_probability": 0.8332545757293701, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8022257114616771, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.29083123803138733, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8739:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3629942834377289, + 0.3296883702278137, + 0.26115748286247253, + 0.8856611847877502, + 0.04415687918663025, + 0.032267436385154724 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8739", + "split": "test", + "style_correct": 0, + "style_probability": 0.9044046998023987, + "style_target": 0, + "support_probability": 0.8909477591514587, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11090005358115461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8030003905296326, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7675215005874634, + 0.3565165102481842, + 0.4788820743560791, + 0.1346248835325241, + 0.09395404160022736, + 0.043122585862874985, + 0.04020845517516136, + 0.07833470404148102, + 0.04540453106164932 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.5754242539405823, + "style_target": 1, + "support_probability": 0.1486881822347641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03871292927756231, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7861523032188416, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8744:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7460525631904602, + 0.3225860595703125, + 0.4606159031391144, + 0.1209130510687828, + 0.07695750147104263 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8744", + "split": "test", + "style_correct": 1, + "style_probability": 0.37526509165763855, + "style_target": 0, + "support_probability": 0.11310608685016632, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02327593862902488, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8908454775810242, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18731257319450378, + 0.9153667092323303, + 0.3173939287662506, + 0.26438358426094055, + 0.13353100419044495, + 0.12707829475402832, + 0.13687509298324585, + 0.11113937199115753, + 0.04291652888059616, + 0.09964001923799515, + 0.10155259072780609 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.5614291429519653, + "style_target": 1, + "support_probability": 0.05463135614991188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012283631626297087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6813346147537231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8745:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.25960472226142883, + 0.9660425186157227, + 0.33035755157470703, + 0.3316817283630371, + 0.21861694753170013, + 0.11126650869846344, + 0.435350239276886, + 0.11130896955728531, + 0.08727685362100601, + 0.1452844887971878 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8745", + "split": "test", + "style_correct": 1, + "style_probability": 0.4100213944911957, + "style_target": 0, + "support_probability": 0.16386429965496063, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7319241726050976, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.19942983984947205, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.45387575030326843, + 0.8040293455123901, + 0.32611265778541565, + 0.5118807554244995, + 0.11410434544086456 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.9160529375076294, + "style_target": 1, + "support_probability": 0.7144938111305237, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2951065158895769, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2695257365703583, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8753:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3694347143173218, + 0.7674222588539124, + 0.2816099226474762, + 0.4271109700202942, + 0.05133466422557831, + 0.10518713295459747, + 0.06725825369358063, + 0.07349348813295364, + 0.046437133103609085 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8753", + "split": "test", + "style_correct": 1, + "style_probability": 0.30550700426101685, + "style_target": 0, + "support_probability": 0.7538325786590576, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8621414945177719, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11998963356018066, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9566900134086609, + 0.11218830943107605, + 0.6388807892799377, + 0.25658243894577026, + 0.0838434174656868, + 0.1054934486746788 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.8517702221870422, + "style_target": 1, + "support_probability": 0.7903227806091309, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.29929058203759246, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1176561638712883, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8772:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9907535314559937, + 0.13604623079299927, + 0.7608475089073181, + 0.43612247705459595, + 0.16808952391147614, + 0.14220882952213287 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:8772", + "split": "test", + "style_correct": 1, + "style_probability": 0.17937062680721283, + "style_target": 0, + "support_probability": 0.8444814085960388, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00041646811459911814, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9926342368125916, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5460404753684998, + 0.21109500527381897, + 0.43038949370384216, + 0.22509731352329254, + 0.03965216502547264, + 0.07396231591701508, + 0.16707776486873627, + 0.05458228290081024, + 0.0628744512796402 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 1, + "style_probability": 0.6907325387001038, + "style_target": 1, + "support_probability": 0.0051537989638745785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003011226072142922, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9114339351654053, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8788:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42694124579429626, + 0.226762056350708, + 0.48107650876045227, + 0.2392023354768753, + 0.10874348133802414, + 0.2024703323841095, + 0.07268602401018143, + 0.1027984470129013, + 0.12012824416160583 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8788", + "split": "test", + "style_correct": 0, + "style_probability": 0.6886862516403198, + "style_target": 0, + "support_probability": 0.01619996875524521, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.039519396551753516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8679227828979492, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3401394188404083, + 0.5488338470458984, + 0.4303586781024933, + 0.3401394188404083, + 0.12252377718687057, + 0.06064861640334129, + 0.05986702814698219, + 0.18896335363388062, + 0.07552770525217056, + 0.08702985942363739 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 1, + "style_probability": 0.7789615988731384, + "style_target": 1, + "support_probability": 0.11583264917135239, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.055142758573988086, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7365273833274841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8793:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3337978422641754, + 0.5859144330024719, + 0.3951731026172638, + 0.3337978422641754, + 0.10146791487932205, + 0.12471290677785873 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8793", + "split": "test", + "style_correct": 1, + "style_probability": 0.4173825979232788, + "style_target": 0, + "support_probability": 0.22122997045516968, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2396170835080511, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7270576357841492, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4608635902404785, + 0.29860007762908936, + 0.16255109012126923, + 0.29860007762908936, + 0.08507560193538666 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.7517762184143066, + "style_target": 1, + "support_probability": 0.6040650010108948, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005740747893335117, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8103345632553101, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8796:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5013208389282227, + 0.3966510593891144, + 0.18169158697128296, + 0.3966510593891144, + 0.14662665128707886 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8796", + "split": "test", + "style_correct": 1, + "style_probability": 0.12830504775047302, + "style_target": 0, + "support_probability": 0.49881595373153687, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4186130570661568, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14100460708141327, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.34006285667419434, + 0.18966935575008392, + 0.2582249641418457, + 0.8126150965690613, + 0.02862619049847126, + 0.06601840257644653 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 1, + "style_probability": 0.8002376556396484, + "style_target": 1, + "support_probability": 0.31411880254745483, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2865091406840108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.17685310542583466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8803:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2960732579231262, + 0.19798068702220917, + 0.3163955509662628, + 0.7548501491546631, + 0.030684715136885643, + 0.08955684304237366 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8803", + "split": "test", + "style_correct": 0, + "style_probability": 0.6495808362960815, + "style_target": 0, + "support_probability": 0.31493398547172546, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8154746037832935, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07614824175834656, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6738079786300659, + 0.2709837555885315, + 0.9410210847854614, + 0.30225124955177307, + 0.029890991747379303, + 0.025590088218450546, + 0.07755924761295319, + 0.050642501562833786, + 0.12134571373462677 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.6607357263565063, + "style_target": 1, + "support_probability": 0.7748306393623352, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4714397009716862, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07113278657197952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8810:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.647951066493988, + 0.2813091278076172, + 0.9250719547271729, + 0.2882947027683258, + 0.23253610730171204 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8810", + "split": "test", + "style_correct": 1, + "style_probability": 0.23561757802963257, + "style_target": 0, + "support_probability": 0.7559061646461487, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.954789094397487, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06077122315764427, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8822:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.09903831034898758, + 0.3214532136917114, + 0.986346423625946, + 0.4083845019340515, + 0.07576830685138702, + 0.05242561921477318, + 0.05735710635781288, + 0.053124427795410156, + 0.16779036819934845, + 0.07544393092393875, + 0.12279247492551804 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.7722072005271912, + "style_target": 1, + "support_probability": 0.8507417440414429, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5048266693829646, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06663793325424194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8822:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15179622173309326, + 0.38468408584594727, + 0.9972891807556152, + 0.33014872670173645, + 0.15548858046531677, + 0.11623068153858185, + 0.09134526550769806, + 0.24919335544109344, + 0.14134107530117035, + 0.1729286164045334 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8822", + "split": "test", + "style_correct": 1, + "style_probability": 0.09731399267911911, + "style_target": 0, + "support_probability": 0.8280612230300903, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.11397695509211307, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24525891244411469, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23150300979614258, + 0.34224388003349304, + 0.5191359519958496, + 0.3036232888698578, + 0.050404276698827744 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 1, + "style_probability": 0.7598130702972412, + "style_target": 1, + "support_probability": 0.34880220890045166, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024689928347102708, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37853574752807617, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.24089966714382172, + 0.29142361879348755, + 0.4537891447544098, + 0.28824374079704285, + 0.13301168382167816 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8828", + "split": "test", + "style_correct": 0, + "style_probability": 0.5171804428100586, + "style_target": 0, + "support_probability": 0.18644288182258606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11456783433734594, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2563438415527344, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32369959354400635, + 0.49055832624435425, + 0.2720627188682556, + 0.32369959354400635, + 0.06678257137537003, + 0.06977585703134537 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.6849316358566284, + "style_target": 1, + "support_probability": 0.26256662607192993, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06413316076468867, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2512624263763428, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8829:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.31838712096214294, + 0.47543060779571533, + 0.24820217490196228, + 0.31838712096214294, + 0.057741083204746246, + 0.09000875800848007, + 0.12248742580413818, + 0.04278352111577988, + 0.17357899248600006, + 0.1367635726928711, + 0.0915917456150055 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8829", + "split": "test", + "style_correct": 1, + "style_probability": 0.3168690800666809, + "style_target": 0, + "support_probability": 0.26897063851356506, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6754364634388567, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2246963530778885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.16651268303394318, + 0.9440892934799194, + 0.17152312397956848, + 0.19949689507484436, + 0.058095499873161316, + 0.055188246071338654, + 0.08961953967809677, + 0.09051001071929932, + 0.11062808334827423 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.8449041247367859, + "style_target": 1, + "support_probability": 0.5152589678764343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3633842625146002, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09495911002159119, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8833:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.18548612296581268, + 0.9861693382263184, + 0.24184520542621613, + 0.23493623733520508, + 0.19458194077014923, + 0.06614242494106293, + 0.04445534572005272, + 0.09555564820766449, + 0.1346125304698944 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8833", + "split": "test", + "style_correct": 1, + "style_probability": 0.17652375996112823, + "style_target": 0, + "support_probability": 0.833869993686676, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9318141703594592, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07630440592765808, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8835:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2680087685585022, + 0.19048967957496643, + 0.21617092192173004, + 0.9476149678230286, + 0.04075014591217041, + 0.05989759787917137, + 0.07059638202190399, + 0.04420670494437218, + 0.09801801294088364, + 0.06491084396839142, + 0.05679646506905556 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 1, + "style_probability": 0.8364131450653076, + "style_target": 1, + "support_probability": 0.9664797186851501, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7605216625234412, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1332162767648697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8835:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.29045000672340393, + 0.26076439023017883, + 0.21901734173297882, + 0.9033617377281189, + 0.0710136890411377, + 0.07552643865346909, + 0.1218554750084877, + 0.14948584139347076, + 0.10654350370168686, + 0.07475212216377258 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8835", + "split": "test", + "style_correct": 1, + "style_probability": 0.399024099111557, + "style_target": 0, + "support_probability": 0.9551610350608826, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.36712709026719637, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5409102439880371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4349396824836731, + 0.469713419675827, + 0.7214659452438354, + 0.30386632680892944, + 0.04197987914085388 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.8000574707984924, + "style_target": 1, + "support_probability": 0.6046650409698486, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017053114159926278, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8035072684288025, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8838:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4032658040523529, + 0.4597017467021942, + 0.6737352013587952, + 0.31021350622177124, + 0.08725868165493011, + 0.06435246020555496, + 0.06119407340884209, + 0.04808714613318443, + 0.08852992206811905 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8838", + "split": "test", + "style_correct": 1, + "style_probability": 0.3920219838619232, + "style_target": 0, + "support_probability": 0.1308302879333496, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07307570533516385, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6451646685600281, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17182709276676178, + 0.4069086015224457, + 0.2940960228443146, + 0.5494153499603271, + 0.16843438148498535, + 0.051811885088682175 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.646817147731781, + "style_target": 1, + "support_probability": 0.24590164422988892, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006020270757966848, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7081362009048462, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8843:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21888385713100433, + 0.4526416063308716, + 0.3740108013153076, + 0.7006834149360657, + 0.1107914075255394, + 0.14797107875347137 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8843", + "split": "test", + "style_correct": 1, + "style_probability": 0.16277346014976501, + "style_target": 0, + "support_probability": 0.24187156558036804, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7765384657357188, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14122648537158966, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.34089744091033936, + 0.8588153123855591, + 0.2177482396364212, + 0.18643955886363983, + 0.02929547242820263, + 0.11103153973817825, + 0.026517855003476143, + 0.020304618403315544, + 0.019111718982458115 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.7780466675758362, + "style_target": 1, + "support_probability": 0.8541143536567688, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5605361234582915, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17426857352256775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8845:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.311088502407074, + 0.7557274103164673, + 0.251547247171402, + 0.1979023665189743, + 0.03780902922153473, + 0.07711397856473923, + 0.05695921182632446, + 0.03522365912795067, + 0.0675678551197052 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8845", + "split": "test", + "style_correct": 1, + "style_probability": 0.48389163613319397, + "style_target": 0, + "support_probability": 0.8552114367485046, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.894387780362468, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12645669281482697, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18843461573123932, + 0.2753957211971283, + 0.4752614498138428, + 0.9644879698753357, + 0.0789729580283165, + 0.09362688660621643, + 0.04711144044995308, + 0.10591837018728256, + 0.08142374455928802, + 0.10193166136741638, + 0.04456113278865814 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.880757749080658, + "style_target": 1, + "support_probability": 0.8407632112503052, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5034045244200378, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.16521908342838287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8851:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17661847174167633, + 0.2540728449821472, + 0.47127285599708557, + 0.9493498206138611, + 0.09745954722166061, + 0.07709161937236786 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8851", + "split": "test", + "style_correct": 1, + "style_probability": 0.44296663999557495, + "style_target": 0, + "support_probability": 0.7991117238998413, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10829931706340544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.7762008309364319, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41382044553756714, + 0.8304352760314941, + 0.1501692682504654, + 0.1875993013381958, + 0.08300485461950302 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.8726120591163635, + "style_target": 1, + "support_probability": 0.14105001091957092, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005434016531461917, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8308076858520508, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8853:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5254737138748169, + 0.8936045169830322, + 0.15383560955524445, + 0.21097442507743835, + 0.12540847063064575 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8853", + "split": "test", + "style_correct": 1, + "style_probability": 0.24606119096279144, + "style_target": 0, + "support_probability": 0.1808302253484726, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10957423208331818, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6025180220603943, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.404509037733078, + 0.19028443098068237, + 0.328647255897522, + 0.7051012516021729, + 0.046972423791885376, + 0.054165229201316833 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.7832512259483337, + "style_target": 1, + "support_probability": 0.1501585692167282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020608937318203565, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6291812658309937, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8854:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3755056858062744, + 0.18349963426589966, + 0.37992194294929504, + 0.6839938759803772, + 0.050787344574928284, + 0.04147159680724144 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8854", + "split": "test", + "style_correct": 1, + "style_probability": 0.2416037768125534, + "style_target": 0, + "support_probability": 0.17891037464141846, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.16677793535657326, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08367844671010971, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2655739486217499, + 0.45179450511932373, + 0.885700523853302, + 0.4795718789100647, + 0.07443121075630188, + 0.06690257042646408, + 0.04113621264696121, + 0.05445531755685806, + 0.07729486376047134 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.5149413347244263, + "style_target": 1, + "support_probability": 0.29087427258491516, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11767883363282469, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.05321800336241722, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8858:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.29084062576293945, + 0.41617491841316223, + 0.8618067502975464, + 0.4624611735343933, + 0.09630025923252106 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8858", + "split": "test", + "style_correct": 1, + "style_probability": 0.2988586127758026, + "style_target": 0, + "support_probability": 0.3587341904640198, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7223713648063743, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20711904764175415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9438757300376892, + 0.37412920594215393, + 0.29691794514656067, + 0.37412920594215393, + 0.09101491421461105, + 0.05033812299370766, + 0.06747113913297653, + 0.12051145732402802, + 0.06124444678425789, + 0.12769228219985962, + 0.08700712025165558 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.7987352013587952, + "style_target": 1, + "support_probability": 0.8087944984436035, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.044460406502615506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24303756654262543, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8864:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9745621681213379, + 0.44933345913887024, + 0.3063157796859741, + 0.44933345913887024, + 0.19403617084026337, + 0.13935987651348114, + 0.13047170639038086, + 0.1702158898115158, + 0.1583489328622818, + 0.1720070093870163 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8864", + "split": "test", + "style_correct": 1, + "style_probability": 0.11901755630970001, + "style_target": 0, + "support_probability": 0.7716784477233887, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6050738555873245, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.28837427496910095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18571197986602783, + 0.5596268177032471, + 0.9827340245246887, + 0.24590682983398438, + 0.090138740837574 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 1, + "style_probability": 0.7666240930557251, + "style_target": 1, + "support_probability": 0.6352212429046631, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7514629977922205, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14260272681713104, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8870:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.196811243891716, + 0.6411910653114319, + 0.958143413066864, + 0.2940327823162079, + 0.10925552994012833 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8870", + "split": "test", + "style_correct": 0, + "style_probability": 0.6874170899391174, + "style_target": 0, + "support_probability": 0.8291012644767761, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8933827680965578, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07815776765346527, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46464964747428894, + 0.1799607127904892, + 0.9654331207275391, + 0.20947426557540894, + 0.09775763750076294, + 0.0999169647693634 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.8731686472892761, + "style_target": 1, + "support_probability": 0.8329179286956787, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5648744018153273, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05515747144818306, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8874:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3635663390159607, + 0.20221707224845886, + 0.9312335848808289, + 0.22114890813827515, + 0.14088086783885956, + 0.08285131305456161, + 0.17701499164104462, + 0.10809390246868134, + 0.05175510048866272, + 0.07379574328660965, + 0.13903649151325226 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8874", + "split": "test", + "style_correct": 1, + "style_probability": 0.40434613823890686, + "style_target": 0, + "support_probability": 0.7804048657417297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8903101031854987, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11177153885364532, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.36326780915260315, + 0.3132055699825287, + 0.9529387950897217, + 0.4376489520072937, + 0.09910216182470322, + 0.11038147658109665, + 0.07678266614675522, + 0.08126676827669144, + 0.09833872318267822 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.8748460412025452, + "style_target": 1, + "support_probability": 0.7547559142112732, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6463034794214124, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09684424847364426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8878:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4439452290534973, + 0.41720810532569885, + 0.9801488518714905, + 0.48864153027534485, + 0.09751607477664948, + 0.1054958626627922, + 0.09828133881092072, + 0.07843898236751556, + 0.08608244359493256 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8878", + "split": "test", + "style_correct": 1, + "style_probability": 0.40395504236221313, + "style_target": 0, + "support_probability": 0.8068155646324158, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22413821839330872, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.37165799736976624, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4245797097682953, + 0.19049429893493652, + 0.2837460935115814, + 0.6643438935279846, + 0.09509196877479553, + 0.06279250234365463, + 0.09990037232637405, + 0.18869389593601227, + 0.0658600851893425, + 0.11191027611494064, + 0.12241725623607635 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.8666665554046631, + "style_target": 1, + "support_probability": 0.34620440006256104, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03980099554254047, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5715104937553406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8879:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.40278884768486023, + 0.16682542860507965, + 0.29485365748405457, + 0.6105462908744812, + 0.0700518935918808, + 0.09576179832220078, + 0.13212929666042328, + 0.11291146278381348, + 0.11584500968456268, + 0.09196510165929794 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8879", + "split": "test", + "style_correct": 1, + "style_probability": 0.4829182028770447, + "style_target": 0, + "support_probability": 0.3068717420101166, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7055943102157833, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13599614799022675, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.854106605052948, + 0.781879186630249, + 0.19726090133190155, + 0.854106605052948, + 0.1331537812948227 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 1, + "style_probability": 0.874424159526825, + "style_target": 1, + "support_probability": 0.5073432326316833, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4335726222461922, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11570414900779724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8881:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8529340624809265, + 0.7589012980461121, + 0.20632268488407135, + 0.8529340624809265, + 0.08222448825836182, + 0.27651548385620117, + 0.07563071697950363, + 0.1474997103214264, + 0.1393667459487915 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:8881", + "split": "test", + "style_correct": 1, + "style_probability": 0.4547324776649475, + "style_target": 0, + "support_probability": 0.5467820167541504, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016400159050613646, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.307973712682724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7104176878929138, + 0.1720285713672638, + 0.44406014680862427, + 0.30625516176223755, + 0.06224289909005165, + 0.16396315395832062 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.5845591425895691, + "style_target": 1, + "support_probability": 0.08698458969593048, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0035552784127680643, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.30787837505340576, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8884:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7540513873100281, + 0.24204131960868835, + 0.48436763882637024, + 0.38138720393180847, + 0.07743886113166809, + 0.1342087984085083 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8884", + "split": "test", + "style_correct": 1, + "style_probability": 0.24419374763965607, + "style_target": 0, + "support_probability": 0.13400469720363617, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8485991539891202, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07469753921031952, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9309512376785278, + 0.6060938835144043, + 0.32670649886131287, + 0.5223104953765869, + 0.03443220630288124, + 0.03874955698847771, + 0.10090535879135132, + 0.06814070791006088, + 0.0628962442278862 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.7445940971374512, + "style_target": 1, + "support_probability": 0.8626652956008911, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6354034832526736, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1504124402999878, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8888:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8921613693237305, + 0.5776245594024658, + 0.35084250569343567, + 0.44709154963493347, + 0.12703976035118103, + 0.06765809655189514, + 0.05755309760570526, + 0.08308704197406769, + 0.10423701256513596 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8888", + "split": "test", + "style_correct": 1, + "style_probability": 0.49988529086112976, + "style_target": 0, + "support_probability": 0.7838667631149292, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21224019197396696, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4239213466644287, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4287196397781372, + 0.8630083799362183, + 0.45775192975997925, + 0.22880035638809204, + 0.04548522084951401, + 0.0378233939409256, + 0.031061191111803055, + 0.051013242453336716, + 0.12223997712135315, + 0.07203354686498642, + 0.05731406807899475 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.8232818245887756, + "style_target": 1, + "support_probability": 0.22818242013454437, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04892021370650923, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3894423246383667, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8889:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3524538576602936, + 0.8307743072509766, + 0.44628840684890747, + 0.2079261690378189, + 0.11797885596752167, + 0.26037997007369995 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8889", + "split": "test", + "style_correct": 1, + "style_probability": 0.4011262059211731, + "style_target": 0, + "support_probability": 0.19745981693267822, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1778278954037269, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41527464985847473, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8122063875198364, + 0.4527021050453186, + 0.3650123178958893, + 0.4527021050453186, + 0.14058515429496765 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 1, + "style_probability": 0.8104516863822937, + "style_target": 1, + "support_probability": 0.11114851385354996, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02861295867884195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3083195686340332, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8896:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8497536182403564, + 0.36759302020072937, + 0.36246585845947266, + 0.36759302020072937, + 0.10184551775455475 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8896", + "split": "test", + "style_correct": 1, + "style_probability": 0.3853314220905304, + "style_target": 0, + "support_probability": 0.12581698596477509, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7949690357581147, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16750967502593994, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20677664875984192, + 0.9706152081489563, + 0.11255498975515366, + 0.24907171726226807, + 0.03549690172076225, + 0.09344430267810822 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 1, + "style_probability": 0.7464398145675659, + "style_target": 1, + "support_probability": 0.7913403511047363, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42144286879700665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2665407359600067, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8901:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21432630717754364, + 0.9569335579872131, + 0.11713694781064987, + 0.22056232392787933, + 0.09501024335622787, + 0.1199088990688324 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8901", + "split": "test", + "style_correct": 1, + "style_probability": 0.3839942514896393, + "style_target": 0, + "support_probability": 0.7093307971954346, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.865427021153522, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08067921549081802, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9324723482131958, + 0.2504592537879944, + 0.1429775059223175, + 0.2504592537879944, + 0.09062863886356354, + 0.08549703657627106, + 0.07549566775560379, + 0.06766381114721298, + 0.05715937167406082 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.8387734293937683, + "style_target": 1, + "support_probability": 0.7781925797462463, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5547318532924805, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06590640544891357, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8908:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9101561307907104, + 0.24032175540924072, + 0.13515080511569977, + 0.24032175540924072, + 0.0818975567817688 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8908", + "split": "test", + "style_correct": 1, + "style_probability": 0.3604157865047455, + "style_target": 0, + "support_probability": 0.8369830250740051, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010660538204579241, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6086207032203674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.22912053763866425, + 0.24298980832099915, + 0.5081824660301208, + 0.5247136354446411, + 0.05770837888121605, + 0.10799847543239594, + 0.08907274901866913, + 0.04543280228972435, + 0.13408538699150085, + 0.12987235188484192, + 0.10907801240682602 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 1, + "style_probability": 0.8798661828041077, + "style_target": 1, + "support_probability": 0.004100474528968334, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002837439811214648, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5408595204353333, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8918:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2296924740076065, + 0.29637324810028076, + 0.596141517162323, + 0.5571208596229553, + 0.05981814116239548, + 0.024752479046583176, + 0.05664030835032463, + 0.10453365743160248, + 0.12438800930976868, + 0.15280082821846008 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8918", + "split": "test", + "style_correct": 0, + "style_probability": 0.5860916376113892, + "style_target": 0, + "support_probability": 0.008263231255114079, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.17154613098724525, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3961690068244934, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.34200042486190796, + 0.237848699092865, + 0.7875616550445557, + 0.34200042486190796, + 0.06799116730690002 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 0, + "style_probability": 0.3327154815196991, + "style_target": 1, + "support_probability": 0.6310631036758423, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5143287025303565, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.311836302280426, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8922:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39011555910110474, + 0.2524707615375519, + 0.7960025668144226, + 0.39011555910110474, + 0.043219760060310364 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8922", + "split": "test", + "style_correct": 0, + "style_probability": 0.5390933752059937, + "style_target": 0, + "support_probability": 0.8024733662605286, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.46026516737669004, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.30196043848991394, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2741468548774719, + 0.5211704969406128, + 0.31218624114990234, + 0.723473072052002, + 0.05723351985216141, + 0.0840449258685112 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.3781997561454773, + "style_target": 1, + "support_probability": 0.8320705890655518, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6801214719084498, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4368150234222412, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8931:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.29153168201446533, + 0.4325322210788727, + 0.385940283536911, + 0.7257216572761536, + 0.06047883629798889, + 0.03203781694173813 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8931", + "split": "test", + "style_correct": 0, + "style_probability": 0.9268014430999756, + "style_target": 0, + "support_probability": 0.7465967535972595, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.157196737212411, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7895172238349915, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6020168662071228, + 0.3674507439136505, + 0.16470329463481903, + 0.27781885862350464, + 0.28632548451423645, + 0.11874581128358841, + 0.1502569615840912, + 0.109407439827919, + 0.1558169275522232 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 0, + "style_probability": 0.40491950511932373, + "style_target": 1, + "support_probability": 0.31240639090538025, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.028049547791789253, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5742907524108887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8936:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8019233345985413, + 0.5477458238601685, + 0.14928878843784332, + 0.23412157595157623, + 0.11209503561258316, + 0.06836780905723572, + 0.04421296343207359, + 0.07262548059225082, + 0.08317679166793823 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8936", + "split": "test", + "style_correct": 1, + "style_probability": 0.06399015337228775, + "style_target": 0, + "support_probability": 0.5448551177978516, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.24643322588338454, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46320104598999023, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2572340965270996, + 0.25228995084762573, + 0.2915325164794922, + 0.7408210635185242, + 0.10180265456438065, + 0.06150725856423378, + 0.06400364637374878, + 0.04108729958534241, + 0.09443309903144836, + 0.061814386397600174, + 0.060375016182661057 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 0, + "style_probability": 0.19285783171653748, + "style_target": 1, + "support_probability": 0.7340365648269653, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3825769390135534, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5055031180381775, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8939:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.25463443994522095, + 0.1873054802417755, + 0.22990375757217407, + 0.770409882068634, + 0.08172130584716797, + 0.05800535902380943, + 0.038457807153463364, + 0.12123210728168488, + 0.03407333418726921, + 0.04540885239839554, + 0.05367211997509003 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8939", + "split": "test", + "style_correct": 1, + "style_probability": 0.30078765749931335, + "style_target": 0, + "support_probability": 0.7615376114845276, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.05586016822143102, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6993070840835571, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7228966355323792, + 0.2858331501483917, + 0.20754235982894897, + 0.4328034222126007, + 0.04280420020222664 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 1, + "style_probability": 0.8772149085998535, + "style_target": 1, + "support_probability": 0.037345465272665024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011085798946650629, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8785728216171265, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8948:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6602323055267334, + 0.26381757855415344, + 0.18420551717281342, + 0.4111845791339874, + 0.08416671305894852, + 0.09230367094278336, + 0.091042160987854, + 0.07053712010383606, + 0.07321686297655106 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8948", + "split": "test", + "style_correct": 0, + "style_probability": 0.5750536918640137, + "style_target": 0, + "support_probability": 0.01628146320581436, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005036886163531375, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6617867350578308, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.40181106328964233, + 0.5012616515159607, + 0.2425888478755951, + 0.8762761950492859, + 0.1474314033985138, + 0.09567075222730637 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.6342714428901672, + "style_target": 1, + "support_probability": 0.020974362269043922, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00160393283275461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6998613476753235, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8951:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4735792577266693, + 0.48349475860595703, + 0.2588057816028595, + 0.9091973900794983, + 0.08453800529241562, + 0.07389172911643982 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8951", + "split": "test", + "style_correct": 1, + "style_probability": 0.3509059250354767, + "style_target": 0, + "support_probability": 0.021244393661618233, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01236635326134694, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7984512448310852, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5006788969039917, + 0.46231842041015625, + 0.487186998128891, + 0.26243841648101807, + 0.04778663069009781, + 0.05108361691236496, + 0.05119777098298073, + 0.08365844935178757, + 0.05154998227953911 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 1, + "style_probability": 0.8460165858268738, + "style_target": 1, + "support_probability": 0.018784504383802414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.057495133358906, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4515991508960724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8960:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5411291718482971, + 0.4952421188354492, + 0.4904165267944336, + 0.35302457213401794, + 0.05655397102236748, + 0.09028502553701401, + 0.03433092683553696, + 0.06965214759111404, + 0.041125841438770294 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8960", + "split": "test", + "style_correct": 0, + "style_probability": 0.5655278563499451, + "style_target": 0, + "support_probability": 0.19674675166606903, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7741694675159749, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09898366034030914, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9463420510292053, + 0.2017630934715271, + 0.3916032314300537, + 0.4262505769729614, + 0.035002004355192184, + 0.05019903928041458, + 0.050031427294015884, + 0.04420754313468933, + 0.08787597715854645, + 0.03154202923178673 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.7500980496406555, + "style_target": 1, + "support_probability": 0.8592947721481323, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4356541451665663, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11835955828428268, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8964:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9177237153053284, + 0.18377317488193512, + 0.4548778831958771, + 0.3808203339576721, + 0.06447035074234009, + 0.06335959583520889 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8964", + "split": "test", + "style_correct": 1, + "style_probability": 0.2891087234020233, + "style_target": 0, + "support_probability": 0.8711068034172058, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15868365521334304, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.16227181255817413, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32211220264434814, + 0.5031591057777405, + 0.32211220264434814, + 0.9704440832138062, + 0.2595241963863373 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 1, + "style_probability": 0.6124394536018372, + "style_target": 1, + "support_probability": 0.1909487545490265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.21308243109220198, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0789967030286789, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8971:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25056272745132446, + 0.4978109300136566, + 0.25056272745132446, + 0.986274778842926, + 0.07519634813070297, + 0.21805664896965027, + 0.07227960228919983, + 0.07804784178733826, + 0.07144545018672943 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8971", + "split": "test", + "style_correct": 1, + "style_probability": 0.3575664162635803, + "style_target": 0, + "support_probability": 0.3775743842124939, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11977512547784386, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6230665445327759, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7768659591674805, + 0.26172351837158203, + 0.47731560468673706, + 0.21695207059383392, + 0.0795929804444313, + 0.394212543964386 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.6901412010192871, + "style_target": 1, + "support_probability": 0.12029355019330978, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003258659354013052, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8618038296699524, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8976:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8428909182548523, + 0.2276092916727066, + 0.5136371850967407, + 0.26719656586647034, + 0.11155958473682404, + 0.11002055555582047 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:8976", + "split": "test", + "style_correct": 1, + "style_probability": 0.21974608302116394, + "style_target": 0, + "support_probability": 0.06329130381345749, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0034853791686473303, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8983448147773743, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.28660210967063904, + 0.2934302091598511, + 0.2198268622159958, + 0.2091147005558014, + 0.10061126202344894, + 0.060884471982717514, + 0.0774678885936737, + 0.08948356658220291, + 0.4293699860572815 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.45384904742240906, + "style_target": 1, + "support_probability": 0.014118709601461887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002142512444578414, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9338918328285217, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8979:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2859649658203125, + 0.27862492203712463, + 0.2285861223936081, + 0.2058076709508896, + 0.08061596751213074, + 0.06164555624127388, + 0.07860331237316132, + 0.10382824391126633, + 0.05956578254699707 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8979", + "split": "test", + "style_correct": 0, + "style_probability": 0.7495949864387512, + "style_target": 0, + "support_probability": 0.007837806828320026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007779242608566547, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4411839544773102, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.21439795196056366, + 0.5906230807304382, + 0.7538135051727295, + 0.1556185483932495, + 0.06727979332208633, + 0.05210862308740616, + 0.10829769819974899, + 0.05681441351771355, + 0.057153429836034775, + 0.11086233705282211, + 0.29731225967407227 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 0, + "style_probability": 0.4422506093978882, + "style_target": 1, + "support_probability": 0.02356477454304695, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014837312613109385, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3059036135673523, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.21031056344509125, + 0.6140722036361694, + 0.7866249680519104, + 0.16630806028842926, + 0.10206733644008636, + 0.21850070357322693 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8990", + "split": "test", + "style_correct": 1, + "style_probability": 0.25532564520835876, + "style_target": 0, + "support_probability": 0.06291121244430542, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8526725848138276, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13285291194915771, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3520120680332184, + 0.9616016745567322, + 0.21040453016757965, + 0.3860245645046234, + 0.04740102216601372 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 1, + "style_probability": 0.5959290266036987, + "style_target": 1, + "support_probability": 0.9155457019805908, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6012798921607114, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09019448608160019, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8993:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3587363660335541, + 0.9916011691093445, + 0.2007666826248169, + 0.3841513991355896, + 0.11979717016220093 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:8993", + "split": "test", + "style_correct": 1, + "style_probability": 0.3775588572025299, + "style_target": 0, + "support_probability": 0.9111414551734924, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9414260809938356, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08455558866262436, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8994:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3549354672431946, + 0.34885454177856445, + 0.9706182479858398, + 0.23247507214546204, + 0.026883205398917198, + 0.046230606734752655 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 1, + "style_probability": 0.640697717666626, + "style_target": 1, + "support_probability": 0.9698960781097412, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9759236119641912, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.08239050954580307, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:8994:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3209594488143921, + 0.329878032207489, + 0.9906888008117676, + 0.26730120182037354, + 0.044753897935152054, + 0.02593194879591465 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:8994", + "split": "test", + "style_correct": 0, + "style_probability": 0.89452064037323, + "style_target": 0, + "support_probability": 0.9412779808044434, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14348692826034612, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5132453441619873, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.22628164291381836, + 0.5635578632354736, + 0.36635157465934753, + 0.6878988146781921, + 0.06501128524541855, + 0.11133020371198654, + 0.0432816781103611, + 0.06316221505403519, + 0.06170867383480072 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.6145063638687134, + "style_target": 1, + "support_probability": 0.33624598383903503, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.260655410129516, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.28199678659439087, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:8995:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2134203463792801, + 0.5239495038986206, + 0.3986016809940338, + 0.6656150817871094, + 0.11666201055049896 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:8995", + "split": "test", + "style_correct": 1, + "style_probability": 0.34299731254577637, + "style_target": 0, + "support_probability": 0.7240095734596252, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5490129241055293, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23977141082286835, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3210010528564453, + 0.29254284501075745, + 0.22733956575393677, + 0.8628661036491394, + 0.038732849061489105, + 0.055290672928094864, + 0.11988255381584167, + 0.03496382012963295, + 0.04251471906900406, + 0.055432189255952835 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 0, + "style_probability": 0.4808253049850464, + "style_target": 1, + "support_probability": 0.6201745271682739, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08017951747700634, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3693683445453644, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:8999:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3929601013660431, + 0.40868720412254333, + 0.3500251770019531, + 0.9287773370742798, + 0.06499075144529343, + 0.07002253085374832, + 0.04300444945693016, + 0.05780642852187157, + 0.03077571839094162, + 0.09354794025421143 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:8999", + "split": "test", + "style_correct": 1, + "style_probability": 0.0956907793879509, + "style_target": 0, + "support_probability": 0.6610507965087891, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1836742575766106, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2925011217594147, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.28323861956596375, + 0.3786054253578186, + 0.18092012405395508, + 0.5275900959968567, + 0.08199265599250793 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.9008846282958984, + "style_target": 1, + "support_probability": 0.24703724682331085, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01036951337219039, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3725857138633728, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9005:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23292149603366852, + 0.39354151487350464, + 0.1322176605463028, + 0.49958205223083496, + 0.11242826282978058, + 0.06988845020532608, + 0.051957108080387115, + 0.0415666289627552, + 0.14126676321029663 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9005", + "split": "test", + "style_correct": 1, + "style_probability": 0.20415742695331573, + "style_target": 0, + "support_probability": 0.12930616736412048, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.15823021144635516, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.14151708781719208, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.37587276101112366, + 0.17874880135059357, + 0.4118749797344208, + 0.952595591545105, + 0.08605840057134628, + 0.18626092374324799 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.8331499099731445, + "style_target": 1, + "support_probability": 0.14613574743270874, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03535848645215878, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.13950394093990326, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9011:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.38942864537239075, + 0.17702023684978485, + 0.46847954392433167, + 0.9669066071510315, + 0.05953415483236313, + 0.08886948972940445 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9011", + "split": "test", + "style_correct": 1, + "style_probability": 0.4300166368484497, + "style_target": 0, + "support_probability": 0.2237778753042221, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04234937303110086, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.749844491481781, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.0880313366651535, + 0.30125099420547485, + 0.7831039428710938, + 0.2381318062543869, + 0.0392502099275589, + 0.09795495867729187, + 0.04225524887442589, + 0.049479890614748, + 0.058065976947546005 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 1, + "style_probability": 0.8131390810012817, + "style_target": 1, + "support_probability": 0.07437041401863098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01851901172560433, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7542580962181091, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9026:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.07973392307758331, + 0.266702264547348, + 0.7941924929618835, + 0.2257317304611206, + 0.117462158203125, + 0.06362983584403992, + 0.14312317967414856, + 0.12643620371818542, + 0.05330280587077141 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9026", + "split": "test", + "style_correct": 1, + "style_probability": 0.4542575180530548, + "style_target": 0, + "support_probability": 0.07734715193510056, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.061383453615704735, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9374198317527771, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.17058387398719788, + 0.4060407876968384, + 0.5522746443748474, + 0.5954428315162659, + 0.03445328772068024, + 0.04886528104543686, + 0.054503265768289566, + 0.06186015531420708, + 0.05602505803108215, + 0.09124478697776794 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.8319982290267944, + "style_target": 1, + "support_probability": 0.15364901721477509, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018842102852882906, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9352157115936279, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9043:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18217740952968597, + 0.40078985691070557, + 0.5013009905815125, + 0.5117652416229248, + 0.0955657884478569, + 0.07799366861581802 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9043", + "split": "test", + "style_correct": 1, + "style_probability": 0.47068461775779724, + "style_target": 0, + "support_probability": 0.10778635740280151, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.512956092439552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10151609778404236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9883732199668884, + 0.3197532594203949, + 0.1579485982656479, + 0.12704429030418396, + 0.06797980517148972 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.6934089660644531, + "style_target": 1, + "support_probability": 0.6513198018074036, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09054489823467902, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1459544599056244, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9044:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9962916374206543, + 0.38243183493614197, + 0.16291013360023499, + 0.16912928223609924, + 0.21503204107284546 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9044", + "split": "test", + "style_correct": 1, + "style_probability": 0.19027012586593628, + "style_target": 0, + "support_probability": 0.727897584438324, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.861366781336983, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13467642664909363, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16728784143924713, + 0.47154563665390015, + 0.9489278793334961, + 0.2442825436592102, + 0.1969325691461563, + 0.04229084774851799 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 1, + "style_probability": 0.7062647342681885, + "style_target": 1, + "support_probability": 0.8954458236694336, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8463860356632225, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.13122805953025818, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9046:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.22642458975315094, + 0.44123584032058716, + 0.9137904047966003, + 0.27741166949272156, + 0.06892812252044678, + 0.07136636227369308 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9046", + "split": "test", + "style_correct": 0, + "style_probability": 0.5804615616798401, + "style_target": 0, + "support_probability": 0.9314277768135071, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.0006351590940332014, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9893335700035095, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3609388470649719, + 0.27468281984329224, + 0.13066251575946808, + 0.27468281984329224, + 0.11404383182525635, + 0.09878279268741608, + 0.10656411945819855, + 0.26755544543266296, + 0.1191384494304657 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.7839096188545227, + "style_target": 1, + "support_probability": 0.007155830506235361, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0005615451675839436, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9650939106941223, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9056:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36543285846710205, + 0.28629574179649353, + 0.13373826444149017, + 0.28629574179649353, + 0.19787220656871796 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9056", + "split": "test", + "style_correct": 1, + "style_probability": 0.31834542751312256, + "style_target": 0, + "support_probability": 0.011071266606450081, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44285637765389657, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2401113212108612, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2871665954589844, + 0.3342207074165344, + 0.3258610665798187, + 0.8435919284820557, + 0.08924005180597305, + 0.19474442303180695, + 0.06352782249450684, + 0.060087960213422775, + 0.07216453552246094, + 0.17301753163337708, + 0.08014529198408127 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.7532979249954224, + "style_target": 1, + "support_probability": 0.478921115398407, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022066614399057327, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16283604502677917, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9063:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.36022061109542847, + 0.35759463906288147, + 0.3621518313884735, + 0.906495213508606, + 0.1303872913122177, + 0.0845155343413353, + 0.10556837171316147, + 0.1104627251625061, + 0.1113443672657013, + 0.17879515886306763 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9063", + "split": "test", + "style_correct": 1, + "style_probability": 0.1353653073310852, + "style_target": 0, + "support_probability": 0.45265570282936096, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.37911661743356007, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07930030673742294, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9660727977752686, + 0.14395643770694733, + 0.14395643770694733, + 0.08743749558925629, + 0.04077152535319328 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 1, + "style_probability": 0.5837683081626892, + "style_target": 1, + "support_probability": 0.4258767366409302, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2385605113476572, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15466850996017456, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9076:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9318022131919861, + 0.17015500366687775, + 0.17015500366687775, + 0.10577370971441269, + 0.04679335281252861 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9076", + "split": "test", + "style_correct": 0, + "style_probability": 0.5301088094711304, + "style_target": 0, + "support_probability": 0.2946193218231201, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.38575696497074935, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.14463824033737183, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.11375938355922699, + 0.20693564414978027, + 0.9646663069725037, + 0.5876545906066895, + 0.04841233789920807, + 0.04282718896865845 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 1, + "style_probability": 0.6101370453834534, + "style_target": 1, + "support_probability": 0.42063072323799133, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2598735149593274, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08195646107196808, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9078:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.11231143772602081, + 0.20367732644081116, + 0.9545992612838745, + 0.5691484212875366, + 0.04067825898528099, + 0.07278348505496979, + 0.13884608447551727, + 0.09060411155223846, + 0.02822992578148842, + 0.045801326632499695 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9078", + "split": "test", + "style_correct": 1, + "style_probability": 0.3887348771095276, + "style_target": 0, + "support_probability": 0.4622440040111542, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004965761065419904, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9607729911804199, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3406098783016205, + 0.3455718755722046, + 0.3582587242126465, + 0.661868691444397, + 0.0908314436674118, + 0.1029370129108429, + 0.1686927080154419, + 0.08341088891029358, + 0.10004255175590515 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.9066885113716125, + "style_target": 1, + "support_probability": 0.013406881131231785, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0002818824882055685, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9232177734375, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9079:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42422035336494446, + 0.3959709107875824, + 0.41295045614242554, + 0.7576521039009094, + 0.0901164710521698, + 0.190542072057724, + 0.1366189569234848, + 0.09510302543640137, + 0.21111422777175903 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9079", + "split": "test", + "style_correct": 1, + "style_probability": 0.21470096707344055, + "style_target": 0, + "support_probability": 0.0325467623770237, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9096926239479969, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14265573024749756, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9080:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9785813689231873, + 0.45576906204223633, + 0.37211352586746216, + 0.26900050044059753, + 0.11173613369464874, + 0.34032365679740906, + 0.05387367680668831, + 0.059604380279779434, + 0.15119197964668274, + 0.08056454360485077, + 0.07470130920410156 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 1, + "style_probability": 0.8702114820480347, + "style_target": 1, + "support_probability": 0.8271678686141968, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6778125709539324, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15763074159622192, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9080:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9619601368904114, + 0.4461079239845276, + 0.3537929952144623, + 0.2721342444419861, + 0.1759052872657776, + 0.10301893949508667, + 0.05977890267968178, + 0.12758396565914154, + 0.1403328776359558, + 0.07780157029628754, + 0.12216056883335114 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9080", + "split": "test", + "style_correct": 0, + "style_probability": 0.5502117276191711, + "style_target": 0, + "support_probability": 0.7962656617164612, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4943090220279167, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3216085433959961, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37619486451148987, + 0.8770290017127991, + 0.37308329343795776, + 0.34649449586868286, + 0.10156597197055817 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.22664116322994232, + "style_target": 1, + "support_probability": 0.8268436789512634, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8578492011339875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.3780631721019745, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9089:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3934580087661743, + 0.9168912768363953, + 0.4155004024505615, + 0.35782885551452637, + 0.04561246559023857 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9089", + "split": "test", + "style_correct": 0, + "style_probability": 0.8386721014976501, + "style_target": 0, + "support_probability": 0.7962292432785034, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4248281267120273, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.33828818798065186, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2121618539094925, + 0.4107571840286255, + 0.5623576641082764, + 0.8575052618980408, + 0.14777642488479614, + 0.1564638912677765 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 0, + "style_probability": 0.3998984098434448, + "style_target": 1, + "support_probability": 0.6878370046615601, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03387366186109641, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.334322452545166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9091:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3091847896575928, + 0.5343794822692871, + 0.7359788417816162, + 0.9432393908500671, + 0.0796402245759964, + 0.126538947224617 + ], + "selected_evidence_indices": [ + 2, + 3 + ], + "speaker_id": "snli:test:9091", + "split": "test", + "style_correct": 1, + "style_probability": 0.12884078919887543, + "style_target": 0, + "support_probability": 0.6638232469558716, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.008940678455285345, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4468003213405609, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7293432950973511, + 0.4004514813423157, + 0.20088525116443634, + 0.24986054003238678, + 0.041095070540905, + 0.050916433334350586, + 0.09418009966611862, + 0.13909608125686646, + 0.061230093240737915 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 0, + "style_probability": 0.2839091122150421, + "style_target": 1, + "support_probability": 0.043638262897729874, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.029105186197825313, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.38909339904785156, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9096:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7465127110481262, + 0.40402328968048096, + 0.24022942781448364, + 0.2810654938220978, + 0.04399232193827629, + 0.06699736416339874, + 0.07908078283071518, + 0.07376261800527573, + 0.08054979890584946 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9096", + "split": "test", + "style_correct": 0, + "style_probability": 0.521641731262207, + "style_target": 0, + "support_probability": 0.06789077818393707, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.058762718024093705, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6607273817062378, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.37191158533096313, + 0.3451324701309204, + 0.4007841646671295, + 0.9227226376533508, + 0.11992113292217255, + 0.0717318207025528, + 0.09362366795539856, + 0.1524486243724823, + 0.2342388927936554, + 0.11234539747238159 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 1, + "style_probability": 0.5370904207229614, + "style_target": 1, + "support_probability": 0.2122366726398468, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.353370585041322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6229403614997864, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9099:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.41406625509262085, + 0.3847236931324005, + 0.4054115116596222, + 0.9484170079231262, + 0.10802415758371353, + 0.04850207641720772, + 0.11614848673343658, + 0.03734132647514343, + 0.06942665576934814, + 0.05062715709209442 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9099", + "split": "test", + "style_correct": 0, + "style_probability": 0.9065316915512085, + "style_target": 0, + "support_probability": 0.3282516598701477, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.007110774179062004, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7430810928344727, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30104637145996094, + 0.31822022795677185, + 0.48671630024909973, + 0.4678550958633423, + 0.14925694465637207 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.8964975476264954, + "style_target": 1, + "support_probability": 0.018224913626909256, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005757160881250557, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6055856943130493, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9102:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36519283056259155, + 0.33703741431236267, + 0.6328256726264954, + 0.5587955713272095, + 0.09082862734794617 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9102", + "split": "test", + "style_correct": 1, + "style_probability": 0.35864049196243286, + "style_target": 0, + "support_probability": 0.06610479205846786, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.44046251110785306, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.07430414855480194, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4241279065608978, + 0.3517308235168457, + 0.4241279065608978, + 0.9975433349609375, + 0.06686993688344955, + 0.04973451793193817 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 1, + "style_probability": 0.6295676827430725, + "style_target": 1, + "support_probability": 0.5838111042976379, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4596039264525445, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.10081164538860321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9103:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4049774408340454, + 0.36088448762893677, + 0.4049774408340454, + 0.9920235872268677, + 0.10147954523563385, + 0.04052990674972534 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9103", + "split": "test", + "style_correct": 1, + "style_probability": 0.4962276816368103, + "style_target": 0, + "support_probability": 0.625980019569397, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.896618866495956, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1592249721288681, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25214630365371704, + 0.9618090391159058, + 0.26293206214904785, + 0.20557379722595215, + 0.03414075821638107, + 0.13048653304576874, + 0.026362761855125427, + 0.08603008836507797, + 0.08663872629404068 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 1, + "style_probability": 0.8764923810958862, + "style_target": 1, + "support_probability": 0.7925657629966736, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7337247819905954, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.10080014914274216, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9113:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23237843811511993, + 0.9511626958847046, + 0.2876642346382141, + 0.22482144832611084, + 0.06604033708572388 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9113", + "split": "test", + "style_correct": 0, + "style_probability": 0.5135526657104492, + "style_target": 0, + "support_probability": 0.8669506907463074, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.07556512324211619, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8763258457183838, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30789700150489807, + 0.8407700061798096, + 0.3759162724018097, + 0.27904731035232544, + 0.07498272508382797, + 0.08449563384056091, + 0.10639310628175735, + 0.12696409225463867, + 0.10676998645067215, + 0.1043742224574089, + 0.08371677249670029 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.8599581718444824, + "style_target": 1, + "support_probability": 0.08198515325784683, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0008606574033149377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9445210695266724, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9121:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.38108599185943604, + 0.8702245354652405, + 0.46158257126808167, + 0.33777570724487305, + 0.14850130677223206, + 0.12711885571479797, + 0.1599370539188385, + 0.13309244811534882, + 0.12745362520217896, + 0.12162166088819504 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9121", + "split": "test", + "style_correct": 1, + "style_probability": 0.1089959591627121, + "style_target": 0, + "support_probability": 0.07026083767414093, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.5591835788164481, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.07226701080799103, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.39918145537376404, + 0.8327121138572693, + 0.26195549964904785, + 0.24992848932743073, + 0.283822238445282 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.6064926385879517, + "style_target": 1, + "support_probability": 0.7222115993499756, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1953007790749517, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.072203129529953, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9125:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.449788898229599, + 0.9302055835723877, + 0.28489723801612854, + 0.2644645869731903, + 0.17043974995613098 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9125", + "split": "test", + "style_correct": 1, + "style_probability": 0.18197114765644073, + "style_target": 0, + "support_probability": 0.8166349530220032, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.49539286968673585, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1727045625448227, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5626394152641296, + 0.9143840670585632, + 0.12934274971485138, + 0.3273197412490845, + 0.06785723567008972, + 0.3010980188846588 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 1, + "style_probability": 0.5830165147781372, + "style_target": 1, + "support_probability": 0.49284690618515015, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6967487761705904, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.23340997099876404, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9129:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4977061450481415, + 0.9613478779792786, + 0.14041700959205627, + 0.2961929738521576, + 0.05823744833469391, + 0.38582906126976013 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9129", + "split": "test", + "style_correct": 0, + "style_probability": 0.8477727770805359, + "style_target": 0, + "support_probability": 0.45033904910087585, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.21917957484728492, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8323181867599487, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7225484251976013, + 0.18055805563926697, + 0.329875648021698, + 0.4146709442138672, + 0.12794016301631927, + 0.1599983274936676, + 0.11617624759674072, + 0.11286034435033798, + 0.2363586574792862 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 0, + "style_probability": 0.4470110833644867, + "style_target": 1, + "support_probability": 0.3585519790649414, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.10022257812114965, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7523877024650574, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9132:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7575033903121948, + 0.20045676827430725, + 0.3238472044467926, + 0.3896539807319641, + 0.23154249787330627 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9132", + "split": "test", + "style_correct": 1, + "style_probability": 0.20301102101802826, + "style_target": 0, + "support_probability": 0.3218255341053009, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6612769711330664, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16380377113819122, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2982560694217682, + 0.135300874710083, + 0.936539351940155, + 0.2982560694217682, + 0.07529585063457489, + 0.15385110676288605, + 0.11855480819940567, + 0.07925767451524734, + 0.10715824365615845, + 0.22501036524772644 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 0, + "style_probability": 0.44180575013160706, + "style_target": 1, + "support_probability": 0.8224388360977173, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.46850767630660606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1330982893705368, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9142:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.411858469247818, + 0.1758064329624176, + 0.9867807030677795, + 0.411858469247818, + 0.06682014465332031, + 0.04583313688635826, + 0.05018841102719307, + 0.08617103844881058, + 0.08731160312891006, + 0.13185974955558777 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9142", + "split": "test", + "style_correct": 1, + "style_probability": 0.18830984830856323, + "style_target": 0, + "support_probability": 0.9211393594741821, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.022608868473845693, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5526289939880371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1571461707353592, + 0.7229639291763306, + 0.13607950508594513, + 0.2432347536087036, + 0.15787464380264282 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.2966122627258301, + "style_target": 1, + "support_probability": 0.0648261159658432, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.25833818300168515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17939414083957672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9157:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1511223018169403, + 0.7596483826637268, + 0.13106943666934967, + 0.19869861006736755, + 0.12766817212104797 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9157", + "split": "test", + "style_correct": 0, + "style_probability": 0.7920365333557129, + "style_target": 0, + "support_probability": 0.3459448516368866, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6138711315532674, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1995469629764557, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.18091759085655212, + 0.31514620780944824, + 0.8911007642745972, + 0.3512756824493408, + 0.07097575813531876, + 0.0846961960196495 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 0, + "style_probability": 0.43271884322166443, + "style_target": 1, + "support_probability": 0.7642092704772949, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.22051611246623357, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.12478534877300262, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9158:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1956867277622223, + 0.4192195236682892, + 0.9642459154129028, + 0.40167227387428284, + 0.07769786566495895, + 0.11870313435792923 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9158", + "split": "test", + "style_correct": 1, + "style_probability": 0.1607564240694046, + "style_target": 0, + "support_probability": 0.8527864813804626, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.001079734873034324, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9797122478485107, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.21051359176635742, + 0.21131238341331482, + 0.468930721282959, + 0.5450932383537292, + 0.14336852729320526, + 0.13130465149879456, + 0.10352751612663269, + 0.11629887670278549, + 0.11984363943338394 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 0, + "style_probability": 0.30751487612724304, + "style_target": 1, + "support_probability": 0.008210636675357819, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.004674414289973061, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9566669464111328, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9171:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23144680261611938, + 0.26686447858810425, + 0.47806474566459656, + 0.5693831443786621, + 0.09128320962190628, + 0.13779094815254211, + 0.15797361731529236, + 0.11941739916801453, + 0.12435086816549301 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9171", + "split": "test", + "style_correct": 1, + "style_probability": 0.46503064036369324, + "style_target": 0, + "support_probability": 0.012510713189840317, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05883684928266674, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48031383752822876, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.14509853720664978, + 0.5985333323478699, + 0.27343571186065674, + 0.5617066621780396, + 0.08196312189102173, + 0.06531807035207748, + 0.05345221236348152, + 0.06900607794523239, + 0.19401372969150543, + 0.0899016410112381, + 0.08673730492591858 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.4439089596271515, + "style_target": 1, + "support_probability": 0.18694981932640076, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.09567442727212949, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.547797679901123, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9177:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.13322940468788147, + 0.587059736251831, + 0.2904825508594513, + 0.5920919179916382, + 0.058961670845746994, + 0.12130042910575867, + 0.07106718420982361, + 0.06468942761421204, + 0.229390487074852, + 0.15328757464885712, + 0.1359148621559143 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9177", + "split": "test", + "style_correct": 0, + "style_probability": 0.859437108039856, + "style_target": 0, + "support_probability": 0.11400985717773438, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3079636900698013, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3586386740207672, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.7252484560012817, + 0.34509557485580444, + 0.20154260098934174, + 0.19967599213123322, + 0.12439844012260437 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.8276380300521851, + "style_target": 1, + "support_probability": 0.3974299430847168, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.020034227049284814, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40525975823402405, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9184:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8168870210647583, + 0.4285776913166046, + 0.25803977251052856, + 0.24713335931301117, + 0.14894628524780273 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9184", + "split": "test", + "style_correct": 1, + "style_probability": 0.17561212182044983, + "style_target": 0, + "support_probability": 0.4176552891731262, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0042239261858273425, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9147001504898071, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1612127125263214, + 0.5217131972312927, + 0.3433152139186859, + 0.4652639329433441, + 0.11863496154546738, + 0.10420932620763779 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 1, + "style_probability": 0.7376631498336792, + "style_target": 1, + "support_probability": 0.031603921204805374, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011978402242470937, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8689776659011841, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9189:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19509786367416382, + 0.5058256387710571, + 0.3309084475040436, + 0.4059748649597168, + 0.13930588960647583, + 0.08933433145284653 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9189", + "split": "test", + "style_correct": 0, + "style_probability": 0.627075731754303, + "style_target": 0, + "support_probability": 0.030455032363533974, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.4203338487392063, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21427716314792633, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.33041128516197205, + 0.27941641211509705, + 0.889315664768219, + 0.3991028666496277, + 0.018674731254577637, + 0.04965256154537201, + 0.05022161826491356, + 0.26273998618125916, + 0.06114983558654785 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.851641833782196, + "style_target": 1, + "support_probability": 0.2967167794704437, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12845234904772518, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.201449453830719, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9194:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.32997584342956543, + 0.2730276584625244, + 0.8663130402565002, + 0.3640035390853882, + 0.06258931010961533 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9194", + "split": "test", + "style_correct": 1, + "style_probability": 0.4202730059623718, + "style_target": 0, + "support_probability": 0.27207064628601074, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.877007456953347, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03940487280488014, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7971270680427551, + 0.4843917489051819, + 0.5750880837440491, + 0.9857242703437805, + 0.02739708311855793, + 0.024495679885149002, + 0.059825118631124496, + 0.060273077338933945, + 0.2033068835735321, + 0.10254544764757156 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.7222959995269775, + "style_target": 1, + "support_probability": 0.9096754789352417, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4881779976140024, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04686635732650757, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9195:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7876201272010803, + 0.4237090051174164, + 0.5525532364845276, + 0.9896140694618225, + 0.07769540697336197, + 0.05673659220337868, + 0.08686373382806778, + 0.11028657853603363, + 0.07266849279403687, + 0.09927497059106827 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9195", + "split": "test", + "style_correct": 1, + "style_probability": 0.19420355558395386, + "style_target": 0, + "support_probability": 0.9412555694580078, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.7157746663257747, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.059273261576890945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18848468363285065, + 0.9721975922584534, + 0.11753951758146286, + 0.420343816280365, + 0.08781658113002777 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 1, + "style_probability": 0.7289982438087463, + "style_target": 1, + "support_probability": 0.7494141459465027, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6647151661620683, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.057409752160310745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9209:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18169404566287994, + 0.9356253147125244, + 0.1371198296546936, + 0.4008898437023163, + 0.10127574950456619 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9209", + "split": "test", + "style_correct": 1, + "style_probability": 0.4999160170555115, + "style_target": 0, + "support_probability": 0.7711378335952759, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06263031450766478, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9239792227745056, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.425869882106781, + 0.5479751825332642, + 0.7503995895385742, + 0.425869882106781, + 0.1562080681324005, + 0.11269841343164444 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.7126195430755615, + "style_target": 1, + "support_probability": 0.07920827716588974, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.016993324271119544, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.919299304485321, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9229:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.36955270171165466, + 0.5400721430778503, + 0.6953754425048828, + 0.36955270171165466, + 0.0866880714893341, + 0.045106031000614166, + 0.08620131015777588, + 0.09195433557033539, + 0.07875189185142517, + 0.13672924041748047, + 0.12086721509695053 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9229", + "split": "test", + "style_correct": 1, + "style_probability": 0.2616272568702698, + "style_target": 0, + "support_probability": 0.06404244899749756, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8626183583732648, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08628629148006439, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.0658264011144638, + 0.28048786520957947, + 0.9634549021720886, + 0.21109934151172638, + 0.031241701915860176, + 0.04890573024749756, + 0.037071116268634796, + 0.02120785228908062, + 0.0264167170971632 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 1, + "style_probability": 0.5250001549720764, + "style_target": 1, + "support_probability": 0.9533722400665283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.40876726116817963, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08806075900793076, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9234:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.09000576287508011, + 0.35076218843460083, + 0.9765081405639648, + 0.29839760065078735, + 0.03831612318754196, + 0.03461868688464165, + 0.07928390055894852, + 0.045953087508678436, + 0.09610822796821594 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9234", + "split": "test", + "style_correct": 1, + "style_probability": 0.1188996359705925, + "style_target": 0, + "support_probability": 0.95823734998703, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6315640582351207, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21984271705150604, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7801693081855774, + 0.23836961388587952, + 0.23050788044929504, + 0.23836961388587952, + 0.06704219430685043, + 0.08126144856214523, + 0.05551093816757202, + 0.06035761162638664, + 0.16057613492012024, + 0.08151388168334961, + 0.06610321253538132 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 1, + "style_probability": 0.8563843965530396, + "style_target": 1, + "support_probability": 0.7337631583213806, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4720298469166835, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2906534671783447, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9236:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7590996623039246, + 0.23355881869792938, + 0.18368220329284668, + 0.23355881869792938, + 0.0513402484357357, + 0.07721433788537979, + 0.07255437225103378, + 0.0760497972369194, + 0.06418811529874802, + 0.0701117292046547 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9236", + "split": "test", + "style_correct": 0, + "style_probability": 0.6488379836082458, + "style_target": 0, + "support_probability": 0.7669119834899902, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1630223219298399, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7014654278755188, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9842485785484314, + 0.2899312973022461, + 0.2544037103652954, + 0.22248519957065582, + 0.1129719465970993 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 1, + "style_probability": 0.5475354194641113, + "style_target": 1, + "support_probability": 0.43063369393348694, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.22440409112756401, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.41613832116127014, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9244:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9762232303619385, + 0.2788701355457306, + 0.23078316450119019, + 0.19129760563373566, + 0.050828367471694946, + 0.04939150810241699, + 0.04330882430076599, + 0.06551619619131088, + 0.0852658674120903 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9244", + "split": "test", + "style_correct": 1, + "style_probability": 0.19775964319705963, + "style_target": 0, + "support_probability": 0.6528348326683044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006883272806248621, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9393562078475952, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3320709764957428, + 0.160798579454422, + 0.2756368815898895, + 0.26465436816215515, + 0.09651408344507217, + 0.15204007923603058 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 1, + "style_probability": 0.8545843958854675, + "style_target": 1, + "support_probability": 0.018964644521474838, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00026389670858179687, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9930632710456848, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9248:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32852351665496826, + 0.2504946291446686, + 0.28983983397483826, + 0.31192177534103394, + 0.1412317305803299, + 0.1279166042804718 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9248", + "split": "test", + "style_correct": 1, + "style_probability": 0.4701688587665558, + "style_target": 0, + "support_probability": 0.005201422143727541, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03460272549794129, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6548921465873718, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23580525815486908, + 0.5542747974395752, + 0.2779398262500763, + 0.577802300453186, + 0.09928882867097855, + 0.12392086535692215, + 0.13783617317676544, + 0.0491766557097435, + 0.20269957184791565 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 1, + "style_probability": 0.7841413021087646, + "style_target": 1, + "support_probability": 0.07229211181402206, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.025927543777994444, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6337903141975403, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9251:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2813246548175812, + 0.4878522753715515, + 0.25307729840278625, + 0.5231955051422119, + 0.24388165771961212, + 0.12530408799648285, + 0.14037886261940002, + 0.08079983294010162, + 0.07766415178775787 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9251", + "split": "test", + "style_correct": 1, + "style_probability": 0.4909052848815918, + "style_target": 0, + "support_probability": 0.12661689519882202, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.3167312653608647, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5049576163291931, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45458194613456726, + 0.2853637635707855, + 0.7615626454353333, + 0.46920329332351685, + 0.06590752303600311, + 0.030419066548347473, + 0.05135947838425636, + 0.24633187055587769, + 0.07073018699884415, + 0.04975791275501251, + 0.11171022057533264 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.6926244497299194, + "style_target": 1, + "support_probability": 0.4953629970550537, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21376847336116, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.39552032947540283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9254:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.41221851110458374, + 0.2504432797431946, + 0.7421308755874634, + 0.45218536257743835, + 0.20958203077316284, + 0.06109393760561943 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9254", + "split": "test", + "style_correct": 1, + "style_probability": 0.24278487265110016, + "style_target": 0, + "support_probability": 0.6056392192840576, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.898794216980118, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08892148733139038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9256:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9518128037452698, + 0.23658746480941772, + 0.1364305019378662, + 0.2411198616027832, + 0.12312281876802444 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.9690873622894287, + "style_target": 1, + "support_probability": 0.663194477558136, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5492419131031454, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.009222484193742275, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9256:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9975622892379761, + 0.3036874830722809, + 0.24985215067863464, + 0.19310268759727478, + 0.22356192767620087 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9256", + "split": "test", + "style_correct": 1, + "style_probability": 0.24252921342849731, + "style_target": 0, + "support_probability": 0.9424454569816589, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.701377254379878, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17026637494564056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6702457666397095, + 0.9504872560501099, + 0.33874616026878357, + 0.3144436180591583, + 0.07575628161430359, + 0.11855831742286682 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 1, + "style_probability": 0.515093207359314, + "style_target": 1, + "support_probability": 0.8381291031837463, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.814382615600195, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13214640319347382, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9265:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5825781226158142, + 0.9283978939056396, + 0.4045378267765045, + 0.2643319070339203, + 0.09843592345714569, + 0.11271540820598602 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9265", + "split": "test", + "style_correct": 0, + "style_probability": 0.5343347787857056, + "style_target": 0, + "support_probability": 0.8928350210189819, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8969167089370629, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1637619286775589, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9849084615707397, + 0.35520103573799133, + 0.5107802748680115, + 0.6229735016822815, + 0.03846216946840286, + 0.04817570000886917, + 0.0343063622713089, + 0.06890139728784561, + 0.08398960530757904 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.8386397957801819, + "style_target": 1, + "support_probability": 0.7677685022354126, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5561122868224615, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17639853060245514, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9268:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.976502537727356, + 0.3367542326450348, + 0.49207770824432373, + 0.5947311520576477, + 0.399985671043396 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9268", + "split": "test", + "style_correct": 1, + "style_probability": 0.4079612195491791, + "style_target": 0, + "support_probability": 0.7654817700386047, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.058259700234337686, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7769156694412231, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.20630241930484772, + 0.7285737991333008, + 0.7111751437187195, + 0.3448089361190796, + 0.11883416026830673, + 0.059003446251153946, + 0.07765226066112518, + 0.06918760389089584, + 0.10475948452949524, + 0.09558571875095367 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.8368134498596191, + "style_target": 1, + "support_probability": 0.1302417367696762, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0021770133652347086, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8448488116264343, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9289:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.2175631821155548, + 0.7857281565666199, + 0.7085354924201965, + 0.44162577390670776, + 0.2160998433828354, + 0.1452013999223709, + 0.18407504260540009, + 0.11820321530103683, + 0.1345614492893219, + 0.1553467959165573 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9289", + "split": "test", + "style_correct": 1, + "style_probability": 0.23610500991344452, + "style_target": 0, + "support_probability": 0.17831648886203766, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10424476448323008, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.39449241757392883, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21701328456401825, + 0.5051876902580261, + 0.28442245721817017, + 0.6946790218353271, + 0.144587442278862 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 0, + "style_probability": 0.3234718143939972, + "style_target": 1, + "support_probability": 0.5224664807319641, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13548733162993543, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.3925393223762512, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9295:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.18623438477516174, + 0.47645068168640137, + 0.22821494936943054, + 0.6954168081283569, + 0.060843102633953094 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9295", + "split": "test", + "style_correct": 1, + "style_probability": 0.39746353030204773, + "style_target": 0, + "support_probability": 0.5599981546401978, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.006070925507866478, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8014377951622009, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5075228810310364, + 0.14291690289974213, + 0.1761806160211563, + 0.34534329175949097, + 0.035865720361471176, + 0.09552726149559021 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.3193131983280182, + "style_target": 1, + "support_probability": 0.0226463433355093, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015868578774908434, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8708621859550476, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9304:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.551174521446228, + 0.13859796524047852, + 0.20127913355827332, + 0.3492316007614136, + 0.054048504680395126, + 0.20446546375751495 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9304", + "split": "test", + "style_correct": 0, + "style_probability": 0.8067196607589722, + "style_target": 0, + "support_probability": 0.02133159525692463, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.1801992444898204, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.28761887550354004, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5078136920928955, + 0.46872416138648987, + 0.3309839963912964, + 0.2844567596912384, + 0.09272397309541702, + 0.08164728432893753, + 0.09315747767686844, + 0.028700876981019974, + 0.1229291707277298 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 1, + "style_probability": 0.5634328722953796, + "style_target": 1, + "support_probability": 0.28610509634017944, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12897827812644372, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2923453748226166, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9305:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5180508494377136, + 0.6587898135185242, + 0.39326566457748413, + 0.23393520712852478, + 0.0410366915166378, + 0.049910206347703934, + 0.06349367648363113, + 0.03889939561486244, + 0.08612918108701706 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9305", + "split": "test", + "style_correct": 1, + "style_probability": 0.3445155620574951, + "style_target": 0, + "support_probability": 0.5081197023391724, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.017115119804405946, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.820529580116272, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.281773179769516, + 0.2696704566478729, + 0.17946091294288635, + 0.9408952593803406, + 0.14988663792610168, + 0.449569433927536, + 0.18050923943519592, + 0.21138699352741241, + 0.2079373300075531, + 0.05206097289919853, + 0.231795072555542 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 0, + "style_probability": 0.22857511043548584, + "style_target": 1, + "support_probability": 0.08689045161008835, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18310626806260544, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6063335537910461, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9311:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3184300363063812, + 0.24620838463306427, + 0.20030778646469116, + 0.9376757144927979, + 0.1243225559592247, + 0.10131487995386124, + 0.11732524633407593, + 0.12892475724220276, + 0.08962520956993103, + 0.17237712442874908 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9311", + "split": "test", + "style_correct": 1, + "style_probability": 0.4338325262069702, + "style_target": 0, + "support_probability": 0.24153341352939606, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06141944243278986, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7573873400688171, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13197322189807892, + 0.2282589077949524, + 0.38565313816070557, + 0.35196107625961304, + 0.2135039120912552 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.3199303448200226, + "style_target": 1, + "support_probability": 0.3516089618206024, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10765050478272543, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8129898309707642, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9314:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.14663852751255035, + 0.23017166554927826, + 0.417006254196167, + 0.3260243535041809, + 0.09206189215183258 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9314", + "split": "test", + "style_correct": 0, + "style_probability": 0.8969044089317322, + "style_target": 0, + "support_probability": 0.1806357353925705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02537270375908895, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8171623349189758, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3418765366077423, + 0.19840799272060394, + 0.6180295348167419, + 0.3275245428085327, + 0.11649816483259201, + 0.1521933525800705 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 0, + "style_probability": 0.48114654421806335, + "style_target": 1, + "support_probability": 0.09305009990930557, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.021415947108390794, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.4916034936904907, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9318:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5038672089576721, + 0.2772683799266815, + 0.8198657631874084, + 0.5225239396095276, + 0.16604392230510712, + 0.1622375100851059 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9318", + "split": "test", + "style_correct": 1, + "style_probability": 0.30652084946632385, + "style_target": 0, + "support_probability": 0.40099191665649414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0031456023851718366, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.923488974571228, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.35069021582603455, + 0.7271535992622375, + 0.559298574924469, + 0.31709858775138855, + 0.1716623157262802, + 0.11326460540294647, + 0.18663735687732697, + 0.10070910304784775, + 0.14937008917331696 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 0, + "style_probability": 0.2018212080001831, + "style_target": 1, + "support_probability": 0.030375201255083084, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002853722167515892, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9390671253204346, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9322:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.44423535466194153, + 0.7690815925598145, + 0.6061291098594666, + 0.3450835347175598, + 0.12092335522174835, + 0.24605518579483032, + 0.22634761035442352, + 0.15675678849220276, + 0.1623786985874176 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9322", + "split": "test", + "style_correct": 1, + "style_probability": 0.4641331136226654, + "style_target": 0, + "support_probability": 0.018500804901123047, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.2688739237616229, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06198876351118088, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.15882381796836853, + 0.2523121237754822, + 0.24518521130084991, + 0.9605770707130432, + 0.10008291155099869, + 0.07440010458230972, + 0.0888112485408783, + 0.15538306534290314, + 0.09261342138051987, + 0.11120814085006714 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.25441184639930725, + "style_target": 1, + "support_probability": 0.6941777467727661, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.6877906946845087, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09464304894208908, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9323:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.17220152914524078, + 0.25655943155288696, + 0.2707234025001526, + 0.9706106185913086, + 0.06461139768362045, + 0.1348438709974289, + 0.06358111649751663, + 0.12128069251775742, + 0.1043008342385292, + 0.10160542279481888, + 0.09562589228153229 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9323", + "split": "test", + "style_correct": 0, + "style_probability": 0.8370977640151978, + "style_target": 0, + "support_probability": 0.6278030872344971, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4441299765576278, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.15179207921028137, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2946741580963135, + 0.27383726835250854, + 0.2838268280029297, + 0.9052643179893494, + 0.0430857352912426 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.7996950149536133, + "style_target": 1, + "support_probability": 0.26611530780792236, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.053178407606019995, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.21873408555984497, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9327:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3047944903373718, + 0.25199365615844727, + 0.30199307203292847, + 0.916959822177887, + 0.10318233072757721 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9327", + "split": "test", + "style_correct": 1, + "style_probability": 0.3254488408565521, + "style_target": 0, + "support_probability": 0.21782803535461426, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00786073626475869, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6389641761779785, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6408473253250122, + 0.6800988912582397, + 0.2777557969093323, + 0.5753194689750671, + 0.19384142756462097, + 0.07944194227457047 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 1, + "style_probability": 0.5798072814941406, + "style_target": 1, + "support_probability": 0.0368976928293705, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011241660956620153, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6455038785934448, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9329:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6179721355438232, + 0.6673610210418701, + 0.3087383210659027, + 0.5804129242897034, + 0.08837305754423141, + 0.0755273848772049 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9329", + "split": "test", + "style_correct": 1, + "style_probability": 0.3996317684650421, + "style_target": 0, + "support_probability": 0.042899131774902344, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.017002767813099066, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6382562518119812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3418225347995758, + 0.4668079614639282, + 0.9405779838562012, + 0.5969311594963074, + 0.08321798592805862, + 0.10489250719547272, + 0.07942894101142883, + 0.1365717500448227, + 0.10515547543764114 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.6145368814468384, + "style_target": 1, + "support_probability": 0.02020474150776863, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01476780919384191, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.343016654253006, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9330:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.35588186979293823, + 0.4438193440437317, + 0.9287527203559875, + 0.5416085720062256, + 0.12318067997694016 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9330", + "split": "test", + "style_correct": 1, + "style_probability": 0.24640975892543793, + "style_target": 0, + "support_probability": 0.02914973720908165, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.14312933122003635, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8369171619415283, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7967342138290405, + 0.12577076256275177, + 0.37781545519828796, + 0.188444584608078, + 0.05173434317111969, + 0.09683645516633987, + 0.08381902426481247, + 0.0900355875492096, + 0.1163877472281456, + 0.1617191582918167, + 0.07103264331817627 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.8648617267608643, + "style_target": 1, + "support_probability": 0.11196025460958481, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04189947750423073, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.48771554231643677, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9332:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.8933908939361572, + 0.15791204571723938, + 0.46631118655204773, + 0.2115674465894699, + 0.1067342758178711, + 0.07827040553092957, + 0.15518780052661896, + 0.06667402386665344, + 0.06039983779191971, + 0.12995021045207977 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9332", + "split": "test", + "style_correct": 1, + "style_probability": 0.3810412585735321, + "style_target": 0, + "support_probability": 0.33896955847740173, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001585498212769084, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.6630683541297913, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38783976435661316, + 0.3924374580383301, + 0.38783976435661316, + 0.4916791617870331, + 0.0650579184293747 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 1, + "style_probability": 0.6818379163742065, + "style_target": 1, + "support_probability": 0.004350551404058933, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005577966659087494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4722771644592285, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9339:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4106142520904541, + 0.3984992206096649, + 0.4106142520904541, + 0.4903184771537781, + 0.2822122871875763 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9339", + "split": "test", + "style_correct": 0, + "style_probability": 0.5549431443214417, + "style_target": 0, + "support_probability": 0.013559240847826004, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8455925781957951, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.03587386757135391, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.8067513704299927, + 0.938254177570343, + 0.4716958999633789, + 0.16473521292209625, + 0.10545402765274048, + 0.08088277280330658 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 1, + "style_probability": 0.8779657483100891, + "style_target": 1, + "support_probability": 0.8113874793052673, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.24863237634001223, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1651500016450882, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9341:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7404496669769287, + 0.8704639673233032, + 0.6032021641731262, + 0.2124778777360916, + 0.17149430513381958, + 0.12015870213508606, + 0.16595558822155, + 0.16906791925430298, + 0.15763650834560394, + 0.1343151330947876 + ], + "selected_evidence_indices": [ + 0, + 1 + ], + "speaker_id": "snli:test:9341", + "split": "test", + "style_correct": 1, + "style_probability": 0.402023047208786, + "style_target": 0, + "support_probability": 0.5869405269622803, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.19306041504571958, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5202081799507141, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3175475299358368, + 0.29963237047195435, + 0.38052666187286377, + 0.563630998134613, + 0.04020855203270912, + 0.020378265529870987, + 0.034677427262067795, + 0.03676619008183479, + 0.04143161326646805 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.7815670371055603, + "style_target": 1, + "support_probability": 0.2621769309043884, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0035605389344510754, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.604418933391571, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9343:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.3381732404232025, + 0.36913204193115234, + 0.45099636912345886, + 0.6048526763916016, + 0.05425636097788811, + 0.058393556624650955, + 0.11165714263916016, + 0.055345457047224045, + 0.11299819499254227 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9343", + "split": "test", + "style_correct": 1, + "style_probability": 0.09975668787956238, + "style_target": 0, + "support_probability": 0.2524506449699402, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9499684352738208, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13282792270183563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9347:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.41641944646835327, + 0.1296190321445465, + 0.22456414997577667, + 0.9763086438179016, + 0.055553242564201355, + 0.05496660992503166, + 0.06189907342195511, + 0.06988119333982468, + 0.06790295988321304, + 0.047586120665073395, + 0.0686490535736084 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 1, + "style_probability": 0.9349054098129272, + "style_target": 1, + "support_probability": 0.8813185095787048, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8666811194060456, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2583726942539215, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9347:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4096551239490509, + 0.12461696565151215, + 0.20246657729148865, + 0.9436766505241394, + 0.055521104484796524, + 0.09062818437814713, + 0.08031066507101059, + 0.1373184621334076, + 0.08244553953409195, + 0.06810217350721359, + 0.3345939517021179 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9347", + "split": "test", + "style_correct": 0, + "style_probability": 0.7737007737159729, + "style_target": 0, + "support_probability": 0.8476391434669495, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9473259669405926, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0905572772026062, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9349:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2725147604942322, + 0.9882082343101501, + 0.5130617618560791, + 0.3054483234882355, + 0.2295653522014618 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.8023216128349304, + "style_target": 1, + "support_probability": 0.9332193732261658, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7963915712641665, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05852353572845459, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9349:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25389108061790466, + 0.9817841053009033, + 0.4750790297985077, + 0.34167471528053284, + 0.3865264058113098, + 0.22923021018505096, + 0.14655983448028564, + 0.08808174729347229, + 0.1525661051273346 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9349", + "split": "test", + "style_correct": 1, + "style_probability": 0.22704440355300903, + "style_target": 0, + "support_probability": 0.9445548057556152, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00400666782428735, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9444355964660645, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3572646379470825, + 0.7527374625205994, + 0.1368655115365982, + 0.07322601974010468, + 0.14488352835178375, + 0.08129037171602249 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.7282412052154541, + "style_target": 1, + "support_probability": 0.015855401754379272, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0007844395411685942, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9316630363464355, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9357:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3912484645843506, + 0.8162497282028198, + 0.1921638548374176, + 0.08969242870807648, + 0.09569103270769119, + 0.1781151294708252 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9357", + "split": "test", + "style_correct": 1, + "style_probability": 0.12909579277038574, + "style_target": 0, + "support_probability": 0.04792005196213722, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7489019642545607, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1878013163805008, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.43121063709259033, + 0.9550408720970154, + 0.1793162077665329, + 0.6314972043037415, + 0.04494420811533928, + 0.0595838837325573, + 0.11092013865709305, + 0.04595734551548958, + 0.04948517307639122 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 1, + "style_probability": 0.7633935213088989, + "style_target": 1, + "support_probability": 0.5251865386962891, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.601064734103133, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35514068603515625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9385:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.4107149541378021, + 0.9369828701019287, + 0.1976468861103058, + 0.6177586913108826, + 0.055115364491939545, + 0.15074165165424347, + 0.06771281361579895, + 0.07539460062980652, + 0.05919588357210159 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9385", + "split": "test", + "style_correct": 0, + "style_probability": 0.6762701272964478, + "style_target": 0, + "support_probability": 0.362958699464798, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1021774875407573, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7574234008789062, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.12261717766523361, + 0.20068424940109253, + 0.34991455078125, + 0.12261717766523361, + 0.12627670168876648, + 0.06470993161201477, + 0.034475646913051605, + 0.0656108558177948, + 0.08428069949150085, + 0.06666018068790436, + 0.04196548834443092 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.894253671169281, + "style_target": 1, + "support_probability": 0.1766137331724167, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07178832980329596, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6322636604309082, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9394:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.12709201872348785, + 0.20545701682567596, + 0.3172968924045563, + 0.12709201872348785, + 0.026626460254192352, + 0.25162628293037415 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9394", + "split": "test", + "style_correct": 1, + "style_probability": 0.39074617624282837, + "style_target": 0, + "support_probability": 0.31014513969421387, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.8071653285750884, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.08859540522098541, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9399:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.1647893190383911, + 0.3104541301727295, + 0.16911067068576813, + 0.9954527616500854, + 0.25881630182266235 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.8715782165527344, + "style_target": 1, + "support_probability": 0.646703839302063, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.30008789099025535, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.10274676233530045, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9399:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.13943544030189514, + 0.35574445128440857, + 0.15319187939167023, + 0.9980182647705078, + 0.0457308255136013 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9399", + "split": "test", + "style_correct": 1, + "style_probability": 0.37109625339508057, + "style_target": 0, + "support_probability": 0.6971402168273926, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.39927424708006143, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1689557284116745, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.956248939037323, + 0.3464116156101227, + 0.3660928010940552, + 0.14056368172168732, + 0.0924311950802803, + 0.14631278812885284 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.6431227922439575, + "style_target": 1, + "support_probability": 0.5537753105163574, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.37371738186255854, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15351426601409912, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9400:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9264739155769348, + 0.4292301535606384, + 0.38760843873023987, + 0.18026255071163177, + 0.09356427937746048, + 0.24954268336296082 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9400", + "split": "test", + "style_correct": 1, + "style_probability": 0.398912638425827, + "style_target": 0, + "support_probability": 0.6789730191230774, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6556705684442399, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19945865869522095, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2556009590625763, + 0.3023046851158142, + 0.9128337502479553, + 0.26908934116363525, + 0.06286334246397018, + 0.10890684276819229, + 0.06233237311244011, + 0.05579907074570656, + 0.13339266180992126 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.6436204314231873, + "style_target": 1, + "support_probability": 0.6983527541160583, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.42246685504230713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.1828886717557907, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9402:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2365380972623825, + 0.38697168231010437, + 0.8895518183708191, + 0.2768732011318207, + 0.2643657922744751 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9402", + "split": "test", + "style_correct": 1, + "style_probability": 0.2570553719997406, + "style_target": 0, + "support_probability": 0.712338924407959, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2930111452253428, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43502527475357056, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.39273199439048767, + 0.6311891078948975, + 0.23705273866653442, + 0.4029313921928406, + 0.03295082226395607, + 0.10521568357944489, + 0.0949423685669899, + 0.09527366608381271, + 0.05658994987607002, + 0.1328948736190796, + 0.2907657325267792 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.9198063611984253, + "style_target": 1, + "support_probability": 0.21316519379615784, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.015476532609537408, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40077561140060425, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9405:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3131667375564575, + 0.7543704509735107, + 0.2796746790409088, + 0.3481011986732483, + 0.06599661707878113, + 0.06799722462892532, + 0.1769682914018631, + 0.0683489516377449, + 0.050515130162239075, + 0.1428481936454773 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9405", + "split": "test", + "style_correct": 1, + "style_probability": 0.16351552307605743, + "style_target": 0, + "support_probability": 0.4927861988544464, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.04814220292293837, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.908658504486084, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4170458912849426, + 0.7035130262374878, + 0.3331145942211151, + 0.2566044330596924, + 0.1086999848484993 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 1, + "style_probability": 0.7648906707763672, + "style_target": 1, + "support_probability": 0.03686908259987831, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07115785688471908, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8397071957588196, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9409:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.42711183428764343, + 0.6845667362213135, + 0.3264196217060089, + 0.2874496877193451, + 0.10656221210956573 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9409", + "split": "test", + "style_correct": 0, + "style_probability": 0.5769522786140442, + "style_target": 0, + "support_probability": 0.06591218709945679, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9358463962116162, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.057564299553632736, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9413:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.06390326470136642, + 0.9881887435913086, + 0.748665452003479, + 0.06390326470136642, + 0.021583538502454758, + 0.030639905482530594 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.8487927913665771, + "style_target": 1, + "support_probability": 0.9240692257881165, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7430304648446651, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0466134287416935, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9413:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.06446854770183563, + 0.9818502068519592, + 0.7620634436607361, + 0.06446854770183563, + 0.07892127335071564, + 0.1294429749250412, + 0.09249301254749298, + 0.057204119861125946, + 0.05555007606744766, + 0.03478533402085304, + 0.03959526866674423 + ], + "selected_evidence_indices": [ + 1, + 2 + ], + "speaker_id": "snli:test:9413", + "split": "test", + "style_correct": 1, + "style_probability": 0.3367512822151184, + "style_target": 0, + "support_probability": 0.9385626316070557, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5035309616575958, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2101074755191803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.553458034992218, + 0.6084626317024231, + 0.6612201929092407, + 0.6084626317024231, + 0.031320348381996155, + 0.0764109343290329, + 0.03255293145775795, + 0.04291900619864464, + 0.044812437146902084 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.8973858952522278, + "style_target": 1, + "support_probability": 0.5878928303718567, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010734656507363494, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5076544284820557, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9448:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5147016644477844, + 0.567692220211029, + 0.6033880114555359, + 0.567692220211029, + 0.0681840181350708, + 0.07721702009439468, + 0.08623579144477844, + 0.08614335209131241, + 0.1505122035741806 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9448", + "split": "test", + "style_correct": 1, + "style_probability": 0.1551084816455841, + "style_target": 0, + "support_probability": 0.5847199559211731, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.44437354113853184, + "accept_target": 1, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39292478561401367, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.3170689344406128, + 0.5852937698364258, + 0.23638229072093964, + 0.3585180938243866, + 0.07999292761087418, + 0.05850980430841446, + 0.03133125975728035, + 0.11316520720720291, + 0.0492466539144516, + 0.057300299406051636, + 0.050680533051490784 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 1, + "style_probability": 0.8586355447769165, + "style_target": 1, + "support_probability": 0.646040141582489, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3207912628515288, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4869600832462311, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9453:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.4063062071800232, + 0.52662593126297, + 0.3342042863368988, + 0.34336286783218384, + 0.16175369918346405, + 0.04949938505887985, + 0.07431977987289429, + 0.04005131870508194, + 0.036354515701532364, + 0.0795513316988945 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9453", + "split": "test", + "style_correct": 0, + "style_probability": 0.6156899333000183, + "style_target": 0, + "support_probability": 0.6544106602668762, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8850145820251427, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11978946626186371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9881133437156677, + 0.42185768485069275, + 0.25353118777275085, + 0.2742437422275543, + 0.04881124198436737 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 1, + "style_probability": 0.6012168526649475, + "style_target": 1, + "support_probability": 0.9049948453903198, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.837909821953872, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12207827717065811, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9472:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9976615905761719, + 0.3231457471847534, + 0.32269078493118286, + 0.3421054780483246, + 0.08524295687675476 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9472", + "split": "test", + "style_correct": 0, + "style_probability": 0.507125735282898, + "style_target": 0, + "support_probability": 0.8881844282150269, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.010467174007906978, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6064419150352478, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17100916802883148, + 0.27342337369918823, + 0.4534599483013153, + 0.6631307601928711, + 0.054980095475912094, + 0.08254876732826233 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.5891132354736328, + "style_target": 1, + "support_probability": 0.03374023735523224, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.01543425450502566, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6067496538162231, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9475:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.1938512623310089, + 0.27977821230888367, + 0.4490508735179901, + 0.6956163048744202, + 0.13875581324100494, + 0.08113671839237213, + 0.0841807872056961, + 0.05606875568628311, + 0.09435475617647171, + 0.12452559918165207 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9475", + "split": "test", + "style_correct": 1, + "style_probability": 0.3802691400051117, + "style_target": 0, + "support_probability": 0.03849606215953827, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0344065663738018, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4177874028682709, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.23788496851921082, + 0.22992828488349915, + 0.33808162808418274, + 0.7528467774391174, + 0.05833891034126282, + 0.08268947899341583, + 0.062412310391664505, + 0.0576426163315773, + 0.07370062917470932 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 0, + "style_probability": 0.4471495449542999, + "style_target": 1, + "support_probability": 0.09966021031141281, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.018963338703538613, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24043519794940948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9479:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.24936242401599884, + 0.2854028642177582, + 0.3256468176841736, + 0.9527767896652222, + 0.10183630138635635, + 0.18493430316448212, + 0.11400797218084335, + 0.08660232275724411, + 0.12345745414495468 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9479", + "split": "test", + "style_correct": 1, + "style_probability": 0.12314587086439133, + "style_target": 0, + "support_probability": 0.23474855720996857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06034885312307161, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4963725209236145, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7479416728019714, + 0.17161519825458527, + 0.25761422514915466, + 0.5164004564285278, + 0.15034250915050507, + 0.15381796658039093, + 0.11066654324531555, + 0.10901401191949844, + 0.15193095803260803, + 0.07264561951160431 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 1, + "style_probability": 0.6080166101455688, + "style_target": 1, + "support_probability": 0.12608787417411804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1801938348553467, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.34028562903404236, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9483:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.8024498820304871, + 0.1708131581544876, + 0.2764367461204529, + 0.6354501843452454, + 0.051951274275779724, + 0.08223321288824081, + 0.09676643460988998, + 0.1084955632686615, + 0.0870094895362854, + 0.09728845953941345, + 0.19366367161273956 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9483", + "split": "test", + "style_correct": 0, + "style_probability": 0.7098397612571716, + "style_target": 0, + "support_probability": 0.21324799954891205, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.053452688549745275, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7447307109832764, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6221767663955688, + 0.2714346945285797, + 0.6430299282073975, + 0.22252845764160156, + 0.07543275505304337 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.5065444707870483, + "style_target": 1, + "support_probability": 0.088937908411026, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.009208199523509653, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.901284396648407, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9484:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5839065909385681, + 0.2599049508571625, + 0.6419321894645691, + 0.2278842329978943, + 0.0920138955116272, + 0.09821771830320358, + 0.10181739181280136, + 0.15043509006500244, + 0.13753321766853333 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9484", + "split": "test", + "style_correct": 1, + "style_probability": 0.33691418170928955, + "style_target": 0, + "support_probability": 0.026899544522166252, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07010308945726429, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5794544816017151, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6753613352775574, + 0.3028421103954315, + 0.6063373684883118, + 0.344186931848526, + 0.07555460929870605, + 0.08243666589260101 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.6618683338165283, + "style_target": 1, + "support_probability": 0.08370861411094666, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.007806857182187376, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6588669419288635, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9488:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7538076043128967, + 0.3078289330005646, + 0.8691588044166565, + 0.27246198058128357, + 0.07671628892421722, + 0.19125892221927643 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9488", + "split": "test", + "style_correct": 1, + "style_probability": 0.38198500871658325, + "style_target": 0, + "support_probability": 0.08135531842708588, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.38019186006362027, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18284963071346283, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20119576156139374, + 0.8628756403923035, + 0.3063376843929291, + 0.3290292024612427, + 0.03996000066399574, + 0.06221027672290802, + 0.08790795505046844, + 0.05600748956203461, + 0.05549701303243637 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 1, + "style_probability": 0.5578128099441528, + "style_target": 1, + "support_probability": 0.33073779940605164, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5612093459548504, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.21054865419864655, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9489:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1506427824497223, + 0.9138172268867493, + 0.31435829401016235, + 0.34702014923095703, + 0.08253347128629684, + 0.031060805544257164, + 0.247950479388237, + 0.07146867364645004, + 0.1485542505979538 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9489", + "split": "test", + "style_correct": 0, + "style_probability": 0.728610098361969, + "style_target": 0, + "support_probability": 0.4139110743999481, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.47406589340061694, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13530786335468292, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.18847475945949554, + 0.9812825918197632, + 0.5872814059257507, + 0.18847475945949554, + 0.03465736657381058, + 0.033604737371206284, + 0.028281720355153084, + 0.12422842532396317, + 0.09521059691905975, + 0.11022517830133438, + 0.05211808159947395 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 1, + "style_probability": 0.6491981744766235, + "style_target": 1, + "support_probability": 0.7318897247314453, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.26544232798869416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14801140129566193, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9493:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19319318234920502, + 0.9819111227989197, + 0.6222249865531921, + 0.19319318234920502, + 0.034918516874313354, + 0.029078572988510132 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9493", + "split": "test", + "style_correct": 1, + "style_probability": 0.42506393790245056, + "style_target": 0, + "support_probability": 0.7345595359802246, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.39904173783491004, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.35988813638687134, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.635280430316925, + 0.6049902439117432, + 0.23485475778579712, + 0.2756959795951843, + 0.03431873396039009 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.6678802371025085, + "style_target": 1, + "support_probability": 0.6156615018844604, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3951445906779725, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4109773337841034, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9524:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5160662531852722, + 0.6425760388374329, + 0.21460208296775818, + 0.2854197025299072, + 0.293549507856369 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9524", + "split": "test", + "style_correct": 1, + "style_probability": 0.4199502170085907, + "style_target": 0, + "support_probability": 0.7159563302993774, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00023404488336659443, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9772692918777466, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2454843819141388, + 0.41977542638778687, + 0.3057387173175812, + 0.2454843819141388, + 0.07741016149520874, + 0.07059550285339355 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 0, + "style_probability": 0.3316945731639862, + "style_target": 1, + "support_probability": 0.003740786574780941, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003978781999651849, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7669267058372498, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9530:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2531917095184326, + 0.40628665685653687, + 0.32672643661499023, + 0.2531917095184326, + 0.05331851914525032, + 0.0887470543384552, + 0.15594634413719177, + 0.0968702882528305, + 0.048921480774879456, + 0.3313978612422943, + 0.12666809558868408 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9530", + "split": "test", + "style_correct": 1, + "style_probability": 0.3537357449531555, + "style_target": 0, + "support_probability": 0.019568802788853645, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3342930157267787, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8304561972618103, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6220918893814087, + 0.34479230642318726, + 0.44768160581588745, + 0.36669889092445374, + 0.05252181738615036, + 0.06447046250104904, + 0.08787072449922562, + 0.05483458936214447, + 0.061082761734724045 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.9609313607215881, + "style_target": 1, + "support_probability": 0.17836181819438934, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10930729549266438, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46217095851898193, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9532:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7948097586631775, + 0.3251423239707947, + 0.5115469098091125, + 0.3872796893119812, + 0.06782757490873337, + 0.04529177024960518, + 0.04364604502916336, + 0.02824077382683754, + 0.08751744776964188 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9532", + "split": "test", + "style_correct": 1, + "style_probability": 0.24292464554309845, + "style_target": 0, + "support_probability": 0.6439029574394226, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8560534243690654, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10343490540981293, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9541:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2465037703514099, + 0.9908503293991089, + 0.2517933249473572, + 0.5828810930252075, + 0.05622673034667969, + 0.10071994364261627, + 0.07394332438707352, + 0.08643926680088043, + 0.14520733058452606, + 0.08608591556549072, + 0.09268639236688614 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 1, + "style_probability": 0.7207174897193909, + "style_target": 1, + "support_probability": 0.9003192782402039, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7499437444489132, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1595863401889801, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9541:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19825442135334015, + 0.9748631715774536, + 0.2884804308414459, + 0.5689293146133423, + 0.11609447747468948, + 0.24733389914035797, + 0.09050549566745758, + 0.14430435001850128, + 0.1251794546842575, + 0.141580268740654, + 0.3088577091693878 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9541", + "split": "test", + "style_correct": 0, + "style_probability": 0.5874066948890686, + "style_target": 0, + "support_probability": 0.8507404327392578, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03598533847832419, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9380345940589905, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4068572521209717, + 0.4073652923107147, + 0.37680020928382874, + 0.39227962493896484, + 0.05826636776328087 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.9071367383003235, + "style_target": 1, + "support_probability": 0.059041135013103485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00572875504744777, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9112526178359985, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9570:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.40625399351119995, + 0.38788148760795593, + 0.39430394768714905, + 0.4165356457233429, + 0.10045021772384644, + 0.13387133181095123, + 0.11108697950839996, + 0.18483872711658478, + 0.11743755638599396 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9570", + "split": "test", + "style_correct": 1, + "style_probability": 0.27349430322647095, + "style_target": 0, + "support_probability": 0.03614319860935211, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8977659606807363, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.17016088962554932, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9577:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19977666437625885, + 0.9864462614059448, + 0.3547254800796509, + 0.2524715065956116, + 0.12565158307552338, + 0.05563032627105713 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.7929593324661255, + "style_target": 1, + "support_probability": 0.9434205889701843, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14338370094323238, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2369813174009323, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9577:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2811867892742157, + 0.9867067933082581, + 0.40356969833374023, + 0.24150137603282928, + 0.0735413134098053, + 0.08150440454483032 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9577", + "split": "test", + "style_correct": 1, + "style_probability": 0.15414729714393616, + "style_target": 0, + "support_probability": 0.9040086269378662, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14133384576560462, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8039918541908264, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7314212322235107, + 0.27011188864707947, + 0.40826278924942017, + 0.5046871304512024, + 0.1294126808643341, + 0.0768325999379158, + 0.10342997312545776, + 0.12177779525518417, + 0.05071362853050232 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 1, + "style_probability": 0.9218961596488953, + "style_target": 1, + "support_probability": 0.047884829342365265, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05646347802677364, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7355796098709106, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9580:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7038223147392273, + 0.25138363242149353, + 0.35676267743110657, + 0.4927244484424591, + 0.08369898051023483, + 0.08095881342887878, + 0.09317419677972794, + 0.18987886607646942, + 0.09615182876586914 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9580", + "split": "test", + "style_correct": 0, + "style_probability": 0.5514193773269653, + "style_target": 0, + "support_probability": 0.10694841295480728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010308598447220377, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.934726357460022, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7202600836753845, + 0.4754289388656616, + 0.3569214642047882, + 0.350574791431427, + 0.1297212690114975, + 0.33352839946746826, + 0.08399952948093414, + 0.1544397920370102, + 0.1754881739616394, + 0.06623229384422302 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.846774160861969, + "style_target": 1, + "support_probability": 0.04554498940706253, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005209376193262641, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9084796905517578, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9588:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6776595711708069, + 0.4362056851387024, + 0.3620462715625763, + 0.3197569251060486, + 0.10335730761289597, + 0.12927600741386414 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9588", + "split": "test", + "style_correct": 1, + "style_probability": 0.35227009654045105, + "style_target": 0, + "support_probability": 0.06251475214958191, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.6759706857769031, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.15063069760799408, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5964178442955017, + 0.7465250492095947, + 0.299050897359848, + 0.4699954092502594, + 0.03148750588297844 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.7672550082206726, + "style_target": 1, + "support_probability": 0.9563601613044739, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03727007186565778, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.2381926327943802, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9601:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6277194023132324, + 0.7981159687042236, + 0.2904299199581146, + 0.5659634470939636, + 0.09156357496976852 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9601", + "split": "test", + "style_correct": 1, + "style_probability": 0.10085882246494293, + "style_target": 0, + "support_probability": 0.9248400330543518, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.9137667053788903, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1425410360097885, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.995226263999939, + 0.3068067729473114, + 0.19706487655639648, + 0.4855441153049469, + 0.04773113504052162, + 0.05375652760267258 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 1, + "style_probability": 0.7831797003746033, + "style_target": 1, + "support_probability": 0.8489871025085449, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8891354704711601, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.13240690529346466, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9604:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9803609848022461, + 0.34186115860939026, + 0.19127874076366425, + 0.4921785295009613, + 0.058221980929374695, + 0.04762997850775719 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9604", + "split": "test", + "style_correct": 0, + "style_probability": 0.6995223760604858, + "style_target": 0, + "support_probability": 0.8726359009742737, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.48667451777578213, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.27047762274742126, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5639266967773438, + 0.5798138976097107, + 0.992067813873291, + 0.4257792830467224, + 0.07371880114078522, + 0.12573286890983582, + 0.15904231369495392, + 0.07160279899835587, + 0.09235410392284393 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 1, + "style_probability": 0.8385390639305115, + "style_target": 1, + "support_probability": 0.4143020808696747, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.15615589305562239, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.24691888689994812, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9606:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.5732596516609192, + 0.5819138288497925, + 0.984413743019104, + 0.3767148554325104, + 0.10894421488046646 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9606", + "split": "test", + "style_correct": 1, + "style_probability": 0.4392169415950775, + "style_target": 0, + "support_probability": 0.43547338247299194, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8870965396825312, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2661372721195221, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5625151991844177, + 0.6265600919723511, + 0.531284749507904, + 0.9747255444526672, + 0.07815025746822357, + 0.036097824573516846, + 0.14649319648742676, + 0.08308090269565582, + 0.08053697645664215, + 0.15477225184440613 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.8923651576042175, + "style_target": 1, + "support_probability": 0.8038361072540283, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2576956815859379, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20396481454372406, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9607:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.7761415243148804, + 0.7247048616409302, + 0.6533585786819458, + 0.9949309229850769, + 0.06631574034690857, + 0.05333489179611206, + 0.11503235995769501, + 0.0846087709069252, + 0.06545354425907135, + 0.12656691670417786 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9607", + "split": "test", + "style_correct": 1, + "style_probability": 0.18025018274784088, + "style_target": 0, + "support_probability": 0.8327770233154297, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4353262753064886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07813622802495956, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9460596442222595, + 0.3643743097782135, + 0.15656980872154236, + 0.2557556927204132, + 0.1702403724193573 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 1, + "style_probability": 0.8517919182777405, + "style_target": 1, + "support_probability": 0.5120203495025635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.35130489801664494, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08737578243017197, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9612:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9142001271247864, + 0.3662366569042206, + 0.21477961540222168, + 0.24749116599559784, + 0.12740114331245422 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9612", + "split": "test", + "style_correct": 0, + "style_probability": 0.7042261958122253, + "style_target": 0, + "support_probability": 0.5256143808364868, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9309870714301737, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05167426913976669, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9617:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.42624539136886597, + 0.9568384289741516, + 0.4255274534225464, + 0.42624539136886597, + 0.08424290269613266, + 0.08492312580347061 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.7505088448524475, + "style_target": 1, + "support_probability": 0.9757819175720215, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7494090629588257, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.08833865821361542, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9617:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4332977533340454, + 0.9242039918899536, + 0.3605108857154846, + 0.4332977533340454, + 0.07108581811189651, + 0.04572935029864311, + 0.08565044403076172, + 0.055748920887708664, + 0.08809557557106018, + 0.08168265223503113, + 0.16309653222560883 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9617", + "split": "test", + "style_correct": 1, + "style_probability": 0.4227895438671112, + "style_target": 0, + "support_probability": 0.9484727382659912, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9781550851664065, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.029726998880505562, + "conflict_target": 0, + "decision": "synthesize", + "episode_id": "snli-compositional:test:9620:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9878151416778564, + 0.17194359004497528, + 0.6389569640159607, + 0.07024939358234406, + 0.04015199840068817, + 0.050582606345415115, + 0.05236407741904259, + 0.059215247631073, + 0.08762689679861069 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.8623328804969788, + "style_target": 1, + "support_probability": 0.9783333539962769, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7684881445952207, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.027924353256821632, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9620:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9974722266197205, + 0.18887153267860413, + 0.7260781526565552, + 0.12780466675758362, + 0.09597659856081009, + 0.07899003475904465, + 0.06736448407173157, + 0.17065027356147766, + 0.1489185392856598 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9620", + "split": "test", + "style_correct": 1, + "style_probability": 0.1424817442893982, + "style_target": 0, + "support_probability": 0.9828534722328186, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.026551150428628437, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7484683990478516, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.44248679280281067, + 0.23144425451755524, + 0.15547409653663635, + 0.14272841811180115, + 0.06079547852277756, + 0.055951859802007675, + 0.052371587604284286, + 0.08430194109678268, + 0.05356735736131668, + 0.09306035935878754, + 0.10116592049598694 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.750755250453949, + "style_target": 1, + "support_probability": 0.12526638805866241, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012287623176406302, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8564106225967407, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9640:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.45087379217147827, + 0.270710825920105, + 0.1524345576763153, + 0.10822781175374985, + 0.04007516801357269, + 0.051043663173913956, + 0.059252072125673294, + 0.0788457989692688, + 0.03572555258870125, + 0.055079385638237, + 0.13378079235553741 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9640", + "split": "test", + "style_correct": 1, + "style_probability": 0.33846473693847656, + "style_target": 0, + "support_probability": 0.13888563215732574, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8769124073611821, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.022437119856476784, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9643:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.30024781823158264, + 0.2938578426837921, + 0.30024781823158264, + 0.9706183671951294, + 0.08905329555273056 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 1, + "style_probability": 0.5565194487571716, + "style_target": 1, + "support_probability": 0.9380377531051636, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9552265044774515, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.024359751492738724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9643:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3109818994998932, + 0.30923739075660706, + 0.3109818994998932, + 0.9779702425003052, + 0.04962954297661781 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9643", + "split": "test", + "style_correct": 0, + "style_probability": 0.8913219571113586, + "style_target": 0, + "support_probability": 0.9405707120895386, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1261283520918033, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.7194138765335083, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.1495659500360489, + 0.5323467254638672, + 0.6057380437850952, + 0.7066980004310608, + 0.16403788328170776, + 0.18438231945037842 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 1, + "style_probability": 0.6431072950363159, + "style_target": 1, + "support_probability": 0.20621871948242188, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07673730021389247, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7037023305892944, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9647:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1731446236371994, + 0.5645434260368347, + 0.5810145735740662, + 0.7680425643920898, + 0.15887388586997986, + 0.0864374190568924, + 0.07019652426242828, + 0.041589222848415375, + 0.0849151536822319, + 0.14968474209308624, + 0.16029198467731476 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9647", + "split": "test", + "style_correct": 1, + "style_probability": 0.4573556184768677, + "style_target": 0, + "support_probability": 0.18107466399669647, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7092237416776221, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11626686900854111, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5007163882255554, + 0.8854933381080627, + 0.5056946277618408, + 0.428415983915329, + 0.026631824672222137, + 0.07524100691080093, + 0.027418674901127815, + 0.03254694864153862, + 0.05798771232366562 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 1, + "style_probability": 0.5726783871650696, + "style_target": 1, + "support_probability": 0.8929912447929382, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12719670759696552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1386079639196396, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9649:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5763643980026245, + 0.940410852432251, + 0.5733752250671387, + 0.5285227298736572, + 0.08137807250022888, + 0.08586861193180084, + 0.04540804401040077, + 0.05821779742836952, + 0.07685298472642899 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9649", + "split": "test", + "style_correct": 1, + "style_probability": 0.15314793586730957, + "style_target": 0, + "support_probability": 0.9026214480400085, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01556717215001946, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8665561676025391, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2753940224647522, + 0.359579861164093, + 0.4859761893749237, + 0.3509693145751953, + 0.10924047231674194, + 0.08833897113800049, + 0.07079827040433884, + 0.06511931866407394, + 0.042754530906677246, + 0.09589165449142456, + 0.10691400617361069 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 1, + "style_probability": 0.608269453048706, + "style_target": 1, + "support_probability": 0.08191772550344467, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08810483240116086, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7725841999053955, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9673:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30806151032447815, + 0.3334694504737854, + 0.5286610722541809, + 0.3316376805305481, + 0.10909974575042725, + 0.13902774453163147, + 0.08756151050329208, + 0.09129630029201508, + 0.28799986839294434, + 0.28266143798828125, + 0.10436363518238068 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9673", + "split": "test", + "style_correct": 0, + "style_probability": 0.876335620880127, + "style_target": 0, + "support_probability": 0.14583748579025269, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.05351087855172487, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6478932499885559, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.23808573186397552, + 0.3073104918003082, + 0.7632787227630615, + 0.23808573186397552, + 0.07584115117788315 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.8557016253471375, + "style_target": 1, + "support_probability": 0.06819020211696625, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011101296281847143, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6942175030708313, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9674:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.25674861669540405, + 0.3556925058364868, + 0.8635815978050232, + 0.25674861669540405, + 0.0847087875008583 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9674", + "split": "test", + "style_correct": 1, + "style_probability": 0.43512409925460815, + "style_target": 0, + "support_probability": 0.06977253407239914, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7009029153753943, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1693200320005417, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.19392122328281403, + 0.7495883107185364, + 0.40383386611938477, + 0.5509470701217651, + 0.045002199709415436, + 0.09276512265205383 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 1, + "style_probability": 0.9005512595176697, + "style_target": 1, + "support_probability": 0.8538157939910889, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4542391196407322, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.29372698068618774, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9681:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.16282916069030762, + 0.7045897245407104, + 0.29008832573890686, + 0.5125002861022949, + 0.187371626496315, + 0.050479769706726074 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9681", + "split": "test", + "style_correct": 0, + "style_probability": 0.5961750149726868, + "style_target": 0, + "support_probability": 0.7990596890449524, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8616634849815341, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06374768912792206, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19936725497245789, + 0.8878109455108643, + 0.3843069076538086, + 0.15417729318141937, + 0.024539198726415634, + 0.04916508495807648, + 0.059264376759529114, + 0.044669318944215775, + 0.047698862850666046 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.5421596765518188, + "style_target": 1, + "support_probability": 0.9644859433174133, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7220480160924581, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.09200489521026611, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9685:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.20984439551830292, + 0.8541348576545715, + 0.43317097425460815, + 0.14321379363536835, + 0.03991715982556343 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9685", + "split": "test", + "style_correct": 1, + "style_probability": 0.17800746858119965, + "style_target": 0, + "support_probability": 0.9583390355110168, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.06585696001406172, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4547799527645111, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.19639387726783752, + 0.18710902333259583, + 0.2802791893482208, + 0.7841913104057312, + 0.03279274329543114, + 0.053568460047245026, + 0.03367127478122711, + 0.055328693240880966, + 0.0409369133412838, + 0.06559285521507263, + 0.08455418795347214 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 1, + "style_probability": 0.7840556502342224, + "style_target": 1, + "support_probability": 0.0642116591334343, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.030586254167607848, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3571236729621887, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9687:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.18440799415111542, + 0.1906481683254242, + 0.30911144614219666, + 0.8443643450737, + 0.07770153880119324, + 0.04625948145985603, + 0.03913445770740509, + 0.06510846316814423, + 0.07548538595438004, + 0.0660015270113945 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9687", + "split": "test", + "style_correct": 1, + "style_probability": 0.38250496983528137, + "style_target": 0, + "support_probability": 0.20172612369060516, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7043548691567914, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.46469154953956604, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9116002917289734, + 0.3044147193431854, + 0.45841142535209656, + 0.2920285165309906, + 0.059433382004499435 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 1, + "style_probability": 0.8250546455383301, + "style_target": 1, + "support_probability": 0.6636278629302979, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.35493794426099606, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4666145443916321, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9688:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8489999771118164, + 0.3081555664539337, + 0.42177271842956543, + 0.31906959414482117, + 0.036405351012945175 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9688", + "split": "test", + "style_correct": 0, + "style_probability": 0.6506540179252625, + "style_target": 0, + "support_probability": 0.4830314815044403, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2570287895465526, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.654171884059906, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3919697403907776, + 0.3712221682071686, + 0.24288104474544525, + 0.776606559753418, + 0.2534473240375519, + 0.09513352066278458 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.9256753325462341, + "style_target": 1, + "support_probability": 0.2937721610069275, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.032745123623124435, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6933503150939941, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9703:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.38733479380607605, + 0.37414875626564026, + 0.21307899057865143, + 0.7630466222763062, + 0.07405129075050354, + 0.08222053200006485, + 0.0689740851521492, + 0.09865672886371613, + 0.07617167383432388, + 0.07935280352830887, + 0.12073316425085068 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9703", + "split": "test", + "style_correct": 1, + "style_probability": 0.42333635687828064, + "style_target": 0, + "support_probability": 0.14814801514148712, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9116806298221363, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18401749432086945, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9712:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9534128904342651, + 0.4637233018875122, + 0.5466952323913574, + 0.2975446879863739, + 0.07268549501895905, + 0.08726498484611511, + 0.043648913502693176, + 0.07142584025859833, + 0.05898214131593704 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.9205957055091858, + "style_target": 1, + "support_probability": 0.9174258708953857, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.12496751536875461, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40370747447013855, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9712:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9784391522407532, + 0.4462756812572479, + 0.6504290103912354, + 0.3309585452079773, + 0.08655665814876556, + 0.1621168851852417, + 0.08303794264793396, + 0.04497317969799042, + 0.1580367237329483 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9712", + "split": "test", + "style_correct": 1, + "style_probability": 0.3130991458892822, + "style_target": 0, + "support_probability": 0.8140832185745239, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04141581632871704, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6799725890159607, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2550312876701355, + 0.40771666169166565, + 0.2550312876701355, + 0.8198850750923157, + 0.0400301031768322, + 0.04049989953637123, + 0.10563555359840393, + 0.057233236730098724, + 0.08861826360225677, + 0.11449480801820755, + 0.07971198856830597 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 1, + "style_probability": 0.8145464062690735, + "style_target": 1, + "support_probability": 0.05492330715060234, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011831945915813376, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7415752410888672, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9727:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.29045790433883667, + 0.4072430729866028, + 0.29045790433883667, + 0.7615888118743896, + 0.039688415825366974, + 0.05318792909383774, + 0.059268489480018616, + 0.06216789782047272, + 0.12748128175735474, + 0.06904750317335129, + 0.07579585909843445 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9727", + "split": "test", + "style_correct": 0, + "style_probability": 0.6463030576705933, + "style_target": 0, + "support_probability": 0.034214507788419724, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8101527073319659, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.048175107687711716, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3008555769920349, + 0.11160877346992493, + 0.36624082922935486, + 0.9658896923065186, + 0.035539478063583374 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.36645442247390747, + "style_target": 1, + "support_probability": 0.951684832572937, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.9094954670849625, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.055920228362083435, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9731:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.33410513401031494, + 0.11325520277023315, + 0.36633390188217163, + 0.9793055653572083, + 0.07289260625839233 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9731", + "split": "test", + "style_correct": 0, + "style_probability": 0.6962351202964783, + "style_target": 0, + "support_probability": 0.9371554851531982, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.1161681638014951, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.33507853746414185, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3758017122745514, + 0.6946603059768677, + 0.40470778942108154, + 0.276821494102478, + 0.18887698650360107, + 0.09935339540243149 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 0, + "style_probability": 0.4522627890110016, + "style_target": 1, + "support_probability": 0.22235463559627533, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10292529567801355, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24143461883068085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9741:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4309934675693512, + 0.8178874850273132, + 0.6176579594612122, + 0.3062998056411743, + 0.12937796115875244, + 0.205891951918602 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9741", + "split": "test", + "style_correct": 1, + "style_probability": 0.31218039989471436, + "style_target": 0, + "support_probability": 0.5190580487251282, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.002948061369940358, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9397828578948975, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2392740696668625, + 0.6591464281082153, + 0.39003488421440125, + 0.41895002126693726, + 0.1288229376077652, + 0.2352655529975891, + 0.09145575761795044, + 0.16374345123767853, + 0.08221793919801712 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 0, + "style_probability": 0.2015564739704132, + "style_target": 1, + "support_probability": 0.011390281841158867, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005362323349683897, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9176791906356812, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9745:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.25668513774871826, + 0.5999513864517212, + 0.39511504769325256, + 0.43291857838630676, + 0.08463647216558456, + 0.19425533711910248, + 0.08259835839271545, + 0.15154649317264557, + 0.07175908982753754 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9745", + "split": "test", + "style_correct": 1, + "style_probability": 0.4919027090072632, + "style_target": 0, + "support_probability": 0.012636872008442879, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.17945115964213887, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5571462512016296, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.44287705421447754, + 0.5589379072189331, + 0.7244122624397278, + 0.13750234246253967, + 0.1117730513215065, + 0.13586951792240143, + 0.175558939576149, + 0.08600238710641861, + 0.09019958972930908, + 0.1562582105398178, + 0.08592956513166428 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 1, + "style_probability": 0.5449693202972412, + "style_target": 1, + "support_probability": 0.12489006668329239, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08303619716900144, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7329512238502502, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9750:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.48393720388412476, + 0.5361036062240601, + 0.7461655735969543, + 0.1348966807126999, + 0.05513245239853859, + 0.06582672148942947, + 0.07727330923080444, + 0.06314489245414734, + 0.06380029767751694, + 0.10149363428354263, + 0.055111728608608246 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9750", + "split": "test", + "style_correct": 0, + "style_probability": 0.8428090214729309, + "style_target": 0, + "support_probability": 0.05522843822836876, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04723932951680475, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.9215093851089478, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.21282236278057098, + 0.41357654333114624, + 0.21721889078617096, + 0.6758689284324646, + 0.2174796760082245 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 1, + "style_probability": 0.5119591951370239, + "style_target": 1, + "support_probability": 0.33236750960350037, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010684871282811039, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.9276963472366333, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9753:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.220807284116745, + 0.2978193759918213, + 0.21560132503509521, + 0.5495191216468811, + 0.08284205943346024, + 0.12929566204547882 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9753", + "split": "test", + "style_correct": 1, + "style_probability": 0.3792531490325928, + "style_target": 0, + "support_probability": 0.05391795560717583, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.250666051323023, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.12889349460601807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.20051436126232147, + 0.28136277198791504, + 0.19730553030967712, + 0.9027659893035889, + 0.15934211015701294, + 0.2102905958890915 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 0, + "style_probability": 0.23589098453521729, + "style_target": 1, + "support_probability": 0.6429523229598999, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.011081874883052236, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.2981524169445038, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9763:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.17698508501052856, + 0.2770285904407501, + 0.22930370271205902, + 0.9649435877799988, + 0.06936726719141006, + 0.08880124986171722 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9763", + "split": "test", + "style_correct": 1, + "style_probability": 0.05842197686433792, + "style_target": 0, + "support_probability": 0.49143967032432556, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04874015660083586, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5411583781242371, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.20702789723873138, + 0.20702789723873138, + 0.757456362247467, + 0.3588797152042389, + 0.0904465913772583, + 0.17917419970035553, + 0.14180077612400055, + 0.09439897537231445, + 0.16334182024002075 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 0, + "style_probability": 0.4891645312309265, + "style_target": 1, + "support_probability": 0.10794918984174728, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.11520641046947058, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.25593486428260803, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9766:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.17516016960144043, + 0.17516016960144043, + 0.7699579000473022, + 0.3422686755657196, + 0.09119381755590439, + 0.1930214911699295, + 0.0939696729183197, + 0.04897184669971466, + 0.0625559538602829 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9766", + "split": "test", + "style_correct": 0, + "style_probability": 0.5022965669631958, + "style_target": 0, + "support_probability": 0.29002174735069275, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7941981099512319, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.11334246397018433, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9538255929946899, + 0.42781680822372437, + 0.20845380425453186, + 0.20595362782478333, + 0.04205361753702164, + 0.1413726657629013, + 0.09495798498392105, + 0.25223734974861145, + 0.16042864322662354, + 0.09433705359697342, + 0.27609360218048096 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 1, + "style_probability": 0.7256381511688232, + "style_target": 1, + "support_probability": 0.880191445350647, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7130712351150379, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04445044696331024, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9768:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9553115367889404, + 0.36974263191223145, + 0.2169046401977539, + 0.18140198290348053, + 0.03950248658657074 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9768", + "split": "test", + "style_correct": 0, + "style_probability": 0.5265313386917114, + "style_target": 0, + "support_probability": 0.9213272333145142, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.00648587744483842, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9744095206260681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.37068358063697815, + 0.5208414793014526, + 0.46613797545433044, + 0.23268555104732513, + 0.1035928726196289 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 1, + "style_probability": 0.6246632933616638, + "style_target": 1, + "support_probability": 0.04660999774932861, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00023576637250189214, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.9929667711257935, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9782:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.2566305100917816, + 0.4995528757572174, + 0.5725198984146118, + 0.2827679514884949, + 0.20260776579380035 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9782", + "split": "test", + "style_correct": 1, + "style_probability": 0.24263998866081238, + "style_target": 0, + "support_probability": 0.01013281662017107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.022490979190772368, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4281170666217804, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.24125638604164124, + 0.7405209541320801, + 0.34384846687316895, + 0.4421418011188507, + 0.24286681413650513, + 0.08363626152276993 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 1, + "style_probability": 0.5830444693565369, + "style_target": 1, + "support_probability": 0.04654986783862114, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011557237096770924, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.708972156047821, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9785:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.2521967589855194, + 0.802767276763916, + 0.40819603204727173, + 0.5126917958259583, + 0.07016053795814514, + 0.05121217668056488 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9785", + "split": "test", + "style_correct": 0, + "style_probability": 0.8663272857666016, + "style_target": 0, + "support_probability": 0.016146810725331306, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.18899179639048033, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.375112384557724, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.2800772786140442, + 0.23721373081207275, + 0.805159866809845, + 0.35617125034332275, + 0.07071669399738312, + 0.07450492680072784, + 0.12077204883098602, + 0.1984325647354126, + 0.07941234111785889 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 1, + "style_probability": 0.5916327238082886, + "style_target": 1, + "support_probability": 0.32818296551704407, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.15807070295387415, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3203815221786499, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9798:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28955894708633423, + 0.20762981474399567, + 0.8095166087150574, + 0.3801637887954712, + 0.06091627851128578, + 0.05165594071149826, + 0.1048596128821373, + 0.09877464920282364, + 0.09879376739263535, + 0.06735977530479431, + 0.07603319734334946 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9798", + "split": "test", + "style_correct": 0, + "style_probability": 0.5071080923080444, + "style_target": 0, + "support_probability": 0.3270605802536011, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.8272469517859946, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.09288493543863297, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.42714375257492065, + 0.9586132764816284, + 0.34798821806907654, + 0.5701504349708557, + 0.09947559237480164, + 0.06888393312692642, + 0.05267098546028137, + 0.061020709574222565, + 0.11135732382535934, + 0.1412503719329834, + 0.10751910507678986 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.5673874020576477, + "style_target": 1, + "support_probability": 0.9130286574363708, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7815896604533776, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.02526918798685074, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9805:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.5411400198936462, + 0.9977318644523621, + 0.5794687271118164, + 0.7760699391365051, + 0.07771584391593933, + 0.03081698715686798, + 0.0574798583984375, + 0.09592228382825851, + 0.051730357110500336, + 0.14496766030788422 + ], + "selected_evidence_indices": [ + 1, + 3 + ], + "speaker_id": "snli:test:9805", + "split": "test", + "style_correct": 1, + "style_probability": 0.2410401850938797, + "style_target": 0, + "support_probability": 0.9730005264282227, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01692713270334823, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.43199244141578674, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41280147433280945, + 0.41714733839035034, + 0.16186632215976715, + 0.7199794054031372, + 0.0777641013264656 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 0, + "style_probability": 0.46029776334762573, + "style_target": 1, + "support_probability": 0.08214609324932098, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0207012366403807, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.4650173485279083, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9807:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41536685824394226, + 0.41594308614730835, + 0.1318323314189911, + 0.7692502737045288, + 0.07882172614336014 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9807", + "split": "test", + "style_correct": 0, + "style_probability": 0.7246841788291931, + "style_target": 0, + "support_probability": 0.06091926246881485, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5002162630682108, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.0485476478934288, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.35594552755355835, + 0.47126665711402893, + 0.9897574186325073, + 0.35594552755355835, + 0.1194685846567154, + 0.07455378025770187 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 0, + "style_probability": 0.41315239667892456, + "style_target": 1, + "support_probability": 0.724237859249115, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.4066116012797493, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.04224524646997452, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9814:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.41682252287864685, + 0.4516679346561432, + 0.9859219193458557, + 0.41682252287864685, + 0.3216501474380493, + 0.06351285427808762, + 0.09064589440822601, + 0.2204194813966751, + 0.060966216027736664 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9814", + "split": "test", + "style_correct": 1, + "style_probability": 0.2594137489795685, + "style_target": 0, + "support_probability": 0.6990382671356201, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1281471973182562, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2836361527442932, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.8697031736373901, + 0.2821306586265564, + 0.3315264582633972, + 0.2883675992488861, + 0.07335545122623444, + 0.1238228976726532, + 0.06178642064332962, + 0.047142211347818375, + 0.06390021741390228 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.5677037835121155, + "style_target": 1, + "support_probability": 0.2505919635295868, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.1228082600086111, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.19553127884864807, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9820:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9529755115509033, + 0.27828532457351685, + 0.34703823924064636, + 0.2713952362537384, + 0.04789453744888306, + 0.05603237822651863, + 0.055953435599803925, + 0.045930538326501846, + 0.14270815253257751 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9820", + "split": "test", + "style_correct": 1, + "style_probability": 0.24035128951072693, + "style_target": 0, + "support_probability": 0.5586593747138977, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18121077010634679, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2660245895385742, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.7349581718444824, + 0.13738901913166046, + 0.7349581718444824, + 0.774526834487915, + 0.09226338565349579, + 0.10549704730510712, + 0.10505751520395279, + 0.13886576890945435, + 0.14600683748722076, + 0.11743420362472534, + 0.051297396421432495 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 1, + "style_probability": 0.5227313041687012, + "style_target": 1, + "support_probability": 0.43091392517089844, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.5973915429762897, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.0873037576675415, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9823:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.797897219657898, + 0.13514170050621033, + 0.797897219657898, + 0.8982695937156677, + 0.0348396971821785, + 0.05043569207191467, + 0.11964396387338638, + 0.05808315426111221, + 0.062467679381370544, + 0.1180475503206253, + 0.0940561443567276 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9823", + "split": "test", + "style_correct": 0, + "style_probability": 0.778656005859375, + "style_target": 0, + "support_probability": 0.6227573752403259, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8549100841157191, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.06200390309095383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9828:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9590979218482971, + 0.4100014269351959, + 0.37036043405532837, + 0.4701755940914154, + 0.05705401673913002 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.8276135325431824, + "style_target": 1, + "support_probability": 0.8316019773483276, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.13037836456534713, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.07766829431056976, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9828:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.9814571142196655, + 0.5167239308357239, + 0.3761504888534546, + 0.638057291507721, + 0.2663290500640869 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9828", + "split": "test", + "style_correct": 1, + "style_probability": 0.12142044305801392, + "style_target": 0, + "support_probability": 0.8147815465927124, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6517375205113999, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.12021336704492569, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7892249226570129, + 0.5021315217018127, + 0.28128302097320557, + 0.41112276911735535, + 0.052352167665958405, + 0.06011861935257912 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 1, + "style_probability": 0.7326784729957581, + "style_target": 1, + "support_probability": 0.6825659275054932, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4536029213559552, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.10744588822126389, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9840:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.734682023525238, + 0.48798254132270813, + 0.29202795028686523, + 0.42603617906570435, + 0.06849861890077591, + 0.05208299681544304 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9840", + "split": "test", + "style_correct": 0, + "style_probability": 0.5622498989105225, + "style_target": 0, + "support_probability": 0.5720475912094116, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.027962448464162407, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.500885009765625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5163887739181519, + 0.46964141726493835, + 0.5777862668037415, + 0.8986364006996155, + 0.043341610580682755, + 0.039014782756567, + 0.04367895796895027, + 0.03209231421351433, + 0.027236154302954674 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.6072708368301392, + "style_target": 1, + "support_probability": 0.04744293540716171, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.01044903374842998, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.47976794838905334, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9843:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4987000823020935, + 0.4048188030719757, + 0.539392352104187, + 0.8593129515647888, + 0.047129809856414795, + 0.07056234776973724, + 0.06409332901239395, + 0.05106129124760628, + 0.05920432507991791, + 0.1386958211660385, + 0.05369339883327484 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9843", + "split": "test", + "style_correct": 1, + "style_probability": 0.26019933819770813, + "style_target": 0, + "support_probability": 0.0448300875723362, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.30982652701229796, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1426728069782257, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.30938395857810974, + 0.8286408185958862, + 0.2805953621864319, + 0.3242162764072418, + 0.12232670933008194, + 0.08073967695236206, + 0.06818156689405441, + 0.09945870190858841, + 0.08556295931339264, + 0.03294574096798897, + 0.09656954556703568 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.8426637649536133, + "style_target": 1, + "support_probability": 0.4243503212928772, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.024764374542114886, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18412452936172485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9845:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.3479083776473999, + 0.8936969637870789, + 0.36341777443885803, + 0.3872934877872467, + 0.17268311977386475, + 0.10418368875980377, + 0.08580504357814789, + 0.11705757677555084, + 0.123806431889534, + 0.10071998089551926 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9845", + "split": "test", + "style_correct": 1, + "style_probability": 0.24486692249774933, + "style_target": 0, + "support_probability": 0.5318329334259033, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.7147659034076632, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.4769916534423828, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.8881691098213196, + 0.11944399029016495, + 0.9597440361976624, + 0.1845354586839676, + 0.06230534240603447 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 1, + "style_probability": 0.9485411643981934, + "style_target": 1, + "support_probability": 0.31194180250167847, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.3336540876659253, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.3446318209171295, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9850:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.852255642414093, + 0.11302544921636581, + 0.9460921883583069, + 0.18864725530147552, + 0.05581502616405487 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9850", + "split": "test", + "style_correct": 0, + "style_probability": 0.6684622168540955, + "style_target": 0, + "support_probability": 0.35561490058898926, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.2077287510561732, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7282506227493286, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9488072395324707, + 0.29464277625083923, + 0.7644333243370056, + 0.5729482173919678, + 0.13031736016273499, + 0.08577442914247513 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 1, + "style_probability": 0.8748486638069153, + "style_target": 1, + "support_probability": 0.1686219722032547, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.13165507239048324, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5775688886642456, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9852:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9385477900505066, + 0.30818620324134827, + 0.7529851198196411, + 0.5564136505126953, + 0.16819891333580017, + 0.18913082778453827, + 0.13587181270122528, + 0.134961798787117, + 0.20182007551193237 + ], + "selected_evidence_indices": [ + 0, + 2 + ], + "speaker_id": "snli:test:9852", + "split": "test", + "style_correct": 0, + "style_probability": 0.6210278272628784, + "style_target": 0, + "support_probability": 0.18414148688316345, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8219866510322156, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14200513064861298, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6699649095535278, + 0.06877651065587997, + 0.9842717051506042, + 0.4261164367198944, + 0.058192454278469086, + 0.044182706624269485, + 0.059432607144117355, + 0.032935455441474915, + 0.07502446323633194 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.7034372091293335, + "style_target": 1, + "support_probability": 0.8586196303367615, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.2257998517103772, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18339671194553375, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9854:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6806209683418274, + 0.07274027913808823, + 0.9932483434677124, + 0.46900978684425354, + 0.13639257848262787, + 0.07832209020853043, + 0.13108448684215546, + 0.07093321532011032, + 0.08976175636053085 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9854", + "split": "test", + "style_correct": 1, + "style_probability": 0.1820836365222931, + "style_target": 0, + "support_probability": 0.8385844826698303, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.6700685465020797, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1456538587808609, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9777919054031372, + 0.30658793449401855, + 0.24873937666416168, + 0.7378658056259155, + 0.11025383323431015, + 0.09115392714738846, + 0.30451086163520813, + 0.09723485261201859, + 0.0969901829957962, + 0.0813552588224411, + 0.05111636593937874 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 1, + "style_probability": 0.6368522047996521, + "style_target": 1, + "support_probability": 0.7105010747909546, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5992521549485907, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.24093586206436157, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9856:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.9517324566841125, + 0.3856247663497925, + 0.27465254068374634, + 0.594606339931488, + 0.08957947045564651, + 0.08417791873216629, + 0.08363516628742218, + 0.11234576255083084, + 0.07913337647914886, + 0.09093299508094788 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9856", + "split": "test", + "style_correct": 0, + "style_probability": 0.6824032664299011, + "style_target": 0, + "support_probability": 0.591938316822052, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08109313584858578, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.575292706489563, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.4460843503475189, + 0.7944415807723999, + 0.25455528497695923, + 0.2430531531572342, + 0.17918848991394043 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.8227419853210449, + "style_target": 1, + "support_probability": 0.09138358384370804, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.005853340959584009, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8378176093101501, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9857:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.46839532256126404, + 0.7388550043106079, + 0.2143537700176239, + 0.23818908631801605, + 0.1513790786266327, + 0.1430283486843109 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9857", + "split": "test", + "style_correct": 1, + "style_probability": 0.2645798325538635, + "style_target": 0, + "support_probability": 0.022179486230015755, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02990550734557429, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8258984684944153, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.45179063081741333, + 0.31444624066352844, + 0.5071134567260742, + 0.5729232430458069, + 0.08198804408311844, + 0.31961724162101746 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.6927505135536194, + "style_target": 1, + "support_probability": 0.07820358872413635, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0017681581978803993, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.8942133188247681, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9858:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.525757908821106, + 0.42033153772354126, + 0.613488495349884, + 0.6507745981216431, + 0.09093543887138367, + 0.16433963179588318 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9858", + "split": "test", + "style_correct": 1, + "style_probability": 0.12883663177490234, + "style_target": 0, + "support_probability": 0.07709047198295593, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.12261958974996823, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7555650472640991, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.7010599970817566, + 0.4382776916027069, + 0.52090984582901, + 0.3149479925632477, + 0.03616678714752197, + 0.05482859909534454, + 0.040094587951898575, + 0.04668240249156952, + 0.07041767984628677 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 1, + "style_probability": 0.9024767279624939, + "style_target": 1, + "support_probability": 0.1256764978170395, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.3189139996917998, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.39541295170783997, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9860:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.6551892161369324, + 0.3872004449367523, + 0.49748098850250244, + 0.34487029910087585, + 0.05561094731092453, + 0.0576586090028286, + 0.06809087097644806, + 0.04996637627482414, + 0.05757623538374901 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9860", + "split": "test", + "style_correct": 0, + "style_probability": 0.5567734837532043, + "style_target": 0, + "support_probability": 0.5611129403114319, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.019603615628033474, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5941601395606995, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:positive", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.32013002038002014, + 0.39933907985687256, + 0.39750775694847107, + 0.34375059604644775, + 0.10007213801145554, + 0.05673109367489815, + 0.08385127037763596, + 0.059987857937812805, + 0.06836649030447006, + 0.12330108880996704 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.8750735521316528, + "style_target": 1, + "support_probability": 0.05734643340110779, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.003890041031706095, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.6916192173957825, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9861:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3338271677494049, + 0.3806234896183014, + 0.3380267322063446, + 0.31827259063720703, + 0.11463800817728043 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9861", + "split": "test", + "style_correct": 1, + "style_probability": 0.36183252930641174, + "style_target": 0, + "support_probability": 0.061825159937143326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.02152563800117946, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8645597696304321, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6483802795410156, + 0.10046129673719406, + 0.16516909003257751, + 0.10046129673719406, + 0.05243634060025215 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 0, + "style_probability": 0.3779021203517914, + "style_target": 1, + "support_probability": 0.19316627085208893, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.014366931409854906, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8880870342254639, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9864:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.7240240573883057, + 0.10305514186620712, + 0.1953817903995514, + 0.10305514186620712, + 0.04146496206521988, + 0.1014958918094635 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9864", + "split": "test", + "style_correct": 1, + "style_probability": 0.2760693430900574, + "style_target": 0, + "support_probability": 0.119304358959198, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.9346055326628786, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.020786507055163383, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3512181341648102, + 0.9844987988471985, + 0.16194656491279602, + 0.3512181341648102, + 0.06343592703342438, + 0.08876344561576843 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 0, + "style_probability": 0.4877282679080963, + "style_target": 1, + "support_probability": 0.9517359137535095, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4338874766252321, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.05937541648745537, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9865:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3694175183773041, + 0.99444180727005, + 0.21449990570545197, + 0.3694175183773041, + 0.11016194522380829, + 0.12130998820066452 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9865", + "split": "test", + "style_correct": 1, + "style_probability": 0.16629157960414886, + "style_target": 0, + "support_probability": 0.9207170605659485, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.4099161658470738, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2150939702987671, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.42023539543151855, + 0.297119677066803, + 0.16445863246917725, + 0.915216863155365, + 0.0343182198703289, + 0.08333364129066467, + 0.03189144283533096, + 0.05555865913629532, + 0.05482788011431694 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 1, + "style_probability": 0.5483185648918152, + "style_target": 1, + "support_probability": 0.4905479848384857, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.376125557904416, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3051939606666565, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9871:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.37179750204086304, + 0.2871954143047333, + 0.1486220359802246, + 0.9219048023223877, + 0.056521352380514145, + 0.02499678172171116, + 0.050267044454813004, + 0.0326051339507103, + 0.055937353521585464 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9871", + "split": "test", + "style_correct": 0, + "style_probability": 0.7180600166320801, + "style_target": 0, + "support_probability": 0.3689740300178528, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.012800177079194569, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8908408284187317, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.4094451367855072, + 0.689045786857605, + 0.6740691661834717, + 0.7261102795600891, + 0.08541528880596161, + 0.09021487087011337, + 0.09029323607683182, + 0.2658941149711609, + 0.06995788216590881, + 0.07248362898826599, + 0.09673872590065002 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 0, + "style_probability": 0.3690548539161682, + "style_target": 1, + "support_probability": 0.07056650519371033, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.010996454922343855, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.8068147897720337, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9882:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.36237025260925293, + 0.6855642199516296, + 0.6615105867385864, + 0.7278088927268982, + 0.12138166278600693 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9882", + "split": "test", + "style_correct": 1, + "style_probability": 0.2674288749694824, + "style_target": 0, + "support_probability": 0.09886536747217178, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6883388619519692, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.1814151555299759, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.31837737560272217, + 0.8818182349205017, + 0.41590550541877747, + 0.2285969853401184, + 0.0361221581697464 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 1, + "style_probability": 0.8754822015762329, + "style_target": 1, + "support_probability": 0.48762908577919006, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.6347391787388545, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.17042897641658783, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9889:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.3491891920566559, + 0.8356041312217712, + 0.41575369238853455, + 0.2802942395210266, + 0.13692741096019745 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9889", + "split": "test", + "style_correct": 0, + "style_probability": 0.7353876233100891, + "style_target": 0, + "support_probability": 0.6706342101097107, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.08249273382210374, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5751505494117737, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.4995763301849365, + 0.35716554522514343, + 0.46550092101097107, + 0.45931804180145264, + 0.0845385417342186, + 0.049346908926963806 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 1, + "style_probability": 0.6639348268508911, + "style_target": 1, + "support_probability": 0.23386910557746887, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.058985548462212385, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5906320214271545, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9898:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.5181248188018799, + 0.3524008095264435, + 0.4251827299594879, + 0.48449933528900146, + 0.1534409523010254, + 0.08808451890945435, + 0.06863673776388168, + 0.09274613857269287, + 0.10673950612545013 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9898", + "split": "test", + "style_correct": 1, + "style_probability": 0.38172009587287903, + "style_target": 0, + "support_probability": 0.1681981384754181, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.060754342427673975, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6681761145591736, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.13682104647159576, + 0.3404775857925415, + 0.2536521255970001, + 0.3410882353782654, + 0.1369943916797638, + 0.07755972445011139, + 0.11846750229597092, + 0.0613236278295517, + 0.043546874076128006 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.7157367467880249, + "style_target": 1, + "support_probability": 0.1473122388124466, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.001981717087272571, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.8810586929321289, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9902:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1467866450548172, + 0.3733188211917877, + 0.27250415086746216, + 0.3861905038356781, + 0.06879177689552307, + 0.09399718046188354, + 0.10462713986635208, + 0.06883060932159424, + 0.16465507447719574 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9902", + "split": "test", + "style_correct": 1, + "style_probability": 0.18019574880599976, + "style_target": 0, + "support_probability": 0.10136816650629044, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.07915999188930199, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.6900232434272766, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.10185849666595459, + 0.39758455753326416, + 0.7085864543914795, + 0.3559363782405853, + 0.130907341837883, + 0.041876234114170074, + 0.054850418120622635, + 0.09145551919937134, + 0.0968008041381836, + 0.058215685188770294, + 0.04100288823246956 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 1, + "style_probability": 0.7843021750450134, + "style_target": 1, + "support_probability": 0.12512561678886414, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.050056069447863574, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.7223572134971619, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9910:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.15039856731891632, + 0.4666588008403778, + 0.6432566046714783, + 0.42073941230773926, + 0.05175098031759262, + 0.13269279897212982, + 0.048668913543224335, + 0.13663676381111145, + 0.09499653428792953, + 0.131378173828125, + 0.3039517104625702 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9910", + "split": "test", + "style_correct": 0, + "style_probability": 0.6221550703048706, + "style_target": 0, + "support_probability": 0.1092880591750145, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.011010258104037217, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0 + ], + "conflict_probability": 0.7944707274436951, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.38455644249916077, + 0.29481634497642517, + 0.2687874734401703, + 0.8074696063995361, + 0.05953895300626755 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.8819844722747803, + "style_target": 1, + "support_probability": 0.008011800236999989, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.0059949785134217875, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.717940628528595, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9919:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.3672551214694977, + 0.3151470124721527, + 0.2727312445640564, + 0.809518039226532, + 0.14261211454868317, + 0.08142535388469696 + ], + "selected_evidence_indices": [ + 3 + ], + "speaker_id": "snli:test:9919", + "split": "test", + "style_correct": 1, + "style_probability": 0.2834632694721222, + "style_target": 0, + "support_probability": 0.010109818540513515, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019477416176624126, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4492833912372589, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9324467778205872, + 0.24049097299575806, + 0.24215713143348694, + 0.6128115653991699, + 0.1843561977148056, + 0.07381104677915573 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.5455095767974854, + "style_target": 1, + "support_probability": 0.0761912539601326, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.00255455191619136, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.5015132427215576, + "conflict_target": 1, + "decision": "abstain", + "episode_id": "snli-compositional:test:9944:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.9425472021102905, + 0.25804948806762695, + 0.23988550901412964, + 0.6860861778259277, + 0.12185543030500412, + 0.09882485866546631 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9944", + "split": "test", + "style_correct": 1, + "style_probability": 0.12181457877159119, + "style_target": 0, + "support_probability": 0.11033898591995239, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.8441283495129994, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.16576407849788666, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1979440301656723, + 0.1581813544034958, + 0.9864382743835449, + 0.26407164335250854, + 0.04062449932098389, + 0.06386735290288925, + 0.044079821556806564, + 0.06988556683063507, + 0.04072529822587967 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.8186461329460144, + "style_target": 1, + "support_probability": 0.7313557267189026, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.5979404627979612, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1295723021030426, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9950:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.1989932656288147, + 0.15258030593395233, + 0.9825435280799866, + 0.27452588081359863, + 0.04514090344309807, + 0.03537135198712349, + 0.04632512107491493, + 0.04303714260458946, + 0.11507446318864822 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9950", + "split": "test", + "style_correct": 1, + "style_probability": 0.3971264958381653, + "style_target": 0, + "support_probability": 0.7027100324630737, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.256030029379275, + "accept_target": 1, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.43488699197769165, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.9681887626647949, + 0.46064069867134094, + 0.5859980583190918, + 0.7161688208580017, + 0.03343988582491875, + 0.05925188586115837, + 0.0569891594350338, + 0.053918514400720596, + 0.07062211632728577, + 0.06145579740405083, + 0.051846228539943695 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 1, + "style_probability": 0.7008741497993469, + "style_target": 1, + "support_probability": 0.27365314960479736, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.14484749149004725, + "accept_target": 0, + "citation_precision": 0.5, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.40138131380081177, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9952:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.963792085647583, + 0.4404143691062927, + 0.5728664398193359, + 0.7188665270805359, + 0.07185935974121094 + ], + "selected_evidence_indices": [ + 0, + 3 + ], + "speaker_id": "snli:test:9952", + "split": "test", + "style_correct": 1, + "style_probability": 0.3304401934146881, + "style_target": 0, + "support_probability": 0.320485383272171, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.04416919848654399, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.5512331128120422, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6585390567779541, + 0.44334712624549866, + 0.928745448589325, + 0.17810620367527008, + 0.09136141836643219 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 1, + "style_probability": 0.7682539820671082, + "style_target": 1, + "support_probability": 0.0597374364733696, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.019067327930746506, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.6217268109321594, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9954:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.6844919919967651, + 0.4650493264198303, + 0.9410969614982605, + 0.19505688548088074, + 0.08624541759490967 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9954", + "split": "test", + "style_correct": 0, + "style_probability": 0.6246160268783569, + "style_target": 0, + "support_probability": 0.05277032405138016, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.10510102046199064, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.5109487175941467, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.32102885842323303, + 0.3661264479160309, + 0.9145312309265137, + 0.2172107696533203, + 0.05246992036700249, + 0.14479826390743256 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 1, + "style_probability": 0.7971264719963074, + "style_target": 1, + "support_probability": 0.08904167264699936, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06749596270200088, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.477521687746048, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9958:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.33735501766204834, + 0.34982722997665405, + 0.9022728204727173, + 0.21034429967403412, + 0.0805957019329071, + 0.07513409107923508 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9958", + "split": "test", + "style_correct": 0, + "style_probability": 0.5418463945388794, + "style_target": 0, + "support_probability": 0.1181817427277565, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.18084481178172493, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.3983445167541504, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.19476406276226044, + 0.3603631258010864, + 0.2767786979675293, + 0.6129969358444214, + 0.10609042644500732, + 0.04420528560876846, + 0.05752270668745041, + 0.027272149920463562, + 0.09981869161128998 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.858819305896759, + "style_target": 1, + "support_probability": 0.12291184812784195, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.03612796924359563, + "accept_target": 0, + "citation_precision": 0.0, + "citation_recall": 0.0, + "citation_targets": [ + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.4962891638278961, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9967:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.1894352287054062, + 0.343136191368103, + 0.28084492683410645, + 0.5841678380966187, + 0.061767350882291794, + 0.07367660850286484, + 0.28279152512550354, + 0.08850729465484619, + 0.1296772062778473, + 0.07389608770608902, + 0.08536908775568008 + ], + "selected_evidence_indices": [], + "speaker_id": "snli:test:9967", + "split": "test", + "style_correct": 1, + "style_probability": 0.3276698887348175, + "style_target": 0, + "support_probability": 0.08201105147600174, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.7869457046881935, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.1635807901620865, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.5784022808074951, + 0.8654810786247253, + 0.5426353812217712, + 0.2882803976535797, + 0.026911400258541107, + 0.07713550329208374, + 0.15980280935764313, + 0.024149214848876, + 0.027994751930236816, + 0.029554596170783043, + 0.09997639060020447 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.7844756841659546, + "style_target": 1, + "support_probability": 0.8951611518859863, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.08702373318625999, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.15924598276615143, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9977:negative", + "evidence_budget": 10, + "evidence_probabilities": [ + 0.675627589225769, + 0.9310099482536316, + 0.577659010887146, + 0.3824514150619507, + 0.05924363061785698, + 0.06823952496051788, + 0.06655193120241165, + 0.06198812648653984, + 0.058936312794685364, + 0.13062675297260284 + ], + "selected_evidence_indices": [ + 1 + ], + "speaker_id": "snli:test:9977", + "split": "test", + "style_correct": 1, + "style_probability": 0.15915237367153168, + "style_target": 0, + "support_probability": 0.8491005301475525, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.03336436682887234, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21228159964084625, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:positive", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.41558006405830383, + 0.39347565174102783, + 0.8628202080726624, + 0.328072190284729, + 0.17957322299480438 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 1, + "style_probability": 0.5092515349388123, + "style_target": 1, + "support_probability": 0.16532200574874878, + "support_target": 0 + }, + { + "accept_correct": 1, + "accept_probability": 0.06523862065673747, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0 + ], + "conflict_probability": 0.21007415652275085, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9982:negative", + "evidence_budget": 5, + "evidence_probabilities": [ + 0.44512930512428284, + 0.44470781087875366, + 0.8410327434539795, + 0.38345766067504883, + 0.11654418706893921 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9982", + "split": "test", + "style_correct": 0, + "style_probability": 0.729746401309967, + "style_target": 0, + "support_probability": 0.14849376678466797, + "support_target": 0 + }, + { + "accept_correct": 0, + "accept_probability": 0.507478583513759, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.058373380452394485, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:positive", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.5576472878456116, + 0.3901766836643219, + 0.9823680520057678, + 0.5576472878456116, + 0.06396277993917465, + 0.11404665559530258 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 0, + "style_probability": 0.46146467328071594, + "style_target": 1, + "support_probability": 0.6810383200645447, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.8604023040439442, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0 + ], + "conflict_probability": 0.04002901166677475, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9990:negative", + "evidence_budget": 6, + "evidence_probabilities": [ + 0.6070640683174133, + 0.4123706817626953, + 0.9922806620597839, + 0.6070640683174133, + 0.13971249759197235, + 0.062088724225759506 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9990", + "split": "test", + "style_correct": 0, + "style_probability": 0.7784810066223145, + "style_target": 0, + "support_probability": 0.7577444314956665, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.37151367339197705, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.14696615934371948, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:positive", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.955898106098175, + 0.46247655153274536, + 0.2844924330711365, + 0.16657844185829163, + 0.035146333277225494, + 0.12682729959487915, + 0.032273657619953156, + 0.07102765887975693, + 0.0769655779004097 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 0, + "style_probability": 0.3205101191997528, + "style_target": 1, + "support_probability": 0.8566847443580627, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.21652221756646473, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.18488067388534546, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9996:negative", + "evidence_budget": 9, + "evidence_probabilities": [ + 0.9781997799873352, + 0.5155919790267944, + 0.2459782510995865, + 0.170034259557724, + 0.10245198756456375, + 0.03144168108701706, + 0.07578767836093903, + 0.07268258929252625, + 0.07785117626190186 + ], + "selected_evidence_indices": [ + 0 + ], + "speaker_id": "snli:test:9996", + "split": "test", + "style_correct": 1, + "style_probability": 0.22416572272777557, + "style_target": 0, + "support_probability": 0.8655814528465271, + "support_target": 1 + }, + { + "accept_correct": 0, + "accept_probability": 0.4641708463462014, + "accept_target": 1, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.2531889081001282, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:positive", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.2930809557437897, + 0.2951121926307678, + 0.9522035717964172, + 0.2930809557437897, + 0.08719397336244583, + 0.16502514481544495, + 0.03843620419502258, + 0.05355321988463402, + 0.08350127190351486, + 0.09847334772348404, + 0.1428031474351883 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.3239496052265167, + "style_target": 1, + "support_probability": 0.8194663524627686, + "support_target": 1 + }, + { + "accept_correct": 1, + "accept_probability": 0.7760884127348645, + "accept_target": 0, + "citation_precision": 1.0, + "citation_recall": 1.0, + "citation_targets": [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "conflict_probability": 0.20010167360305786, + "conflict_target": 0, + "decision": "abstain", + "episode_id": "snli-compositional:test:9998:negative", + "evidence_budget": 11, + "evidence_probabilities": [ + 0.28940337896347046, + 0.27688953280448914, + 0.9409515857696533, + 0.28940337896347046, + 0.10000686347484589, + 0.056590352207422256, + 0.04368332028388977, + 0.1278190314769745, + 0.1496146023273468, + 0.038937970995903015, + 0.06902509927749634 + ], + "selected_evidence_indices": [ + 2 + ], + "speaker_id": "snli:test:9998", + "split": "test", + "style_correct": 0, + "style_probability": 0.5998783111572266, + "style_target": 0, + "support_probability": 0.8935000896453857, + "support_target": 1 + } + ], + "schema_version": "personavoice-language-evaluation-v1", + "status": "measured_evaluation_run", + "summary": { + "accept_accuracy": { + "lower": 0.8223333333333334, + "mean": 0.8343333333333334, + "speaker_count": 1500.0, + "upper": 0.8466666666666667 + }, + "accept_brier": 0.13082480013211986, + "accept_ece": 0.13087622263856757, + "area_under_risk_coverage": 0.3504821872165865, + "budget_breakdown": { + "10": { + "acceptance_rate": 0.0, + "count": 277, + "f1_macro": 0.7392092982301665, + "false_abstention": 1.0, + "false_acceptance": 0.0, + "precision_macro": 0.7148014440433214, + "recall_macro": 0.7653429602888087 + }, + "11": { + "acceptance_rate": 0.0, + "count": 463, + "f1_macro": 0.6925789818342202, + "false_abstention": 1.0, + "false_acceptance": 0.0, + "precision_macro": 0.6814254859611231, + "recall_macro": 0.7041036717062635 + }, + "12": { + "acceptance_rate": 0.0, + "count": 4, + "f1_macro": 0.25, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.25, + "recall_macro": 0.25 + }, + "14": { + "acceptance_rate": 0.0, + "count": 2, + "f1_macro": 0.5, + "false_abstention": 0.0, + "false_acceptance": 0.0, + "precision_macro": 0.5, + "recall_macro": 0.5 + }, + "5": { + "acceptance_rate": 0.0, + "count": 747, + "f1_macro": 0.7276000553939898, + "false_abstention": 1.0, + "false_acceptance": 0.0, + "precision_macro": 0.7128514056224899, + "recall_macro": 0.7429718875502008 + }, + "6": { + "acceptance_rate": 0.00261437908496732, + "count": 765, + "f1_macro": 0.723557444046091, + "false_abstention": 0.9921259842519685, + "false_acceptance": 0.5, + "precision_macro": 0.7091503267973857, + "recall_macro": 0.738562091503268 + }, + "9": { + "acceptance_rate": 0.004043126684636119, + "count": 742, + "f1_macro": 0.7113835456995835, + "false_abstention": 0.9761904761904762, + "false_acceptance": 0.0, + "precision_macro": 0.6994609164420486, + "recall_macro": 0.7237196765498652 + } + }, + "citation_macro": { + "f1_macro": 0.7174536940284066, + "precision_macro": 0.7031666666666667, + "recall_macro": 0.7323333333333333 + }, + "citation_precision_mean": 0.7031666666666667, + "citation_precision_micro": 0.9044874433923425, + "citation_recall_mean": 0.7323333333333333, + "citation_recall_micro": 0.7323333333333333, + "conflict": { + "accuracy": 0.7836666666666666, + "brier": 0.15301865175840587, + "count": 3000, + "ece": 0.07719582872480774, + "f1": 0.6887290167865707, + "precision": 0.6617511520737327, + "recall": 0.718 + }, + "false_abstention": 0.992, + "false_acceptance": 0.2, + "positive_coverage": 0.008, + "risk_coverage": [ + { + "coverage": 0.0003333333333333333, + "risk": 0.0, + "threshold": 0.987034491640742 + }, + { + "coverage": 0.0006666666666666666, + "risk": 0.0, + "threshold": 0.9781550851664065 + }, + { + "coverage": 0.001, + "risk": 0.3333333333333333, + "threshold": 0.9759236119641912 + }, + { + "coverage": 0.0013333333333333333, + "risk": 0.25, + "threshold": 0.9741064625707168 + }, + { + "coverage": 0.0016666666666666668, + "risk": 0.2, + "threshold": 0.9723574151286093 + }, + { + "coverage": 0.002, + "risk": 0.16666666666666666, + "threshold": 0.963871882971123 + }, + { + "coverage": 0.0023333333333333335, + "risk": 0.2857142857142857, + "threshold": 0.9636998967017049 + }, + { + "coverage": 0.0026666666666666666, + "risk": 0.375, + "threshold": 0.9590390185915253 + }, + { + "coverage": 0.003, + "risk": 0.4444444444444444, + "threshold": 0.9568814751841792 + }, + { + "coverage": 0.0033333333333333335, + "risk": 0.4, + "threshold": 0.9552265044774515 + }, + { + "coverage": 0.0036666666666666666, + "risk": 0.45454545454545453, + "threshold": 0.9551828993030149 + }, + { + "coverage": 0.004, + "risk": 0.5, + "threshold": 0.9549125205790361 + }, + { + "coverage": 0.004333333333333333, + "risk": 0.5384615384615384, + "threshold": 0.954789094397487 + }, + { + "coverage": 0.004666666666666667, + "risk": 0.5714285714285714, + "threshold": 0.9499684352738208 + }, + { + "coverage": 0.005, + "risk": 0.5333333333333333, + "threshold": 0.9497876229343589 + }, + { + "coverage": 0.005333333333333333, + "risk": 0.5625, + "threshold": 0.9473259669405926 + }, + { + "coverage": 0.005666666666666667, + "risk": 0.5294117647058824, + "threshold": 0.9471066365663248 + }, + { + "coverage": 0.006, + "risk": 0.5555555555555556, + "threshold": 0.9434698060816871 + }, + { + "coverage": 0.006333333333333333, + "risk": 0.5789473684210527, + "threshold": 0.9426440754559557 + }, + { + "coverage": 0.006666666666666667, + "risk": 0.6, + "threshold": 0.9414260809938356 + }, + { + "coverage": 0.007, + "risk": 0.6190476190476191, + "threshold": 0.9358463962116162 + }, + { + "coverage": 0.007333333333333333, + "risk": 0.6363636363636364, + "threshold": 0.9353267105098481 + }, + { + "coverage": 0.007666666666666666, + "risk": 0.6521739130434783, + "threshold": 0.9346055326628786 + }, + { + "coverage": 0.008, + "risk": 0.625, + "threshold": 0.9322025068677461 + }, + { + "coverage": 0.008333333333333333, + "risk": 0.64, + "threshold": 0.9318141703594592 + }, + { + "coverage": 0.008666666666666666, + "risk": 0.6153846153846154, + "threshold": 0.931786254762109 + }, + { + "coverage": 0.009, + "risk": 0.6296296296296297, + "threshold": 0.9309870714301737 + }, + { + "coverage": 0.009333333333333334, + "risk": 0.6428571428571429, + "threshold": 0.9299527883945657 + }, + { + "coverage": 0.009666666666666667, + "risk": 0.6551724137931034, + "threshold": 0.9291971962786363 + }, + { + "coverage": 0.01, + "risk": 0.6666666666666666, + "threshold": 0.9251395327544394 + }, + { + "coverage": 0.010333333333333333, + "risk": 0.6774193548387096, + "threshold": 0.9242936999093309 + }, + { + "coverage": 0.010666666666666666, + "risk": 0.6875, + "threshold": 0.923792810025315 + }, + { + "coverage": 0.011, + "risk": 0.696969696969697, + "threshold": 0.9216630418537477 + }, + { + "coverage": 0.011333333333333334, + "risk": 0.7058823529411765, + "threshold": 0.9214014940317128 + }, + { + "coverage": 0.011666666666666667, + "risk": 0.7142857142857143, + "threshold": 0.9207775460435385 + }, + { + "coverage": 0.012, + "risk": 0.6944444444444444, + "threshold": 0.9195867064731473 + }, + { + "coverage": 0.012333333333333333, + "risk": 0.7027027027027027, + "threshold": 0.9189374614883867 + }, + { + "coverage": 0.012666666666666666, + "risk": 0.7105263157894737, + "threshold": 0.9186256210548827 + }, + { + "coverage": 0.013, + "risk": 0.717948717948718, + "threshold": 0.9175815719072489 + }, + { + "coverage": 0.013333333333333334, + "risk": 0.7, + "threshold": 0.9174071897542467 + }, + { + "coverage": 0.013666666666666667, + "risk": 0.7073170731707317, + "threshold": 0.9167684028642817 + }, + { + "coverage": 0.014, + "risk": 0.7142857142857143, + "threshold": 0.9145340487213929 + }, + { + "coverage": 0.014333333333333333, + "risk": 0.7209302325581395, + "threshold": 0.9141335685924085 + }, + { + "coverage": 0.014666666666666666, + "risk": 0.7272727272727273, + "threshold": 0.9137667053788903 + }, + { + "coverage": 0.015, + "risk": 0.7333333333333333, + "threshold": 0.9137184272041701 + }, + { + "coverage": 0.015333333333333332, + "risk": 0.7391304347826086, + "threshold": 0.9135067200522825 + }, + { + "coverage": 0.015666666666666666, + "risk": 0.723404255319149, + "threshold": 0.9134774345067208 + }, + { + "coverage": 0.016, + "risk": 0.7291666666666666, + "threshold": 0.91253541001862 + }, + { + "coverage": 0.01633333333333333, + "risk": 0.7346938775510204, + "threshold": 0.9118926912803573 + }, + { + "coverage": 0.016666666666666666, + "risk": 0.74, + "threshold": 0.9116806298221363 + }, + { + "coverage": 0.017, + "risk": 0.7450980392156863, + "threshold": 0.9096926239479969 + }, + { + "coverage": 0.017333333333333333, + "risk": 0.7307692307692307, + "threshold": 0.9094954670849625 + }, + { + "coverage": 0.017666666666666667, + "risk": 0.7358490566037735, + "threshold": 0.9091279625810459 + }, + { + "coverage": 0.018, + "risk": 0.7222222222222222, + "threshold": 0.9076270034036504 + }, + { + "coverage": 0.018333333333333333, + "risk": 0.7272727272727273, + "threshold": 0.9052291323262409 + }, + { + "coverage": 0.018666666666666668, + "risk": 0.7321428571428571, + "threshold": 0.9049957006261061 + }, + { + "coverage": 0.019, + "risk": 0.7368421052631579, + "threshold": 0.9048756627920439 + }, + { + "coverage": 0.019333333333333334, + "risk": 0.7413793103448276, + "threshold": 0.9045813885534669 + }, + { + "coverage": 0.019666666666666666, + "risk": 0.7288135593220338, + "threshold": 0.9034598813547065 + }, + { + "coverage": 0.02, + "risk": 0.7333333333333333, + "threshold": 0.9032629961396879 + }, + { + "coverage": 0.02033333333333333, + "risk": 0.7377049180327869, + "threshold": 0.9025385749002269 + }, + { + "coverage": 0.020666666666666667, + "risk": 0.7419354838709677, + "threshold": 0.901523773144028 + }, + { + "coverage": 0.021, + "risk": 0.7301587301587301, + "threshold": 0.9010265786922212 + }, + { + "coverage": 0.021333333333333333, + "risk": 0.734375, + "threshold": 0.8995110713870779 + }, + { + "coverage": 0.021666666666666667, + "risk": 0.7384615384615385, + "threshold": 0.898794216980118 + }, + { + "coverage": 0.022, + "risk": 0.7424242424242424, + "threshold": 0.8986279106916442 + }, + { + "coverage": 0.022333333333333334, + "risk": 0.7313432835820896, + "threshold": 0.8982117128173321 + }, + { + "coverage": 0.02266666666666667, + "risk": 0.7352941176470589, + "threshold": 0.8977659606807363 + }, + { + "coverage": 0.023, + "risk": 0.7391304347826086, + "threshold": 0.8969365997968666 + }, + { + "coverage": 0.023333333333333334, + "risk": 0.7428571428571429, + "threshold": 0.8969167089370629 + }, + { + "coverage": 0.023666666666666666, + "risk": 0.7464788732394366, + "threshold": 0.896618866495956 + }, + { + "coverage": 0.024, + "risk": 0.75, + "threshold": 0.8965219407379265 + }, + { + "coverage": 0.024333333333333332, + "risk": 0.7397260273972602, + "threshold": 0.8950230099574803 + }, + { + "coverage": 0.024666666666666667, + "risk": 0.7297297297297297, + "threshold": 0.894880526370141 + }, + { + "coverage": 0.025, + "risk": 0.7333333333333333, + "threshold": 0.8946284627715784 + }, + { + "coverage": 0.025333333333333333, + "risk": 0.7368421052631579, + "threshold": 0.8946220672040894 + }, + { + "coverage": 0.025666666666666667, + "risk": 0.7402597402597403, + "threshold": 0.894387780362468 + }, + { + "coverage": 0.026, + "risk": 0.7435897435897436, + "threshold": 0.8935809388310929 + }, + { + "coverage": 0.026333333333333334, + "risk": 0.7468354430379747, + "threshold": 0.8933827680965578 + }, + { + "coverage": 0.02666666666666667, + "risk": 0.75, + "threshold": 0.893081636465876 + }, + { + "coverage": 0.027, + "risk": 0.7530864197530864, + "threshold": 0.8929861293649991 + }, + { + "coverage": 0.027333333333333334, + "risk": 0.7560975609756098, + "threshold": 0.892581495716616 + }, + { + "coverage": 0.027666666666666666, + "risk": 0.7469879518072289, + "threshold": 0.8921102273902797 + }, + { + "coverage": 0.028, + "risk": 0.7380952380952381, + "threshold": 0.8903101031854987 + }, + { + "coverage": 0.028333333333333332, + "risk": 0.7294117647058823, + "threshold": 0.8899162772679466 + }, + { + "coverage": 0.028666666666666667, + "risk": 0.7325581395348837, + "threshold": 0.8899114617239653 + }, + { + "coverage": 0.029, + "risk": 0.735632183908046, + "threshold": 0.8896943797996109 + }, + { + "coverage": 0.029333333333333333, + "risk": 0.7272727272727273, + "threshold": 0.8891354704711601 + }, + { + "coverage": 0.029666666666666668, + "risk": 0.7303370786516854, + "threshold": 0.8883968232276162 + }, + { + "coverage": 0.03, + "risk": 0.7333333333333333, + "threshold": 0.8870965396825312 + }, + { + "coverage": 0.030333333333333334, + "risk": 0.7252747252747253, + "threshold": 0.8860337575535198 + }, + { + "coverage": 0.030666666666666665, + "risk": 0.7282608695652174, + "threshold": 0.8858506268350119 + }, + { + "coverage": 0.031, + "risk": 0.7311827956989247, + "threshold": 0.8850145820251427 + }, + { + "coverage": 0.03133333333333333, + "risk": 0.7340425531914894, + "threshold": 0.8839480649583108 + }, + { + "coverage": 0.03166666666666667, + "risk": 0.7263157894736842, + "threshold": 0.8826370797475392 + }, + { + "coverage": 0.032, + "risk": 0.7291666666666666, + "threshold": 0.8825669965249006 + }, + { + "coverage": 0.03233333333333333, + "risk": 0.7319587628865979, + "threshold": 0.8823244292586878 + }, + { + "coverage": 0.03266666666666666, + "risk": 0.7346938775510204, + "threshold": 0.8817411396154995 + }, + { + "coverage": 0.033, + "risk": 0.7272727272727273, + "threshold": 0.8816932459909061 + }, + { + "coverage": 0.03333333333333333, + "risk": 0.73, + "threshold": 0.8799094236907194 + }, + { + "coverage": 0.033666666666666664, + "risk": 0.7326732673267327, + "threshold": 0.8795146399520385 + }, + { + "coverage": 0.034, + "risk": 0.7352941176470589, + "threshold": 0.8793309642867797 + }, + { + "coverage": 0.034333333333333334, + "risk": 0.7378640776699029, + "threshold": 0.8784528207842471 + }, + { + "coverage": 0.034666666666666665, + "risk": 0.7403846153846154, + "threshold": 0.8777444097590673 + }, + { + "coverage": 0.035, + "risk": 0.7428571428571429, + "threshold": 0.877007456953347 + }, + { + "coverage": 0.035333333333333335, + "risk": 0.7452830188679245, + "threshold": 0.8769124073611821 + }, + { + "coverage": 0.035666666666666666, + "risk": 0.7476635514018691, + "threshold": 0.8766987348783069 + }, + { + "coverage": 0.036, + "risk": 0.7407407407407407, + "threshold": 0.8755336889315402 + }, + { + "coverage": 0.036333333333333336, + "risk": 0.7431192660550459, + "threshold": 0.8754713522269598 + }, + { + "coverage": 0.03666666666666667, + "risk": 0.7363636363636363, + "threshold": 0.8750414040765265 + }, + { + "coverage": 0.037, + "risk": 0.7387387387387387, + "threshold": 0.8747217352452344 + }, + { + "coverage": 0.037333333333333336, + "risk": 0.7410714285714286, + "threshold": 0.8727909481119768 + }, + { + "coverage": 0.03766666666666667, + "risk": 0.7345132743362832, + "threshold": 0.8719437936153972 + }, + { + "coverage": 0.038, + "risk": 0.7368421052631579, + "threshold": 0.8716880125288284 + }, + { + "coverage": 0.03833333333333333, + "risk": 0.7391304347826086, + "threshold": 0.871044620421216 + }, + { + "coverage": 0.03866666666666667, + "risk": 0.7327586206896551, + "threshold": 0.8710138317199032 + }, + { + "coverage": 0.039, + "risk": 0.7264957264957265, + "threshold": 0.8708101349863994 + }, + { + "coverage": 0.03933333333333333, + "risk": 0.7203389830508474, + "threshold": 0.8706424387490337 + }, + { + "coverage": 0.03966666666666667, + "risk": 0.7142857142857143, + "threshold": 0.8706192833670023 + }, + { + "coverage": 0.04, + "risk": 0.7166666666666667, + "threshold": 0.8701476460936419 + }, + { + "coverage": 0.04033333333333333, + "risk": 0.71900826446281, + "threshold": 0.8691336458469934 + }, + { + "coverage": 0.04066666666666666, + "risk": 0.7213114754098361, + "threshold": 0.8690330297283566 + }, + { + "coverage": 0.041, + "risk": 0.7235772357723578, + "threshold": 0.8675724120067905 + }, + { + "coverage": 0.04133333333333333, + "risk": 0.717741935483871, + "threshold": 0.8669672328895928 + }, + { + "coverage": 0.041666666666666664, + "risk": 0.712, + "threshold": 0.8666811194060456 + }, + { + "coverage": 0.042, + "risk": 0.7063492063492064, + "threshold": 0.866384603287866 + }, + { + "coverage": 0.042333333333333334, + "risk": 0.7007874015748031, + "threshold": 0.8657525724168557 + }, + { + "coverage": 0.042666666666666665, + "risk": 0.6953125, + "threshold": 0.865427021153522 + }, + { + "coverage": 0.043, + "risk": 0.6976744186046512, + "threshold": 0.8652499891841288 + }, + { + "coverage": 0.043333333333333335, + "risk": 0.6923076923076923, + "threshold": 0.8650493893559279 + }, + { + "coverage": 0.043666666666666666, + "risk": 0.6946564885496184, + "threshold": 0.8646075799386495 + }, + { + "coverage": 0.044, + "risk": 0.696969696969697, + "threshold": 0.8642536754813791 + }, + { + "coverage": 0.044333333333333336, + "risk": 0.6992481203007519, + "threshold": 0.8640223312283979 + }, + { + "coverage": 0.04466666666666667, + "risk": 0.7014925373134329, + "threshold": 0.8639770078620933 + }, + { + "coverage": 0.045, + "risk": 0.7037037037037037, + "threshold": 0.8636966416235161 + }, + { + "coverage": 0.04533333333333334, + "risk": 0.7058823529411765, + "threshold": 0.863592482922389 + }, + { + "coverage": 0.04566666666666667, + "risk": 0.708029197080292, + "threshold": 0.8633638177498391 + }, + { + "coverage": 0.046, + "risk": 0.7101449275362319, + "threshold": 0.8627595236517536 + }, + { + "coverage": 0.04633333333333333, + "risk": 0.7122302158273381, + "threshold": 0.8626501230303419 + }, + { + "coverage": 0.04666666666666667, + "risk": 0.7142857142857143, + "threshold": 0.8626183583732648 + }, + { + "coverage": 0.047, + "risk": 0.7092198581560284, + "threshold": 0.8626160781323904 + }, + { + "coverage": 0.04733333333333333, + "risk": 0.704225352112676, + "threshold": 0.8623190467024954 + }, + { + "coverage": 0.04766666666666667, + "risk": 0.7062937062937062, + "threshold": 0.8621517115592391 + }, + { + "coverage": 0.048, + "risk": 0.7083333333333334, + "threshold": 0.8621414945177719 + }, + { + "coverage": 0.04833333333333333, + "risk": 0.7103448275862069, + "threshold": 0.8616634849815341 + }, + { + "coverage": 0.048666666666666664, + "risk": 0.7123287671232876, + "threshold": 0.861366781336983 + }, + { + "coverage": 0.049, + "risk": 0.7142857142857143, + "threshold": 0.8612836590516934 + }, + { + "coverage": 0.04933333333333333, + "risk": 0.7094594594594594, + "threshold": 0.8604023040439442 + }, + { + "coverage": 0.049666666666666665, + "risk": 0.7114093959731543, + "threshold": 0.8592463931286716 + }, + { + "coverage": 0.05, + "risk": 0.7066666666666667, + "threshold": 0.8589700421436888 + }, + { + "coverage": 0.050333333333333334, + "risk": 0.7086092715231788, + "threshold": 0.8583723942086685 + }, + { + "coverage": 0.050666666666666665, + "risk": 0.7039473684210527, + "threshold": 0.8578492011339875 + }, + { + "coverage": 0.051, + "risk": 0.6993464052287581, + "threshold": 0.8577412430773278 + }, + { + "coverage": 0.051333333333333335, + "risk": 0.6948051948051948, + "threshold": 0.8562037793925086 + }, + { + "coverage": 0.051666666666666666, + "risk": 0.6967741935483871, + "threshold": 0.8560534243690654 + }, + { + "coverage": 0.052, + "risk": 0.6987179487179487, + "threshold": 0.8559215960158325 + }, + { + "coverage": 0.052333333333333336, + "risk": 0.7006369426751592, + "threshold": 0.8555533806741379 + }, + { + "coverage": 0.05266666666666667, + "risk": 0.6962025316455697, + "threshold": 0.8552167092136286 + }, + { + "coverage": 0.053, + "risk": 0.6981132075471698, + "threshold": 0.8549100841157191 + }, + { + "coverage": 0.05333333333333334, + "risk": 0.7, + "threshold": 0.8545136396216968 + }, + { + "coverage": 0.05366666666666667, + "risk": 0.6956521739130435, + "threshold": 0.8542023281263234 + }, + { + "coverage": 0.054, + "risk": 0.6975308641975309, + "threshold": 0.8539414879807764 + }, + { + "coverage": 0.05433333333333333, + "risk": 0.6932515337423313, + "threshold": 0.853906708790138 + }, + { + "coverage": 0.05466666666666667, + "risk": 0.6951219512195121, + "threshold": 0.8526725848138276 + }, + { + "coverage": 0.055, + "risk": 0.696969696969697, + "threshold": 0.852561296213814 + }, + { + "coverage": 0.05533333333333333, + "risk": 0.6987951807228916, + "threshold": 0.8517099684192715 + }, + { + "coverage": 0.05566666666666667, + "risk": 0.7005988023952096, + "threshold": 0.8509599116654343 + }, + { + "coverage": 0.056, + "risk": 0.7023809523809523, + "threshold": 0.8493455429572805 + }, + { + "coverage": 0.05633333333333333, + "risk": 0.7041420118343196, + "threshold": 0.8485991539891202 + }, + { + "coverage": 0.056666666666666664, + "risk": 0.7, + "threshold": 0.848594044314723 + }, + { + "coverage": 0.057, + "risk": 0.695906432748538, + "threshold": 0.8465944578385347 + }, + { + "coverage": 0.05733333333333333, + "risk": 0.6918604651162791, + "threshold": 0.8463860356632225 + }, + { + "coverage": 0.057666666666666665, + "risk": 0.6878612716763006, + "threshold": 0.846324825297586 + }, + { + "coverage": 0.058, + "risk": 0.6839080459770115, + "threshold": 0.845804448822065 + }, + { + "coverage": 0.058333333333333334, + "risk": 0.68, + "threshold": 0.8455925781957951 + }, + { + "coverage": 0.058666666666666666, + "risk": 0.6818181818181818, + "threshold": 0.845176991306719 + }, + { + "coverage": 0.059, + "risk": 0.6779661016949152, + "threshold": 0.8447323685370448 + }, + { + "coverage": 0.059333333333333335, + "risk": 0.6741573033707865, + "threshold": 0.8442573820865107 + }, + { + "coverage": 0.059666666666666666, + "risk": 0.6759776536312849, + "threshold": 0.8441283495129994 + }, + { + "coverage": 0.06, + "risk": 0.6777777777777778, + "threshold": 0.8441133732226546 + }, + { + "coverage": 0.060333333333333336, + "risk": 0.6795580110497238, + "threshold": 0.8435629799268582 + }, + { + "coverage": 0.06066666666666667, + "risk": 0.6758241758241759, + "threshold": 0.8428655479942222 + }, + { + "coverage": 0.061, + "risk": 0.6721311475409836, + "threshold": 0.8421733744179108 + }, + { + "coverage": 0.06133333333333333, + "risk": 0.6739130434782609, + "threshold": 0.8418988471242911 + }, + { + "coverage": 0.06166666666666667, + "risk": 0.6756756756756757, + "threshold": 0.8417100317946975 + }, + { + "coverage": 0.062, + "risk": 0.6774193548387096, + "threshold": 0.8415517612610728 + }, + { + "coverage": 0.06233333333333333, + "risk": 0.6737967914438503, + "threshold": 0.8414898636419216 + }, + { + "coverage": 0.06266666666666666, + "risk": 0.6702127659574468, + "threshold": 0.8413953886159219 + }, + { + "coverage": 0.063, + "risk": 0.6666666666666666, + "threshold": 0.8404841945332181 + }, + { + "coverage": 0.06333333333333334, + "risk": 0.6684210526315789, + "threshold": 0.8398510246941129 + }, + { + "coverage": 0.06366666666666666, + "risk": 0.6701570680628273, + "threshold": 0.8388155963101553 + }, + { + "coverage": 0.064, + "risk": 0.6666666666666666, + "threshold": 0.8387955276728577 + }, + { + "coverage": 0.06433333333333334, + "risk": 0.6632124352331606, + "threshold": 0.8384687144844634 + }, + { + "coverage": 0.06466666666666666, + "risk": 0.6597938144329897, + "threshold": 0.8380211101295743 + }, + { + "coverage": 0.065, + "risk": 0.6564102564102564, + "threshold": 0.8380050584889888 + }, + { + "coverage": 0.06533333333333333, + "risk": 0.6530612244897959, + "threshold": 0.837909821953872 + }, + { + "coverage": 0.06566666666666666, + "risk": 0.6548223350253807, + "threshold": 0.8373790127787117 + }, + { + "coverage": 0.066, + "risk": 0.6565656565656566, + "threshold": 0.8356215087541939 + }, + { + "coverage": 0.06633333333333333, + "risk": 0.6582914572864321, + "threshold": 0.8349624109614524 + }, + { + "coverage": 0.06666666666666667, + "risk": 0.655, + "threshold": 0.8329028328888376 + }, + { + "coverage": 0.067, + "risk": 0.6517412935323383, + "threshold": 0.8318080186221835 + }, + { + "coverage": 0.06733333333333333, + "risk": 0.6534653465346535, + "threshold": 0.831694427928369 + }, + { + "coverage": 0.06766666666666667, + "risk": 0.6551724137931034, + "threshold": 0.8316110987260253 + }, + { + "coverage": 0.068, + "risk": 0.6519607843137255, + "threshold": 0.8313071965320447 + }, + { + "coverage": 0.06833333333333333, + "risk": 0.6487804878048781, + "threshold": 0.8311430436824594 + }, + { + "coverage": 0.06866666666666667, + "risk": 0.6504854368932039, + "threshold": 0.8310615648110442 + }, + { + "coverage": 0.069, + "risk": 0.6473429951690821, + "threshold": 0.8291581826502216 + }, + { + "coverage": 0.06933333333333333, + "risk": 0.6442307692307693, + "threshold": 0.8284519379359807 + }, + { + "coverage": 0.06966666666666667, + "risk": 0.6411483253588517, + "threshold": 0.8276541885290101 + }, + { + "coverage": 0.07, + "risk": 0.6428571428571429, + "threshold": 0.8272469517859946 + }, + { + "coverage": 0.07033333333333333, + "risk": 0.6445497630331753, + "threshold": 0.827110038117368 + }, + { + "coverage": 0.07066666666666667, + "risk": 0.6462264150943396, + "threshold": 0.8259862893148124 + }, + { + "coverage": 0.071, + "risk": 0.647887323943662, + "threshold": 0.8256180007070668 + }, + { + "coverage": 0.07133333333333333, + "risk": 0.6495327102803738, + "threshold": 0.8249298795060298 + }, + { + "coverage": 0.07166666666666667, + "risk": 0.6465116279069767, + "threshold": 0.8241612138685498 + }, + { + "coverage": 0.072, + "risk": 0.6481481481481481, + "threshold": 0.823089532779541 + }, + { + "coverage": 0.07233333333333333, + "risk": 0.6497695852534562, + "threshold": 0.8230173257274487 + }, + { + "coverage": 0.07266666666666667, + "risk": 0.6513761467889908, + "threshold": 0.8219866510322156 + }, + { + "coverage": 0.073, + "risk": 0.6484018264840182, + "threshold": 0.819840361458926 + }, + { + "coverage": 0.07333333333333333, + "risk": 0.6454545454545455, + "threshold": 0.8186525327675084 + }, + { + "coverage": 0.07366666666666667, + "risk": 0.6470588235294118, + "threshold": 0.8184162116547604 + }, + { + "coverage": 0.074, + "risk": 0.6441441441441441, + "threshold": 0.8181515846748937 + }, + { + "coverage": 0.07433333333333333, + "risk": 0.6412556053811659, + "threshold": 0.816880958818551 + }, + { + "coverage": 0.07466666666666667, + "risk": 0.6428571428571429, + "threshold": 0.8168436370075048 + }, + { + "coverage": 0.075, + "risk": 0.6444444444444445, + "threshold": 0.8155711697060964 + }, + { + "coverage": 0.07533333333333334, + "risk": 0.6460176991150443, + "threshold": 0.8154746037832935 + }, + { + "coverage": 0.07566666666666666, + "risk": 0.6431718061674009, + "threshold": 0.814382615600195 + }, + { + "coverage": 0.076, + "risk": 0.6403508771929824, + "threshold": 0.8135544879269041 + }, + { + "coverage": 0.07633333333333334, + "risk": 0.6419213973799127, + "threshold": 0.8118395724167158 + }, + { + "coverage": 0.07666666666666666, + "risk": 0.6391304347826087, + "threshold": 0.8117405637563307 + }, + { + "coverage": 0.077, + "risk": 0.6406926406926406, + "threshold": 0.8111033345742982 + }, + { + "coverage": 0.07733333333333334, + "risk": 0.6422413793103449, + "threshold": 0.8110221946830087 + }, + { + "coverage": 0.07766666666666666, + "risk": 0.6394849785407726, + "threshold": 0.8107749743099077 + }, + { + "coverage": 0.078, + "risk": 0.6367521367521367, + "threshold": 0.8106610686261461 + }, + { + "coverage": 0.07833333333333334, + "risk": 0.6382978723404256, + "threshold": 0.8101527073319659 + }, + { + "coverage": 0.07866666666666666, + "risk": 0.6398305084745762, + "threshold": 0.8074282158873914 + }, + { + "coverage": 0.079, + "risk": 0.6371308016877637, + "threshold": 0.8073570899099992 + }, + { + "coverage": 0.07933333333333334, + "risk": 0.634453781512605, + "threshold": 0.8071653285750884 + }, + { + "coverage": 0.07966666666666666, + "risk": 0.6317991631799164, + "threshold": 0.806580362036371 + }, + { + "coverage": 0.08, + "risk": 0.6333333333333333, + "threshold": 0.8055614301107781 + }, + { + "coverage": 0.08033333333333334, + "risk": 0.6307053941908713, + "threshold": 0.8052148964860066 + }, + { + "coverage": 0.08066666666666666, + "risk": 0.6322314049586777, + "threshold": 0.8045084399719542 + }, + { + "coverage": 0.081, + "risk": 0.6337448559670782, + "threshold": 0.8043839431306843 + }, + { + "coverage": 0.08133333333333333, + "risk": 0.6311475409836066, + "threshold": 0.8036197786801865 + }, + { + "coverage": 0.08166666666666667, + "risk": 0.6285714285714286, + "threshold": 0.8034797147135606 + }, + { + "coverage": 0.082, + "risk": 0.6260162601626016, + "threshold": 0.8022257114616771 + }, + { + "coverage": 0.08233333333333333, + "risk": 0.6234817813765182, + "threshold": 0.8019302795006027 + }, + { + "coverage": 0.08266666666666667, + "risk": 0.625, + "threshold": 0.8010226826280586 + }, + { + "coverage": 0.083, + "risk": 0.6224899598393574, + "threshold": 0.8009278483910975 + }, + { + "coverage": 0.08333333333333333, + "risk": 0.624, + "threshold": 0.8008949006843373 + }, + { + "coverage": 0.08366666666666667, + "risk": 0.6215139442231076, + "threshold": 0.8004750543645045 + }, + { + "coverage": 0.084, + "risk": 0.6190476190476191, + "threshold": 0.8000977381594332 + }, + { + "coverage": 0.08433333333333333, + "risk": 0.616600790513834, + "threshold": 0.7994470933329438 + }, + { + "coverage": 0.08466666666666667, + "risk": 0.6141732283464567, + "threshold": 0.7987269134388839 + }, + { + "coverage": 0.085, + "risk": 0.611764705882353, + "threshold": 0.79869516531177 + }, + { + "coverage": 0.08533333333333333, + "risk": 0.61328125, + "threshold": 0.7985962585732622 + }, + { + "coverage": 0.08566666666666667, + "risk": 0.6108949416342413, + "threshold": 0.7979742117005628 + }, + { + "coverage": 0.086, + "risk": 0.6085271317829457, + "threshold": 0.7968125740090812 + }, + { + "coverage": 0.08633333333333333, + "risk": 0.6061776061776062, + "threshold": 0.7963915712641665 + }, + { + "coverage": 0.08666666666666667, + "risk": 0.6076923076923076, + "threshold": 0.79497794101691 + }, + { + "coverage": 0.087, + "risk": 0.6091954022988506, + "threshold": 0.7949690357581147 + }, + { + "coverage": 0.08733333333333333, + "risk": 0.6106870229007634, + "threshold": 0.7948551032960012 + }, + { + "coverage": 0.08766666666666667, + "risk": 0.6121673003802282, + "threshold": 0.7946391108052495 + }, + { + "coverage": 0.088, + "risk": 0.6098484848484849, + "threshold": 0.7945020756923142 + }, + { + "coverage": 0.08833333333333333, + "risk": 0.6075471698113207, + "threshold": 0.7942416490177067 + }, + { + "coverage": 0.08866666666666667, + "risk": 0.6090225563909775, + "threshold": 0.7941981099512319 + }, + { + "coverage": 0.089, + "risk": 0.6104868913857678, + "threshold": 0.7938112776214652 + }, + { + "coverage": 0.08933333333333333, + "risk": 0.6119402985074627, + "threshold": 0.7929530940584292 + }, + { + "coverage": 0.08966666666666667, + "risk": 0.6096654275092936, + "threshold": 0.7915960069896676 + }, + { + "coverage": 0.09, + "risk": 0.6111111111111112, + "threshold": 0.7914836784973289 + }, + { + "coverage": 0.09033333333333333, + "risk": 0.6125461254612546, + "threshold": 0.7905246667231579 + }, + { + "coverage": 0.09066666666666667, + "risk": 0.6102941176470589, + "threshold": 0.7898600427047882 + }, + { + "coverage": 0.091, + "risk": 0.6117216117216118, + "threshold": 0.7887739812267434 + }, + { + "coverage": 0.09133333333333334, + "risk": 0.6094890510948905, + "threshold": 0.7883596099104189 + }, + { + "coverage": 0.09166666666666666, + "risk": 0.610909090909091, + "threshold": 0.7873112528281646 + }, + { + "coverage": 0.092, + "risk": 0.6086956521739131, + "threshold": 0.7872151823833207 + }, + { + "coverage": 0.09233333333333334, + "risk": 0.6101083032490975, + "threshold": 0.7869457046881935 + }, + { + "coverage": 0.09266666666666666, + "risk": 0.6079136690647482, + "threshold": 0.7859786982932064 + }, + { + "coverage": 0.093, + "risk": 0.6057347670250897, + "threshold": 0.7847315872216392 + }, + { + "coverage": 0.09333333333333334, + "risk": 0.6071428571428571, + "threshold": 0.78441269227989 + }, + { + "coverage": 0.09366666666666666, + "risk": 0.608540925266904, + "threshold": 0.7835257778789355 + }, + { + "coverage": 0.094, + "risk": 0.6063829787234043, + "threshold": 0.7829331906084747 + }, + { + "coverage": 0.09433333333333334, + "risk": 0.6042402826855123, + "threshold": 0.7826355530912288 + }, + { + "coverage": 0.09466666666666666, + "risk": 0.602112676056338, + "threshold": 0.7825430494888829 + }, + { + "coverage": 0.095, + "risk": 0.6, + "threshold": 0.7815896604533776 + }, + { + "coverage": 0.09533333333333334, + "risk": 0.5979020979020979, + "threshold": 0.781459405147924 + }, + { + "coverage": 0.09566666666666666, + "risk": 0.5958188153310104, + "threshold": 0.7812262624765154 + }, + { + "coverage": 0.096, + "risk": 0.59375, + "threshold": 0.7796684205419594 + }, + { + "coverage": 0.09633333333333334, + "risk": 0.5916955017301038, + "threshold": 0.7791080154989488 + }, + { + "coverage": 0.09666666666666666, + "risk": 0.5896551724137931, + "threshold": 0.7790338685886269 + }, + { + "coverage": 0.097, + "risk": 0.5910652920962199, + "threshold": 0.7779866461894471 + }, + { + "coverage": 0.09733333333333333, + "risk": 0.589041095890411, + "threshold": 0.7779035712394773 + }, + { + "coverage": 0.09766666666666667, + "risk": 0.590443686006826, + "threshold": 0.7778528849300592 + }, + { + "coverage": 0.098, + "risk": 0.5918367346938775, + "threshold": 0.7773939216897077 + }, + { + "coverage": 0.09833333333333333, + "risk": 0.5932203389830508, + "threshold": 0.7766875123201409 + }, + { + "coverage": 0.09866666666666667, + "risk": 0.5945945945945946, + "threshold": 0.7765384657357188 + }, + { + "coverage": 0.099, + "risk": 0.5959595959595959, + "threshold": 0.7765064827911486 + }, + { + "coverage": 0.09933333333333333, + "risk": 0.5973154362416108, + "threshold": 0.776458631061729 + }, + { + "coverage": 0.09966666666666667, + "risk": 0.5953177257525084, + "threshold": 0.7761967162177826 + }, + { + "coverage": 0.1, + "risk": 0.5933333333333334, + "threshold": 0.7760884127348645 + }, + { + "coverage": 0.10033333333333333, + "risk": 0.5946843853820598, + "threshold": 0.7758065178806817 + }, + { + "coverage": 0.10066666666666667, + "risk": 0.5960264900662252, + "threshold": 0.775656027885752 + }, + { + "coverage": 0.101, + "risk": 0.5973597359735974, + "threshold": 0.7756004957363913 + }, + { + "coverage": 0.10133333333333333, + "risk": 0.5953947368421053, + "threshold": 0.774389212897777 + }, + { + "coverage": 0.10166666666666667, + "risk": 0.5967213114754099, + "threshold": 0.7741694675159749 + }, + { + "coverage": 0.102, + "risk": 0.5947712418300654, + "threshold": 0.7729638892569142 + }, + { + "coverage": 0.10233333333333333, + "risk": 0.5960912052117264, + "threshold": 0.772924505988648 + }, + { + "coverage": 0.10266666666666667, + "risk": 0.5974025974025974, + "threshold": 0.7728935110677959 + }, + { + "coverage": 0.103, + "risk": 0.598705501618123, + "threshold": 0.7728676944800483 + }, + { + "coverage": 0.10333333333333333, + "risk": 0.5967741935483871, + "threshold": 0.7721643940962508 + }, + { + "coverage": 0.10366666666666667, + "risk": 0.594855305466238, + "threshold": 0.771359901265768 + }, + { + "coverage": 0.104, + "risk": 0.592948717948718, + "threshold": 0.7707288245991961 + }, + { + "coverage": 0.10433333333333333, + "risk": 0.5942492012779552, + "threshold": 0.7701818178007187 + }, + { + "coverage": 0.10466666666666667, + "risk": 0.5923566878980892, + "threshold": 0.7700419288044102 + }, + { + "coverage": 0.105, + "risk": 0.5904761904761905, + "threshold": 0.7696822045231038 + }, + { + "coverage": 0.10533333333333333, + "risk": 0.5886075949367089, + "threshold": 0.7684881445952207 + }, + { + "coverage": 0.10566666666666667, + "risk": 0.5867507886435331, + "threshold": 0.7680107828134588 + }, + { + "coverage": 0.106, + "risk": 0.5849056603773585, + "threshold": 0.7678548260585778 + }, + { + "coverage": 0.10633333333333334, + "risk": 0.5830721003134797, + "threshold": 0.7677532323963163 + }, + { + "coverage": 0.10666666666666667, + "risk": 0.58125, + "threshold": 0.7667214014963406 + }, + { + "coverage": 0.107, + "risk": 0.5794392523364486, + "threshold": 0.7656808953989724 + }, + { + "coverage": 0.10733333333333334, + "risk": 0.5807453416149069, + "threshold": 0.7650325118341073 + }, + { + "coverage": 0.10766666666666666, + "risk": 0.5820433436532507, + "threshold": 0.7640802798558362 + }, + { + "coverage": 0.108, + "risk": 0.5802469135802469, + "threshold": 0.7635866207186126 + }, + { + "coverage": 0.10833333333333334, + "risk": 0.5784615384615385, + "threshold": 0.7630044761002317 + }, + { + "coverage": 0.10866666666666666, + "risk": 0.5766871165644172, + "threshold": 0.7625805169411173 + }, + { + "coverage": 0.109, + "risk": 0.5749235474006116, + "threshold": 0.7621570726244897 + }, + { + "coverage": 0.10933333333333334, + "risk": 0.573170731707317, + "threshold": 0.7613669938760382 + }, + { + "coverage": 0.10966666666666666, + "risk": 0.5714285714285714, + "threshold": 0.7608269781178479 + }, + { + "coverage": 0.11, + "risk": 0.5696969696969697, + "threshold": 0.7605216625234412 + }, + { + "coverage": 0.11033333333333334, + "risk": 0.56797583081571, + "threshold": 0.7604073914961873 + }, + { + "coverage": 0.11066666666666666, + "risk": 0.5692771084337349, + "threshold": 0.7596763858134068 + }, + { + "coverage": 0.111, + "risk": 0.5675675675675675, + "threshold": 0.7585933604011431 + }, + { + "coverage": 0.11133333333333334, + "risk": 0.5688622754491018, + "threshold": 0.7582218931211031 + }, + { + "coverage": 0.11166666666666666, + "risk": 0.5671641791044776, + "threshold": 0.7546806199752609 + }, + { + "coverage": 0.112, + "risk": 0.5684523809523809, + "threshold": 0.7539579824366456 + }, + { + "coverage": 0.11233333333333333, + "risk": 0.5667655786350149, + "threshold": 0.7532928078989762 + }, + { + "coverage": 0.11266666666666666, + "risk": 0.5680473372781065, + "threshold": 0.753128542899411 + }, + { + "coverage": 0.113, + "risk": 0.5693215339233039, + "threshold": 0.7526143633798625 + }, + { + "coverage": 0.11333333333333333, + "risk": 0.5705882352941176, + "threshold": 0.7515218713469807 + }, + { + "coverage": 0.11366666666666667, + "risk": 0.5689149560117303, + "threshold": 0.7514653862578768 + }, + { + "coverage": 0.114, + "risk": 0.5672514619883041, + "threshold": 0.7514629977922205 + }, + { + "coverage": 0.11433333333333333, + "risk": 0.565597667638484, + "threshold": 0.7504996280404796 + }, + { + "coverage": 0.11466666666666667, + "risk": 0.563953488372093, + "threshold": 0.7499437444489132 + }, + { + "coverage": 0.115, + "risk": 0.5623188405797102, + "threshold": 0.7494090629588257 + }, + { + "coverage": 0.11533333333333333, + "risk": 0.5635838150289018, + "threshold": 0.7489019642545607 + }, + { + "coverage": 0.11566666666666667, + "risk": 0.5648414985590778, + "threshold": 0.7477965585523734 + }, + { + "coverage": 0.116, + "risk": 0.5660919540229885, + "threshold": 0.7468658863826754 + }, + { + "coverage": 0.11633333333333333, + "risk": 0.5673352435530086, + "threshold": 0.7467596813464077 + }, + { + "coverage": 0.11666666666666667, + "risk": 0.5685714285714286, + "threshold": 0.7454464724936365 + }, + { + "coverage": 0.117, + "risk": 0.5698005698005698, + "threshold": 0.7452305377843871 + }, + { + "coverage": 0.11733333333333333, + "risk": 0.5681818181818182, + "threshold": 0.7451914126828285 + }, + { + "coverage": 0.11766666666666667, + "risk": 0.5694050991501416, + "threshold": 0.7449596703326123 + }, + { + "coverage": 0.118, + "risk": 0.5706214689265536, + "threshold": 0.744639845030477 + }, + { + "coverage": 0.11833333333333333, + "risk": 0.5690140845070423, + "threshold": 0.7433361984534949 + }, + { + "coverage": 0.11866666666666667, + "risk": 0.5674157303370787, + "threshold": 0.7430304648446651 + }, + { + "coverage": 0.119, + "risk": 0.5686274509803921, + "threshold": 0.7428918342409645 + }, + { + "coverage": 0.11933333333333333, + "risk": 0.5698324022346368, + "threshold": 0.742848202415356 + }, + { + "coverage": 0.11966666666666667, + "risk": 0.5710306406685237, + "threshold": 0.7419106245556294 + }, + { + "coverage": 0.12, + "risk": 0.5694444444444444, + "threshold": 0.7411248099369877 + }, + { + "coverage": 0.12033333333333333, + "risk": 0.5678670360110804, + "threshold": 0.7405092667589059 + }, + { + "coverage": 0.12066666666666667, + "risk": 0.569060773480663, + "threshold": 0.7404502895516956 + }, + { + "coverage": 0.121, + "risk": 0.5702479338842975, + "threshold": 0.7403628882666168 + }, + { + "coverage": 0.12133333333333333, + "risk": 0.5714285714285714, + "threshold": 0.740199248462929 + }, + { + "coverage": 0.12166666666666667, + "risk": 0.5726027397260274, + "threshold": 0.7398309574964657 + }, + { + "coverage": 0.122, + "risk": 0.5737704918032787, + "threshold": 0.7393871255758812 + }, + { + "coverage": 0.12233333333333334, + "risk": 0.5722070844686649, + "threshold": 0.7390190175563305 + }, + { + "coverage": 0.12266666666666666, + "risk": 0.5706521739130435, + "threshold": 0.7372901290138428 + }, + { + "coverage": 0.123, + "risk": 0.5718157181571816, + "threshold": 0.7372166404856629 + }, + { + "coverage": 0.12333333333333334, + "risk": 0.5702702702702702, + "threshold": 0.7360547385830998 + }, + { + "coverage": 0.12366666666666666, + "risk": 0.568733153638814, + "threshold": 0.7358789007713717 + }, + { + "coverage": 0.124, + "risk": 0.5698924731182796, + "threshold": 0.7347421102724141 + }, + { + "coverage": 0.12433333333333334, + "risk": 0.5683646112600537, + "threshold": 0.7339105322443659 + }, + { + "coverage": 0.12466666666666666, + "risk": 0.5668449197860963, + "threshold": 0.7337247819905954 + }, + { + "coverage": 0.125, + "risk": 0.5653333333333334, + "threshold": 0.7325860841227083 + }, + { + "coverage": 0.12533333333333332, + "risk": 0.5664893617021277, + "threshold": 0.7323756079343298 + }, + { + "coverage": 0.12566666666666668, + "risk": 0.5676392572944297, + "threshold": 0.7320771680810445 + }, + { + "coverage": 0.126, + "risk": 0.5687830687830688, + "threshold": 0.7319241726050976 + }, + { + "coverage": 0.12633333333333333, + "risk": 0.5672823218997362, + "threshold": 0.7307615089432851 + }, + { + "coverage": 0.12666666666666668, + "risk": 0.5657894736842105, + "threshold": 0.73037938640951 + }, + { + "coverage": 0.127, + "risk": 0.5669291338582677, + "threshold": 0.7298559346320133 + }, + { + "coverage": 0.12733333333333333, + "risk": 0.5680628272251309, + "threshold": 0.7296778377780129 + }, + { + "coverage": 0.12766666666666668, + "risk": 0.566579634464752, + "threshold": 0.7288497425409872 + }, + { + "coverage": 0.128, + "risk": 0.5677083333333334, + "threshold": 0.7287627978697842 + }, + { + "coverage": 0.12833333333333333, + "risk": 0.5688311688311688, + "threshold": 0.7286383432282761 + }, + { + "coverage": 0.12866666666666668, + "risk": 0.5673575129533679, + "threshold": 0.7280420676644184 + }, + { + "coverage": 0.129, + "risk": 0.5684754521963824, + "threshold": 0.7276668245619929 + }, + { + "coverage": 0.12933333333333333, + "risk": 0.5670103092783505, + "threshold": 0.7259273709255147 + }, + { + "coverage": 0.12966666666666668, + "risk": 0.5681233933161953, + "threshold": 0.7247065431404209 + }, + { + "coverage": 0.13, + "risk": 0.5692307692307692, + "threshold": 0.7230808895331435 + }, + { + "coverage": 0.13033333333333333, + "risk": 0.5703324808184144, + "threshold": 0.7223713648063743 + }, + { + "coverage": 0.13066666666666665, + "risk": 0.5688775510204082, + "threshold": 0.7223247370991075 + }, + { + "coverage": 0.131, + "risk": 0.5674300254452926, + "threshold": 0.7221066847872493 + }, + { + "coverage": 0.13133333333333333, + "risk": 0.565989847715736, + "threshold": 0.7220480160924581 + }, + { + "coverage": 0.13166666666666665, + "risk": 0.5645569620253165, + "threshold": 0.7219057854684934 + }, + { + "coverage": 0.132, + "risk": 0.5656565656565656, + "threshold": 0.7212356608434796 + }, + { + "coverage": 0.13233333333333333, + "risk": 0.5667506297229219, + "threshold": 0.7210283552410601 + }, + { + "coverage": 0.13266666666666665, + "risk": 0.5653266331658291, + "threshold": 0.7195572722357788 + }, + { + "coverage": 0.133, + "risk": 0.5639097744360902, + "threshold": 0.7195245619865817 + }, + { + "coverage": 0.13333333333333333, + "risk": 0.5625, + "threshold": 0.7188926420276687 + }, + { + "coverage": 0.13366666666666666, + "risk": 0.5635910224438903, + "threshold": 0.7184971263761923 + }, + { + "coverage": 0.134, + "risk": 0.5621890547263682, + "threshold": 0.7182862137666177 + }, + { + "coverage": 0.13433333333333333, + "risk": 0.5607940446650124, + "threshold": 0.7182385939478237 + }, + { + "coverage": 0.13466666666666666, + "risk": 0.5594059405940595, + "threshold": 0.7180569075243923 + }, + { + "coverage": 0.135, + "risk": 0.5604938271604938, + "threshold": 0.7175999737514722 + }, + { + "coverage": 0.13533333333333333, + "risk": 0.5591133004926109, + "threshold": 0.7175898688235449 + }, + { + "coverage": 0.13566666666666666, + "risk": 0.5601965601965602, + "threshold": 0.717569089888638 + }, + { + "coverage": 0.136, + "risk": 0.5588235294117647, + "threshold": 0.7171825709013859 + }, + { + "coverage": 0.13633333333333333, + "risk": 0.5574572127139364, + "threshold": 0.7163677159757107 + }, + { + "coverage": 0.13666666666666666, + "risk": 0.5560975609756098, + "threshold": 0.715777329530578 + }, + { + "coverage": 0.137, + "risk": 0.5571776155717761, + "threshold": 0.7157746663257747 + }, + { + "coverage": 0.13733333333333334, + "risk": 0.558252427184466, + "threshold": 0.7155869920255284 + }, + { + "coverage": 0.13766666666666666, + "risk": 0.5569007263922519, + "threshold": 0.7155387358510444 + }, + { + "coverage": 0.138, + "risk": 0.5579710144927537, + "threshold": 0.7151737774106337 + }, + { + "coverage": 0.13833333333333334, + "risk": 0.5590361445783133, + "threshold": 0.7150627569598811 + }, + { + "coverage": 0.13866666666666666, + "risk": 0.5576923076923077, + "threshold": 0.7149723939546035 + }, + { + "coverage": 0.139, + "risk": 0.5587529976019184, + "threshold": 0.7149302867448313 + }, + { + "coverage": 0.13933333333333334, + "risk": 0.5574162679425837, + "threshold": 0.7147659034076632 + }, + { + "coverage": 0.13966666666666666, + "risk": 0.5560859188544153, + "threshold": 0.7130712351150379 + }, + { + "coverage": 0.14, + "risk": 0.5547619047619048, + "threshold": 0.712133942530798 + }, + { + "coverage": 0.14033333333333334, + "risk": 0.5558194774346793, + "threshold": 0.710455709256408 + }, + { + "coverage": 0.14066666666666666, + "risk": 0.5545023696682464, + "threshold": 0.7101853510796197 + }, + { + "coverage": 0.141, + "risk": 0.5555555555555556, + "threshold": 0.7098775704152587 + }, + { + "coverage": 0.14133333333333334, + "risk": 0.5542452830188679, + "threshold": 0.7098687560100602 + }, + { + "coverage": 0.14166666666666666, + "risk": 0.5552941176470588, + "threshold": 0.7097649794501031 + }, + { + "coverage": 0.142, + "risk": 0.5563380281690141, + "threshold": 0.7092237416776221 + }, + { + "coverage": 0.14233333333333334, + "risk": 0.5550351288056206, + "threshold": 0.7091401417969578 + }, + { + "coverage": 0.14266666666666666, + "risk": 0.5537383177570093, + "threshold": 0.7089892081939553 + }, + { + "coverage": 0.143, + "risk": 0.5524475524475524, + "threshold": 0.7086675984385771 + }, + { + "coverage": 0.14333333333333334, + "risk": 0.5534883720930233, + "threshold": 0.7082303566090324 + }, + { + "coverage": 0.14366666666666666, + "risk": 0.5522041763341067, + "threshold": 0.708071064707891 + }, + { + "coverage": 0.144, + "risk": 0.5509259259259259, + "threshold": 0.7062418706761129 + }, + { + "coverage": 0.14433333333333334, + "risk": 0.5496535796766744, + "threshold": 0.7055943102157833 + }, + { + "coverage": 0.14466666666666667, + "risk": 0.5483870967741935, + "threshold": 0.7055499101288376 + }, + { + "coverage": 0.145, + "risk": 0.5471264367816092, + "threshold": 0.7052982080117971 + }, + { + "coverage": 0.14533333333333334, + "risk": 0.5458715596330275, + "threshold": 0.7052248753424948 + }, + { + "coverage": 0.14566666666666667, + "risk": 0.5469107551487414, + "threshold": 0.7051444291696782 + }, + { + "coverage": 0.146, + "risk": 0.545662100456621, + "threshold": 0.7051379485543698 + }, + { + "coverage": 0.14633333333333334, + "risk": 0.5444191343963554, + "threshold": 0.7049644540419919 + }, + { + "coverage": 0.14666666666666667, + "risk": 0.5454545454545454, + "threshold": 0.7043548691567914 + }, + { + "coverage": 0.147, + "risk": 0.54421768707483, + "threshold": 0.7039165584664957 + }, + { + "coverage": 0.14733333333333334, + "risk": 0.5429864253393665, + "threshold": 0.702685082109916 + }, + { + "coverage": 0.14766666666666667, + "risk": 0.5417607223476298, + "threshold": 0.7025394124451374 + }, + { + "coverage": 0.148, + "risk": 0.5405405405405406, + "threshold": 0.7024772745960632 + }, + { + "coverage": 0.14833333333333334, + "risk": 0.5415730337078651, + "threshold": 0.7019382654248146 + }, + { + "coverage": 0.14866666666666667, + "risk": 0.5426008968609866, + "threshold": 0.701377254379878 + }, + { + "coverage": 0.149, + "risk": 0.5436241610738255, + "threshold": 0.7012846548872887 + }, + { + "coverage": 0.14933333333333335, + "risk": 0.5446428571428571, + "threshold": 0.701071852438929 + }, + { + "coverage": 0.14966666666666667, + "risk": 0.5456570155902004, + "threshold": 0.7009029153753943 + }, + { + "coverage": 0.15, + "risk": 0.5466666666666666, + "threshold": 0.6997170987316252 + }, + { + "coverage": 0.15033333333333335, + "risk": 0.5476718403547672, + "threshold": 0.6981018420864018 + }, + { + "coverage": 0.15066666666666667, + "risk": 0.5464601769911505, + "threshold": 0.6980820542205706 + }, + { + "coverage": 0.151, + "risk": 0.5452538631346578, + "threshold": 0.6980763714458327 + }, + { + "coverage": 0.15133333333333332, + "risk": 0.5440528634361234, + "threshold": 0.6978226820902194 + }, + { + "coverage": 0.15166666666666667, + "risk": 0.545054945054945, + "threshold": 0.6973083694558326 + }, + { + "coverage": 0.152, + "risk": 0.543859649122807, + "threshold": 0.6967487761705904 + }, + { + "coverage": 0.15233333333333332, + "risk": 0.5448577680525164, + "threshold": 0.696665349128118 + }, + { + "coverage": 0.15266666666666667, + "risk": 0.5458515283842795, + "threshold": 0.696456600801188 + }, + { + "coverage": 0.153, + "risk": 0.5446623093681917, + "threshold": 0.6955709931493245 + }, + { + "coverage": 0.15333333333333332, + "risk": 0.5434782608695652, + "threshold": 0.6946663904356427 + }, + { + "coverage": 0.15366666666666667, + "risk": 0.5422993492407809, + "threshold": 0.6937951650129222 + }, + { + "coverage": 0.154, + "risk": 0.5411255411255411, + "threshold": 0.6932619284494876 + }, + { + "coverage": 0.15433333333333332, + "risk": 0.5399568034557235, + "threshold": 0.6932359163107478 + }, + { + "coverage": 0.15466666666666667, + "risk": 0.5387931034482759, + "threshold": 0.6931926073139323 + }, + { + "coverage": 0.155, + "risk": 0.5397849462365591, + "threshold": 0.6924176850522556 + }, + { + "coverage": 0.15533333333333332, + "risk": 0.5386266094420601, + "threshold": 0.6919561119046165 + }, + { + "coverage": 0.15566666666666668, + "risk": 0.5374732334047109, + "threshold": 0.6918189160701793 + }, + { + "coverage": 0.156, + "risk": 0.5363247863247863, + "threshold": 0.6913591411192463 + }, + { + "coverage": 0.15633333333333332, + "risk": 0.535181236673774, + "threshold": 0.6904715789008335 + }, + { + "coverage": 0.15666666666666668, + "risk": 0.5340425531914894, + "threshold": 0.6903230260076967 + }, + { + "coverage": 0.157, + "risk": 0.5329087048832272, + "threshold": 0.6898141699372146 + }, + { + "coverage": 0.15733333333333333, + "risk": 0.5338983050847458, + "threshold": 0.6885128135307355 + }, + { + "coverage": 0.15766666666666668, + "risk": 0.53276955602537, + "threshold": 0.6883388619519692 + }, + { + "coverage": 0.158, + "risk": 0.5316455696202531, + "threshold": 0.6879554600150186 + }, + { + "coverage": 0.15833333333333333, + "risk": 0.5305263157894737, + "threshold": 0.6877906946845087 + }, + { + "coverage": 0.15866666666666668, + "risk": 0.5294117647058824, + "threshold": 0.6877447601798963 + }, + { + "coverage": 0.159, + "risk": 0.5303983228511531, + "threshold": 0.6875389509150942 + }, + { + "coverage": 0.15933333333333333, + "risk": 0.5292887029288703, + "threshold": 0.687073556456788 + }, + { + "coverage": 0.15966666666666668, + "risk": 0.5281837160751566, + "threshold": 0.6859908640078629 + }, + { + "coverage": 0.16, + "risk": 0.5270833333333333, + "threshold": 0.6825506018556524 + }, + { + "coverage": 0.16033333333333333, + "risk": 0.5280665280665281, + "threshold": 0.6815964196837072 + }, + { + "coverage": 0.16066666666666668, + "risk": 0.5269709543568465, + "threshold": 0.6815428392522733 + }, + { + "coverage": 0.161, + "risk": 0.5279503105590062, + "threshold": 0.6809577111810157 + }, + { + "coverage": 0.16133333333333333, + "risk": 0.5289256198347108, + "threshold": 0.6804055172473689 + }, + { + "coverage": 0.16166666666666665, + "risk": 0.5278350515463918, + "threshold": 0.6801214719084498 + }, + { + "coverage": 0.162, + "risk": 0.5288065843621399, + "threshold": 0.6799760668357919 + }, + { + "coverage": 0.16233333333333333, + "risk": 0.5277207392197125, + "threshold": 0.6786197531392197 + }, + { + "coverage": 0.16266666666666665, + "risk": 0.5266393442622951, + "threshold": 0.6784825762215405 + }, + { + "coverage": 0.163, + "risk": 0.5276073619631901, + "threshold": 0.6782045882017016 + }, + { + "coverage": 0.16333333333333333, + "risk": 0.5265306122448979, + "threshold": 0.6778125709539324 + }, + { + "coverage": 0.16366666666666665, + "risk": 0.5274949083503055, + "threshold": 0.6777640099041803 + }, + { + "coverage": 0.164, + "risk": 0.5264227642276422, + "threshold": 0.676225108715677 + }, + { + "coverage": 0.16433333333333333, + "risk": 0.5273833671399595, + "threshold": 0.6759706857769031 + }, + { + "coverage": 0.16466666666666666, + "risk": 0.5263157894736842, + "threshold": 0.6757991245822126 + }, + { + "coverage": 0.165, + "risk": 0.5252525252525253, + "threshold": 0.6754644662106912 + }, + { + "coverage": 0.16533333333333333, + "risk": 0.5241935483870968, + "threshold": 0.6754364634388567 + }, + { + "coverage": 0.16566666666666666, + "risk": 0.5231388329979879, + "threshold": 0.6746920759908228 + }, + { + "coverage": 0.166, + "risk": 0.5240963855421686, + "threshold": 0.6742142826650976 + }, + { + "coverage": 0.16633333333333333, + "risk": 0.5230460921843687, + "threshold": 0.673894189793316 + }, + { + "coverage": 0.16666666666666666, + "risk": 0.524, + "threshold": 0.673400878912009 + }, + { + "coverage": 0.167, + "risk": 0.5229540918163673, + "threshold": 0.6727063081941083 + }, + { + "coverage": 0.16733333333333333, + "risk": 0.5219123505976095, + "threshold": 0.6726440755901422 + }, + { + "coverage": 0.16766666666666666, + "risk": 0.5208747514910537, + "threshold": 0.6720124182393619 + }, + { + "coverage": 0.168, + "risk": 0.5218253968253969, + "threshold": 0.6719738577099008 + }, + { + "coverage": 0.16833333333333333, + "risk": 0.5207920792079208, + "threshold": 0.6715128111822821 + }, + { + "coverage": 0.16866666666666666, + "risk": 0.5197628458498024, + "threshold": 0.6704105344137902 + }, + { + "coverage": 0.169, + "risk": 0.5187376725838264, + "threshold": 0.6703390550084798 + }, + { + "coverage": 0.16933333333333334, + "risk": 0.5196850393700787, + "threshold": 0.6700685465020797 + }, + { + "coverage": 0.16966666666666666, + "risk": 0.518664047151277, + "threshold": 0.6684243597595838 + }, + { + "coverage": 0.17, + "risk": 0.5196078431372549, + "threshold": 0.668235630095495 + }, + { + "coverage": 0.17033333333333334, + "risk": 0.5185909980430529, + "threshold": 0.6679124579125961 + }, + { + "coverage": 0.17066666666666666, + "risk": 0.51953125, + "threshold": 0.6670138292579146 + }, + { + "coverage": 0.171, + "risk": 0.5185185185185185, + "threshold": 0.6669987225991747 + }, + { + "coverage": 0.17133333333333334, + "risk": 0.519455252918288, + "threshold": 0.6668390571502414 + }, + { + "coverage": 0.17166666666666666, + "risk": 0.5184466019417475, + "threshold": 0.666054457889377 + }, + { + "coverage": 0.172, + "risk": 0.5174418604651163, + "threshold": 0.6658203879244411 + }, + { + "coverage": 0.17233333333333334, + "risk": 0.5183752417794971, + "threshold": 0.6651936531644084 + }, + { + "coverage": 0.17266666666666666, + "risk": 0.5173745173745173, + "threshold": 0.6648422012778319 + }, + { + "coverage": 0.173, + "risk": 0.5163776493256262, + "threshold": 0.6647151661620683 + }, + { + "coverage": 0.17333333333333334, + "risk": 0.5153846153846153, + "threshold": 0.6646238370101436 + }, + { + "coverage": 0.17366666666666666, + "risk": 0.5143953934740882, + "threshold": 0.6645932110261192 + }, + { + "coverage": 0.174, + "risk": 0.5153256704980843, + "threshold": 0.6640087149064584 + }, + { + "coverage": 0.17433333333333334, + "risk": 0.51434034416826, + "threshold": 0.6639724750304055 + }, + { + "coverage": 0.17466666666666666, + "risk": 0.5152671755725191, + "threshold": 0.6636317614133986 + }, + { + "coverage": 0.175, + "risk": 0.5142857142857142, + "threshold": 0.6635612291335562 + }, + { + "coverage": 0.17533333333333334, + "risk": 0.5133079847908745, + "threshold": 0.6633307556311472 + }, + { + "coverage": 0.17566666666666667, + "risk": 0.5123339658444023, + "threshold": 0.6632881463090925 + }, + { + "coverage": 0.176, + "risk": 0.5113636363636364, + "threshold": 0.6632150036113416 + }, + { + "coverage": 0.17633333333333334, + "risk": 0.5103969754253308, + "threshold": 0.6628874125041371 + }, + { + "coverage": 0.17666666666666667, + "risk": 0.5094339622641509, + "threshold": 0.6626485648584435 + }, + { + "coverage": 0.177, + "risk": 0.5103578154425612, + "threshold": 0.6625744103915281 + }, + { + "coverage": 0.17733333333333334, + "risk": 0.5093984962406015, + "threshold": 0.6622833658163998 + }, + { + "coverage": 0.17766666666666667, + "risk": 0.5084427767354597, + "threshold": 0.6616828703939781 + }, + { + "coverage": 0.178, + "risk": 0.5093632958801498, + "threshold": 0.6612769711330664 + }, + { + "coverage": 0.17833333333333334, + "risk": 0.508411214953271, + "threshold": 0.6609209990676135 + }, + { + "coverage": 0.17866666666666667, + "risk": 0.5093283582089553, + "threshold": 0.660788503446895 + }, + { + "coverage": 0.179, + "risk": 0.5083798882681564, + "threshold": 0.6607039584860246 + }, + { + "coverage": 0.17933333333333334, + "risk": 0.5092936802973977, + "threshold": 0.6603479232351788 + }, + { + "coverage": 0.17966666666666667, + "risk": 0.5102040816326531, + "threshold": 0.6598909948402714 + }, + { + "coverage": 0.18, + "risk": 0.5092592592592593, + "threshold": 0.6574314241204887 + }, + { + "coverage": 0.18033333333333335, + "risk": 0.5101663585951941, + "threshold": 0.6556705684442399 + }, + { + "coverage": 0.18066666666666667, + "risk": 0.511070110701107, + "threshold": 0.6549810241360063 + }, + { + "coverage": 0.181, + "risk": 0.5119705340699816, + "threshold": 0.6537716391285642 + }, + { + "coverage": 0.18133333333333335, + "risk": 0.5128676470588235, + "threshold": 0.6530980186223303 + }, + { + "coverage": 0.18166666666666667, + "risk": 0.5137614678899083, + "threshold": 0.6529662017820783 + }, + { + "coverage": 0.182, + "risk": 0.5128205128205128, + "threshold": 0.652825137263946 + }, + { + "coverage": 0.18233333333333332, + "risk": 0.5137111517367459, + "threshold": 0.6517375205113999 + }, + { + "coverage": 0.18266666666666667, + "risk": 0.5145985401459854, + "threshold": 0.6515509840832117 + }, + { + "coverage": 0.183, + "risk": 0.5136612021857924, + "threshold": 0.6513446006507309 + }, + { + "coverage": 0.18333333333333332, + "risk": 0.5127272727272727, + "threshold": 0.6513356105467665 + }, + { + "coverage": 0.18366666666666667, + "risk": 0.5136116152450091, + "threshold": 0.6509725583946988 + }, + { + "coverage": 0.184, + "risk": 0.5126811594202898, + "threshold": 0.6508883693312484 + }, + { + "coverage": 0.18433333333333332, + "risk": 0.5135623869801085, + "threshold": 0.650683229461464 + }, + { + "coverage": 0.18466666666666667, + "risk": 0.5126353790613718, + "threshold": 0.6503303788292243 + }, + { + "coverage": 0.185, + "risk": 0.5117117117117117, + "threshold": 0.649484511381047 + }, + { + "coverage": 0.18533333333333332, + "risk": 0.5107913669064749, + "threshold": 0.6486930177261634 + }, + { + "coverage": 0.18566666666666667, + "risk": 0.5098743267504489, + "threshold": 0.6482042906659018 + }, + { + "coverage": 0.186, + "risk": 0.510752688172043, + "threshold": 0.6478547619294659 + }, + { + "coverage": 0.18633333333333332, + "risk": 0.5098389982110912, + "threshold": 0.6477663281161199 + }, + { + "coverage": 0.18666666666666668, + "risk": 0.5089285714285714, + "threshold": 0.6472034123471175 + }, + { + "coverage": 0.187, + "risk": 0.5080213903743316, + "threshold": 0.6471992730933778 + }, + { + "coverage": 0.18733333333333332, + "risk": 0.5088967971530249, + "threshold": 0.6469642698729743 + }, + { + "coverage": 0.18766666666666668, + "risk": 0.5097690941385435, + "threshold": 0.6468705297850619 + }, + { + "coverage": 0.188, + "risk": 0.5088652482269503, + "threshold": 0.6466002311366026 + }, + { + "coverage": 0.18833333333333332, + "risk": 0.5079646017699115, + "threshold": 0.6463034794214124 + }, + { + "coverage": 0.18866666666666668, + "risk": 0.508833922261484, + "threshold": 0.6440224183103355 + }, + { + "coverage": 0.189, + "risk": 0.5079365079365079, + "threshold": 0.6428544438408075 + }, + { + "coverage": 0.18933333333333333, + "risk": 0.5070422535211268, + "threshold": 0.642151691825049 + }, + { + "coverage": 0.18966666666666668, + "risk": 0.5061511423550088, + "threshold": 0.6421108045876521 + }, + { + "coverage": 0.19, + "risk": 0.5052631578947369, + "threshold": 0.6416054712587993 + }, + { + "coverage": 0.19033333333333333, + "risk": 0.5043782837127846, + "threshold": 0.6415778376064303 + }, + { + "coverage": 0.19066666666666668, + "risk": 0.5052447552447552, + "threshold": 0.6415352717640701 + }, + { + "coverage": 0.191, + "risk": 0.5043630017452007, + "threshold": 0.6409699462980356 + }, + { + "coverage": 0.19133333333333333, + "risk": 0.5034843205574913, + "threshold": 0.6402793468229362 + }, + { + "coverage": 0.19166666666666668, + "risk": 0.5026086956521739, + "threshold": 0.6396289582831969 + }, + { + "coverage": 0.192, + "risk": 0.5034722222222222, + "threshold": 0.638818676905931 + }, + { + "coverage": 0.19233333333333333, + "risk": 0.5025996533795494, + "threshold": 0.6378657307538553 + }, + { + "coverage": 0.19266666666666668, + "risk": 0.5034602076124568, + "threshold": 0.6371102808010677 + }, + { + "coverage": 0.193, + "risk": 0.5043177892918825, + "threshold": 0.6361383380450611 + }, + { + "coverage": 0.19333333333333333, + "risk": 0.503448275862069, + "threshold": 0.6359809853481657 + }, + { + "coverage": 0.19366666666666665, + "risk": 0.5025817555938038, + "threshold": 0.6357792572107428 + }, + { + "coverage": 0.194, + "risk": 0.5017182130584192, + "threshold": 0.6354892769697571 + }, + { + "coverage": 0.19433333333333333, + "risk": 0.5008576329331046, + "threshold": 0.6354034832526736 + }, + { + "coverage": 0.19466666666666665, + "risk": 0.5017123287671232, + "threshold": 0.6352490436870113 + }, + { + "coverage": 0.195, + "risk": 0.5008547008547009, + "threshold": 0.6351201416787866 + }, + { + "coverage": 0.19533333333333333, + "risk": 0.5, + "threshold": 0.6347391787388545 + }, + { + "coverage": 0.19566666666666666, + "risk": 0.5008517887563884, + "threshold": 0.6338648424943649 + }, + { + "coverage": 0.196, + "risk": 0.5, + "threshold": 0.6338160342470616 + }, + { + "coverage": 0.19633333333333333, + "risk": 0.5008488964346349, + "threshold": 0.6329290426876284 + }, + { + "coverage": 0.19666666666666666, + "risk": 0.5, + "threshold": 0.6323955296158452 + }, + { + "coverage": 0.197, + "risk": 0.49915397631133673, + "threshold": 0.6322328504102082 + }, + { + "coverage": 0.19733333333333333, + "risk": 0.4983108108108108, + "threshold": 0.6315640582351207 + }, + { + "coverage": 0.19766666666666666, + "risk": 0.4974704890387858, + "threshold": 0.630471326556048 + }, + { + "coverage": 0.198, + "risk": 0.49663299663299665, + "threshold": 0.6304332051113235 + }, + { + "coverage": 0.19833333333333333, + "risk": 0.4957983193277311, + "threshold": 0.6298173974908181 + }, + { + "coverage": 0.19866666666666666, + "risk": 0.4949664429530201, + "threshold": 0.62971621315965 + }, + { + "coverage": 0.199, + "risk": 0.49581239530988275, + "threshold": 0.6295762653642213 + }, + { + "coverage": 0.19933333333333333, + "risk": 0.49498327759197325, + "threshold": 0.6294091195234159 + }, + { + "coverage": 0.19966666666666666, + "risk": 0.4941569282136895, + "threshold": 0.6275352607659279 + }, + { + "coverage": 0.2, + "risk": 0.49333333333333335, + "threshold": 0.6268982034666557 + }, + { + "coverage": 0.20033333333333334, + "risk": 0.49417637271214643, + "threshold": 0.6255995440237996 + }, + { + "coverage": 0.20066666666666666, + "risk": 0.49335548172757476, + "threshold": 0.625130351314624 + }, + { + "coverage": 0.201, + "risk": 0.4925373134328358, + "threshold": 0.6246469188861958 + }, + { + "coverage": 0.20133333333333334, + "risk": 0.49337748344370863, + "threshold": 0.6244878260371962 + }, + { + "coverage": 0.20166666666666666, + "risk": 0.4925619834710744, + "threshold": 0.6240807869382365 + }, + { + "coverage": 0.202, + "risk": 0.49174917491749176, + "threshold": 0.6240330405537263 + }, + { + "coverage": 0.20233333333333334, + "risk": 0.49093904448105435, + "threshold": 0.6232540886847993 + }, + { + "coverage": 0.20266666666666666, + "risk": 0.4917763157894737, + "threshold": 0.6230094023154464 + }, + { + "coverage": 0.203, + "risk": 0.4909688013136289, + "threshold": 0.6226418288189706 + }, + { + "coverage": 0.20333333333333334, + "risk": 0.4918032786885246, + "threshold": 0.6224725810810625 + }, + { + "coverage": 0.20366666666666666, + "risk": 0.4909983633387889, + "threshold": 0.6223058724115127 + }, + { + "coverage": 0.204, + "risk": 0.49019607843137253, + "threshold": 0.6222816201725933 + }, + { + "coverage": 0.20433333333333334, + "risk": 0.4893964110929853, + "threshold": 0.6222315653360511 + }, + { + "coverage": 0.20466666666666666, + "risk": 0.48859934853420195, + "threshold": 0.6218465145620867 + }, + { + "coverage": 0.205, + "risk": 0.4894308943089431, + "threshold": 0.6203296391257851 + }, + { + "coverage": 0.20533333333333334, + "risk": 0.4902597402597403, + "threshold": 0.6191950621633211 + }, + { + "coverage": 0.20566666666666666, + "risk": 0.49108589951377635, + "threshold": 0.6160278543736125 + }, + { + "coverage": 0.206, + "risk": 0.4919093851132686, + "threshold": 0.6159180433681684 + }, + { + "coverage": 0.20633333333333334, + "risk": 0.4911147011308562, + "threshold": 0.615726724356378 + }, + { + "coverage": 0.20666666666666667, + "risk": 0.49032258064516127, + "threshold": 0.6154205011236868 + }, + { + "coverage": 0.207, + "risk": 0.49114331723027377, + "threshold": 0.6143154115974561 + }, + { + "coverage": 0.20733333333333334, + "risk": 0.4919614147909968, + "threshold": 0.6138711315532674 + }, + { + "coverage": 0.20766666666666667, + "risk": 0.492776886035313, + "threshold": 0.6134626887440362 + }, + { + "coverage": 0.208, + "risk": 0.49198717948717946, + "threshold": 0.6130420108637828 + }, + { + "coverage": 0.20833333333333334, + "risk": 0.4928, + "threshold": 0.61291816868534 + }, + { + "coverage": 0.20866666666666667, + "risk": 0.49201277955271566, + "threshold": 0.6124435674433926 + }, + { + "coverage": 0.209, + "risk": 0.49282296650717705, + "threshold": 0.6117131224508512 + }, + { + "coverage": 0.20933333333333334, + "risk": 0.49203821656050956, + "threshold": 0.6107252778119943 + }, + { + "coverage": 0.20966666666666667, + "risk": 0.492845786963434, + "threshold": 0.6096988532897427 + }, + { + "coverage": 0.21, + "risk": 0.4936507936507937, + "threshold": 0.6090869249188486 + }, + { + "coverage": 0.21033333333333334, + "risk": 0.49445324881141045, + "threshold": 0.6090510253368961 + }, + { + "coverage": 0.21066666666666667, + "risk": 0.4936708860759494, + "threshold": 0.6084340066991238 + }, + { + "coverage": 0.211, + "risk": 0.4928909952606635, + "threshold": 0.6065852917581438 + }, + { + "coverage": 0.21133333333333335, + "risk": 0.49369085173501576, + "threshold": 0.6064247234983213 + }, + { + "coverage": 0.21166666666666667, + "risk": 0.49291338582677163, + "threshold": 0.6059878180505684 + }, + { + "coverage": 0.212, + "risk": 0.4921383647798742, + "threshold": 0.6059234424313047 + }, + { + "coverage": 0.21233333333333335, + "risk": 0.4913657770800628, + "threshold": 0.6050738555873245 + }, + { + "coverage": 0.21266666666666667, + "risk": 0.49216300940438873, + "threshold": 0.6048117251054173 + }, + { + "coverage": 0.213, + "risk": 0.49139280125195617, + "threshold": 0.6042689801244698 + }, + { + "coverage": 0.21333333333333335, + "risk": 0.490625, + "threshold": 0.6030610796390761 + }, + { + "coverage": 0.21366666666666667, + "risk": 0.4914196567862715, + "threshold": 0.6027335123417998 + }, + { + "coverage": 0.214, + "risk": 0.49221183800623053, + "threshold": 0.6024056806965383 + }, + { + "coverage": 0.21433333333333332, + "risk": 0.49300155520995337, + "threshold": 0.601393745185831 + }, + { + "coverage": 0.21466666666666667, + "risk": 0.4922360248447205, + "threshold": 0.6012798921607114 + }, + { + "coverage": 0.215, + "risk": 0.49147286821705427, + "threshold": 0.601064734103133 + }, + { + "coverage": 0.21533333333333332, + "risk": 0.4907120743034056, + "threshold": 0.600768541488911 + }, + { + "coverage": 0.21566666666666667, + "risk": 0.4899536321483771, + "threshold": 0.5999694545780585 + }, + { + "coverage": 0.216, + "risk": 0.48919753086419754, + "threshold": 0.5998707698916235 + }, + { + "coverage": 0.21633333333333332, + "risk": 0.4884437596302003, + "threshold": 0.5996746717016777 + }, + { + "coverage": 0.21666666666666667, + "risk": 0.4876923076923077, + "threshold": 0.5992521549485907 + }, + { + "coverage": 0.217, + "risk": 0.48847926267281105, + "threshold": 0.599200108515987 + }, + { + "coverage": 0.21733333333333332, + "risk": 0.48773006134969327, + "threshold": 0.5979404627979612 + }, + { + "coverage": 0.21766666666666667, + "risk": 0.4869831546707504, + "threshold": 0.5978732916833671 + }, + { + "coverage": 0.218, + "risk": 0.48623853211009177, + "threshold": 0.5973915429762897 + }, + { + "coverage": 0.21833333333333332, + "risk": 0.4854961832061069, + "threshold": 0.59692559623306 + }, + { + "coverage": 0.21866666666666668, + "risk": 0.4847560975609756, + "threshold": 0.596227857561356 + }, + { + "coverage": 0.219, + "risk": 0.4840182648401826, + "threshold": 0.5960930736301506 + }, + { + "coverage": 0.21933333333333332, + "risk": 0.48328267477203646, + "threshold": 0.5960100375453976 + }, + { + "coverage": 0.21966666666666668, + "risk": 0.48406676783004554, + "threshold": 0.5955944790906642 + }, + { + "coverage": 0.22, + "risk": 0.48333333333333334, + "threshold": 0.595019673826851 + }, + { + "coverage": 0.22033333333333333, + "risk": 0.4826021180030257, + "threshold": 0.5936685697649722 + }, + { + "coverage": 0.22066666666666668, + "risk": 0.4818731117824773, + "threshold": 0.5934025730124297 + }, + { + "coverage": 0.221, + "risk": 0.48265460030165913, + "threshold": 0.5931302141359538 + }, + { + "coverage": 0.22133333333333333, + "risk": 0.4819277108433735, + "threshold": 0.5929083431403299 + }, + { + "coverage": 0.22166666666666668, + "risk": 0.48270676691729325, + "threshold": 0.5928410310273122 + }, + { + "coverage": 0.222, + "risk": 0.481981981981982, + "threshold": 0.5928108789985167 + }, + { + "coverage": 0.22233333333333333, + "risk": 0.48125937031484256, + "threshold": 0.5922892844473182 + }, + { + "coverage": 0.22266666666666668, + "risk": 0.4805389221556886, + "threshold": 0.592257571933196 + }, + { + "coverage": 0.223, + "risk": 0.4798206278026906, + "threshold": 0.5914271199724226 + }, + { + "coverage": 0.22333333333333333, + "risk": 0.48059701492537316, + "threshold": 0.5913235329940818 + }, + { + "coverage": 0.22366666666666668, + "risk": 0.481371087928465, + "threshold": 0.5912068419611778 + }, + { + "coverage": 0.224, + "risk": 0.48214285714285715, + "threshold": 0.5911055395788409 + }, + { + "coverage": 0.22433333333333333, + "risk": 0.48291233283803864, + "threshold": 0.5897910727969702 + }, + { + "coverage": 0.22466666666666665, + "risk": 0.4821958456973294, + "threshold": 0.5885141588150589 + }, + { + "coverage": 0.225, + "risk": 0.48148148148148145, + "threshold": 0.5884119670657384 + }, + { + "coverage": 0.22533333333333333, + "risk": 0.4807692307692308, + "threshold": 0.5869601114321092 + }, + { + "coverage": 0.22566666666666665, + "risk": 0.48005908419497784, + "threshold": 0.5868312126966089 + }, + { + "coverage": 0.226, + "risk": 0.47935103244837757, + "threshold": 0.586644474959769 + }, + { + "coverage": 0.22633333333333333, + "risk": 0.4801178203240059, + "threshold": 0.5858237862775795 + }, + { + "coverage": 0.22666666666666666, + "risk": 0.4808823529411765, + "threshold": 0.5852724725938856 + }, + { + "coverage": 0.227, + "risk": 0.4801762114537445, + "threshold": 0.5851673659336569 + }, + { + "coverage": 0.22733333333333333, + "risk": 0.4809384164222874, + "threshold": 0.5848630078098331 + }, + { + "coverage": 0.22766666666666666, + "risk": 0.4816983894582723, + "threshold": 0.5837596537341484 + }, + { + "coverage": 0.228, + "risk": 0.48099415204678364, + "threshold": 0.5826733621413283 + }, + { + "coverage": 0.22833333333333333, + "risk": 0.4802919708029197, + "threshold": 0.5816272191728302 + }, + { + "coverage": 0.22866666666666666, + "risk": 0.47959183673469385, + "threshold": 0.5814614431722731 + }, + { + "coverage": 0.229, + "risk": 0.47889374090247455, + "threshold": 0.5811303551267102 + }, + { + "coverage": 0.22933333333333333, + "risk": 0.47819767441860467, + "threshold": 0.5805758684102962 + }, + { + "coverage": 0.22966666666666666, + "risk": 0.47750362844702465, + "threshold": 0.5804637839587201 + }, + { + "coverage": 0.23, + "risk": 0.47681159420289854, + "threshold": 0.5802220846416688 + }, + { + "coverage": 0.23033333333333333, + "risk": 0.4761215629522431, + "threshold": 0.5796290853067774 + }, + { + "coverage": 0.23066666666666666, + "risk": 0.4754335260115607, + "threshold": 0.579497423324483 + }, + { + "coverage": 0.231, + "risk": 0.47474747474747475, + "threshold": 0.5784925331465728 + }, + { + "coverage": 0.23133333333333334, + "risk": 0.47406340057636887, + "threshold": 0.5774188865215208 + }, + { + "coverage": 0.23166666666666666, + "risk": 0.47338129496402875, + "threshold": 0.575818268512478 + }, + { + "coverage": 0.232, + "risk": 0.47413793103448276, + "threshold": 0.5743324439834296 + }, + { + "coverage": 0.23233333333333334, + "risk": 0.4734576757532281, + "threshold": 0.5742351247146971 + }, + { + "coverage": 0.23266666666666666, + "risk": 0.47277936962750716, + "threshold": 0.5740455633319819 + }, + { + "coverage": 0.233, + "risk": 0.47353361945636624, + "threshold": 0.5737667546956366 + }, + { + "coverage": 0.23333333333333334, + "risk": 0.47285714285714286, + "threshold": 0.5723085363802483 + }, + { + "coverage": 0.23366666666666666, + "risk": 0.47218259629101283, + "threshold": 0.5713420997066158 + }, + { + "coverage": 0.234, + "risk": 0.47150997150997154, + "threshold": 0.5713134655129295 + }, + { + "coverage": 0.23433333333333334, + "risk": 0.4722617354196302, + "threshold": 0.5707055170225175 + }, + { + "coverage": 0.23466666666666666, + "risk": 0.4715909090909091, + "threshold": 0.5706404814521875 + }, + { + "coverage": 0.235, + "risk": 0.4723404255319149, + "threshold": 0.570444192706745 + }, + { + "coverage": 0.23533333333333334, + "risk": 0.471671388101983, + "threshold": 0.5702021237426003 + }, + { + "coverage": 0.23566666666666666, + "risk": 0.471004243281471, + "threshold": 0.5700508973165058 + }, + { + "coverage": 0.236, + "risk": 0.4703389830508475, + "threshold": 0.569834754341457 + }, + { + "coverage": 0.23633333333333334, + "risk": 0.4696755994358251, + "threshold": 0.5698069510968296 + }, + { + "coverage": 0.23666666666666666, + "risk": 0.4704225352112676, + "threshold": 0.5678044053243916 + }, + { + "coverage": 0.237, + "risk": 0.4711673699015471, + "threshold": 0.5675524491421413 + }, + { + "coverage": 0.23733333333333334, + "risk": 0.4705056179775281, + "threshold": 0.5674808843943886 + }, + { + "coverage": 0.23766666666666666, + "risk": 0.46984572230014027, + "threshold": 0.5654632827434725 + }, + { + "coverage": 0.238, + "risk": 0.469187675070028, + "threshold": 0.5648744018153273 + }, + { + "coverage": 0.23833333333333334, + "risk": 0.4699300699300699, + "threshold": 0.564669477492469 + }, + { + "coverage": 0.23866666666666667, + "risk": 0.47067039106145253, + "threshold": 0.5644397357663706 + }, + { + "coverage": 0.239, + "risk": 0.4700139470013947, + "threshold": 0.5637529636939007 + }, + { + "coverage": 0.23933333333333334, + "risk": 0.46935933147632314, + "threshold": 0.5631584248390027 + }, + { + "coverage": 0.23966666666666667, + "risk": 0.46870653685674546, + "threshold": 0.5620905684509836 + }, + { + "coverage": 0.24, + "risk": 0.46944444444444444, + "threshold": 0.5620247581321074 + }, + { + "coverage": 0.24033333333333334, + "risk": 0.46879334257975036, + "threshold": 0.5612093459548504 + }, + { + "coverage": 0.24066666666666667, + "risk": 0.46814404432132967, + "threshold": 0.5611308880202444 + }, + { + "coverage": 0.241, + "risk": 0.46749654218533887, + "threshold": 0.5605361234582915 + }, + { + "coverage": 0.24133333333333334, + "risk": 0.46685082872928174, + "threshold": 0.5604187481630944 + }, + { + "coverage": 0.24166666666666667, + "risk": 0.46620689655172415, + "threshold": 0.5600588444033576 + }, + { + "coverage": 0.242, + "risk": 0.4669421487603306, + "threshold": 0.5591835788164481 + }, + { + "coverage": 0.24233333333333335, + "risk": 0.4676753782668501, + "threshold": 0.559164124771101 + }, + { + "coverage": 0.24266666666666667, + "risk": 0.4684065934065934, + "threshold": 0.5562394621854594 + }, + { + "coverage": 0.243, + "risk": 0.46776406035665297, + "threshold": 0.5561122868224615 + }, + { + "coverage": 0.24333333333333335, + "risk": 0.4671232876712329, + "threshold": 0.5547481139889179 + }, + { + "coverage": 0.24366666666666667, + "risk": 0.466484268125855, + "threshold": 0.5547318532924805 + }, + { + "coverage": 0.244, + "risk": 0.4672131147540984, + "threshold": 0.5542804077726977 + }, + { + "coverage": 0.24433333333333335, + "risk": 0.4665757162346521, + "threshold": 0.5539043059899313 + }, + { + "coverage": 0.24466666666666667, + "risk": 0.4659400544959128, + "threshold": 0.5525103951971815 + }, + { + "coverage": 0.245, + "risk": 0.4666666666666667, + "threshold": 0.5506025695441394 + }, + { + "coverage": 0.24533333333333332, + "risk": 0.46603260869565216, + "threshold": 0.5504694627305575 + }, + { + "coverage": 0.24566666666666667, + "risk": 0.4654002713704206, + "threshold": 0.5500066705206166 + }, + { + "coverage": 0.246, + "risk": 0.46476964769647694, + "threshold": 0.5499829005117433 + }, + { + "coverage": 0.24633333333333332, + "risk": 0.46414073071718537, + "threshold": 0.5495672196412613 + }, + { + "coverage": 0.24666666666666667, + "risk": 0.4635135135135135, + "threshold": 0.5492419131031454 + }, + { + "coverage": 0.247, + "risk": 0.46288798920377866, + "threshold": 0.5492302265940481 + }, + { + "coverage": 0.24733333333333332, + "risk": 0.4636118598382749, + "threshold": 0.5490129241055293 + }, + { + "coverage": 0.24766666666666667, + "risk": 0.46433378196500674, + "threshold": 0.5471179067745027 + }, + { + "coverage": 0.248, + "risk": 0.4650537634408602, + "threshold": 0.5469126532027566 + }, + { + "coverage": 0.24833333333333332, + "risk": 0.46442953020134226, + "threshold": 0.5466722169989247 + }, + { + "coverage": 0.24866666666666667, + "risk": 0.46380697050938335, + "threshold": 0.5462409326174367 + }, + { + "coverage": 0.249, + "risk": 0.46318607764390896, + "threshold": 0.5461655773047269 + }, + { + "coverage": 0.24933333333333332, + "risk": 0.4625668449197861, + "threshold": 0.5455717402950029 + }, + { + "coverage": 0.24966666666666668, + "risk": 0.46194926568758343, + "threshold": 0.5453856766890297 + }, + { + "coverage": 0.25, + "risk": 0.46266666666666667, + "threshold": 0.5448618577153184 + }, + { + "coverage": 0.25033333333333335, + "risk": 0.4620505992010652, + "threshold": 0.5448309178888415 + }, + { + "coverage": 0.25066666666666665, + "risk": 0.4627659574468085, + "threshold": 0.5442210921609139 + }, + { + "coverage": 0.251, + "risk": 0.46215139442231074, + "threshold": 0.5441838296326829 + }, + { + "coverage": 0.25133333333333335, + "risk": 0.46153846153846156, + "threshold": 0.5432006805382059 + }, + { + "coverage": 0.25166666666666665, + "risk": 0.4609271523178808, + "threshold": 0.5420978536619784 + }, + { + "coverage": 0.252, + "risk": 0.46164021164021163, + "threshold": 0.5418597515468875 + }, + { + "coverage": 0.25233333333333335, + "risk": 0.4610303830911493, + "threshold": 0.5417393995229745 + }, + { + "coverage": 0.25266666666666665, + "risk": 0.4604221635883905, + "threshold": 0.5414446124945798 + }, + { + "coverage": 0.253, + "risk": 0.4598155467720685, + "threshold": 0.5398870990130255 + }, + { + "coverage": 0.25333333333333335, + "risk": 0.45921052631578946, + "threshold": 0.5382653612111241 + }, + { + "coverage": 0.25366666666666665, + "risk": 0.45860709592641263, + "threshold": 0.5378822313139544 + }, + { + "coverage": 0.254, + "risk": 0.45931758530183725, + "threshold": 0.5370349391891516 + }, + { + "coverage": 0.25433333333333336, + "risk": 0.45871559633027525, + "threshold": 0.5370117978419843 + }, + { + "coverage": 0.25466666666666665, + "risk": 0.4581151832460733, + "threshold": 0.5369079538275514 + }, + { + "coverage": 0.255, + "risk": 0.45751633986928103, + "threshold": 0.5365525069808328 + }, + { + "coverage": 0.25533333333333336, + "risk": 0.45691906005221933, + "threshold": 0.5364463780166915 + }, + { + "coverage": 0.25566666666666665, + "risk": 0.45632333767926986, + "threshold": 0.5360858246421785 + }, + { + "coverage": 0.256, + "risk": 0.4557291666666667, + "threshold": 0.5356762580898433 + }, + { + "coverage": 0.25633333333333336, + "risk": 0.45513654096228867, + "threshold": 0.534184606666404 + }, + { + "coverage": 0.25666666666666665, + "risk": 0.45454545454545453, + "threshold": 0.5341359791546694 + }, + { + "coverage": 0.257, + "risk": 0.45525291828793774, + "threshold": 0.5327201095701071 + }, + { + "coverage": 0.25733333333333336, + "risk": 0.45466321243523317, + "threshold": 0.5302552077685618 + }, + { + "coverage": 0.25766666666666665, + "risk": 0.4540750323415265, + "threshold": 0.5302193366787549 + }, + { + "coverage": 0.258, + "risk": 0.45348837209302323, + "threshold": 0.5301226243714865 + }, + { + "coverage": 0.25833333333333336, + "risk": 0.4529032258064516, + "threshold": 0.5297443715262204 + }, + { + "coverage": 0.25866666666666666, + "risk": 0.4536082474226804, + "threshold": 0.5288315350684343 + }, + { + "coverage": 0.259, + "risk": 0.45302445302445304, + "threshold": 0.5269434285567306 + }, + { + "coverage": 0.25933333333333336, + "risk": 0.4524421593830334, + "threshold": 0.5258798104554501 + }, + { + "coverage": 0.25966666666666666, + "risk": 0.45314505776636715, + "threshold": 0.5256293989991718 + }, + { + "coverage": 0.26, + "risk": 0.45256410256410257, + "threshold": 0.5255725634644826 + }, + { + "coverage": 0.26033333333333336, + "risk": 0.4519846350832266, + "threshold": 0.5249778862546393 + }, + { + "coverage": 0.26066666666666666, + "risk": 0.4514066496163683, + "threshold": 0.5242944782647498 + }, + { + "coverage": 0.261, + "risk": 0.4521072796934866, + "threshold": 0.5237832523986441 + }, + { + "coverage": 0.2613333333333333, + "risk": 0.4528061224489796, + "threshold": 0.5221942972477933 + }, + { + "coverage": 0.26166666666666666, + "risk": 0.45222929936305734, + "threshold": 0.521552514802277 + }, + { + "coverage": 0.262, + "risk": 0.45165394402035625, + "threshold": 0.52127918167578 + }, + { + "coverage": 0.2623333333333333, + "risk": 0.45108005082592123, + "threshold": 0.5204281474811125 + }, + { + "coverage": 0.26266666666666666, + "risk": 0.45050761421319796, + "threshold": 0.5196690351594171 + }, + { + "coverage": 0.263, + "risk": 0.4512040557667934, + "threshold": 0.5192777966602087 + }, + { + "coverage": 0.2633333333333333, + "risk": 0.4506329113924051, + "threshold": 0.5187343445323833 + }, + { + "coverage": 0.26366666666666666, + "risk": 0.450063211125158, + "threshold": 0.5181904174511942 + }, + { + "coverage": 0.264, + "risk": 0.4494949494949495, + "threshold": 0.5181695023681243 + }, + { + "coverage": 0.2643333333333333, + "risk": 0.4489281210592686, + "threshold": 0.5178833890836252 + }, + { + "coverage": 0.26466666666666666, + "risk": 0.44836272040302266, + "threshold": 0.5177336720313974 + }, + { + "coverage": 0.265, + "risk": 0.4490566037735849, + "threshold": 0.5153285887212569 + }, + { + "coverage": 0.2653333333333333, + "risk": 0.4484924623115578, + "threshold": 0.5144045325882445 + }, + { + "coverage": 0.26566666666666666, + "risk": 0.4479297365119197, + "threshold": 0.5143287025303565 + }, + { + "coverage": 0.266, + "risk": 0.4473684210526316, + "threshold": 0.5134085050381145 + }, + { + "coverage": 0.2663333333333333, + "risk": 0.44680851063829785, + "threshold": 0.5131127496190402 + }, + { + "coverage": 0.26666666666666666, + "risk": 0.44625, + "threshold": 0.512956092439552 + }, + { + "coverage": 0.267, + "risk": 0.44569288389513106, + "threshold": 0.5116513127939442 + }, + { + "coverage": 0.2673333333333333, + "risk": 0.4451371571072319, + "threshold": 0.5111179528287693 + }, + { + "coverage": 0.26766666666666666, + "risk": 0.44458281444582815, + "threshold": 0.5106001370428288 + }, + { + "coverage": 0.268, + "risk": 0.44402985074626866, + "threshold": 0.5087419856261874 + }, + { + "coverage": 0.2683333333333333, + "risk": 0.4434782608695652, + "threshold": 0.5078090483052442 + }, + { + "coverage": 0.26866666666666666, + "risk": 0.44292803970223327, + "threshold": 0.5075754837705944 + }, + { + "coverage": 0.269, + "risk": 0.4436183395291202, + "threshold": 0.507478583513759 + }, + { + "coverage": 0.2693333333333333, + "risk": 0.4430693069306931, + "threshold": 0.5070915512770924 + }, + { + "coverage": 0.26966666666666667, + "risk": 0.44252163164400493, + "threshold": 0.5065395443883887 + }, + { + "coverage": 0.27, + "risk": 0.4419753086419753, + "threshold": 0.5064572183856342 + }, + { + "coverage": 0.2703333333333333, + "risk": 0.44143033292231815, + "threshold": 0.5056537625383761 + }, + { + "coverage": 0.27066666666666667, + "risk": 0.4408866995073892, + "threshold": 0.5052795400972698 + }, + { + "coverage": 0.271, + "risk": 0.4403444034440344, + "threshold": 0.5048266693829646 + }, + { + "coverage": 0.2713333333333333, + "risk": 0.4398034398034398, + "threshold": 0.5046284650698816 + }, + { + "coverage": 0.27166666666666667, + "risk": 0.4392638036809816, + "threshold": 0.503872884171443 + }, + { + "coverage": 0.272, + "risk": 0.4387254901960784, + "threshold": 0.5035309616575958 + }, + { + "coverage": 0.2723333333333333, + "risk": 0.43818849449204406, + "threshold": 0.5034045244200378 + }, + { + "coverage": 0.27266666666666667, + "risk": 0.43765281173594134, + "threshold": 0.5024208518131701 + }, + { + "coverage": 0.273, + "risk": 0.4371184371184371, + "threshold": 0.5002192266572258 + }, + { + "coverage": 0.2733333333333333, + "risk": 0.43658536585365854, + "threshold": 0.5002162630682108 + }, + { + "coverage": 0.27366666666666667, + "risk": 0.43605359317904996, + "threshold": 0.49998591273016757 + }, + { + "coverage": 0.274, + "risk": 0.4367396593673966, + "threshold": 0.4998781381724626 + }, + { + "coverage": 0.2743333333333333, + "risk": 0.4362089914945322, + "threshold": 0.4996529059150312 + }, + { + "coverage": 0.27466666666666667, + "risk": 0.4356796116504854, + "threshold": 0.4991475180324514 + }, + { + "coverage": 0.275, + "risk": 0.43636363636363634, + "threshold": 0.4988709729875644 + }, + { + "coverage": 0.2753333333333333, + "risk": 0.437046004842615, + "threshold": 0.4987008969563789 + }, + { + "coverage": 0.27566666666666667, + "risk": 0.4365175332527207, + "threshold": 0.49572004406600806 + }, + { + "coverage": 0.276, + "risk": 0.4359903381642512, + "threshold": 0.49539286968673585 + }, + { + "coverage": 0.2763333333333333, + "risk": 0.43546441495778043, + "threshold": 0.4943090220279167 + }, + { + "coverage": 0.27666666666666667, + "risk": 0.43493975903614457, + "threshold": 0.4939015857956974 + }, + { + "coverage": 0.277, + "risk": 0.43561973525872444, + "threshold": 0.493726144827356 + }, + { + "coverage": 0.2773333333333333, + "risk": 0.4362980769230769, + "threshold": 0.4928319917311228 + }, + { + "coverage": 0.2776666666666667, + "risk": 0.43577430972388953, + "threshold": 0.4919856299563535 + }, + { + "coverage": 0.278, + "risk": 0.4352517985611511, + "threshold": 0.49134787743762226 + }, + { + "coverage": 0.2783333333333333, + "risk": 0.4347305389221557, + "threshold": 0.4900779364247824 + }, + { + "coverage": 0.2786666666666667, + "risk": 0.4342105263157895, + "threshold": 0.4900672925577308 + }, + { + "coverage": 0.279, + "risk": 0.4336917562724014, + "threshold": 0.49000334533809564 + }, + { + "coverage": 0.2793333333333333, + "risk": 0.4331742243436754, + "threshold": 0.48999523631591657 + }, + { + "coverage": 0.2796666666666667, + "risk": 0.432657926102503, + "threshold": 0.48966499178224865 + }, + { + "coverage": 0.28, + "risk": 0.43214285714285716, + "threshold": 0.4891233003788855 + }, + { + "coverage": 0.2803333333333333, + "risk": 0.43162901307966706, + "threshold": 0.4881779976140024 + }, + { + "coverage": 0.2806666666666667, + "risk": 0.4311163895486936, + "threshold": 0.4879289956607628 + }, + { + "coverage": 0.281, + "risk": 0.4306049822064057, + "threshold": 0.48743497857515194 + }, + { + "coverage": 0.2813333333333333, + "risk": 0.43009478672985785, + "threshold": 0.48667451777578213 + }, + { + "coverage": 0.2816666666666667, + "risk": 0.42958579881656805, + "threshold": 0.4866662446295855 + }, + { + "coverage": 0.282, + "risk": 0.42907801418439717, + "threshold": 0.4859856661786139 + }, + { + "coverage": 0.2823333333333333, + "risk": 0.42857142857142855, + "threshold": 0.48561575246603544 + }, + { + "coverage": 0.2826666666666667, + "risk": 0.42924528301886794, + "threshold": 0.4846324405800715 + }, + { + "coverage": 0.283, + "risk": 0.4299175500588928, + "threshold": 0.48449828011225193 + }, + { + "coverage": 0.2833333333333333, + "risk": 0.4294117647058823, + "threshold": 0.48351272626716835 + }, + { + "coverage": 0.2836666666666667, + "risk": 0.4289071680376028, + "threshold": 0.4833795572437083 + }, + { + "coverage": 0.284, + "risk": 0.4284037558685446, + "threshold": 0.482809928689683 + }, + { + "coverage": 0.2843333333333333, + "risk": 0.42790152403282533, + "threshold": 0.4824388471844294 + }, + { + "coverage": 0.2846666666666667, + "risk": 0.42857142857142855, + "threshold": 0.48111076534922503 + }, + { + "coverage": 0.285, + "risk": 0.4280701754385965, + "threshold": 0.4809491849772413 + }, + { + "coverage": 0.2853333333333333, + "risk": 0.42757009345794394, + "threshold": 0.48059296398781015 + }, + { + "coverage": 0.2856666666666667, + "risk": 0.42707117852975496, + "threshold": 0.47964265652218785 + }, + { + "coverage": 0.286, + "risk": 0.42657342657342656, + "threshold": 0.4795355991973338 + }, + { + "coverage": 0.28633333333333333, + "risk": 0.42724097788125726, + "threshold": 0.47949659075567297 + }, + { + "coverage": 0.2866666666666667, + "risk": 0.42674418604651165, + "threshold": 0.479313856959692 + }, + { + "coverage": 0.287, + "risk": 0.4262485481997677, + "threshold": 0.4784942942990185 + }, + { + "coverage": 0.28733333333333333, + "risk": 0.425754060324826, + "threshold": 0.4783115359960668 + }, + { + "coverage": 0.2876666666666667, + "risk": 0.4264194669756663, + "threshold": 0.4777029367964936 + }, + { + "coverage": 0.288, + "risk": 0.42592592592592593, + "threshold": 0.4763197703617621 + }, + { + "coverage": 0.28833333333333333, + "risk": 0.42658959537572255, + "threshold": 0.4762537246927537 + }, + { + "coverage": 0.2886666666666667, + "risk": 0.42609699769053117, + "threshold": 0.4760772828118149 + }, + { + "coverage": 0.289, + "risk": 0.42675893886966554, + "threshold": 0.47578536160375506 + }, + { + "coverage": 0.28933333333333333, + "risk": 0.42626728110599077, + "threshold": 0.47406589340061694 + }, + { + "coverage": 0.2896666666666667, + "risk": 0.42577675489067895, + "threshold": 0.4720298469166835 + }, + { + "coverage": 0.29, + "risk": 0.42528735632183906, + "threshold": 0.4714397009716862 + }, + { + "coverage": 0.29033333333333333, + "risk": 0.42479908151549944, + "threshold": 0.47143760306383337 + }, + { + "coverage": 0.2906666666666667, + "risk": 0.4243119266055046, + "threshold": 0.4712726790764386 + }, + { + "coverage": 0.291, + "risk": 0.4238258877434135, + "threshold": 0.47080143509477773 + }, + { + "coverage": 0.29133333333333333, + "risk": 0.4233409610983982, + "threshold": 0.47066120676266365 + }, + { + "coverage": 0.2916666666666667, + "risk": 0.4228571428571429, + "threshold": 0.47062965187878447 + }, + { + "coverage": 0.292, + "risk": 0.4223744292237443, + "threshold": 0.4702901243608757 + }, + { + "coverage": 0.29233333333333333, + "risk": 0.4218928164196123, + "threshold": 0.4696420456725719 + }, + { + "coverage": 0.2926666666666667, + "risk": 0.4214123006833713, + "threshold": 0.46928955709995895 + }, + { + "coverage": 0.293, + "risk": 0.42093287827076226, + "threshold": 0.46853423094345453 + }, + { + "coverage": 0.29333333333333333, + "risk": 0.42045454545454547, + "threshold": 0.46850767630660606 + }, + { + "coverage": 0.2936666666666667, + "risk": 0.4199772985244041, + "threshold": 0.4682037492331937 + }, + { + "coverage": 0.294, + "risk": 0.42063492063492064, + "threshold": 0.46816873035363576 + }, + { + "coverage": 0.29433333333333334, + "risk": 0.420158550396376, + "threshold": 0.46770185016549704 + }, + { + "coverage": 0.2946666666666667, + "risk": 0.41968325791855204, + "threshold": 0.46733033377940814 + }, + { + "coverage": 0.295, + "risk": 0.4192090395480226, + "threshold": 0.46672447908258086 + }, + { + "coverage": 0.29533333333333334, + "risk": 0.4187358916478555, + "threshold": 0.4665691133086405 + }, + { + "coverage": 0.2956666666666667, + "risk": 0.4182638105975197, + "threshold": 0.4654828671659939 + }, + { + "coverage": 0.296, + "risk": 0.4177927927927928, + "threshold": 0.4649349507271978 + }, + { + "coverage": 0.29633333333333334, + "risk": 0.41732283464566927, + "threshold": 0.4646511636003083 + }, + { + "coverage": 0.2966666666666667, + "risk": 0.41797752808988764, + "threshold": 0.4641708463462014 + }, + { + "coverage": 0.297, + "risk": 0.4175084175084175, + "threshold": 0.46415010374767 + }, + { + "coverage": 0.29733333333333334, + "risk": 0.4170403587443946, + "threshold": 0.46412366138719463 + }, + { + "coverage": 0.2976666666666667, + "risk": 0.4165733482642777, + "threshold": 0.46396117935656633 + }, + { + "coverage": 0.298, + "risk": 0.4172259507829978, + "threshold": 0.4638391479834248 + }, + { + "coverage": 0.29833333333333334, + "risk": 0.41675977653631285, + "threshold": 0.4636841248829526 + }, + { + "coverage": 0.2986666666666667, + "risk": 0.41629464285714285, + "threshold": 0.46337421621887737 + }, + { + "coverage": 0.299, + "risk": 0.4158305462653289, + "threshold": 0.4625887606733876 + }, + { + "coverage": 0.29933333333333334, + "risk": 0.41648106904231624, + "threshold": 0.462323156534552 + }, + { + "coverage": 0.2996666666666667, + "risk": 0.41601779755283647, + "threshold": 0.46214476709580404 + }, + { + "coverage": 0.3, + "risk": 0.41555555555555557, + "threshold": 0.4620841736819869 + }, + { + "coverage": 0.30033333333333334, + "risk": 0.41509433962264153, + "threshold": 0.46094114073787656 + }, + { + "coverage": 0.3006666666666667, + "risk": 0.4146341463414634, + "threshold": 0.46066613136828494 + }, + { + "coverage": 0.301, + "risk": 0.4152823920265781, + "threshold": 0.46026516737669004 + }, + { + "coverage": 0.30133333333333334, + "risk": 0.41482300884955753, + "threshold": 0.45996430361050505 + }, + { + "coverage": 0.3016666666666667, + "risk": 0.4143646408839779, + "threshold": 0.4596039264525445 + }, + { + "coverage": 0.302, + "risk": 0.4139072847682119, + "threshold": 0.4593340114414133 + }, + { + "coverage": 0.30233333333333334, + "risk": 0.41345093715545755, + "threshold": 0.4586690116385163 + }, + { + "coverage": 0.30266666666666664, + "risk": 0.41409691629955947, + "threshold": 0.458617968461752 + }, + { + "coverage": 0.303, + "risk": 0.4136413641364136, + "threshold": 0.45839710311108206 + }, + { + "coverage": 0.30333333333333334, + "risk": 0.41318681318681316, + "threshold": 0.458003350800542 + }, + { + "coverage": 0.30366666666666664, + "risk": 0.4127332601536773, + "threshold": 0.4566966014029106 + }, + { + "coverage": 0.304, + "risk": 0.4133771929824561, + "threshold": 0.45668866323773294 + }, + { + "coverage": 0.30433333333333334, + "risk": 0.4129244249726177, + "threshold": 0.45621876013152374 + }, + { + "coverage": 0.30466666666666664, + "risk": 0.412472647702407, + "threshold": 0.45537588705300885 + }, + { + "coverage": 0.305, + "risk": 0.41202185792349727, + "threshold": 0.4546327665864922 + }, + { + "coverage": 0.30533333333333335, + "risk": 0.4115720524017467, + "threshold": 0.4542391196407322 + }, + { + "coverage": 0.30566666666666664, + "risk": 0.411123227917121, + "threshold": 0.45423578350430205 + }, + { + "coverage": 0.306, + "risk": 0.4117647058823529, + "threshold": 0.45409033699304047 + }, + { + "coverage": 0.30633333333333335, + "risk": 0.41131664853101196, + "threshold": 0.4536029213559552 + }, + { + "coverage": 0.30666666666666664, + "risk": 0.4108695652173913, + "threshold": 0.452044771337448 + }, + { + "coverage": 0.307, + "risk": 0.41042345276872966, + "threshold": 0.45144682430578703 + }, + { + "coverage": 0.30733333333333335, + "risk": 0.40997830802603036, + "threshold": 0.4501920446802288 + }, + { + "coverage": 0.30766666666666664, + "risk": 0.409534127843987, + "threshold": 0.44897729582404006 + }, + { + "coverage": 0.308, + "risk": 0.4090909090909091, + "threshold": 0.4487969544755987 + }, + { + "coverage": 0.30833333333333335, + "risk": 0.4097297297297297, + "threshold": 0.44879509130491085 + }, + { + "coverage": 0.30866666666666664, + "risk": 0.40928725701943847, + "threshold": 0.4480480031759981 + }, + { + "coverage": 0.309, + "risk": 0.40884573894282633, + "threshold": 0.44770043756017947 + }, + { + "coverage": 0.30933333333333335, + "risk": 0.4084051724137931, + "threshold": 0.4471780479300662 + }, + { + "coverage": 0.30966666666666665, + "risk": 0.40904198062432723, + "threshold": 0.44635833624602134 + }, + { + "coverage": 0.31, + "risk": 0.40860215053763443, + "threshold": 0.445883568979486 + }, + { + "coverage": 0.31033333333333335, + "risk": 0.4092373791621912, + "threshold": 0.4458009057329182 + }, + { + "coverage": 0.31066666666666665, + "risk": 0.40987124463519314, + "threshold": 0.44437354113853184 + }, + { + "coverage": 0.311, + "risk": 0.40943193997856375, + "threshold": 0.4442851802612502 + }, + { + "coverage": 0.31133333333333335, + "risk": 0.4100642398286938, + "threshold": 0.4441299765576278 + }, + { + "coverage": 0.31166666666666665, + "risk": 0.40962566844919784, + "threshold": 0.44285637765389657 + }, + { + "coverage": 0.312, + "risk": 0.4091880341880342, + "threshold": 0.44251527370846944 + }, + { + "coverage": 0.31233333333333335, + "risk": 0.4087513340448239, + "threshold": 0.4423706882623222 + }, + { + "coverage": 0.31266666666666665, + "risk": 0.40831556503198296, + "threshold": 0.44189396795251307 + }, + { + "coverage": 0.313, + "risk": 0.4078807241746539, + "threshold": 0.4418495738886819 + }, + { + "coverage": 0.31333333333333335, + "risk": 0.4074468085106383, + "threshold": 0.44179851261808517 + }, + { + "coverage": 0.31366666666666665, + "risk": 0.4070138150903294, + "threshold": 0.4413414374644233 + }, + { + "coverage": 0.314, + "risk": 0.40658174097664546, + "threshold": 0.4405306245794129 + }, + { + "coverage": 0.31433333333333335, + "risk": 0.4061505832449629, + "threshold": 0.44046251110785306 + }, + { + "coverage": 0.31466666666666665, + "risk": 0.4057203389830508, + "threshold": 0.4403312041172817 + }, + { + "coverage": 0.315, + "risk": 0.4052910052910053, + "threshold": 0.44027116867725974 + }, + { + "coverage": 0.31533333333333335, + "risk": 0.4048625792811839, + "threshold": 0.44021528049431596 + }, + { + "coverage": 0.31566666666666665, + "risk": 0.40549102428722283, + "threshold": 0.4399593045285712 + }, + { + "coverage": 0.316, + "risk": 0.4061181434599156, + "threshold": 0.4396578390492003 + }, + { + "coverage": 0.31633333333333336, + "risk": 0.4067439409905163, + "threshold": 0.43965451152133994 + }, + { + "coverage": 0.31666666666666665, + "risk": 0.4063157894736842, + "threshold": 0.4391819241184969 + }, + { + "coverage": 0.317, + "risk": 0.4058885383806519, + "threshold": 0.4389609190003725 + }, + { + "coverage": 0.31733333333333336, + "risk": 0.4065126050420168, + "threshold": 0.43724209919046686 + }, + { + "coverage": 0.31766666666666665, + "risk": 0.4060860440713536, + "threshold": 0.43653597020504264 + }, + { + "coverage": 0.318, + "risk": 0.4056603773584906, + "threshold": 0.43653461933090115 + }, + { + "coverage": 0.31833333333333336, + "risk": 0.406282722513089, + "threshold": 0.4362427479001596 + }, + { + "coverage": 0.31866666666666665, + "risk": 0.40585774058577406, + "threshold": 0.43594469112260725 + }, + { + "coverage": 0.319, + "risk": 0.4054336468129572, + "threshold": 0.4359380776877515 + }, + { + "coverage": 0.31933333333333336, + "risk": 0.40501043841336115, + "threshold": 0.4356541451665663 + }, + { + "coverage": 0.31966666666666665, + "risk": 0.4056308654848801, + "threshold": 0.43542891468183953 + }, + { + "coverage": 0.32, + "risk": 0.40520833333333334, + "threshold": 0.4354034406599982 + }, + { + "coverage": 0.32033333333333336, + "risk": 0.404786680541103, + "threshold": 0.4353262753064886 + }, + { + "coverage": 0.32066666666666666, + "risk": 0.40436590436590436, + "threshold": 0.4338874766252321 + }, + { + "coverage": 0.321, + "risk": 0.4039460020768432, + "threshold": 0.4335726222461922 + }, + { + "coverage": 0.32133333333333336, + "risk": 0.40352697095435686, + "threshold": 0.4333577946805889 + }, + { + "coverage": 0.32166666666666666, + "risk": 0.40310880829015544, + "threshold": 0.4330373804959464 + }, + { + "coverage": 0.322, + "risk": 0.4026915113871636, + "threshold": 0.42961845083511113 + }, + { + "coverage": 0.32233333333333336, + "risk": 0.40227507755946224, + "threshold": 0.42921978434393365 + }, + { + "coverage": 0.32266666666666666, + "risk": 0.4018595041322314, + "threshold": 0.4284722397986905 + }, + { + "coverage": 0.323, + "risk": 0.40144478844169246, + "threshold": 0.4284365494198461 + }, + { + "coverage": 0.3233333333333333, + "risk": 0.40103092783505156, + "threshold": 0.4283289269015657 + }, + { + "coverage": 0.32366666666666666, + "risk": 0.40061791967044286, + "threshold": 0.42815515509229835 + }, + { + "coverage": 0.324, + "risk": 0.4012345679012346, + "threshold": 0.4263366219231273 + }, + { + "coverage": 0.3243333333333333, + "risk": 0.40082219938335045, + "threshold": 0.42561375593643896 + }, + { + "coverage": 0.32466666666666666, + "risk": 0.4004106776180698, + "threshold": 0.42509377324330444 + }, + { + "coverage": 0.325, + "risk": 0.40102564102564103, + "threshold": 0.4248281267120273 + }, + { + "coverage": 0.3253333333333333, + "risk": 0.40061475409836067, + "threshold": 0.4246624239778285 + }, + { + "coverage": 0.32566666666666666, + "risk": 0.4002047082906858, + "threshold": 0.42458443350205494 + }, + { + "coverage": 0.326, + "risk": 0.3997955010224949, + "threshold": 0.424543062101474 + }, + { + "coverage": 0.3263333333333333, + "risk": 0.4004085801838611, + "threshold": 0.4242925628639149 + }, + { + "coverage": 0.32666666666666666, + "risk": 0.4010204081632653, + "threshold": 0.4241061359546627 + }, + { + "coverage": 0.327, + "risk": 0.4016309887869521, + "threshold": 0.42368105680195356 + }, + { + "coverage": 0.3273333333333333, + "risk": 0.40122199592668023, + "threshold": 0.4224995161635831 + }, + { + "coverage": 0.32766666666666666, + "risk": 0.4008138351983723, + "threshold": 0.42246685504230713 + }, + { + "coverage": 0.328, + "risk": 0.40040650406504064, + "threshold": 0.42223763710305023 + }, + { + "coverage": 0.3283333333333333, + "risk": 0.4, + "threshold": 0.422069184463453 + }, + { + "coverage": 0.32866666666666666, + "risk": 0.40060851926977686, + "threshold": 0.4219396889962468 + }, + { + "coverage": 0.329, + "risk": 0.4002026342451874, + "threshold": 0.42156694767900044 + }, + { + "coverage": 0.3293333333333333, + "risk": 0.39979757085020245, + "threshold": 0.42144286879700665 + }, + { + "coverage": 0.32966666666666666, + "risk": 0.40040444893832156, + "threshold": 0.42099956517209586 + }, + { + "coverage": 0.33, + "risk": 0.401010101010101, + "threshold": 0.4203338487392063 + }, + { + "coverage": 0.3303333333333333, + "risk": 0.40060544904137235, + "threshold": 0.4200641314239526 + }, + { + "coverage": 0.33066666666666666, + "risk": 0.4002016129032258, + "threshold": 0.41984269797683466 + }, + { + "coverage": 0.331, + "risk": 0.3997985901309164, + "threshold": 0.4186130570661568 + }, + { + "coverage": 0.3313333333333333, + "risk": 0.3993963782696177, + "threshold": 0.4179929547973078 + }, + { + "coverage": 0.33166666666666667, + "risk": 0.39899497487437185, + "threshold": 0.4179644216796614 + }, + { + "coverage": 0.332, + "risk": 0.3985943775100402, + "threshold": 0.4176300840423224 + }, + { + "coverage": 0.3323333333333333, + "risk": 0.399197592778335, + "threshold": 0.417498923632742 + }, + { + "coverage": 0.33266666666666667, + "risk": 0.39879759519038077, + "threshold": 0.41680017099273664 + }, + { + "coverage": 0.333, + "risk": 0.3993993993993994, + "threshold": 0.4158713524731467 + }, + { + "coverage": 0.3333333333333333, + "risk": 0.399, + "threshold": 0.41533160992806906 + }, + { + "coverage": 0.33366666666666667, + "risk": 0.3986013986013986, + "threshold": 0.41438527727460084 + }, + { + "coverage": 0.334, + "risk": 0.3992015968063872, + "threshold": 0.4141399660873901 + }, + { + "coverage": 0.3343333333333333, + "risk": 0.3988035892323031, + "threshold": 0.4138260740271349 + }, + { + "coverage": 0.33466666666666667, + "risk": 0.398406374501992, + "threshold": 0.4135128617250068 + }, + { + "coverage": 0.335, + "risk": 0.39800995024875624, + "threshold": 0.41122045867366097 + }, + { + "coverage": 0.3353333333333333, + "risk": 0.3976143141153082, + "threshold": 0.41047078090457134 + }, + { + "coverage": 0.33566666666666667, + "risk": 0.3972194637537239, + "threshold": 0.4102683634080008 + }, + { + "coverage": 0.336, + "risk": 0.3968253968253968, + "threshold": 0.4099161658470738 + }, + { + "coverage": 0.3363333333333333, + "risk": 0.39643211100099107, + "threshold": 0.4097250463036061 + }, + { + "coverage": 0.33666666666666667, + "risk": 0.39603960396039606, + "threshold": 0.4090815858909345 + }, + { + "coverage": 0.337, + "risk": 0.39564787339268054, + "threshold": 0.40876726116817963 + }, + { + "coverage": 0.3373333333333333, + "risk": 0.3952569169960474, + "threshold": 0.4078763957289197 + }, + { + "coverage": 0.33766666666666667, + "risk": 0.39486673247778875, + "threshold": 0.40744244893188863 + }, + { + "coverage": 0.338, + "risk": 0.39447731755424065, + "threshold": 0.40731487489127716 + }, + { + "coverage": 0.3383333333333333, + "risk": 0.39408866995073893, + "threshold": 0.4066116012797493 + }, + { + "coverage": 0.33866666666666667, + "risk": 0.3937007874015748, + "threshold": 0.404606634887017 + }, + { + "coverage": 0.339, + "risk": 0.39331366764995085, + "threshold": 0.4044456516695334 + }, + { + "coverage": 0.3393333333333333, + "risk": 0.3929273084479371, + "threshold": 0.4038794713582377 + }, + { + "coverage": 0.3396666666666667, + "risk": 0.39254170755642787, + "threshold": 0.4038457744426758 + }, + { + "coverage": 0.34, + "risk": 0.39215686274509803, + "threshold": 0.40379549163940703 + }, + { + "coverage": 0.3403333333333333, + "risk": 0.39275220372184133, + "threshold": 0.40374927568237595 + }, + { + "coverage": 0.3406666666666667, + "risk": 0.3923679060665362, + "threshold": 0.4029471787176941 + }, + { + "coverage": 0.341, + "risk": 0.3919843597262952, + "threshold": 0.4027216369408926 + }, + { + "coverage": 0.3413333333333333, + "risk": 0.392578125, + "threshold": 0.40257008839471814 + }, + { + "coverage": 0.3416666666666667, + "risk": 0.3921951219512195, + "threshold": 0.4024538992642981 + }, + { + "coverage": 0.342, + "risk": 0.39278752436647174, + "threshold": 0.4007808312677863 + }, + { + "coverage": 0.3423333333333333, + "risk": 0.3924050632911392, + "threshold": 0.4006583897898075 + }, + { + "coverage": 0.3426666666666667, + "risk": 0.39202334630350194, + "threshold": 0.3997224617892386 + }, + { + "coverage": 0.343, + "risk": 0.391642371234208, + "threshold": 0.3994710682287277 + }, + { + "coverage": 0.3433333333333333, + "risk": 0.39223300970873787, + "threshold": 0.39934065562282406 + }, + { + "coverage": 0.3436666666666667, + "risk": 0.39282250242483024, + "threshold": 0.39927424708006143 + }, + { + "coverage": 0.344, + "risk": 0.39244186046511625, + "threshold": 0.39904173783491004 + }, + { + "coverage": 0.3443333333333333, + "risk": 0.3920619554695063, + "threshold": 0.3988423568714303 + }, + { + "coverage": 0.3446666666666667, + "risk": 0.3916827852998066, + "threshold": 0.3988421587223883 + }, + { + "coverage": 0.345, + "risk": 0.391304347826087, + "threshold": 0.3974300783734108 + }, + { + "coverage": 0.3453333333333333, + "risk": 0.3909266409266409, + "threshold": 0.3967727348765564 + }, + { + "coverage": 0.3456666666666667, + "risk": 0.390549662487946, + "threshold": 0.3965948614522929 + }, + { + "coverage": 0.346, + "risk": 0.3901734104046243, + "threshold": 0.39586501420567544 + }, + { + "coverage": 0.3463333333333333, + "risk": 0.3897978825794033, + "threshold": 0.3958445499279577 + }, + { + "coverage": 0.3466666666666667, + "risk": 0.3894230769230769, + "threshold": 0.39517162811012696 + }, + { + "coverage": 0.347, + "risk": 0.38904899135446686, + "threshold": 0.3951445906779725 + }, + { + "coverage": 0.3473333333333333, + "risk": 0.3886756238003839, + "threshold": 0.39463056635469584 + }, + { + "coverage": 0.3476666666666667, + "risk": 0.38926174496644295, + "threshold": 0.39404732643134105 + }, + { + "coverage": 0.348, + "risk": 0.3888888888888889, + "threshold": 0.39294166019618615 + }, + { + "coverage": 0.34833333333333333, + "risk": 0.38851674641148326, + "threshold": 0.39269776005005497 + }, + { + "coverage": 0.3486666666666667, + "risk": 0.38910133843212236, + "threshold": 0.3907022974720036 + }, + { + "coverage": 0.349, + "risk": 0.38872970391595035, + "threshold": 0.38997557228273294 + }, + { + "coverage": 0.34933333333333333, + "risk": 0.3883587786259542, + "threshold": 0.3861202908401395 + }, + { + "coverage": 0.3496666666666667, + "risk": 0.38798856053384173, + "threshold": 0.38575696497074935 + }, + { + "coverage": 0.35, + "risk": 0.38761904761904764, + "threshold": 0.3854210739491126 + }, + { + "coverage": 0.35033333333333333, + "risk": 0.3872502378686965, + "threshold": 0.38476507299761303 + }, + { + "coverage": 0.3506666666666667, + "risk": 0.38688212927756654, + "threshold": 0.3833805301911468 + }, + { + "coverage": 0.351, + "risk": 0.3865147198480532, + "threshold": 0.3827177685580434 + }, + { + "coverage": 0.35133333333333333, + "risk": 0.38614800759013285, + "threshold": 0.3825769390135534 + }, + { + "coverage": 0.3516666666666667, + "risk": 0.385781990521327, + "threshold": 0.3814523708988986 + }, + { + "coverage": 0.352, + "risk": 0.3854166666666667, + "threshold": 0.38066306338380684 + }, + { + "coverage": 0.35233333333333333, + "risk": 0.3859981078524125, + "threshold": 0.38019186006362027 + }, + { + "coverage": 0.3526666666666667, + "risk": 0.3856332703213611, + "threshold": 0.37986162896316067 + }, + { + "coverage": 0.353, + "risk": 0.3862134088762984, + "threshold": 0.37911661743356007 + }, + { + "coverage": 0.35333333333333333, + "risk": 0.3858490566037736, + "threshold": 0.37724592449520783 + }, + { + "coverage": 0.3536666666666667, + "risk": 0.38548539114043356, + "threshold": 0.37674415604437606 + }, + { + "coverage": 0.354, + "risk": 0.3851224105461394, + "threshold": 0.376125557904416 + }, + { + "coverage": 0.35433333333333333, + "risk": 0.3847601128880527, + "threshold": 0.3746224565445466 + }, + { + "coverage": 0.3546666666666667, + "risk": 0.3843984962406015, + "threshold": 0.37371738186255854 + }, + { + "coverage": 0.355, + "risk": 0.38403755868544603, + "threshold": 0.372942168455932 + }, + { + "coverage": 0.35533333333333333, + "risk": 0.38367729831144465, + "threshold": 0.37293544881045193 + }, + { + "coverage": 0.3556666666666667, + "risk": 0.38331771321462044, + "threshold": 0.37253351287781733 + }, + { + "coverage": 0.356, + "risk": 0.38295880149812733, + "threshold": 0.3719288452858365 + }, + { + "coverage": 0.35633333333333334, + "risk": 0.38353601496725914, + "threshold": 0.3715390336171503 + }, + { + "coverage": 0.3566666666666667, + "risk": 0.3841121495327103, + "threshold": 0.37151367339197705 + }, + { + "coverage": 0.357, + "risk": 0.38375350140056025, + "threshold": 0.3712127987823166 + }, + { + "coverage": 0.35733333333333334, + "risk": 0.3843283582089552, + "threshold": 0.3711143502896021 + }, + { + "coverage": 0.3576666666666667, + "risk": 0.38397017707362535, + "threshold": 0.3707090775085575 + }, + { + "coverage": 0.358, + "risk": 0.38361266294227186, + "threshold": 0.3706801702307772 + }, + { + "coverage": 0.35833333333333334, + "risk": 0.38325581395348834, + "threshold": 0.3691017160779175 + }, + { + "coverage": 0.3586666666666667, + "risk": 0.3828996282527881, + "threshold": 0.36854297381534096 + }, + { + "coverage": 0.359, + "risk": 0.38347260909935005, + "threshold": 0.3682361404772223 + }, + { + "coverage": 0.35933333333333334, + "risk": 0.38311688311688313, + "threshold": 0.3682117404538499 + }, + { + "coverage": 0.3596666666666667, + "risk": 0.38368860055607046, + "threshold": 0.3677418786523816 + }, + { + "coverage": 0.36, + "risk": 0.38333333333333336, + "threshold": 0.36712709026719637 + }, + { + "coverage": 0.36033333333333334, + "risk": 0.38390379278445885, + "threshold": 0.3670768286889903 + }, + { + "coverage": 0.3606666666666667, + "risk": 0.3835489833641405, + "threshold": 0.36690720391380444 + }, + { + "coverage": 0.361, + "risk": 0.3831948291782087, + "threshold": 0.3661382808753686 + }, + { + "coverage": 0.36133333333333334, + "risk": 0.3828413284132841, + "threshold": 0.3659943276194966 + }, + { + "coverage": 0.3616666666666667, + "risk": 0.38341013824884795, + "threshold": 0.36585299277429506 + }, + { + "coverage": 0.362, + "risk": 0.3830570902394107, + "threshold": 0.3657475353170003 + }, + { + "coverage": 0.36233333333333334, + "risk": 0.3827046918123275, + "threshold": 0.36552019383459466 + }, + { + "coverage": 0.3626666666666667, + "risk": 0.38235294117647056, + "threshold": 0.3653860746484731 + }, + { + "coverage": 0.363, + "risk": 0.3820018365472911, + "threshold": 0.3652978917403403 + }, + { + "coverage": 0.36333333333333334, + "risk": 0.381651376146789, + "threshold": 0.36462307319994386 + }, + { + "coverage": 0.3636666666666667, + "risk": 0.38221814848762603, + "threshold": 0.3641285003123027 + }, + { + "coverage": 0.364, + "risk": 0.38186813186813184, + "threshold": 0.3633842625146002 + }, + { + "coverage": 0.36433333333333334, + "risk": 0.38151875571820676, + "threshold": 0.3631262316886392 + }, + { + "coverage": 0.36466666666666664, + "risk": 0.3820840950639854, + "threshold": 0.36198885022812094 + }, + { + "coverage": 0.365, + "risk": 0.3817351598173516, + "threshold": 0.36159541991368366 + }, + { + "coverage": 0.36533333333333334, + "risk": 0.3813868613138686, + "threshold": 0.3586449573624506 + }, + { + "coverage": 0.36566666666666664, + "risk": 0.3810391978122151, + "threshold": 0.3584336710464342 + }, + { + "coverage": 0.366, + "risk": 0.38160291438979965, + "threshold": 0.35800623488410105 + }, + { + "coverage": 0.36633333333333334, + "risk": 0.3812556869881711, + "threshold": 0.357731238155619 + }, + { + "coverage": 0.36666666666666664, + "risk": 0.3809090909090909, + "threshold": 0.3576205703509654 + }, + { + "coverage": 0.367, + "risk": 0.38056312443233425, + "threshold": 0.3559928961791119 + }, + { + "coverage": 0.36733333333333335, + "risk": 0.38021778584392013, + "threshold": 0.3553049175715977 + }, + { + "coverage": 0.36766666666666664, + "risk": 0.3798730734360834, + "threshold": 0.355101000940099 + }, + { + "coverage": 0.368, + "risk": 0.3795289855072464, + "threshold": 0.35493794426099606 + }, + { + "coverage": 0.36833333333333335, + "risk": 0.37918552036199094, + "threshold": 0.3547764803031476 + }, + { + "coverage": 0.36866666666666664, + "risk": 0.37884267631103075, + "threshold": 0.353370585041322 + }, + { + "coverage": 0.369, + "risk": 0.3785004516711834, + "threshold": 0.3531622407418915 + }, + { + "coverage": 0.36933333333333335, + "risk": 0.37815884476534295, + "threshold": 0.35288905827834327 + }, + { + "coverage": 0.36966666666666664, + "risk": 0.37781785392245265, + "threshold": 0.35130489801664494 + }, + { + "coverage": 0.37, + "risk": 0.37747747747747745, + "threshold": 0.3512076672945277 + }, + { + "coverage": 0.37033333333333335, + "risk": 0.3771377137713771, + "threshold": 0.35091095102312236 + }, + { + "coverage": 0.37066666666666664, + "risk": 0.3767985611510791, + "threshold": 0.35067913605915635 + }, + { + "coverage": 0.371, + "risk": 0.37735849056603776, + "threshold": 0.349985547317587 + }, + { + "coverage": 0.37133333333333335, + "risk": 0.3779174147217235, + "threshold": 0.3494700922678699 + }, + { + "coverage": 0.37166666666666665, + "risk": 0.3775784753363229, + "threshold": 0.347219877409401 + }, + { + "coverage": 0.372, + "risk": 0.3772401433691756, + "threshold": 0.3471735886829351 + }, + { + "coverage": 0.37233333333333335, + "risk": 0.37690241718889883, + "threshold": 0.3448796842897828 + }, + { + "coverage": 0.37266666666666665, + "risk": 0.37656529516994636, + "threshold": 0.3440504231888166 + }, + { + "coverage": 0.373, + "risk": 0.3762287756925827, + "threshold": 0.3424365026230092 + }, + { + "coverage": 0.37333333333333335, + "risk": 0.37589285714285714, + "threshold": 0.3416490042222975 + }, + { + "coverage": 0.37366666666666665, + "risk": 0.37555753791257807, + "threshold": 0.3389221805184697 + }, + { + "coverage": 0.374, + "risk": 0.375222816399287, + "threshold": 0.33889365968028073 + }, + { + "coverage": 0.37433333333333335, + "risk": 0.3748886910062333, + "threshold": 0.3383941342238721 + }, + { + "coverage": 0.37466666666666665, + "risk": 0.37455516014234874, + "threshold": 0.3379626688535673 + }, + { + "coverage": 0.375, + "risk": 0.37422222222222223, + "threshold": 0.3377761524141647 + }, + { + "coverage": 0.37533333333333335, + "risk": 0.3738898756660746, + "threshold": 0.336890283383422 + }, + { + "coverage": 0.37566666666666665, + "risk": 0.3744454303460515, + "threshold": 0.33538380652745836 + }, + { + "coverage": 0.376, + "risk": 0.374113475177305, + "threshold": 0.3352320107213984 + }, + { + "coverage": 0.37633333333333335, + "risk": 0.37378210806023027, + "threshold": 0.3351138156371008 + }, + { + "coverage": 0.37666666666666665, + "risk": 0.3734513274336283, + "threshold": 0.33452838864113943 + }, + { + "coverage": 0.377, + "risk": 0.3731211317418214, + "threshold": 0.3343687576639862 + }, + { + "coverage": 0.37733333333333335, + "risk": 0.37279151943462896, + "threshold": 0.3342930157267787 + }, + { + "coverage": 0.37766666666666665, + "risk": 0.3733451015004413, + "threshold": 0.33378191102357846 + }, + { + "coverage": 0.378, + "risk": 0.373015873015873, + "threshold": 0.33369673470007477 + }, + { + "coverage": 0.37833333333333335, + "risk": 0.3726872246696035, + "threshold": 0.3336540876659253 + }, + { + "coverage": 0.37866666666666665, + "risk": 0.37235915492957744, + "threshold": 0.33254471254424445 + }, + { + "coverage": 0.379, + "risk": 0.3729111697449428, + "threshold": 0.3323666626015296 + }, + { + "coverage": 0.37933333333333336, + "risk": 0.37258347978910367, + "threshold": 0.33166258657055364 + }, + { + "coverage": 0.37966666666666665, + "risk": 0.3722563652326602, + "threshold": 0.3316380243857555 + }, + { + "coverage": 0.38, + "risk": 0.3719298245614035, + "threshold": 0.33150189437033184 + }, + { + "coverage": 0.38033333333333336, + "risk": 0.37160385626643294, + "threshold": 0.3314134168198475 + }, + { + "coverage": 0.38066666666666665, + "risk": 0.3712784588441331, + "threshold": 0.33115951850331976 + }, + { + "coverage": 0.381, + "risk": 0.37182852143482065, + "threshold": 0.3310606872157784 + }, + { + "coverage": 0.38133333333333336, + "risk": 0.3723776223776224, + "threshold": 0.3300886278629953 + }, + { + "coverage": 0.38166666666666665, + "risk": 0.3720524017467249, + "threshold": 0.32951466697916487 + }, + { + "coverage": 0.382, + "risk": 0.3717277486910995, + "threshold": 0.32925212041081764 + }, + { + "coverage": 0.38233333333333336, + "risk": 0.3714036617262424, + "threshold": 0.32802325587362624 + }, + { + "coverage": 0.38266666666666665, + "risk": 0.3710801393728223, + "threshold": 0.3278472630470617 + }, + { + "coverage": 0.383, + "risk": 0.370757180156658, + "threshold": 0.32773319344896207 + }, + { + "coverage": 0.38333333333333336, + "risk": 0.37043478260869567, + "threshold": 0.32738869209305577 + }, + { + "coverage": 0.38366666666666666, + "risk": 0.37011294526498695, + "threshold": 0.326638813894192 + }, + { + "coverage": 0.384, + "risk": 0.3697916666666667, + "threshold": 0.32633586945816795 + }, + { + "coverage": 0.38433333333333336, + "risk": 0.36947094535993064, + "threshold": 0.326024140712164 + }, + { + "coverage": 0.38466666666666666, + "risk": 0.36915077989601386, + "threshold": 0.3250763561465817 + }, + { + "coverage": 0.385, + "risk": 0.36883116883116884, + "threshold": 0.32337789387857363 + }, + { + "coverage": 0.38533333333333336, + "risk": 0.3685121107266436, + "threshold": 0.3224708816932332 + }, + { + "coverage": 0.38566666666666666, + "risk": 0.3681936041486603, + "threshold": 0.32201384846581865 + }, + { + "coverage": 0.386, + "risk": 0.36787564766839376, + "threshold": 0.3216258931250753 + }, + { + "coverage": 0.3863333333333333, + "risk": 0.36755823986194996, + "threshold": 0.32157508620161934 + }, + { + "coverage": 0.38666666666666666, + "risk": 0.36724137931034484, + "threshold": 0.32121535457102723 + }, + { + "coverage": 0.387, + "risk": 0.3669250645994832, + "threshold": 0.3207912628515288 + }, + { + "coverage": 0.3873333333333333, + "risk": 0.36660929432013767, + "threshold": 0.3197163689485194 + }, + { + "coverage": 0.38766666666666666, + "risk": 0.3662940670679278, + "threshold": 0.3195698251221941 + }, + { + "coverage": 0.388, + "risk": 0.36597938144329895, + "threshold": 0.31895954602663856 + }, + { + "coverage": 0.3883333333333333, + "risk": 0.36566523605150214, + "threshold": 0.3189139996917998 + }, + { + "coverage": 0.38866666666666666, + "risk": 0.3653516295025729, + "threshold": 0.31890128854802846 + }, + { + "coverage": 0.389, + "risk": 0.36503856041131105, + "threshold": 0.3186211252699973 + }, + { + "coverage": 0.3893333333333333, + "risk": 0.3647260273972603, + "threshold": 0.31851356996964597 + }, + { + "coverage": 0.38966666666666666, + "risk": 0.3652694610778443, + "threshold": 0.3167312653608647 + }, + { + "coverage": 0.39, + "risk": 0.36495726495726494, + "threshold": 0.3163243637105211 + }, + { + "coverage": 0.3903333333333333, + "risk": 0.36464560204953034, + "threshold": 0.3153079601462423 + }, + { + "coverage": 0.39066666666666666, + "risk": 0.3651877133105802, + "threshold": 0.31514691032092834 + }, + { + "coverage": 0.391, + "risk": 0.3648763853367434, + "threshold": 0.3151222089314558 + }, + { + "coverage": 0.3913333333333333, + "risk": 0.3645655877342419, + "threshold": 0.3134932169387188 + }, + { + "coverage": 0.39166666666666666, + "risk": 0.36425531914893616, + "threshold": 0.3127777462226007 + }, + { + "coverage": 0.392, + "risk": 0.36394557823129253, + "threshold": 0.3126342581266144 + }, + { + "coverage": 0.3923333333333333, + "risk": 0.36363636363636365, + "threshold": 0.3125210122301496 + }, + { + "coverage": 0.39266666666666666, + "risk": 0.3633276740237691, + "threshold": 0.31131635375336586 + }, + { + "coverage": 0.393, + "risk": 0.363019508057676, + "threshold": 0.31104231444676245 + }, + { + "coverage": 0.3933333333333333, + "risk": 0.36271186440677966, + "threshold": 0.3104516397092425 + }, + { + "coverage": 0.39366666666666666, + "risk": 0.3624047417442845, + "threshold": 0.30982652701229796 + }, + { + "coverage": 0.394, + "risk": 0.36209813874788493, + "threshold": 0.30981791839539063 + }, + { + "coverage": 0.3943333333333333, + "risk": 0.3617920540997464, + "threshold": 0.30947127742649627 + }, + { + "coverage": 0.39466666666666667, + "risk": 0.3614864864864865, + "threshold": 0.30919902835433355 + }, + { + "coverage": 0.395, + "risk": 0.36118143459915614, + "threshold": 0.30851682037295447 + }, + { + "coverage": 0.3953333333333333, + "risk": 0.3608768971332209, + "threshold": 0.3079764562924652 + }, + { + "coverage": 0.39566666666666667, + "risk": 0.36057287278854255, + "threshold": 0.3079636900698013 + }, + { + "coverage": 0.396, + "risk": 0.3602693602693603, + "threshold": 0.3077002779465232 + }, + { + "coverage": 0.3963333333333333, + "risk": 0.3599663582842725, + "threshold": 0.30673821401965284 + }, + { + "coverage": 0.39666666666666667, + "risk": 0.3596638655462185, + "threshold": 0.3062501343816848 + }, + { + "coverage": 0.397, + "risk": 0.3602015113350126, + "threshold": 0.3055277191536021 + }, + { + "coverage": 0.3973333333333333, + "risk": 0.3598993288590604, + "threshold": 0.30546835865040656 + }, + { + "coverage": 0.39766666666666667, + "risk": 0.3595976529756915, + "threshold": 0.3054136373606795 + }, + { + "coverage": 0.398, + "risk": 0.3592964824120603, + "threshold": 0.3044335204157118 + }, + { + "coverage": 0.3983333333333333, + "risk": 0.35899581589958157, + "threshold": 0.3039388385025106 + }, + { + "coverage": 0.39866666666666667, + "risk": 0.358695652173913, + "threshold": 0.30315931372735433 + }, + { + "coverage": 0.399, + "risk": 0.3583959899749373, + "threshold": 0.3031475520557745 + }, + { + "coverage": 0.3993333333333333, + "risk": 0.3589315525876461, + "threshold": 0.30314496773652877 + }, + { + "coverage": 0.39966666666666667, + "risk": 0.359466221851543, + "threshold": 0.303038455293283 + }, + { + "coverage": 0.4, + "risk": 0.3591666666666667, + "threshold": 0.30008789099025535 + }, + { + "coverage": 0.4003333333333333, + "risk": 0.3588676103247294, + "threshold": 0.29929058203759246 + }, + { + "coverage": 0.40066666666666667, + "risk": 0.35856905158069885, + "threshold": 0.29897515070877745 + }, + { + "coverage": 0.401, + "risk": 0.35827098919368244, + "threshold": 0.2980915703813025 + }, + { + "coverage": 0.4013333333333333, + "risk": 0.3579734219269103, + "threshold": 0.29804623930839824 + }, + { + "coverage": 0.40166666666666667, + "risk": 0.35767634854771785, + "threshold": 0.29618944651883794 + }, + { + "coverage": 0.402, + "risk": 0.357379767827529, + "threshold": 0.29547576486237404 + }, + { + "coverage": 0.4023333333333333, + "risk": 0.35708367854183926, + "threshold": 0.2951065158895769 + }, + { + "coverage": 0.4026666666666667, + "risk": 0.3567880794701987, + "threshold": 0.2939262783928367 + }, + { + "coverage": 0.403, + "risk": 0.3564929693961952, + "threshold": 0.2932809218070709 + }, + { + "coverage": 0.4033333333333333, + "risk": 0.35619834710743803, + "threshold": 0.2930111452253428 + }, + { + "coverage": 0.4036666666666667, + "risk": 0.35590421139554085, + "threshold": 0.2917354156641281 + }, + { + "coverage": 0.404, + "risk": 0.3564356435643564, + "threshold": 0.29064717901594683 + }, + { + "coverage": 0.4043333333333333, + "risk": 0.35614179719703215, + "threshold": 0.2894847945090895 + }, + { + "coverage": 0.4046666666666667, + "risk": 0.35667215815486, + "threshold": 0.2891121165877273 + }, + { + "coverage": 0.405, + "risk": 0.35720164609053495, + "threshold": 0.2882063656144851 + }, + { + "coverage": 0.4053333333333333, + "risk": 0.3569078947368421, + "threshold": 0.28777627302055464 + }, + { + "coverage": 0.4056666666666667, + "risk": 0.35661462612982747, + "threshold": 0.28725684717284317 + }, + { + "coverage": 0.406, + "risk": 0.3563218390804598, + "threshold": 0.2871778015646438 + }, + { + "coverage": 0.4063333333333333, + "risk": 0.3560295324036095, + "threshold": 0.2870837535062165 + }, + { + "coverage": 0.4066666666666667, + "risk": 0.3557377049180328, + "threshold": 0.2865091406840108 + }, + { + "coverage": 0.407, + "risk": 0.35544635544635544, + "threshold": 0.28567403637772776 + }, + { + "coverage": 0.4073333333333333, + "risk": 0.35515548281505727, + "threshold": 0.28559708415566654 + }, + { + "coverage": 0.4076666666666667, + "risk": 0.35486508585445625, + "threshold": 0.285417065623232 + }, + { + "coverage": 0.408, + "risk": 0.3545751633986928, + "threshold": 0.28460581003663776 + }, + { + "coverage": 0.4083333333333333, + "risk": 0.35428571428571426, + "threshold": 0.2836273271508567 + }, + { + "coverage": 0.4086666666666667, + "risk": 0.3539967373572594, + "threshold": 0.2832285346811769 + }, + { + "coverage": 0.409, + "risk": 0.3537082314588427, + "threshold": 0.2831900316286884 + }, + { + "coverage": 0.4093333333333333, + "risk": 0.3542345276872964, + "threshold": 0.28313696318441123 + }, + { + "coverage": 0.4096666666666667, + "risk": 0.35394629780309195, + "threshold": 0.28281885875744095 + }, + { + "coverage": 0.41, + "risk": 0.35365853658536583, + "threshold": 0.2825975608728726 + }, + { + "coverage": 0.4103333333333333, + "risk": 0.35337124289195776, + "threshold": 0.28220187183766576 + }, + { + "coverage": 0.4106666666666667, + "risk": 0.3530844155844156, + "threshold": 0.2821924111655427 + }, + { + "coverage": 0.411, + "risk": 0.35279805352798055, + "threshold": 0.28218518520549907 + }, + { + "coverage": 0.41133333333333333, + "risk": 0.3525121555915721, + "threshold": 0.2818346587895153 + }, + { + "coverage": 0.4116666666666667, + "risk": 0.3522267206477733, + "threshold": 0.2814653211011411 + }, + { + "coverage": 0.412, + "risk": 0.35194174757281554, + "threshold": 0.2812068512108503 + }, + { + "coverage": 0.41233333333333333, + "risk": 0.3516572352465643, + "threshold": 0.28043347265750285 + }, + { + "coverage": 0.4126666666666667, + "risk": 0.35137318255250405, + "threshold": 0.28027524779742835 + }, + { + "coverage": 0.413, + "risk": 0.35189669087974174, + "threshold": 0.2801643411000754 + }, + { + "coverage": 0.41333333333333333, + "risk": 0.35161290322580646, + "threshold": 0.27981286380721393 + }, + { + "coverage": 0.4136666666666667, + "risk": 0.35132957292506045, + "threshold": 0.279347949471731 + }, + { + "coverage": 0.414, + "risk": 0.35104669887278583, + "threshold": 0.27823444961973287 + }, + { + "coverage": 0.41433333333333333, + "risk": 0.3507642799678198, + "threshold": 0.27808926055008953 + }, + { + "coverage": 0.4146666666666667, + "risk": 0.3504823151125402, + "threshold": 0.2775668939505209 + }, + { + "coverage": 0.415, + "risk": 0.351004016064257, + "threshold": 0.27620191714746767 + }, + { + "coverage": 0.41533333333333333, + "risk": 0.3507223113964687, + "threshold": 0.2759414107152942 + }, + { + "coverage": 0.4156666666666667, + "risk": 0.3504410585404972, + "threshold": 0.27590622094006 + }, + { + "coverage": 0.416, + "risk": 0.3501602564102564, + "threshold": 0.2758254273639693 + }, + { + "coverage": 0.41633333333333333, + "risk": 0.3498799039231385, + "threshold": 0.275758007745198 + }, + { + "coverage": 0.4166666666666667, + "risk": 0.3496, + "threshold": 0.2747961689094644 + }, + { + "coverage": 0.417, + "risk": 0.34932054356514786, + "threshold": 0.27410539275948415 + }, + { + "coverage": 0.41733333333333333, + "risk": 0.3490415335463259, + "threshold": 0.273510284187104 + }, + { + "coverage": 0.4176666666666667, + "risk": 0.3487629688747007, + "threshold": 0.27324605432512783 + }, + { + "coverage": 0.418, + "risk": 0.3484848484848485, + "threshold": 0.27281383370803525 + }, + { + "coverage": 0.41833333333333333, + "risk": 0.34820717131474105, + "threshold": 0.2721717568259532 + }, + { + "coverage": 0.4186666666666667, + "risk": 0.3479299363057325, + "threshold": 0.27206677964616865 + }, + { + "coverage": 0.419, + "risk": 0.34765314240254575, + "threshold": 0.27191031941026544 + }, + { + "coverage": 0.41933333333333334, + "risk": 0.34737678855325915, + "threshold": 0.2715982319782011 + }, + { + "coverage": 0.4196666666666667, + "risk": 0.3471008737092931, + "threshold": 0.27097254647979163 + }, + { + "coverage": 0.42, + "risk": 0.3468253968253968, + "threshold": 0.27074031563971346 + }, + { + "coverage": 0.42033333333333334, + "risk": 0.3465503568596352, + "threshold": 0.2700848710227035 + }, + { + "coverage": 0.4206666666666667, + "risk": 0.347068145800317, + "threshold": 0.26954434311637243 + }, + { + "coverage": 0.421, + "risk": 0.34679334916864607, + "threshold": 0.2695150724876427 + }, + { + "coverage": 0.42133333333333334, + "risk": 0.34651898734177217, + "threshold": 0.269207837992953 + }, + { + "coverage": 0.4216666666666667, + "risk": 0.34624505928853755, + "threshold": 0.2691435517897389 + }, + { + "coverage": 0.422, + "risk": 0.3459715639810427, + "threshold": 0.2689457928121648 + }, + { + "coverage": 0.42233333333333334, + "risk": 0.34648776637726914, + "threshold": 0.2688739237616229 + }, + { + "coverage": 0.4226666666666667, + "risk": 0.3462145110410095, + "threshold": 0.2678516969050832 + }, + { + "coverage": 0.423, + "risk": 0.3459416863672183, + "threshold": 0.26730130314109424 + }, + { + "coverage": 0.42333333333333334, + "risk": 0.34566929133858265, + "threshold": 0.26702415940679103 + }, + { + "coverage": 0.4236666666666667, + "risk": 0.34618410700236035, + "threshold": 0.2662742084742149 + }, + { + "coverage": 0.424, + "risk": 0.34591194968553457, + "threshold": 0.26624840166761216 + }, + { + "coverage": 0.42433333333333334, + "risk": 0.34564021995286726, + "threshold": 0.26565710852107394 + }, + { + "coverage": 0.4246666666666667, + "risk": 0.3453689167974882, + "threshold": 0.26561386395946585 + }, + { + "coverage": 0.425, + "risk": 0.34509803921568627, + "threshold": 0.26544232798869416 + }, + { + "coverage": 0.42533333333333334, + "risk": 0.3448275862068966, + "threshold": 0.2653420313399124 + }, + { + "coverage": 0.4256666666666667, + "risk": 0.3445575567736883, + "threshold": 0.2651469765958847 + }, + { + "coverage": 0.426, + "risk": 0.3442879499217527, + "threshold": 0.2650381592166031 + }, + { + "coverage": 0.42633333333333334, + "risk": 0.34401876465989056, + "threshold": 0.26474370278526044 + }, + { + "coverage": 0.4266666666666667, + "risk": 0.34375, + "threshold": 0.26421193680561395 + }, + { + "coverage": 0.427, + "risk": 0.3434816549570648, + "threshold": 0.26350486696707875 + }, + { + "coverage": 0.42733333333333334, + "risk": 0.34321372854914195, + "threshold": 0.26333799447011863 + }, + { + "coverage": 0.42766666666666664, + "risk": 0.34294621979734996, + "threshold": 0.2632685112951627 + }, + { + "coverage": 0.428, + "risk": 0.3426791277258567, + "threshold": 0.2629449658526877 + }, + { + "coverage": 0.42833333333333334, + "risk": 0.3424124513618677, + "threshold": 0.2625584216697803 + }, + { + "coverage": 0.42866666666666664, + "risk": 0.3421461897356143, + "threshold": 0.26232472732933587 + }, + { + "coverage": 0.429, + "risk": 0.3418803418803419, + "threshold": 0.26228751322619814 + }, + { + "coverage": 0.42933333333333334, + "risk": 0.3416149068322981, + "threshold": 0.2622659819319573 + }, + { + "coverage": 0.42966666666666664, + "risk": 0.3413498836307215, + "threshold": 0.26187297562221096 + }, + { + "coverage": 0.43, + "risk": 0.34108527131782945, + "threshold": 0.260655410129516 + }, + { + "coverage": 0.43033333333333335, + "risk": 0.34082106893880715, + "threshold": 0.2605595410994699 + }, + { + "coverage": 0.43066666666666664, + "risk": 0.34055727554179566, + "threshold": 0.2604288322236492 + }, + { + "coverage": 0.431, + "risk": 0.3402938901778809, + "threshold": 0.26007378264061387 + }, + { + "coverage": 0.43133333333333335, + "risk": 0.3400309119010819, + "threshold": 0.2598735149593274 + }, + { + "coverage": 0.43166666666666664, + "risk": 0.33976833976833976, + "threshold": 0.25901143692905393 + }, + { + "coverage": 0.432, + "risk": 0.3395061728395062, + "threshold": 0.2584047351029266 + }, + { + "coverage": 0.43233333333333335, + "risk": 0.3392444101773323, + "threshold": 0.25833818300168515 + }, + { + "coverage": 0.43266666666666664, + "risk": 0.3389830508474576, + "threshold": 0.258337327724357 + }, + { + "coverage": 0.433, + "risk": 0.33872209391839875, + "threshold": 0.2576956815859379 + }, + { + "coverage": 0.43333333333333335, + "risk": 0.3384615384615385, + "threshold": 0.2572141312769549 + }, + { + "coverage": 0.43366666666666664, + "risk": 0.3382013835511145, + "threshold": 0.25707154262400067 + }, + { + "coverage": 0.434, + "risk": 0.3379416282642089, + "threshold": 0.2570287895465526 + }, + { + "coverage": 0.43433333333333335, + "risk": 0.33844973138910206, + "threshold": 0.2569436625197561 + }, + { + "coverage": 0.43466666666666665, + "risk": 0.33819018404907975, + "threshold": 0.25612358081389175 + }, + { + "coverage": 0.435, + "risk": 0.33869731800766284, + "threshold": 0.256030029379275 + }, + { + "coverage": 0.43533333333333335, + "risk": 0.33843797856049007, + "threshold": 0.25575754841499787 + }, + { + "coverage": 0.43566666666666665, + "risk": 0.33817903596021426, + "threshold": 0.25487875692192397 + }, + { + "coverage": 0.436, + "risk": 0.33868501529051986, + "threshold": 0.25345902146612076 + }, + { + "coverage": 0.43633333333333335, + "risk": 0.3384262796027502, + "threshold": 0.2517040428115642 + }, + { + "coverage": 0.43666666666666665, + "risk": 0.3381679389312977, + "threshold": 0.25133786092777655 + }, + { + "coverage": 0.437, + "risk": 0.33867276887871856, + "threshold": 0.250666051323023 + }, + { + "coverage": 0.43733333333333335, + "risk": 0.3384146341463415, + "threshold": 0.24972884040625587 + }, + { + "coverage": 0.43766666666666665, + "risk": 0.33815689261233817, + "threshold": 0.24945833497189088 + }, + { + "coverage": 0.438, + "risk": 0.3378995433789954, + "threshold": 0.24863237634001223 + }, + { + "coverage": 0.43833333333333335, + "risk": 0.33840304182509506, + "threshold": 0.2477380987148017 + }, + { + "coverage": 0.43866666666666665, + "risk": 0.33814589665653494, + "threshold": 0.24729197806174066 + }, + { + "coverage": 0.439, + "risk": 0.33788914198936976, + "threshold": 0.24663129599629124 + }, + { + "coverage": 0.43933333333333335, + "risk": 0.33839150227617604, + "threshold": 0.24643322588338454 + }, + { + "coverage": 0.43966666666666665, + "risk": 0.33813495072024263, + "threshold": 0.24621256274987321 + }, + { + "coverage": 0.44, + "risk": 0.3378787878787879, + "threshold": 0.24604221646357868 + }, + { + "coverage": 0.44033333333333335, + "risk": 0.3376230128690386, + "threshold": 0.24328968571387222 + }, + { + "coverage": 0.44066666666666665, + "risk": 0.3373676248108926, + "threshold": 0.24279317035046155 + }, + { + "coverage": 0.441, + "risk": 0.3371126228269085, + "threshold": 0.2418531376242683 + }, + { + "coverage": 0.44133333333333336, + "risk": 0.3368580060422961, + "threshold": 0.2414350044911284 + }, + { + "coverage": 0.44166666666666665, + "risk": 0.3366037735849057, + "threshold": 0.24110535412606812 + }, + { + "coverage": 0.442, + "risk": 0.33634992458521873, + "threshold": 0.24041081854261398 + }, + { + "coverage": 0.44233333333333336, + "risk": 0.3360964581763376, + "threshold": 0.2396170835080511 + }, + { + "coverage": 0.44266666666666665, + "risk": 0.3358433734939759, + "threshold": 0.23906658396587058 + }, + { + "coverage": 0.443, + "risk": 0.33559066967644846, + "threshold": 0.2385605113476572 + }, + { + "coverage": 0.44333333333333336, + "risk": 0.33533834586466166, + "threshold": 0.23855694040552503 + }, + { + "coverage": 0.44366666666666665, + "risk": 0.3350864012021037, + "threshold": 0.23843702835379685 + }, + { + "coverage": 0.444, + "risk": 0.3355855855855856, + "threshold": 0.2383490290009789 + }, + { + "coverage": 0.44433333333333336, + "risk": 0.3353338334583646, + "threshold": 0.23830457205797909 + }, + { + "coverage": 0.44466666666666665, + "risk": 0.3350824587706147, + "threshold": 0.23819392725110866 + }, + { + "coverage": 0.445, + "risk": 0.3348314606741573, + "threshold": 0.23714024761535232 + }, + { + "coverage": 0.44533333333333336, + "risk": 0.3345808383233533, + "threshold": 0.23708409313814727 + }, + { + "coverage": 0.44566666666666666, + "risk": 0.3343305908750935, + "threshold": 0.2368550316604452 + }, + { + "coverage": 0.446, + "risk": 0.33408071748878926, + "threshold": 0.23675553806645772 + }, + { + "coverage": 0.44633333333333336, + "risk": 0.33383121732636295, + "threshold": 0.2364661384399689 + }, + { + "coverage": 0.44666666666666666, + "risk": 0.3335820895522388, + "threshold": 0.23629883398001542 + }, + { + "coverage": 0.447, + "risk": 0.3333333333333333, + "threshold": 0.23599297653629492 + }, + { + "coverage": 0.44733333333333336, + "risk": 0.3330849478390462, + "threshold": 0.23587777623326645 + }, + { + "coverage": 0.44766666666666666, + "risk": 0.33283693224125094, + "threshold": 0.23552587212056864 + }, + { + "coverage": 0.448, + "risk": 0.3325892857142857, + "threshold": 0.23501490844096648 + }, + { + "coverage": 0.4483333333333333, + "risk": 0.33234200743494424, + "threshold": 0.2345542422791258 + }, + { + "coverage": 0.44866666666666666, + "risk": 0.33209509658246655, + "threshold": 0.23435476704233513 + }, + { + "coverage": 0.449, + "risk": 0.3325909428359317, + "threshold": 0.23424326603502624 + }, + { + "coverage": 0.4493333333333333, + "risk": 0.3330860534124629, + "threshold": 0.2337976600811054 + }, + { + "coverage": 0.44966666666666666, + "risk": 0.3328391401037806, + "threshold": 0.2331128260237847 + }, + { + "coverage": 0.45, + "risk": 0.3325925925925926, + "threshold": 0.2305759666150777 + }, + { + "coverage": 0.4503333333333333, + "risk": 0.3323464100666173, + "threshold": 0.23056714967022887 + }, + { + "coverage": 0.45066666666666666, + "risk": 0.3328402366863905, + "threshold": 0.2296820014779755 + }, + { + "coverage": 0.451, + "risk": 0.3325942350332594, + "threshold": 0.22943619676590998 + }, + { + "coverage": 0.4513333333333333, + "risk": 0.33234859675036926, + "threshold": 0.22938096623036583 + }, + { + "coverage": 0.45166666666666666, + "risk": 0.33210332103321033, + "threshold": 0.22926653899104646 + }, + { + "coverage": 0.452, + "risk": 0.33185840707964603, + "threshold": 0.2288835803858689 + }, + { + "coverage": 0.4523333333333333, + "risk": 0.3316138540899042, + "threshold": 0.22816316807427486 + }, + { + "coverage": 0.45266666666666666, + "risk": 0.33136966126656847, + "threshold": 0.22808902772451847 + }, + { + "coverage": 0.453, + "risk": 0.33112582781456956, + "threshold": 0.22776492936975504 + }, + { + "coverage": 0.4533333333333333, + "risk": 0.33088235294117646, + "threshold": 0.22705196509755685 + }, + { + "coverage": 0.45366666666666666, + "risk": 0.33063923585598826, + "threshold": 0.2269930189169553 + }, + { + "coverage": 0.454, + "risk": 0.3303964757709251, + "threshold": 0.22662183087313895 + }, + { + "coverage": 0.4543333333333333, + "risk": 0.3301540719002201, + "threshold": 0.226609098896823 + }, + { + "coverage": 0.45466666666666666, + "risk": 0.32991202346041054, + "threshold": 0.2264096065630898 + }, + { + "coverage": 0.455, + "risk": 0.32967032967032966, + "threshold": 0.22634573525727045 + }, + { + "coverage": 0.4553333333333333, + "risk": 0.3301610541727672, + "threshold": 0.22631380633951798 + }, + { + "coverage": 0.45566666666666666, + "risk": 0.3306510607168983, + "threshold": 0.22580599518333597 + }, + { + "coverage": 0.456, + "risk": 0.3304093567251462, + "threshold": 0.2257998517103772 + }, + { + "coverage": 0.4563333333333333, + "risk": 0.3301680058436815, + "threshold": 0.2257141193083143 + }, + { + "coverage": 0.45666666666666667, + "risk": 0.32992700729927005, + "threshold": 0.22541640833792803 + }, + { + "coverage": 0.457, + "risk": 0.3296863603209336, + "threshold": 0.2249443708741694 + }, + { + "coverage": 0.4573333333333333, + "risk": 0.3294460641399417, + "threshold": 0.2248853004967941 + }, + { + "coverage": 0.45766666666666667, + "risk": 0.32920611798980337, + "threshold": 0.22466840620372608 + }, + { + "coverage": 0.458, + "risk": 0.3289665211062591, + "threshold": 0.22440409112756401 + }, + { + "coverage": 0.4583333333333333, + "risk": 0.3287272727272727, + "threshold": 0.22431650880995768 + }, + { + "coverage": 0.45866666666666667, + "risk": 0.32848837209302323, + "threshold": 0.22413821839330872 + }, + { + "coverage": 0.459, + "risk": 0.3282498184458969, + "threshold": 0.22372246048857036 + }, + { + "coverage": 0.4593333333333333, + "risk": 0.32801161103047893, + "threshold": 0.22344425933419834 + }, + { + "coverage": 0.45966666666666667, + "risk": 0.32777374909354606, + "threshold": 0.2234431394731704 + }, + { + "coverage": 0.46, + "risk": 0.32753623188405795, + "threshold": 0.22338634353715744 + }, + { + "coverage": 0.4603333333333333, + "risk": 0.3272990586531499, + "threshold": 0.22307483748769918 + }, + { + "coverage": 0.46066666666666667, + "risk": 0.32706222865412443, + "threshold": 0.2221141373642368 + }, + { + "coverage": 0.461, + "risk": 0.32682574114244395, + "threshold": 0.22157767748741897 + }, + { + "coverage": 0.4613333333333333, + "risk": 0.3265895953757225, + "threshold": 0.22124914036703824 + }, + { + "coverage": 0.46166666666666667, + "risk": 0.3263537906137184, + "threshold": 0.22051611246623357 + }, + { + "coverage": 0.462, + "risk": 0.32611832611832614, + "threshold": 0.21987715958495446 + }, + { + "coverage": 0.4623333333333333, + "risk": 0.32660418168709443, + "threshold": 0.21940854186176395 + }, + { + "coverage": 0.46266666666666667, + "risk": 0.3270893371757925, + "threshold": 0.21917957484728492 + }, + { + "coverage": 0.463, + "risk": 0.32685385169186465, + "threshold": 0.2184836492900644 + }, + { + "coverage": 0.4633333333333333, + "risk": 0.32661870503597124, + "threshold": 0.21837317003222445 + }, + { + "coverage": 0.46366666666666667, + "risk": 0.32638389647735444, + "threshold": 0.2179393504093596 + }, + { + "coverage": 0.464, + "risk": 0.32686781609195403, + "threshold": 0.2171907912017659 + }, + { + "coverage": 0.4643333333333333, + "risk": 0.32663316582914576, + "threshold": 0.21697354103827138 + }, + { + "coverage": 0.4646666666666667, + "risk": 0.32639885222381637, + "threshold": 0.2168778003080574 + }, + { + "coverage": 0.465, + "risk": 0.32616487455197135, + "threshold": 0.21652221756646473 + }, + { + "coverage": 0.4653333333333333, + "risk": 0.32593123209169056, + "threshold": 0.21633134208906668 + }, + { + "coverage": 0.4656666666666667, + "risk": 0.325697924123121, + "threshold": 0.21566063215509776 + }, + { + "coverage": 0.466, + "risk": 0.32546494992846925, + "threshold": 0.2153248941265673 + }, + { + "coverage": 0.4663333333333333, + "risk": 0.3252323087919943, + "threshold": 0.21526772089074694 + }, + { + "coverage": 0.4666666666666667, + "risk": 0.325, + "threshold": 0.21497133900775015 + }, + { + "coverage": 0.467, + "risk": 0.32476802284082795, + "threshold": 0.2149310911262852 + }, + { + "coverage": 0.4673333333333333, + "risk": 0.32453637660485024, + "threshold": 0.21485934958665445 + }, + { + "coverage": 0.4676666666666667, + "risk": 0.32430506058446185, + "threshold": 0.21442369624153734 + }, + { + "coverage": 0.468, + "risk": 0.32407407407407407, + "threshold": 0.21436749148580866 + }, + { + "coverage": 0.4683333333333333, + "risk": 0.32455516014234875, + "threshold": 0.21403358322870325 + }, + { + "coverage": 0.4686666666666667, + "risk": 0.32432432432432434, + "threshold": 0.21376847336116 + }, + { + "coverage": 0.469, + "risk": 0.32409381663113007, + "threshold": 0.21362772201230665 + }, + { + "coverage": 0.4693333333333333, + "risk": 0.32386363636363635, + "threshold": 0.21319586568511542 + }, + { + "coverage": 0.4696666666666667, + "risk": 0.32363378282469835, + "threshold": 0.21308243109220198 + }, + { + "coverage": 0.47, + "risk": 0.32340425531914896, + "threshold": 0.21224019197396696 + }, + { + "coverage": 0.4703333333333333, + "risk": 0.32317505315379164, + "threshold": 0.2120882308687655 + }, + { + "coverage": 0.4706666666666667, + "risk": 0.32294617563739375, + "threshold": 0.21192397670731294 + }, + { + "coverage": 0.471, + "risk": 0.3227176220806794, + "threshold": 0.21090381105653755 + }, + { + "coverage": 0.4713333333333333, + "risk": 0.3231966053748232, + "threshold": 0.21074894924100998 + }, + { + "coverage": 0.4716666666666667, + "risk": 0.32296819787985864, + "threshold": 0.21062740811824202 + }, + { + "coverage": 0.472, + "risk": 0.3227401129943503, + "threshold": 0.2103331472425775 + }, + { + "coverage": 0.4723333333333333, + "risk": 0.32251235003528583, + "threshold": 0.2102703762353908 + }, + { + "coverage": 0.4726666666666667, + "risk": 0.3229901269393512, + "threshold": 0.2100721689944933 + }, + { + "coverage": 0.473, + "risk": 0.3227625088090204, + "threshold": 0.20942561308133634 + }, + { + "coverage": 0.47333333333333333, + "risk": 0.32253521126760565, + "threshold": 0.2093295098120862 + }, + { + "coverage": 0.4736666666666667, + "risk": 0.3223082336382829, + "threshold": 0.20913650312873064 + }, + { + "coverage": 0.474, + "risk": 0.32208157524613223, + "threshold": 0.20838293736554017 + }, + { + "coverage": 0.47433333333333333, + "risk": 0.3218552354181307, + "threshold": 0.20828891529826518 + }, + { + "coverage": 0.4746666666666667, + "risk": 0.3223314606741573, + "threshold": 0.20789899569947143 + }, + { + "coverage": 0.475, + "risk": 0.32210526315789473, + "threshold": 0.2077287510561732 + }, + { + "coverage": 0.47533333333333333, + "risk": 0.32187938288920054, + "threshold": 0.20685220858409178 + }, + { + "coverage": 0.4756666666666667, + "risk": 0.32165381920112124, + "threshold": 0.20582578518430772 + }, + { + "coverage": 0.476, + "risk": 0.32142857142857145, + "threshold": 0.20574571476152492 + }, + { + "coverage": 0.47633333333333333, + "risk": 0.3212036389083275, + "threshold": 0.20526230610817645 + }, + { + "coverage": 0.4766666666666667, + "risk": 0.320979020979021, + "threshold": 0.2047358635892138 + }, + { + "coverage": 0.477, + "risk": 0.32075471698113206, + "threshold": 0.20469562796392887 + }, + { + "coverage": 0.47733333333333333, + "risk": 0.32053072625698326, + "threshold": 0.20442349879039234 + }, + { + "coverage": 0.4776666666666667, + "risk": 0.32030704815073274, + "threshold": 0.2043890385856386 + }, + { + "coverage": 0.478, + "risk": 0.3200836820083682, + "threshold": 0.20367139780936755 + }, + { + "coverage": 0.47833333333333333, + "risk": 0.31986062717770036, + "threshold": 0.2035643208038094 + }, + { + "coverage": 0.4786666666666667, + "risk": 0.31963788300835655, + "threshold": 0.20351799918251512 + }, + { + "coverage": 0.479, + "risk": 0.31941544885177453, + "threshold": 0.20339939136917448 + }, + { + "coverage": 0.47933333333333333, + "risk": 0.3191933240611961, + "threshold": 0.2030638997020979 + }, + { + "coverage": 0.4796666666666667, + "risk": 0.31897150799166085, + "threshold": 0.20304262838611467 + }, + { + "coverage": 0.48, + "risk": 0.31875, + "threshold": 0.20292549817584063 + }, + { + "coverage": 0.48033333333333333, + "risk": 0.31852879944483, + "threshold": 0.20281525257764257 + }, + { + "coverage": 0.4806666666666667, + "risk": 0.31900138696255204, + "threshold": 0.20256070376653756 + }, + { + "coverage": 0.481, + "risk": 0.31878031878031876, + "threshold": 0.2019002807873278 + }, + { + "coverage": 0.48133333333333334, + "risk": 0.3192520775623269, + "threshold": 0.20167863981694015 + }, + { + "coverage": 0.4816666666666667, + "risk": 0.3190311418685121, + "threshold": 0.20164012176625692 + }, + { + "coverage": 0.482, + "risk": 0.3188105117565698, + "threshold": 0.20058073140636526 + }, + { + "coverage": 0.48233333333333334, + "risk": 0.31928127159640635, + "threshold": 0.19867852309140588 + }, + { + "coverage": 0.4826666666666667, + "risk": 0.319060773480663, + "threshold": 0.1986761916621731 + }, + { + "coverage": 0.483, + "risk": 0.3195307108350587, + "threshold": 0.198250926279 + }, + { + "coverage": 0.48333333333333334, + "risk": 0.3193103448275862, + "threshold": 0.19714873718767834 + }, + { + "coverage": 0.4836666666666667, + "risk": 0.31909028256374916, + "threshold": 0.19561479882290553 + }, + { + "coverage": 0.484, + "risk": 0.31887052341597794, + "threshold": 0.19544569706545548 + }, + { + "coverage": 0.48433333333333334, + "risk": 0.31865106675843086, + "threshold": 0.19541418896608587 + }, + { + "coverage": 0.4846666666666667, + "risk": 0.3184319119669876, + "threshold": 0.1953007790749517 + }, + { + "coverage": 0.485, + "risk": 0.31821305841924397, + "threshold": 0.19519881786677828 + }, + { + "coverage": 0.48533333333333334, + "risk": 0.3179945054945055, + "threshold": 0.19472690853897046 + }, + { + "coverage": 0.4856666666666667, + "risk": 0.3177762525737817, + "threshold": 0.19443900884191476 + }, + { + "coverage": 0.486, + "risk": 0.3175582990397805, + "threshold": 0.19416500681482066 + }, + { + "coverage": 0.48633333333333334, + "risk": 0.317340644276902, + "threshold": 0.1939857635338349 + }, + { + "coverage": 0.4866666666666667, + "risk": 0.3171232876712329, + "threshold": 0.19362821701526478 + }, + { + "coverage": 0.487, + "risk": 0.31690622861054074, + "threshold": 0.19346252080690718 + }, + { + "coverage": 0.48733333333333334, + "risk": 0.3166894664842681, + "threshold": 0.19334000973768362 + }, + { + "coverage": 0.4876666666666667, + "risk": 0.316473000683527, + "threshold": 0.19306041504571958 + }, + { + "coverage": 0.488, + "risk": 0.3162568306010929, + "threshold": 0.19200389713660115 + }, + { + "coverage": 0.48833333333333334, + "risk": 0.31604095563139933, + "threshold": 0.19143870531673102 + }, + { + "coverage": 0.4886666666666667, + "risk": 0.31582537517053205, + "threshold": 0.1911970846940019 + }, + { + "coverage": 0.489, + "risk": 0.3156100886162236, + "threshold": 0.19099082865001282 + }, + { + "coverage": 0.48933333333333334, + "risk": 0.31539509536784743, + "threshold": 0.19036441016556438 + }, + { + "coverage": 0.48966666666666664, + "risk": 0.3151803948264125, + "threshold": 0.19012106736450815 + }, + { + "coverage": 0.49, + "risk": 0.3149659863945578, + "threshold": 0.18944094358913066 + }, + { + "coverage": 0.49033333333333334, + "risk": 0.31475186947654654, + "threshold": 0.1892604936425997 + }, + { + "coverage": 0.49066666666666664, + "risk": 0.31521739130434784, + "threshold": 0.1891953410360081 + }, + { + "coverage": 0.491, + "risk": 0.31568228105906315, + "threshold": 0.18913778859544195 + }, + { + "coverage": 0.49133333333333334, + "risk": 0.31614654002713705, + "threshold": 0.18899179639048033 + }, + { + "coverage": 0.49166666666666664, + "risk": 0.3159322033898305, + "threshold": 0.18886101810980802 + }, + { + "coverage": 0.492, + "risk": 0.3157181571815718, + "threshold": 0.18857078740526492 + }, + { + "coverage": 0.49233333333333335, + "risk": 0.3155044008124577, + "threshold": 0.18843023017909655 + }, + { + "coverage": 0.49266666666666664, + "risk": 0.3152909336941813, + "threshold": 0.18657557860340293 + }, + { + "coverage": 0.493, + "risk": 0.31507775524002707, + "threshold": 0.1862243860697177 + }, + { + "coverage": 0.49333333333333335, + "risk": 0.31486486486486487, + "threshold": 0.18602441296845296 + }, + { + "coverage": 0.49366666666666664, + "risk": 0.3146522619851452, + "threshold": 0.18595653730564976 + }, + { + "coverage": 0.494, + "risk": 0.3144399460188934, + "threshold": 0.18563671935419726 + }, + { + "coverage": 0.49433333333333335, + "risk": 0.31422791638570463, + "threshold": 0.1844398038364012 + }, + { + "coverage": 0.49466666666666664, + "risk": 0.3140161725067385, + "threshold": 0.18442559900387678 + }, + { + "coverage": 0.495, + "risk": 0.3138047138047138, + "threshold": 0.18415673048884426 + }, + { + "coverage": 0.49533333333333335, + "risk": 0.3135935397039031, + "threshold": 0.18411746099990098 + }, + { + "coverage": 0.49566666666666664, + "risk": 0.3133826496301278, + "threshold": 0.18405666770040785 + }, + { + "coverage": 0.496, + "risk": 0.3131720430107527, + "threshold": 0.1836742575766106 + }, + { + "coverage": 0.49633333333333335, + "risk": 0.31296171927468097, + "threshold": 0.18330179880617692 + }, + { + "coverage": 0.49666666666666665, + "risk": 0.312751677852349, + "threshold": 0.18323893177425357 + }, + { + "coverage": 0.497, + "risk": 0.31254191817572097, + "threshold": 0.18310626806260544 + }, + { + "coverage": 0.49733333333333335, + "risk": 0.31233243967828417, + "threshold": 0.18177547398859772 + }, + { + "coverage": 0.49766666666666665, + "risk": 0.3121232417950435, + "threshold": 0.1815046315156037 + }, + { + "coverage": 0.498, + "risk": 0.3119143239625167, + "threshold": 0.1814408173991584 + }, + { + "coverage": 0.49833333333333335, + "risk": 0.3117056856187291, + "threshold": 0.1814294456865753 + }, + { + "coverage": 0.49866666666666665, + "risk": 0.3114973262032086, + "threshold": 0.18121077010634679 + }, + { + "coverage": 0.499, + "risk": 0.31128924515698064, + "threshold": 0.18084481178172493 + }, + { + "coverage": 0.49933333333333335, + "risk": 0.31108144192256343, + "threshold": 0.18052693146206256 + }, + { + "coverage": 0.49966666666666665, + "risk": 0.3115410273515677, + "threshold": 0.1801992444898204 + }, + { + "coverage": 0.5, + "risk": 0.31133333333333335, + "threshold": 0.1801938348553467 + }, + { + "coverage": 0.5003333333333333, + "risk": 0.3111259160559627, + "threshold": 0.1797700053548512 + }, + { + "coverage": 0.5006666666666667, + "risk": 0.31091877496671105, + "threshold": 0.17960888440113632 + }, + { + "coverage": 0.501, + "risk": 0.3107119095143047, + "threshold": 0.17947208371709425 + }, + { + "coverage": 0.5013333333333333, + "risk": 0.31117021276595747, + "threshold": 0.17945115964213887 + }, + { + "coverage": 0.5016666666666667, + "risk": 0.3109634551495017, + "threshold": 0.17879845829720864 + }, + { + "coverage": 0.502, + "risk": 0.3114209827357238, + "threshold": 0.1783133092100524 + }, + { + "coverage": 0.5023333333333333, + "risk": 0.3112143331121433, + "threshold": 0.17820892897201546 + }, + { + "coverage": 0.5026666666666667, + "risk": 0.3110079575596817, + "threshold": 0.17812372054009987 + }, + { + "coverage": 0.503, + "risk": 0.3108018555334659, + "threshold": 0.1778278954037269 + }, + { + "coverage": 0.5033333333333333, + "risk": 0.31059602649006623, + "threshold": 0.17779573436191495 + }, + { + "coverage": 0.5036666666666667, + "risk": 0.31039046988749175, + "threshold": 0.17764067724960242 + }, + { + "coverage": 0.504, + "risk": 0.3101851851851852, + "threshold": 0.17732569826945976 + }, + { + "coverage": 0.5043333333333333, + "risk": 0.3099801718440185, + "threshold": 0.17696215351686106 + }, + { + "coverage": 0.5046666666666667, + "risk": 0.309775429326288, + "threshold": 0.1765979602614998 + }, + { + "coverage": 0.505, + "risk": 0.3095709570957096, + "threshold": 0.17649734552201934 + }, + { + "coverage": 0.5053333333333333, + "risk": 0.30936675461741425, + "threshold": 0.17494229337156852 + }, + { + "coverage": 0.5056666666666667, + "risk": 0.3091628213579433, + "threshold": 0.1746736245586753 + }, + { + "coverage": 0.506, + "risk": 0.3089591567852437, + "threshold": 0.1744065034740991 + }, + { + "coverage": 0.5063333333333333, + "risk": 0.3087557603686636, + "threshold": 0.17412573257293393 + }, + { + "coverage": 0.5066666666666667, + "risk": 0.30855263157894736, + "threshold": 0.17409490800287752 + }, + { + "coverage": 0.507, + "risk": 0.30834976988823143, + "threshold": 0.17272937642151834 + }, + { + "coverage": 0.5073333333333333, + "risk": 0.30814717477003944, + "threshold": 0.1724882113930164 + }, + { + "coverage": 0.5076666666666667, + "risk": 0.3079448456992777, + "threshold": 0.17239549213423902 + }, + { + "coverage": 0.508, + "risk": 0.307742782152231, + "threshold": 0.17184488828726963 + }, + { + "coverage": 0.5083333333333333, + "risk": 0.30754098360655735, + "threshold": 0.17175033064197986 + }, + { + "coverage": 0.5086666666666667, + "risk": 0.3073394495412844, + "threshold": 0.17164570431247658 + }, + { + "coverage": 0.509, + "risk": 0.3071381794368042, + "threshold": 0.17154613098724525 + }, + { + "coverage": 0.5093333333333333, + "risk": 0.3069371727748691, + "threshold": 0.17123672538787105 + }, + { + "coverage": 0.5096666666666667, + "risk": 0.30673642903858733, + "threshold": 0.17042643375071886 + }, + { + "coverage": 0.51, + "risk": 0.3065359477124183, + "threshold": 0.17002563877828614 + }, + { + "coverage": 0.5103333333333333, + "risk": 0.30633572828216854, + "threshold": 0.16995467062969635 + }, + { + "coverage": 0.5106666666666667, + "risk": 0.30613577023498695, + "threshold": 0.16949372912414162 + }, + { + "coverage": 0.511, + "risk": 0.3059360730593607, + "threshold": 0.16947670761011047 + }, + { + "coverage": 0.5113333333333333, + "risk": 0.3057366362451108, + "threshold": 0.16927481427038593 + }, + { + "coverage": 0.5116666666666667, + "risk": 0.3055374592833876, + "threshold": 0.168907006905739 + }, + { + "coverage": 0.512, + "risk": 0.3053385416666667, + "threshold": 0.1683118203949414 + }, + { + "coverage": 0.5123333333333333, + "risk": 0.30513988288874433, + "threshold": 0.16809782825552236 + }, + { + "coverage": 0.5126666666666667, + "risk": 0.3049414824447334, + "threshold": 0.1674713817011835 + }, + { + "coverage": 0.513, + "risk": 0.3047433398310591, + "threshold": 0.1669788784740014 + }, + { + "coverage": 0.5133333333333333, + "risk": 0.30454545454545456, + "threshold": 0.16677793535657326 + }, + { + "coverage": 0.5136666666666667, + "risk": 0.30434782608695654, + "threshold": 0.16676515076486 + }, + { + "coverage": 0.514, + "risk": 0.3041504539559014, + "threshold": 0.16669203754835138 + }, + { + "coverage": 0.5143333333333333, + "risk": 0.3039533376539209, + "threshold": 0.16659887562455292 + }, + { + "coverage": 0.5146666666666667, + "risk": 0.30375647668393785, + "threshold": 0.1664607084634895 + }, + { + "coverage": 0.515, + "risk": 0.3035598705501618, + "threshold": 0.1657939331064231 + }, + { + "coverage": 0.5153333333333333, + "risk": 0.30336351875808537, + "threshold": 0.16523934582046174 + }, + { + "coverage": 0.5156666666666667, + "risk": 0.3031674208144796, + "threshold": 0.16512005987214987 + }, + { + "coverage": 0.516, + "risk": 0.3029715762273902, + "threshold": 0.16507498153644826 + }, + { + "coverage": 0.5163333333333333, + "risk": 0.30277598450613297, + "threshold": 0.1648063817066001 + }, + { + "coverage": 0.5166666666666667, + "risk": 0.3025806451612903, + "threshold": 0.16475125577080085 + }, + { + "coverage": 0.517, + "risk": 0.30303030303030304, + "threshold": 0.16419860295442845 + }, + { + "coverage": 0.5173333333333333, + "risk": 0.30283505154639173, + "threshold": 0.16351905699927194 + }, + { + "coverage": 0.5176666666666667, + "risk": 0.3026400515132003, + "threshold": 0.1630223219298399 + }, + { + "coverage": 0.518, + "risk": 0.30244530244530243, + "threshold": 0.1629567354766266 + }, + { + "coverage": 0.5183333333333333, + "risk": 0.3022508038585209, + "threshold": 0.1621988182355771 + }, + { + "coverage": 0.5186666666666667, + "risk": 0.30205655526992287, + "threshold": 0.1620047063276564 + }, + { + "coverage": 0.519, + "risk": 0.3018625561978163, + "threshold": 0.16108619069424535 + }, + { + "coverage": 0.5193333333333333, + "risk": 0.30166880616174585, + "threshold": 0.16092667092424406 + }, + { + "coverage": 0.5196666666666667, + "risk": 0.3014753046824888, + "threshold": 0.1603821498791591 + }, + { + "coverage": 0.52, + "risk": 0.30128205128205127, + "threshold": 0.1602417606515978 + }, + { + "coverage": 0.5203333333333333, + "risk": 0.30108904548366433, + "threshold": 0.16012918262527262 + }, + { + "coverage": 0.5206666666666667, + "risk": 0.30089628681177977, + "threshold": 0.15948648550364108 + }, + { + "coverage": 0.521, + "risk": 0.30070377479206656, + "threshold": 0.15946080426368248 + }, + { + "coverage": 0.5213333333333333, + "risk": 0.30051150895140666, + "threshold": 0.15930826764843078 + }, + { + "coverage": 0.5216666666666666, + "risk": 0.3003194888178914, + "threshold": 0.15911938203350218 + }, + { + "coverage": 0.522, + "risk": 0.3001277139208174, + "threshold": 0.1589180070973697 + }, + { + "coverage": 0.5223333333333333, + "risk": 0.2999361837906828, + "threshold": 0.15868365521334304 + }, + { + "coverage": 0.5226666666666666, + "risk": 0.2997448979591837, + "threshold": 0.15845079302478382 + }, + { + "coverage": 0.523, + "risk": 0.2995538559592097, + "threshold": 0.15827291896347795 + }, + { + "coverage": 0.5233333333333333, + "risk": 0.3, + "threshold": 0.15823021144635516 + }, + { + "coverage": 0.5236666666666666, + "risk": 0.29980903882877147, + "threshold": 0.15821616757007465 + }, + { + "coverage": 0.524, + "risk": 0.29961832061068705, + "threshold": 0.15807070295387415 + }, + { + "coverage": 0.5243333333333333, + "risk": 0.2994278448823903, + "threshold": 0.157489151995442 + }, + { + "coverage": 0.5246666666666666, + "risk": 0.29923761118170267, + "threshold": 0.15747616482743257 + }, + { + "coverage": 0.525, + "risk": 0.29904761904761906, + "threshold": 0.15725842357035089 + }, + { + "coverage": 0.5253333333333333, + "risk": 0.29949238578680204, + "threshold": 0.157196737212411 + }, + { + "coverage": 0.5256666666666666, + "risk": 0.2993024730500951, + "threshold": 0.15712805245712502 + }, + { + "coverage": 0.526, + "risk": 0.2991128010139417, + "threshold": 0.15696453969970928 + }, + { + "coverage": 0.5263333333333333, + "risk": 0.29892336922102597, + "threshold": 0.15654374464932985 + }, + { + "coverage": 0.5266666666666666, + "risk": 0.29873417721518986, + "threshold": 0.1563613257601446 + }, + { + "coverage": 0.527, + "risk": 0.29854522454142945, + "threshold": 0.15625363067618195 + }, + { + "coverage": 0.5273333333333333, + "risk": 0.29835651074589126, + "threshold": 0.15615589305562239 + }, + { + "coverage": 0.5276666666666666, + "risk": 0.2981680353758686, + "threshold": 0.15572101455553877 + }, + { + "coverage": 0.528, + "risk": 0.29797979797979796, + "threshold": 0.1544927151281481 + }, + { + "coverage": 0.5283333333333333, + "risk": 0.29779179810725553, + "threshold": 0.15433952894353453 + }, + { + "coverage": 0.5286666666666666, + "risk": 0.29760403530895335, + "threshold": 0.15386478605942955 + }, + { + "coverage": 0.529, + "risk": 0.297416509136736, + "threshold": 0.15349492742558546 + }, + { + "coverage": 0.5293333333333333, + "risk": 0.2972292191435768, + "threshold": 0.15318222402798387 + }, + { + "coverage": 0.5296666666666666, + "risk": 0.2976714915040906, + "threshold": 0.1530826970610022 + }, + { + "coverage": 0.53, + "risk": 0.29748427672955974, + "threshold": 0.15284506505552817 + }, + { + "coverage": 0.5303333333333333, + "risk": 0.2972972972972973, + "threshold": 0.152218067288187 + }, + { + "coverage": 0.5306666666666666, + "risk": 0.2971105527638191, + "threshold": 0.15201585439130308 + }, + { + "coverage": 0.531, + "risk": 0.29692404268675454, + "threshold": 0.15196872264920724 + }, + { + "coverage": 0.5313333333333333, + "risk": 0.29673776662484314, + "threshold": 0.1519256619123217 + }, + { + "coverage": 0.5316666666666666, + "risk": 0.296551724137931, + "threshold": 0.15084337759017075 + }, + { + "coverage": 0.532, + "risk": 0.29699248120300753, + "threshold": 0.15075549135413982 + }, + { + "coverage": 0.5323333333333333, + "risk": 0.2968065122103945, + "threshold": 0.1505823348550539 + }, + { + "coverage": 0.5326666666666666, + "risk": 0.29662077596996245, + "threshold": 0.14891894366472513 + }, + { + "coverage": 0.533, + "risk": 0.2964352720450281, + "threshold": 0.14889702651981196 + }, + { + "coverage": 0.5333333333333333, + "risk": 0.29625, + "threshold": 0.14823376073704875 + }, + { + "coverage": 0.5336666666666666, + "risk": 0.29606495940037475, + "threshold": 0.14724725764273305 + }, + { + "coverage": 0.534, + "risk": 0.2958801498127341, + "threshold": 0.14713163369513357 + }, + { + "coverage": 0.5343333333333333, + "risk": 0.2956955708047411, + "threshold": 0.14665718099426517 + }, + { + "coverage": 0.5346666666666666, + "risk": 0.2955112219451372, + "threshold": 0.14644229571013057 + }, + { + "coverage": 0.535, + "risk": 0.2953271028037383, + "threshold": 0.1464230120191985 + }, + { + "coverage": 0.5353333333333333, + "risk": 0.29514321295143214, + "threshold": 0.1460905450550385 + }, + { + "coverage": 0.5356666666666666, + "risk": 0.29495955196017426, + "threshold": 0.14573001621781856 + }, + { + "coverage": 0.536, + "risk": 0.2947761194029851, + "threshold": 0.14563639052234123 + }, + { + "coverage": 0.5363333333333333, + "risk": 0.29459291485394656, + "threshold": 0.14508277324516072 + }, + { + "coverage": 0.5366666666666666, + "risk": 0.29440993788819875, + "threshold": 0.14484749149004725 + }, + { + "coverage": 0.537, + "risk": 0.2948479205462446, + "threshold": 0.14386286397141745 + }, + { + "coverage": 0.5373333333333333, + "risk": 0.29466501240694787, + "threshold": 0.14348692826034612 + }, + { + "coverage": 0.5376666666666666, + "risk": 0.2944823310601364, + "threshold": 0.14338370094323238 + }, + { + "coverage": 0.538, + "risk": 0.2942998760842627, + "threshold": 0.14312933122003635 + }, + { + "coverage": 0.5383333333333333, + "risk": 0.29411764705882354, + "threshold": 0.1429604726794089 + }, + { + "coverage": 0.5386666666666666, + "risk": 0.2939356435643564, + "threshold": 0.14217711270056113 + }, + { + "coverage": 0.539, + "risk": 0.2937538651824366, + "threshold": 0.1420075009643167 + }, + { + "coverage": 0.5393333333333333, + "risk": 0.29357231149567364, + "threshold": 0.141864325979963 + }, + { + "coverage": 0.5396666666666666, + "risk": 0.2933909820877085, + "threshold": 0.14153108571561113 + }, + { + "coverage": 0.54, + "risk": 0.2932098765432099, + "threshold": 0.14133384576560462 + }, + { + "coverage": 0.5403333333333333, + "risk": 0.29302899444787167, + "threshold": 0.13931386601188578 + }, + { + "coverage": 0.5406666666666666, + "risk": 0.2928483353884094, + "threshold": 0.13913919307642963 + }, + { + "coverage": 0.541, + "risk": 0.292667898952557, + "threshold": 0.13911894798916172 + }, + { + "coverage": 0.5413333333333333, + "risk": 0.29248768472906406, + "threshold": 0.13893682739083957 + }, + { + "coverage": 0.5416666666666666, + "risk": 0.2923076923076923, + "threshold": 0.13874781027815253 + }, + { + "coverage": 0.542, + "risk": 0.2921279212792128, + "threshold": 0.13859629514055613 + }, + { + "coverage": 0.5423333333333333, + "risk": 0.29194837123540257, + "threshold": 0.13857128584049858 + }, + { + "coverage": 0.5426666666666666, + "risk": 0.2917690417690418, + "threshold": 0.13855494102155497 + }, + { + "coverage": 0.543, + "risk": 0.29158993247391035, + "threshold": 0.13840655536237098 + }, + { + "coverage": 0.5433333333333333, + "risk": 0.29141104294478526, + "threshold": 0.13756389269807826 + }, + { + "coverage": 0.5436666666666666, + "risk": 0.29123237277743713, + "threshold": 0.13748053328497534 + }, + { + "coverage": 0.544, + "risk": 0.29105392156862747, + "threshold": 0.13746981431658423 + }, + { + "coverage": 0.5443333333333333, + "risk": 0.2908756889161053, + "threshold": 0.13735209219441025 + }, + { + "coverage": 0.5446666666666666, + "risk": 0.29069767441860467, + "threshold": 0.13691361724644047 + }, + { + "coverage": 0.545, + "risk": 0.290519877675841, + "threshold": 0.13613180495383403 + }, + { + "coverage": 0.5453333333333333, + "risk": 0.29034229828850855, + "threshold": 0.13596401801997612 + }, + { + "coverage": 0.5456666666666666, + "risk": 0.29016493585827735, + "threshold": 0.13557794708015788 + }, + { + "coverage": 0.546, + "risk": 0.28998778998779, + "threshold": 0.13548733162993543 + }, + { + "coverage": 0.5463333333333333, + "risk": 0.28981086028065894, + "threshold": 0.13527257755419075 + }, + { + "coverage": 0.5466666666666666, + "risk": 0.2896341463414634, + "threshold": 0.13525579911601415 + }, + { + "coverage": 0.547, + "risk": 0.2894576477757465, + "threshold": 0.13486809011531076 + }, + { + "coverage": 0.5473333333333333, + "risk": 0.28989037758830694, + "threshold": 0.13464280338563128 + }, + { + "coverage": 0.5476666666666666, + "risk": 0.28971393791844186, + "threshold": 0.13463038552729412 + }, + { + "coverage": 0.548, + "risk": 0.2895377128953771, + "threshold": 0.13456023669432324 + }, + { + "coverage": 0.5483333333333333, + "risk": 0.28936170212765955, + "threshold": 0.13424722572174463 + }, + { + "coverage": 0.5486666666666666, + "risk": 0.2891859052247874, + "threshold": 0.13419894842599867 + }, + { + "coverage": 0.549, + "risk": 0.28901032179720704, + "threshold": 0.134167952304769 + }, + { + "coverage": 0.5493333333333333, + "risk": 0.28883495145631066, + "threshold": 0.13406429445666923 + }, + { + "coverage": 0.5496666666666666, + "risk": 0.28865979381443296, + "threshold": 0.13383088943023758 + }, + { + "coverage": 0.55, + "risk": 0.2884848484848485, + "threshold": 0.1337360011075397 + }, + { + "coverage": 0.5503333333333333, + "risk": 0.2883101150817686, + "threshold": 0.1336553631878985 + }, + { + "coverage": 0.5506666666666666, + "risk": 0.288135593220339, + "threshold": 0.13354561621925765 + }, + { + "coverage": 0.551, + "risk": 0.28796128251663644, + "threshold": 0.13316417666349345 + }, + { + "coverage": 0.5513333333333333, + "risk": 0.2877871825876663, + "threshold": 0.13292869213439576 + }, + { + "coverage": 0.5516666666666666, + "risk": 0.2876132930513595, + "threshold": 0.1328908406324603 + }, + { + "coverage": 0.552, + "risk": 0.28743961352657005, + "threshold": 0.13285349315569572 + }, + { + "coverage": 0.5523333333333333, + "risk": 0.2872661436330718, + "threshold": 0.13165507239048324 + }, + { + "coverage": 0.5526666666666666, + "risk": 0.2870928829915561, + "threshold": 0.13163901929556232 + }, + { + "coverage": 0.553, + "risk": 0.2869198312236287, + "threshold": 0.1315167684856462 + }, + { + "coverage": 0.5533333333333333, + "risk": 0.28674698795180725, + "threshold": 0.1312064203459995 + }, + { + "coverage": 0.5536666666666666, + "risk": 0.2865743527995184, + "threshold": 0.13087582559682964 + }, + { + "coverage": 0.554, + "risk": 0.28640192539109505, + "threshold": 0.13074565482139633 + }, + { + "coverage": 0.5543333333333333, + "risk": 0.2862297053517739, + "threshold": 0.13044129718939512 + }, + { + "coverage": 0.5546666666666666, + "risk": 0.2860576923076923, + "threshold": 0.13037836456534713 + }, + { + "coverage": 0.555, + "risk": 0.28588588588588587, + "threshold": 0.1296655663740731 + }, + { + "coverage": 0.5553333333333333, + "risk": 0.2857142857142857, + "threshold": 0.12962605502047977 + }, + { + "coverage": 0.5556666666666666, + "risk": 0.2855428914217157, + "threshold": 0.12935868881059506 + }, + { + "coverage": 0.556, + "risk": 0.2853717026378897, + "threshold": 0.12905989789634464 + }, + { + "coverage": 0.5563333333333333, + "risk": 0.2852007189934092, + "threshold": 0.12897827812644372 + }, + { + "coverage": 0.5566666666666666, + "risk": 0.28502994011976046, + "threshold": 0.12880995772498238 + }, + { + "coverage": 0.557, + "risk": 0.2848593656493118, + "threshold": 0.12866789166199943 + }, + { + "coverage": 0.5573333333333333, + "risk": 0.284688995215311, + "threshold": 0.12845234904772518 + }, + { + "coverage": 0.5576666666666666, + "risk": 0.28451882845188287, + "threshold": 0.12833516882749407 + }, + { + "coverage": 0.558, + "risk": 0.2843488649940263, + "threshold": 0.1281471973182562 + }, + { + "coverage": 0.5583333333333333, + "risk": 0.28417910447761197, + "threshold": 0.12755712779481954 + }, + { + "coverage": 0.5586666666666666, + "risk": 0.2840095465393795, + "threshold": 0.12752201125256057 + }, + { + "coverage": 0.559, + "risk": 0.283840190816935, + "threshold": 0.12719670759696552 + }, + { + "coverage": 0.5593333333333333, + "risk": 0.2836710369487485, + "threshold": 0.12696675712487604 + }, + { + "coverage": 0.5596666666666666, + "risk": 0.28350208457415127, + "threshold": 0.12631800770991933 + }, + { + "coverage": 0.56, + "risk": 0.2833333333333333, + "threshold": 0.1261283520918033 + }, + { + "coverage": 0.5603333333333333, + "risk": 0.28316478286734087, + "threshold": 0.1260920216039665 + }, + { + "coverage": 0.5606666666666666, + "risk": 0.28299643281807374, + "threshold": 0.12595758385057734 + }, + { + "coverage": 0.561, + "risk": 0.2828282828282828, + "threshold": 0.1255711379240303 + }, + { + "coverage": 0.5613333333333334, + "risk": 0.2826603325415677, + "threshold": 0.1250143747216832 + }, + { + "coverage": 0.5616666666666666, + "risk": 0.2824925816023739, + "threshold": 0.12496751536875461 + }, + { + "coverage": 0.562, + "risk": 0.2823250296559905, + "threshold": 0.12485224628564139 + }, + { + "coverage": 0.5623333333333334, + "risk": 0.2821576763485477, + "threshold": 0.12470561678566039 + }, + { + "coverage": 0.5626666666666666, + "risk": 0.28199052132701424, + "threshold": 0.12446744083506364 + }, + { + "coverage": 0.563, + "risk": 0.2818235642391948, + "threshold": 0.12425458954986368 + }, + { + "coverage": 0.5633333333333334, + "risk": 0.2816568047337278, + "threshold": 0.12402811630536095 + }, + { + "coverage": 0.5636666666666666, + "risk": 0.2814902424600828, + "threshold": 0.1239182503894698 + }, + { + "coverage": 0.564, + "risk": 0.28132387706855794, + "threshold": 0.12382983380978539 + }, + { + "coverage": 0.5643333333333334, + "risk": 0.28115770821027763, + "threshold": 0.1228082600086111 + }, + { + "coverage": 0.5646666666666667, + "risk": 0.28158205430932703, + "threshold": 0.12275775426566152 + }, + { + "coverage": 0.565, + "risk": 0.2820058997050148, + "threshold": 0.12261958974996823 + }, + { + "coverage": 0.5653333333333334, + "risk": 0.2818396226415094, + "threshold": 0.12255805475724915 + }, + { + "coverage": 0.5656666666666667, + "risk": 0.281673541543901, + "threshold": 0.1225573979806769 + }, + { + "coverage": 0.566, + "risk": 0.28150765606595995, + "threshold": 0.12231815378307155 + }, + { + "coverage": 0.5663333333333334, + "risk": 0.28134196586227195, + "threshold": 0.12195093476333164 + }, + { + "coverage": 0.5666666666666667, + "risk": 0.2811764705882353, + "threshold": 0.12187610248636817 + }, + { + "coverage": 0.567, + "risk": 0.2810111699000588, + "threshold": 0.12109675435463617 + }, + { + "coverage": 0.5673333333333334, + "risk": 0.2808460634547591, + "threshold": 0.12078764294076816 + }, + { + "coverage": 0.5676666666666667, + "risk": 0.28068115091015855, + "threshold": 0.12065284893461835 + }, + { + "coverage": 0.568, + "risk": 0.2805164319248826, + "threshold": 0.12042832319367312 + }, + { + "coverage": 0.5683333333333334, + "risk": 0.28035190615835776, + "threshold": 0.12032395531003104 + }, + { + "coverage": 0.5686666666666667, + "risk": 0.2801875732708089, + "threshold": 0.12031631480949213 + }, + { + "coverage": 0.569, + "risk": 0.28002343292325715, + "threshold": 0.11984342675182802 + }, + { + "coverage": 0.5693333333333334, + "risk": 0.27985948477751754, + "threshold": 0.11982109111429651 + }, + { + "coverage": 0.5696666666666667, + "risk": 0.2796957284961966, + "threshold": 0.11979408209220566 + }, + { + "coverage": 0.57, + "risk": 0.2795321637426901, + "threshold": 0.11977512547784386 + }, + { + "coverage": 0.5703333333333334, + "risk": 0.2793687901811806, + "threshold": 0.11962711936465721 + }, + { + "coverage": 0.5706666666666667, + "risk": 0.27920560747663553, + "threshold": 0.11950004707091688 + }, + { + "coverage": 0.571, + "risk": 0.27904261529480445, + "threshold": 0.11947362354931546 + }, + { + "coverage": 0.5713333333333334, + "risk": 0.27887981330221706, + "threshold": 0.11943996182212759 + }, + { + "coverage": 0.5716666666666667, + "risk": 0.27871720116618076, + "threshold": 0.11927186594900263 + }, + { + "coverage": 0.572, + "risk": 0.27855477855477856, + "threshold": 0.11892179514629221 + }, + { + "coverage": 0.5723333333333334, + "risk": 0.2783925451368666, + "threshold": 0.1189178350434617 + }, + { + "coverage": 0.5726666666666667, + "risk": 0.27823050058207216, + "threshold": 0.11863091060292422 + }, + { + "coverage": 0.573, + "risk": 0.27806864456079117, + "threshold": 0.1183055026879583 + }, + { + "coverage": 0.5733333333333334, + "risk": 0.2779069767441861, + "threshold": 0.11809478430133068 + }, + { + "coverage": 0.5736666666666667, + "risk": 0.2777454968041836, + "threshold": 0.11767883363282469 + }, + { + "coverage": 0.574, + "risk": 0.2775842044134727, + "threshold": 0.11761622616461763 + }, + { + "coverage": 0.5743333333333334, + "risk": 0.277423099245502, + "threshold": 0.11712155220055731 + }, + { + "coverage": 0.5746666666666667, + "risk": 0.27726218097447797, + "threshold": 0.11694618922897602 + }, + { + "coverage": 0.575, + "risk": 0.27710144927536234, + "threshold": 0.11678632986881608 + }, + { + "coverage": 0.5753333333333334, + "risk": 0.2769409038238702, + "threshold": 0.11676130008268615 + }, + { + "coverage": 0.5756666666666667, + "risk": 0.27678054429646787, + "threshold": 0.116513679078432 + }, + { + "coverage": 0.576, + "risk": 0.27662037037037035, + "threshold": 0.1161681638014951 + }, + { + "coverage": 0.5763333333333334, + "risk": 0.2764603817235396, + "threshold": 0.11586256439668573 + }, + { + "coverage": 0.5766666666666667, + "risk": 0.27630057803468205, + "threshold": 0.11530389075858326 + }, + { + "coverage": 0.577, + "risk": 0.2761409589832467, + "threshold": 0.11520641046947058 + }, + { + "coverage": 0.5773333333333334, + "risk": 0.2759815242494226, + "threshold": 0.11492288985924137 + }, + { + "coverage": 0.5776666666666667, + "risk": 0.27582227351413735, + "threshold": 0.11475186986412654 + }, + { + "coverage": 0.578, + "risk": 0.27566320645905423, + "threshold": 0.11468772522066573 + }, + { + "coverage": 0.5783333333333334, + "risk": 0.2755043227665706, + "threshold": 0.11456783433734594 + }, + { + "coverage": 0.5786666666666667, + "risk": 0.27534562211981567, + "threshold": 0.1141541870400591 + }, + { + "coverage": 0.579, + "risk": 0.27518710420264825, + "threshold": 0.11411666553682666 + }, + { + "coverage": 0.5793333333333334, + "risk": 0.27502876869965476, + "threshold": 0.11397695509211307 + }, + { + "coverage": 0.5796666666666667, + "risk": 0.2748706152961472, + "threshold": 0.11377981055032382 + }, + { + "coverage": 0.58, + "risk": 0.2747126436781609, + "threshold": 0.11362622040394396 + }, + { + "coverage": 0.5803333333333334, + "risk": 0.2745548535324526, + "threshold": 0.11322029503098463 + }, + { + "coverage": 0.5806666666666667, + "risk": 0.2743972445464983, + "threshold": 0.113155719336806 + }, + { + "coverage": 0.581, + "risk": 0.2742398164084911, + "threshold": 0.11311181483499409 + }, + { + "coverage": 0.5813333333333334, + "risk": 0.27408256880733944, + "threshold": 0.11295369532144141 + }, + { + "coverage": 0.5816666666666667, + "risk": 0.27392550143266475, + "threshold": 0.1127510908123606 + }, + { + "coverage": 0.582, + "risk": 0.27376861397479957, + "threshold": 0.11257377423407933 + }, + { + "coverage": 0.5823333333333334, + "risk": 0.27361190612478536, + "threshold": 0.11232600877832866 + }, + { + "coverage": 0.5826666666666667, + "risk": 0.2734553775743707, + "threshold": 0.11190763665733795 + }, + { + "coverage": 0.583, + "risk": 0.27329902801600914, + "threshold": 0.11184947996539131 + }, + { + "coverage": 0.5833333333333334, + "risk": 0.27314285714285713, + "threshold": 0.11144467964610529 + }, + { + "coverage": 0.5836666666666667, + "risk": 0.27298686464877214, + "threshold": 0.11090005358115461 + }, + { + "coverage": 0.584, + "risk": 0.2728310502283105, + "threshold": 0.11088525246360702 + }, + { + "coverage": 0.5843333333333334, + "risk": 0.2726754135767256, + "threshold": 0.11059266413785983 + }, + { + "coverage": 0.5846666666666667, + "risk": 0.2725199543899658, + "threshold": 0.11041641617462394 + }, + { + "coverage": 0.585, + "risk": 0.2723646723646724, + "threshold": 0.11039104702310798 + }, + { + "coverage": 0.5853333333333334, + "risk": 0.2722095671981777, + "threshold": 0.11015894039657637 + }, + { + "coverage": 0.5856666666666667, + "risk": 0.27205463858850315, + "threshold": 0.11009308830172418 + }, + { + "coverage": 0.586, + "risk": 0.27189988623435724, + "threshold": 0.10985147085293434 + }, + { + "coverage": 0.5863333333333334, + "risk": 0.2717453098351336, + "threshold": 0.10957450948456682 + }, + { + "coverage": 0.5866666666666667, + "risk": 0.2715909090909091, + "threshold": 0.10957423208331818 + }, + { + "coverage": 0.587, + "risk": 0.2714366837024418, + "threshold": 0.10943458398047624 + }, + { + "coverage": 0.5873333333333334, + "risk": 0.27128263337116915, + "threshold": 0.10930729549266438 + }, + { + "coverage": 0.5876666666666667, + "risk": 0.2711287577992059, + "threshold": 0.10924453893444142 + }, + { + "coverage": 0.588, + "risk": 0.2709750566893424, + "threshold": 0.10900675606000122 + }, + { + "coverage": 0.5883333333333334, + "risk": 0.2708215297450425, + "threshold": 0.10889049987155944 + }, + { + "coverage": 0.5886666666666667, + "risk": 0.2706681766704417, + "threshold": 0.10882169052452184 + }, + { + "coverage": 0.589, + "risk": 0.2705149971703452, + "threshold": 0.10868876699152709 + }, + { + "coverage": 0.5893333333333334, + "risk": 0.27092760180995473, + "threshold": 0.10843681837727015 + }, + { + "coverage": 0.5896666666666667, + "risk": 0.2707744488411532, + "threshold": 0.10829931706340544 + }, + { + "coverage": 0.59, + "risk": 0.2706214689265537, + "threshold": 0.10814270266020998 + }, + { + "coverage": 0.5903333333333334, + "risk": 0.2704686617730096, + "threshold": 0.1079945517264953 + }, + { + "coverage": 0.5906666666666667, + "risk": 0.2703160270880361, + "threshold": 0.10795880913010514 + }, + { + "coverage": 0.591, + "risk": 0.2701635645798082, + "threshold": 0.10772493569676114 + }, + { + "coverage": 0.5913333333333334, + "risk": 0.27001127395715896, + "threshold": 0.10765050478272543 + }, + { + "coverage": 0.5916666666666667, + "risk": 0.26985915492957746, + "threshold": 0.10761498348004218 + }, + { + "coverage": 0.592, + "risk": 0.2697072072072072, + "threshold": 0.1075294025200435 + }, + { + "coverage": 0.5923333333333334, + "risk": 0.26955543050084413, + "threshold": 0.10749184580662607 + }, + { + "coverage": 0.5926666666666667, + "risk": 0.26940382452193473, + "threshold": 0.10655431808468839 + }, + { + "coverage": 0.593, + "risk": 0.2692523889825745, + "threshold": 0.10641120782615497 + }, + { + "coverage": 0.5933333333333334, + "risk": 0.26910112359550564, + "threshold": 0.10619331151181298 + }, + { + "coverage": 0.5936666666666667, + "risk": 0.26895002807411567, + "threshold": 0.10576413729450172 + }, + { + "coverage": 0.594, + "risk": 0.26879910213243546, + "threshold": 0.10574094585112803 + }, + { + "coverage": 0.5943333333333334, + "risk": 0.2686483454851374, + "threshold": 0.10562606361269242 + }, + { + "coverage": 0.5946666666666667, + "risk": 0.26849775784753366, + "threshold": 0.10535396281763586 + }, + { + "coverage": 0.595, + "risk": 0.2683473389355742, + "threshold": 0.10531610359731314 + }, + { + "coverage": 0.5953333333333334, + "risk": 0.26819708846584545, + "threshold": 0.10511101259061695 + }, + { + "coverage": 0.5956666666666667, + "risk": 0.268047006155568, + "threshold": 0.10510102046199064 + }, + { + "coverage": 0.596, + "risk": 0.26789709172259507, + "threshold": 0.10496075529626395 + }, + { + "coverage": 0.5963333333333334, + "risk": 0.26774734488541085, + "threshold": 0.10451164853801485 + }, + { + "coverage": 0.5966666666666667, + "risk": 0.2675977653631285, + "threshold": 0.10450794868923197 + }, + { + "coverage": 0.597, + "risk": 0.26744835287548857, + "threshold": 0.10424476448323008 + }, + { + "coverage": 0.5973333333333334, + "risk": 0.26729910714285715, + "threshold": 0.10390256667906739 + }, + { + "coverage": 0.5976666666666667, + "risk": 0.2671500278862242, + "threshold": 0.10310919634916615 + }, + { + "coverage": 0.598, + "risk": 0.2670011148272018, + "threshold": 0.10292529567801355 + }, + { + "coverage": 0.5983333333333334, + "risk": 0.2668523676880223, + "threshold": 0.10286756243764653 + }, + { + "coverage": 0.5986666666666667, + "risk": 0.26670378619153673, + "threshold": 0.10256545378253358 + }, + { + "coverage": 0.599, + "risk": 0.26655537006121316, + "threshold": 0.10241124537432268 + }, + { + "coverage": 0.5993333333333334, + "risk": 0.2664071190211346, + "threshold": 0.10231747259672373 + }, + { + "coverage": 0.5996666666666667, + "risk": 0.2662590327959978, + "threshold": 0.1021774875407573 + }, + { + "coverage": 0.6, + "risk": 0.26611111111111113, + "threshold": 0.10180025117323102 + }, + { + "coverage": 0.6003333333333334, + "risk": 0.2659633536923931, + "threshold": 0.10170235609484737 + }, + { + "coverage": 0.6006666666666667, + "risk": 0.2658157602663707, + "threshold": 0.10156085468967556 + }, + { + "coverage": 0.601, + "risk": 0.26566833056017747, + "threshold": 0.10156069626527219 + }, + { + "coverage": 0.6013333333333334, + "risk": 0.2655210643015521, + "threshold": 0.10150487429996247 + }, + { + "coverage": 0.6016666666666667, + "risk": 0.26537396121883655, + "threshold": 0.10136708204522284 + }, + { + "coverage": 0.602, + "risk": 0.2652270210409745, + "threshold": 0.10129945986586009 + }, + { + "coverage": 0.6023333333333334, + "risk": 0.2650802434975097, + "threshold": 0.10070112409671594 + }, + { + "coverage": 0.6026666666666667, + "risk": 0.26548672566371684, + "threshold": 0.10046925081859116 + }, + { + "coverage": 0.603, + "risk": 0.26533996683250416, + "threshold": 0.10022257812114965 + }, + { + "coverage": 0.6033333333333334, + "risk": 0.26519337016574585, + "threshold": 0.09997942719214238 + }, + { + "coverage": 0.6036666666666667, + "risk": 0.2650469353948095, + "threshold": 0.09995928701585656 + }, + { + "coverage": 0.604, + "risk": 0.26490066225165565, + "threshold": 0.09977677627087189 + }, + { + "coverage": 0.6043333333333333, + "risk": 0.26475455046883617, + "threshold": 0.09967036083111933 + }, + { + "coverage": 0.6046666666666667, + "risk": 0.26460859977949286, + "threshold": 0.09950968940553828 + }, + { + "coverage": 0.605, + "risk": 0.2650137741046832, + "threshold": 0.09944964957147252 + }, + { + "coverage": 0.6053333333333333, + "risk": 0.2648678414096916, + "threshold": 0.09934303040824942 + }, + { + "coverage": 0.6056666666666667, + "risk": 0.2647220693450743, + "threshold": 0.09927530307214916 + }, + { + "coverage": 0.606, + "risk": 0.2645764576457646, + "threshold": 0.099167731094127 + }, + { + "coverage": 0.6063333333333333, + "risk": 0.2644310060472787, + "threshold": 0.09857265191567065 + }, + { + "coverage": 0.6066666666666667, + "risk": 0.2642857142857143, + "threshold": 0.09840558765078634 + }, + { + "coverage": 0.607, + "risk": 0.2641405820977485, + "threshold": 0.0982458078840791 + }, + { + "coverage": 0.6073333333333333, + "risk": 0.26399560922063664, + "threshold": 0.09809677673453267 + }, + { + "coverage": 0.6076666666666667, + "risk": 0.2638507953922106, + "threshold": 0.0977290635886612 + }, + { + "coverage": 0.608, + "risk": 0.2642543859649123, + "threshold": 0.09697951406344087 + }, + { + "coverage": 0.6083333333333333, + "risk": 0.2641095890410959, + "threshold": 0.0968074634382061 + }, + { + "coverage": 0.6086666666666667, + "risk": 0.26396495071193865, + "threshold": 0.09673259095409394 + }, + { + "coverage": 0.609, + "risk": 0.2638204707170224, + "threshold": 0.09668279371283008 + }, + { + "coverage": 0.6093333333333333, + "risk": 0.2636761487964989, + "threshold": 0.09633651012141974 + }, + { + "coverage": 0.6096666666666667, + "risk": 0.263531984691088, + "threshold": 0.09631749767922672 + }, + { + "coverage": 0.61, + "risk": 0.2633879781420765, + "threshold": 0.09623705449928367 + }, + { + "coverage": 0.6103333333333333, + "risk": 0.2632441288913162, + "threshold": 0.09609505999039093 + }, + { + "coverage": 0.6106666666666667, + "risk": 0.2631004366812227, + "threshold": 0.09604672510250989 + }, + { + "coverage": 0.611, + "risk": 0.2629569012547736, + "threshold": 0.09567442727212949 + }, + { + "coverage": 0.6113333333333333, + "risk": 0.2628135223555071, + "threshold": 0.09548603005127011 + }, + { + "coverage": 0.6116666666666667, + "risk": 0.26267029972752043, + "threshold": 0.09539903196471756 + }, + { + "coverage": 0.612, + "risk": 0.2625272331154684, + "threshold": 0.0952535748270229 + }, + { + "coverage": 0.6123333333333333, + "risk": 0.26238432226456176, + "threshold": 0.09511696742460171 + }, + { + "coverage": 0.6126666666666667, + "risk": 0.2622415669205658, + "threshold": 0.09492491288029292 + }, + { + "coverage": 0.613, + "risk": 0.26209896682979883, + "threshold": 0.09483146985347135 + }, + { + "coverage": 0.6133333333333333, + "risk": 0.2619565217391304, + "threshold": 0.09481234293243766 + }, + { + "coverage": 0.6136666666666667, + "risk": 0.2618142313959804, + "threshold": 0.09413471183123744 + }, + { + "coverage": 0.614, + "risk": 0.26167209554831705, + "threshold": 0.09381507509919196 + }, + { + "coverage": 0.6143333333333333, + "risk": 0.2615301139446555, + "threshold": 0.09376763656688693 + }, + { + "coverage": 0.6146666666666667, + "risk": 0.2613882863340564, + "threshold": 0.09352217782578622 + }, + { + "coverage": 0.615, + "risk": 0.2612466124661247, + "threshold": 0.0930869379178101 + }, + { + "coverage": 0.6153333333333333, + "risk": 0.2611050920910076, + "threshold": 0.09301468669624491 + }, + { + "coverage": 0.6156666666666667, + "risk": 0.26096372495939363, + "threshold": 0.092334978492004 + }, + { + "coverage": 0.616, + "risk": 0.26082251082251084, + "threshold": 0.0918690002778213 + }, + { + "coverage": 0.6163333333333333, + "risk": 0.26068144943212546, + "threshold": 0.09163243449048064 + }, + { + "coverage": 0.6166666666666667, + "risk": 0.26054054054054054, + "threshold": 0.09152135560697472 + }, + { + "coverage": 0.617, + "risk": 0.26039978390059426, + "threshold": 0.09141457361449934 + }, + { + "coverage": 0.6173333333333333, + "risk": 0.2602591792656587, + "threshold": 0.09137168756126357 + }, + { + "coverage": 0.6176666666666667, + "risk": 0.2601187263896384, + "threshold": 0.09092304201988634 + }, + { + "coverage": 0.618, + "risk": 0.2599784250269687, + "threshold": 0.09074649752428511 + }, + { + "coverage": 0.6183333333333333, + "risk": 0.2598382749326146, + "threshold": 0.09072722358463992 + }, + { + "coverage": 0.6186666666666667, + "risk": 0.25969827586206895, + "threshold": 0.09054489823467902 + }, + { + "coverage": 0.619, + "risk": 0.25955842757135167, + "threshold": 0.09019149110836411 + }, + { + "coverage": 0.6193333333333333, + "risk": 0.25941872981700753, + "threshold": 0.089893357280495 + }, + { + "coverage": 0.6196666666666667, + "risk": 0.2592791823561054, + "threshold": 0.08986748029025547 + }, + { + "coverage": 0.62, + "risk": 0.2591397849462366, + "threshold": 0.08977784842804226 + }, + { + "coverage": 0.6203333333333333, + "risk": 0.25900053734551315, + "threshold": 0.08973592053223012 + }, + { + "coverage": 0.6206666666666667, + "risk": 0.2588614393125671, + "threshold": 0.08972758376700617 + }, + { + "coverage": 0.621, + "risk": 0.25872249060654856, + "threshold": 0.08952236976822237 + }, + { + "coverage": 0.6213333333333333, + "risk": 0.25858369098712447, + "threshold": 0.08937593005871049 + }, + { + "coverage": 0.6216666666666667, + "risk": 0.25844504021447723, + "threshold": 0.08810483240116086 + }, + { + "coverage": 0.622, + "risk": 0.2583065380493033, + "threshold": 0.08772491180275656 + }, + { + "coverage": 0.6223333333333333, + "risk": 0.258168184252812, + "threshold": 0.08772057802858325 + }, + { + "coverage": 0.6226666666666667, + "risk": 0.2580299785867238, + "threshold": 0.0873301136504905 + }, + { + "coverage": 0.623, + "risk": 0.2578919208132691, + "threshold": 0.08705446691476125 + }, + { + "coverage": 0.6233333333333333, + "risk": 0.25775401069518716, + "threshold": 0.08702373318625999 + }, + { + "coverage": 0.6236666666666667, + "risk": 0.2576162479957242, + "threshold": 0.08694597965952512 + }, + { + "coverage": 0.624, + "risk": 0.25747863247863245, + "threshold": 0.08646171216935185 + }, + { + "coverage": 0.6243333333333333, + "risk": 0.2573411639081687, + "threshold": 0.08609102915916457 + }, + { + "coverage": 0.6246666666666667, + "risk": 0.25720384204909286, + "threshold": 0.08570322219276072 + }, + { + "coverage": 0.625, + "risk": 0.25706666666666667, + "threshold": 0.08566690047282677 + }, + { + "coverage": 0.6253333333333333, + "risk": 0.25692963752665243, + "threshold": 0.08525988368973536 + }, + { + "coverage": 0.6256666666666667, + "risk": 0.2567927543953117, + "threshold": 0.08524226345829788 + }, + { + "coverage": 0.626, + "risk": 0.2566560170394036, + "threshold": 0.0849021981736371 + }, + { + "coverage": 0.6263333333333333, + "risk": 0.25651942522618415, + "threshold": 0.08472992331630015 + }, + { + "coverage": 0.6266666666666667, + "risk": 0.25638297872340426, + "threshold": 0.08472722310956372 + }, + { + "coverage": 0.627, + "risk": 0.2562466772993089, + "threshold": 0.08448387707933724 + }, + { + "coverage": 0.6273333333333333, + "risk": 0.2561105207226355, + "threshold": 0.084222930211868 + }, + { + "coverage": 0.6276666666666667, + "risk": 0.25597450876261285, + "threshold": 0.08382037747284654 + }, + { + "coverage": 0.628, + "risk": 0.25583864118895966, + "threshold": 0.08362783852185185 + }, + { + "coverage": 0.6283333333333333, + "risk": 0.2557029177718833, + "threshold": 0.08330113486998149 + }, + { + "coverage": 0.6286666666666667, + "risk": 0.25556733828207845, + "threshold": 0.08321597098424773 + }, + { + "coverage": 0.629, + "risk": 0.255431902490726, + "threshold": 0.08303619716900144 + }, + { + "coverage": 0.6293333333333333, + "risk": 0.2552966101694915, + "threshold": 0.08249546180355694 + }, + { + "coverage": 0.6296666666666667, + "risk": 0.25516146109052407, + "threshold": 0.08249273382210374 + }, + { + "coverage": 0.63, + "risk": 0.25502645502645505, + "threshold": 0.08223508119195701 + }, + { + "coverage": 0.6303333333333333, + "risk": 0.2548915917503966, + "threshold": 0.08196932246734522 + }, + { + "coverage": 0.6306666666666667, + "risk": 0.2547568710359408, + "threshold": 0.08171829064906433 + }, + { + "coverage": 0.631, + "risk": 0.25462229265715797, + "threshold": 0.08148279996877504 + }, + { + "coverage": 0.6313333333333333, + "risk": 0.25448785638859556, + "threshold": 0.08147133542843799 + }, + { + "coverage": 0.6316666666666667, + "risk": 0.25435356200527703, + "threshold": 0.08129031658079588 + }, + { + "coverage": 0.632, + "risk": 0.2542194092827004, + "threshold": 0.08109313584858578 + }, + { + "coverage": 0.6323333333333333, + "risk": 0.2540853979968371, + "threshold": 0.08104152720226308 + }, + { + "coverage": 0.6326666666666667, + "risk": 0.25395152792413067, + "threshold": 0.08103069225762737 + }, + { + "coverage": 0.633, + "risk": 0.2538177988414955, + "threshold": 0.080845057633665 + }, + { + "coverage": 0.6333333333333333, + "risk": 0.2536842105263158, + "threshold": 0.08081661799034424 + }, + { + "coverage": 0.6336666666666667, + "risk": 0.253550762756444, + "threshold": 0.0805509277302452 + }, + { + "coverage": 0.634, + "risk": 0.25341745531019977, + "threshold": 0.08049461519009336 + }, + { + "coverage": 0.6343333333333333, + "risk": 0.25328428796636887, + "threshold": 0.08039475864023295 + }, + { + "coverage": 0.6346666666666667, + "risk": 0.25315126050420167, + "threshold": 0.08017951747700634 + }, + { + "coverage": 0.635, + "risk": 0.2530183727034121, + "threshold": 0.080103913887001 + }, + { + "coverage": 0.6353333333333333, + "risk": 0.2528856243441763, + "threshold": 0.08008804490395958 + }, + { + "coverage": 0.6356666666666667, + "risk": 0.2527530152071316, + "threshold": 0.08002412608765574 + }, + { + "coverage": 0.636, + "risk": 0.2526205450733753, + "threshold": 0.0795882981821876 + }, + { + "coverage": 0.6363333333333333, + "risk": 0.25248821372446306, + "threshold": 0.07946181224553238 + }, + { + "coverage": 0.6366666666666667, + "risk": 0.2523560209424084, + "threshold": 0.07915999188930199 + }, + { + "coverage": 0.637, + "risk": 0.2522239665096808, + "threshold": 0.07905662516006283 + }, + { + "coverage": 0.6373333333333333, + "risk": 0.252092050209205, + "threshold": 0.0789719659667014 + }, + { + "coverage": 0.6376666666666667, + "risk": 0.25196027182435965, + "threshold": 0.07883390667071444 + }, + { + "coverage": 0.638, + "risk": 0.25182863113897597, + "threshold": 0.07862643491340914 + }, + { + "coverage": 0.6383333333333333, + "risk": 0.25169712793733684, + "threshold": 0.07816904679560864 + }, + { + "coverage": 0.6386666666666667, + "risk": 0.25156576200417535, + "threshold": 0.07804714489592125 + }, + { + "coverage": 0.639, + "risk": 0.25143453312467395, + "threshold": 0.07673730021389247 + }, + { + "coverage": 0.6393333333333333, + "risk": 0.251303441084463, + "threshold": 0.07663978282312378 + }, + { + "coverage": 0.6396666666666667, + "risk": 0.2511724856696196, + "threshold": 0.07662803936945295 + }, + { + "coverage": 0.64, + "risk": 0.25104166666666666, + "threshold": 0.0763979958601806 + }, + { + "coverage": 0.6403333333333333, + "risk": 0.2509109838625716, + "threshold": 0.07630537372497415 + }, + { + "coverage": 0.6406666666666667, + "risk": 0.25078043704474506, + "threshold": 0.07589250003788983 + }, + { + "coverage": 0.641, + "risk": 0.25065002600104, + "threshold": 0.07586694034794574 + }, + { + "coverage": 0.6413333333333333, + "risk": 0.2505197505197505, + "threshold": 0.07562862088365208 + }, + { + "coverage": 0.6416666666666667, + "risk": 0.2503896103896104, + "threshold": 0.07556512324211619 + }, + { + "coverage": 0.642, + "risk": 0.2502596053997923, + "threshold": 0.075499461697329 + }, + { + "coverage": 0.6423333333333333, + "risk": 0.25012973533990657, + "threshold": 0.07475771805281407 + }, + { + "coverage": 0.6426666666666667, + "risk": 0.25, + "threshold": 0.07462168623717513 + }, + { + "coverage": 0.643, + "risk": 0.2498703991705547, + "threshold": 0.07435212539022552 + }, + { + "coverage": 0.6433333333333333, + "risk": 0.24974093264248703, + "threshold": 0.07406101615031833 + }, + { + "coverage": 0.6436666666666667, + "risk": 0.24961160020714657, + "threshold": 0.07404842788984213 + }, + { + "coverage": 0.644, + "risk": 0.2494824016563147, + "threshold": 0.07403618113254938 + }, + { + "coverage": 0.6443333333333333, + "risk": 0.24935333678220384, + "threshold": 0.07373556276837957 + }, + { + "coverage": 0.6446666666666667, + "risk": 0.24922440537745605, + "threshold": 0.07358891504438113 + }, + { + "coverage": 0.645, + "risk": 0.24909560723514212, + "threshold": 0.07335090585917213 + }, + { + "coverage": 0.6453333333333333, + "risk": 0.24896694214876033, + "threshold": 0.07321054435597539 + }, + { + "coverage": 0.6456666666666667, + "risk": 0.2488384099122354, + "threshold": 0.07312816338756933 + }, + { + "coverage": 0.646, + "risk": 0.24871001031991744, + "threshold": 0.07307570533516385 + }, + { + "coverage": 0.6463333333333333, + "risk": 0.24858174316658072, + "threshold": 0.07258376788491462 + }, + { + "coverage": 0.6466666666666666, + "risk": 0.24845360824742269, + "threshold": 0.0723651046073215 + }, + { + "coverage": 0.647, + "risk": 0.24832560535806286, + "threshold": 0.07213734655687888 + }, + { + "coverage": 0.6473333333333333, + "risk": 0.2481977342945417, + "threshold": 0.07213659410652708 + }, + { + "coverage": 0.6476666666666666, + "risk": 0.2480699948533196, + "threshold": 0.07178832980329596 + }, + { + "coverage": 0.648, + "risk": 0.24845679012345678, + "threshold": 0.071700199836755 + }, + { + "coverage": 0.6483333333333333, + "risk": 0.24832904884318766, + "threshold": 0.07133795368303854 + }, + { + "coverage": 0.6486666666666666, + "risk": 0.24820143884892087, + "threshold": 0.07133162126391479 + }, + { + "coverage": 0.649, + "risk": 0.24807395993836673, + "threshold": 0.07115785688471908 + }, + { + "coverage": 0.6493333333333333, + "risk": 0.24794661190965092, + "threshold": 0.07110196529661163 + }, + { + "coverage": 0.6496666666666666, + "risk": 0.2478193945613135, + "threshold": 0.07093583156399495 + }, + { + "coverage": 0.65, + "risk": 0.24769230769230768, + "threshold": 0.07052538117456565 + }, + { + "coverage": 0.6503333333333333, + "risk": 0.24807790876473604, + "threshold": 0.0705235345535762 + }, + { + "coverage": 0.6506666666666666, + "risk": 0.24795081967213115, + "threshold": 0.07048140494145817 + }, + { + "coverage": 0.651, + "risk": 0.24782386072708654, + "threshold": 0.07035394360697377 + }, + { + "coverage": 0.6513333333333333, + "risk": 0.24769703172978505, + "threshold": 0.0703180681136182 + }, + { + "coverage": 0.6516666666666666, + "risk": 0.24757033248081842, + "threshold": 0.07010308945726429 + }, + { + "coverage": 0.652, + "risk": 0.2474437627811861, + "threshold": 0.06989296168826799 + }, + { + "coverage": 0.6523333333333333, + "risk": 0.24731732243229432, + "threshold": 0.06970520765612598 + }, + { + "coverage": 0.6526666666666666, + "risk": 0.24719101123595505, + "threshold": 0.06940297832247344 + }, + { + "coverage": 0.653, + "risk": 0.2470648289943849, + "threshold": 0.0690556466385579 + }, + { + "coverage": 0.6533333333333333, + "risk": 0.24693877551020407, + "threshold": 0.06869723619520939 + }, + { + "coverage": 0.6536666666666666, + "risk": 0.24681285058643548, + "threshold": 0.06866539322672516 + }, + { + "coverage": 0.654, + "risk": 0.24668705402650357, + "threshold": 0.0682854178945913 + }, + { + "coverage": 0.6543333333333333, + "risk": 0.24656138563423333, + "threshold": 0.0682279096458238 + }, + { + "coverage": 0.6546666666666666, + "risk": 0.24643584521384929, + "threshold": 0.06800711944515436 + }, + { + "coverage": 0.655, + "risk": 0.24631043256997456, + "threshold": 0.06796792761049096 + }, + { + "coverage": 0.6553333333333333, + "risk": 0.2461851475076297, + "threshold": 0.06771378356041424 + }, + { + "coverage": 0.6556666666666666, + "risk": 0.24605998983223182, + "threshold": 0.06755596772745422 + }, + { + "coverage": 0.656, + "risk": 0.2459349593495935, + "threshold": 0.06749596270200088 + }, + { + "coverage": 0.6563333333333333, + "risk": 0.24581005586592178, + "threshold": 0.06744983606522982 + }, + { + "coverage": 0.6566666666666666, + "risk": 0.24568527918781727, + "threshold": 0.06722969039440585 + }, + { + "coverage": 0.657, + "risk": 0.24556062912227294, + "threshold": 0.06721618783610593 + }, + { + "coverage": 0.6573333333333333, + "risk": 0.24543610547667344, + "threshold": 0.0671768972260658 + }, + { + "coverage": 0.6576666666666666, + "risk": 0.2453117080587937, + "threshold": 0.0671474438488562 + }, + { + "coverage": 0.658, + "risk": 0.24518743667679838, + "threshold": 0.06701129137846885 + }, + { + "coverage": 0.6583333333333333, + "risk": 0.2450632911392405, + "threshold": 0.06697792469886554 + }, + { + "coverage": 0.6586666666666666, + "risk": 0.24493927125506074, + "threshold": 0.06696404414530122 + }, + { + "coverage": 0.659, + "risk": 0.24481537683358623, + "threshold": 0.06672053858378034 + }, + { + "coverage": 0.6593333333333333, + "risk": 0.24469160768452983, + "threshold": 0.06650032054514865 + }, + { + "coverage": 0.6596666666666666, + "risk": 0.24456796361798888, + "threshold": 0.06629931462871587 + }, + { + "coverage": 0.66, + "risk": 0.24494949494949494, + "threshold": 0.06618147633840543 + }, + { + "coverage": 0.6603333333333333, + "risk": 0.24482584553255932, + "threshold": 0.06606266860534671 + }, + { + "coverage": 0.6606666666666666, + "risk": 0.24470232088799193, + "threshold": 0.065873921996193 + }, + { + "coverage": 0.661, + "risk": 0.24457892082702976, + "threshold": 0.06586556379846797 + }, + { + "coverage": 0.6613333333333333, + "risk": 0.2444556451612903, + "threshold": 0.06585696001406172 + }, + { + "coverage": 0.6616666666666666, + "risk": 0.24433249370277077, + "threshold": 0.0656707462836774 + }, + { + "coverage": 0.662, + "risk": 0.24420946626384693, + "threshold": 0.06559685456163278 + }, + { + "coverage": 0.6623333333333333, + "risk": 0.24408656265727227, + "threshold": 0.06549142165817522 + }, + { + "coverage": 0.6626666666666666, + "risk": 0.24396378269617705, + "threshold": 0.06545318901834474 + }, + { + "coverage": 0.663, + "risk": 0.24384112619406736, + "threshold": 0.06543647257189678 + }, + { + "coverage": 0.6633333333333333, + "risk": 0.24371859296482412, + "threshold": 0.06523862065673747 + }, + { + "coverage": 0.6636666666666666, + "risk": 0.24359618282270215, + "threshold": 0.06473146787970308 + }, + { + "coverage": 0.664, + "risk": 0.24347389558232932, + "threshold": 0.06453935671566312 + }, + { + "coverage": 0.6643333333333333, + "risk": 0.24335173105870547, + "threshold": 0.06422531575622159 + }, + { + "coverage": 0.6646666666666666, + "risk": 0.2432296890672016, + "threshold": 0.06413316076468867 + }, + { + "coverage": 0.665, + "risk": 0.24310776942355888, + "threshold": 0.06395926206236435 + }, + { + "coverage": 0.6653333333333333, + "risk": 0.24298597194388777, + "threshold": 0.0636577029442549 + }, + { + "coverage": 0.6656666666666666, + "risk": 0.242864296444667, + "threshold": 0.06354221472717292 + }, + { + "coverage": 0.666, + "risk": 0.24274274274274274, + "threshold": 0.06299502826870727 + }, + { + "coverage": 0.6663333333333333, + "risk": 0.24262131065532766, + "threshold": 0.06299316388370979 + }, + { + "coverage": 0.6666666666666666, + "risk": 0.2425, + "threshold": 0.06294405810049053 + }, + { + "coverage": 0.667, + "risk": 0.24237881059470265, + "threshold": 0.06263031450766478 + }, + { + "coverage": 0.6673333333333333, + "risk": 0.24225774225774227, + "threshold": 0.06262408163034161 + }, + { + "coverage": 0.6676666666666666, + "risk": 0.24213679480778832, + "threshold": 0.06234659045802893 + }, + { + "coverage": 0.668, + "risk": 0.24201596806387227, + "threshold": 0.062205110659244944 + }, + { + "coverage": 0.6683333333333333, + "risk": 0.24189526184538654, + "threshold": 0.06216831865497509 + }, + { + "coverage": 0.6686666666666666, + "risk": 0.24177467597208374, + "threshold": 0.061976353667255055 + }, + { + "coverage": 0.669, + "risk": 0.24165421026407574, + "threshold": 0.061651016426291796 + }, + { + "coverage": 0.6693333333333333, + "risk": 0.24153386454183268, + "threshold": 0.06142449691774044 + }, + { + "coverage": 0.6696666666666666, + "risk": 0.24141363862618218, + "threshold": 0.06141944243278986 + }, + { + "coverage": 0.67, + "risk": 0.24129353233830847, + "threshold": 0.061383453615704735 + }, + { + "coverage": 0.6703333333333333, + "risk": 0.24117354549975137, + "threshold": 0.06135059966847666 + }, + { + "coverage": 0.6706666666666666, + "risk": 0.24105367793240556, + "threshold": 0.06123786701958523 + }, + { + "coverage": 0.671, + "risk": 0.24093392945851963, + "threshold": 0.06113355612262917 + }, + { + "coverage": 0.6713333333333333, + "risk": 0.24081429990069514, + "threshold": 0.060754342427673975 + }, + { + "coverage": 0.6716666666666666, + "risk": 0.24069478908188585, + "threshold": 0.06067713099510863 + }, + { + "coverage": 0.672, + "risk": 0.24057539682539683, + "threshold": 0.06041380330746194 + }, + { + "coverage": 0.6723333333333333, + "risk": 0.2404561229548835, + "threshold": 0.06035303166738081 + }, + { + "coverage": 0.6726666666666666, + "risk": 0.24033696729435083, + "threshold": 0.06034885312307161 + }, + { + "coverage": 0.673, + "risk": 0.24021792966815256, + "threshold": 0.06012570462169185 + }, + { + "coverage": 0.6733333333333333, + "risk": 0.2400990099009901, + "threshold": 0.0601208181047781 + }, + { + "coverage": 0.6736666666666666, + "risk": 0.23998020781791193, + "threshold": 0.05994121288681243 + }, + { + "coverage": 0.674, + "risk": 0.23986152324431256, + "threshold": 0.05985270357363959 + }, + { + "coverage": 0.6743333333333333, + "risk": 0.2397429560059318, + "threshold": 0.05953515352604022 + }, + { + "coverage": 0.6746666666666666, + "risk": 0.23962450592885376, + "threshold": 0.05952861752989724 + }, + { + "coverage": 0.675, + "risk": 0.23950617283950618, + "threshold": 0.059527872943430093 + }, + { + "coverage": 0.6753333333333333, + "risk": 0.23938795656465942, + "threshold": 0.059427200366850556 + }, + { + "coverage": 0.6756666666666666, + "risk": 0.23926985693142575, + "threshold": 0.05937712505749194 + }, + { + "coverage": 0.676, + "risk": 0.23915187376725838, + "threshold": 0.05913688386875738 + }, + { + "coverage": 0.6763333333333333, + "risk": 0.23903400689995072, + "threshold": 0.058985548462212385 + }, + { + "coverage": 0.6766666666666666, + "risk": 0.23891625615763548, + "threshold": 0.05885059442882668 + }, + { + "coverage": 0.677, + "risk": 0.23879862136878385, + "threshold": 0.05883684928266674 + }, + { + "coverage": 0.6773333333333333, + "risk": 0.23868110236220472, + "threshold": 0.05883017572873861 + }, + { + "coverage": 0.6776666666666666, + "risk": 0.23856369896704377, + "threshold": 0.05882509168304749 + }, + { + "coverage": 0.678, + "risk": 0.2384464110127827, + "threshold": 0.05877777349655843 + }, + { + "coverage": 0.6783333333333333, + "risk": 0.2388206388206388, + "threshold": 0.058762718024093705 + }, + { + "coverage": 0.6786666666666666, + "risk": 0.2387033398821218, + "threshold": 0.058675243579648914 + }, + { + "coverage": 0.679, + "risk": 0.23858615611192932, + "threshold": 0.058260683172948184 + }, + { + "coverage": 0.6793333333333333, + "risk": 0.23846908734052993, + "threshold": 0.058259700234337686 + }, + { + "coverage": 0.6796666666666666, + "risk": 0.23835213339872485, + "threshold": 0.05816211908384217 + }, + { + "coverage": 0.68, + "risk": 0.23823529411764705, + "threshold": 0.058134053698606365 + }, + { + "coverage": 0.6803333333333333, + "risk": 0.2381185693287604, + "threshold": 0.058071422190055795 + }, + { + "coverage": 0.6806666666666666, + "risk": 0.23800195886385897, + "threshold": 0.057985754757015055 + }, + { + "coverage": 0.681, + "risk": 0.23788546255506607, + "threshold": 0.05768636827365369 + }, + { + "coverage": 0.6813333333333333, + "risk": 0.23776908023483365, + "threshold": 0.05763554809487426 + }, + { + "coverage": 0.6816666666666666, + "risk": 0.23765281173594133, + "threshold": 0.05751616880587119 + }, + { + "coverage": 0.682, + "risk": 0.2375366568914956, + "threshold": 0.057495133358906 + }, + { + "coverage": 0.6823333333333333, + "risk": 0.23742061553492916, + "threshold": 0.057331947684658546 + }, + { + "coverage": 0.6826666666666666, + "risk": 0.2373046875, + "threshold": 0.05715619498759228 + }, + { + "coverage": 0.683, + "risk": 0.23718887262079064, + "threshold": 0.05667041352668711 + }, + { + "coverage": 0.6833333333333333, + "risk": 0.2370731707317073, + "threshold": 0.05649866491112344 + }, + { + "coverage": 0.6836666666666666, + "risk": 0.23695758166747927, + "threshold": 0.05646347802677364 + }, + { + "coverage": 0.684, + "risk": 0.23684210526315788, + "threshold": 0.05592258307573595 + }, + { + "coverage": 0.6843333333333333, + "risk": 0.23672674135411592, + "threshold": 0.05586016822143102 + }, + { + "coverage": 0.6846666666666666, + "risk": 0.23661148977604674, + "threshold": 0.055756556828694404 + }, + { + "coverage": 0.685, + "risk": 0.2364963503649635, + "threshold": 0.05560062937151856 + }, + { + "coverage": 0.6853333333333333, + "risk": 0.23638132295719844, + "threshold": 0.05550617424218762 + }, + { + "coverage": 0.6856666666666666, + "risk": 0.23626640738940205, + "threshold": 0.05534049942135244 + }, + { + "coverage": 0.686, + "risk": 0.23615160349854228, + "threshold": 0.05518144865688689 + }, + { + "coverage": 0.6863333333333334, + "risk": 0.23603691112190384, + "threshold": 0.055142758573988086 + }, + { + "coverage": 0.6866666666666666, + "risk": 0.23592233009708738, + "threshold": 0.05468564860475867 + }, + { + "coverage": 0.687, + "risk": 0.23580786026200873, + "threshold": 0.05453226763103557 + }, + { + "coverage": 0.6873333333333334, + "risk": 0.23569350145489815, + "threshold": 0.05415755597212701 + }, + { + "coverage": 0.6876666666666666, + "risk": 0.23557925351429956, + "threshold": 0.05409685101107929 + }, + { + "coverage": 0.688, + "risk": 0.23546511627906977, + "threshold": 0.053587433338222475 + }, + { + "coverage": 0.6883333333333334, + "risk": 0.23535108958837772, + "threshold": 0.05351087855172487 + }, + { + "coverage": 0.6886666666666666, + "risk": 0.23523717328170377, + "threshold": 0.053452688549745275 + }, + { + "coverage": 0.689, + "risk": 0.2351233671988389, + "threshold": 0.05340693028941245 + }, + { + "coverage": 0.6893333333333334, + "risk": 0.23549323017408125, + "threshold": 0.05320748071832025 + }, + { + "coverage": 0.6896666666666667, + "risk": 0.23537941034316096, + "threshold": 0.053178407606019995 + }, + { + "coverage": 0.69, + "risk": 0.2357487922705314, + "threshold": 0.053125264346902686 + }, + { + "coverage": 0.6903333333333334, + "risk": 0.2356349589570256, + "threshold": 0.05300336763977143 + }, + { + "coverage": 0.6906666666666667, + "risk": 0.23552123552123552, + "threshold": 0.052755623746158616 + }, + { + "coverage": 0.691, + "risk": 0.23589001447178004, + "threshold": 0.052418732988297016 + }, + { + "coverage": 0.6913333333333334, + "risk": 0.23577627772420442, + "threshold": 0.05238862436078695 + }, + { + "coverage": 0.6916666666666667, + "risk": 0.23566265060240965, + "threshold": 0.0523405733640847 + }, + { + "coverage": 0.692, + "risk": 0.23554913294797689, + "threshold": 0.05229958358832977 + }, + { + "coverage": 0.6923333333333334, + "risk": 0.2354357246027925, + "threshold": 0.052158467746863835 + }, + { + "coverage": 0.6926666666666667, + "risk": 0.23532242540904716, + "threshold": 0.05199113235883956 + }, + { + "coverage": 0.693, + "risk": 0.2352092352092352, + "threshold": 0.05164635811192307 + }, + { + "coverage": 0.6933333333333334, + "risk": 0.23509615384615384, + "threshold": 0.051604452897251456 + }, + { + "coverage": 0.6936666666666667, + "risk": 0.23498318116290245, + "threshold": 0.0515528012466282 + }, + { + "coverage": 0.694, + "risk": 0.23487031700288186, + "threshold": 0.05152791606545958 + }, + { + "coverage": 0.6943333333333334, + "risk": 0.23475756120979357, + "threshold": 0.05147250592918783 + }, + { + "coverage": 0.6946666666666667, + "risk": 0.23464491362763915, + "threshold": 0.05139979576468346 + }, + { + "coverage": 0.695, + "risk": 0.23453237410071942, + "threshold": 0.05111029011612445 + }, + { + "coverage": 0.6953333333333334, + "risk": 0.23441994247363374, + "threshold": 0.05083673973441449 + }, + { + "coverage": 0.6956666666666667, + "risk": 0.23430761859127935, + "threshold": 0.050635979883452815 + }, + { + "coverage": 0.696, + "risk": 0.23419540229885058, + "threshold": 0.05045698428066459 + }, + { + "coverage": 0.6963333333333334, + "risk": 0.2340832934418382, + "threshold": 0.05006109619999367 + }, + { + "coverage": 0.6966666666666667, + "risk": 0.23397129186602872, + "threshold": 0.050056069447863574 + }, + { + "coverage": 0.697, + "risk": 0.23385939741750358, + "threshold": 0.04999117432257806 + }, + { + "coverage": 0.6973333333333334, + "risk": 0.23374760994263863, + "threshold": 0.04970430186190456 + }, + { + "coverage": 0.6976666666666667, + "risk": 0.2336359292881032, + "threshold": 0.04967197477549301 + }, + { + "coverage": 0.698, + "risk": 0.2335243553008596, + "threshold": 0.04962787349249176 + }, + { + "coverage": 0.6983333333333334, + "risk": 0.2334128878281623, + "threshold": 0.04958201274464535 + }, + { + "coverage": 0.6986666666666667, + "risk": 0.23330152671755724, + "threshold": 0.04931832234052158 + }, + { + "coverage": 0.699, + "risk": 0.23319027181688126, + "threshold": 0.049125749442485045 + }, + { + "coverage": 0.6993333333333334, + "risk": 0.2330791229742612, + "threshold": 0.04897652269932306 + }, + { + "coverage": 0.6996666666666667, + "risk": 0.2329680800381134, + "threshold": 0.048956602691880846 + }, + { + "coverage": 0.7, + "risk": 0.23285714285714285, + "threshold": 0.04892021370650923 + }, + { + "coverage": 0.7003333333333334, + "risk": 0.2327463112803427, + "threshold": 0.04874015660083586 + }, + { + "coverage": 0.7006666666666667, + "risk": 0.23263558515699334, + "threshold": 0.04854207347411066 + }, + { + "coverage": 0.701, + "risk": 0.23252496433666192, + "threshold": 0.048460793092453 + }, + { + "coverage": 0.7013333333333334, + "risk": 0.23241444866920152, + "threshold": 0.04838253438267781 + }, + { + "coverage": 0.7016666666666667, + "risk": 0.2323040380047506, + "threshold": 0.04830068150007971 + }, + { + "coverage": 0.702, + "risk": 0.23219373219373218, + "threshold": 0.04816840185273107 + }, + { + "coverage": 0.7023333333333334, + "risk": 0.23208353108685334, + "threshold": 0.04814220292293837 + }, + { + "coverage": 0.7026666666666667, + "risk": 0.23197343453510436, + "threshold": 0.048076432433683536 + }, + { + "coverage": 0.703, + "risk": 0.23186344238975817, + "threshold": 0.04804712386832444 + }, + { + "coverage": 0.7033333333333334, + "risk": 0.23175355450236967, + "threshold": 0.04801334923860968 + }, + { + "coverage": 0.7036666666666667, + "risk": 0.231643770724775, + "threshold": 0.04798723491245837 + }, + { + "coverage": 0.704, + "risk": 0.2315340909090909, + "threshold": 0.04791184882398271 + }, + { + "coverage": 0.7043333333333334, + "risk": 0.23142451490771415, + "threshold": 0.04779102286334737 + }, + { + "coverage": 0.7046666666666667, + "risk": 0.23131504257332072, + "threshold": 0.047780292470795516 + }, + { + "coverage": 0.705, + "risk": 0.23120567375886525, + "threshold": 0.04765234426478271 + }, + { + "coverage": 0.7053333333333334, + "risk": 0.23109640831758035, + "threshold": 0.047511300775175076 + }, + { + "coverage": 0.7056666666666667, + "risk": 0.2309872461029759, + "threshold": 0.047245898387849206 + }, + { + "coverage": 0.706, + "risk": 0.23087818696883852, + "threshold": 0.04723932951680475 + }, + { + "coverage": 0.7063333333333334, + "risk": 0.23076923076923078, + "threshold": 0.04669831824556515 + }, + { + "coverage": 0.7066666666666667, + "risk": 0.23066037735849057, + "threshold": 0.04666903121418377 + }, + { + "coverage": 0.707, + "risk": 0.23055162659123055, + "threshold": 0.0466192800025822 + }, + { + "coverage": 0.7073333333333334, + "risk": 0.23044297832233743, + "threshold": 0.04655797227062109 + }, + { + "coverage": 0.7076666666666667, + "risk": 0.23033443240697127, + "threshold": 0.04651744611619969 + }, + { + "coverage": 0.708, + "risk": 0.23022598870056496, + "threshold": 0.046490109087743654 + }, + { + "coverage": 0.7083333333333334, + "risk": 0.23058823529411765, + "threshold": 0.04635805828875284 + }, + { + "coverage": 0.7086666666666667, + "risk": 0.23047977422389465, + "threshold": 0.046331097730014634 + }, + { + "coverage": 0.709, + "risk": 0.230371415138693, + "threshold": 0.046049932627510344 + }, + { + "coverage": 0.7093333333333334, + "risk": 0.23026315789473684, + "threshold": 0.04603620920324336 + }, + { + "coverage": 0.7096666666666667, + "risk": 0.23015500234852043, + "threshold": 0.045533030798288195 + }, + { + "coverage": 0.71, + "risk": 0.2300469483568075, + "threshold": 0.04537002790578271 + }, + { + "coverage": 0.7103333333333334, + "risk": 0.22993899577663068, + "threshold": 0.045189155337395086 + }, + { + "coverage": 0.7106666666666667, + "risk": 0.2298311444652908, + "threshold": 0.04513759541125184 + }, + { + "coverage": 0.711, + "risk": 0.22972339428035632, + "threshold": 0.045128657465263844 + }, + { + "coverage": 0.7113333333333334, + "risk": 0.2296157450796626, + "threshold": 0.044827365713253194 + }, + { + "coverage": 0.7116666666666667, + "risk": 0.22950819672131148, + "threshold": 0.044761044288104575 + }, + { + "coverage": 0.712, + "risk": 0.22940074906367042, + "threshold": 0.0447094970562778 + }, + { + "coverage": 0.7123333333333334, + "risk": 0.229293401965372, + "threshold": 0.044460406502615506 + }, + { + "coverage": 0.7126666666666667, + "risk": 0.2291861552853134, + "threshold": 0.04442577344260012 + }, + { + "coverage": 0.713, + "risk": 0.22907900888265545, + "threshold": 0.044395995466232734 + }, + { + "coverage": 0.7133333333333334, + "risk": 0.22897196261682243, + "threshold": 0.0443783887245086 + }, + { + "coverage": 0.7136666666666667, + "risk": 0.22886501634750117, + "threshold": 0.044369574815385164 + }, + { + "coverage": 0.714, + "risk": 0.22875816993464052, + "threshold": 0.044300859993184886 + }, + { + "coverage": 0.7143333333333334, + "risk": 0.22865142323845078, + "threshold": 0.044174408785406356 + }, + { + "coverage": 0.7146666666666667, + "risk": 0.228544776119403, + "threshold": 0.04416919848654399 + }, + { + "coverage": 0.715, + "risk": 0.22843822843822845, + "threshold": 0.04406579689545143 + }, + { + "coverage": 0.7153333333333334, + "risk": 0.22833178005591798, + "threshold": 0.0440652763983378 + }, + { + "coverage": 0.7156666666666667, + "risk": 0.22822543083372146, + "threshold": 0.0440526449412153 + }, + { + "coverage": 0.716, + "risk": 0.22811918063314712, + "threshold": 0.04379144128194845 + }, + { + "coverage": 0.7163333333333334, + "risk": 0.2280130293159609, + "threshold": 0.043738647606020635 + }, + { + "coverage": 0.7166666666666667, + "risk": 0.22790697674418606, + "threshold": 0.04343068100527247 + }, + { + "coverage": 0.717, + "risk": 0.22780102278010228, + "threshold": 0.043411138279875416 + }, + { + "coverage": 0.7173333333333334, + "risk": 0.22769516728624536, + "threshold": 0.043203472930466454 + }, + { + "coverage": 0.7176666666666667, + "risk": 0.2275894101254064, + "threshold": 0.04317620497004482 + }, + { + "coverage": 0.718, + "risk": 0.22748375116063138, + "threshold": 0.04252989296720621 + }, + { + "coverage": 0.7183333333333334, + "risk": 0.2273781902552204, + "threshold": 0.04238497096195016 + }, + { + "coverage": 0.7186666666666667, + "risk": 0.22727272727272727, + "threshold": 0.04234937303110086 + }, + { + "coverage": 0.719, + "risk": 0.22716736207695873, + "threshold": 0.042065958731515966 + }, + { + "coverage": 0.7193333333333334, + "risk": 0.22706209453197404, + "threshold": 0.04191911968624296 + }, + { + "coverage": 0.7196666666666667, + "risk": 0.2269569245020843, + "threshold": 0.04190302827269531 + }, + { + "coverage": 0.72, + "risk": 0.22685185185185186, + "threshold": 0.04189947750423073 + }, + { + "coverage": 0.7203333333333334, + "risk": 0.22674687644608976, + "threshold": 0.04189029841768365 + }, + { + "coverage": 0.7206666666666667, + "risk": 0.22664199814986125, + "threshold": 0.04166247511060761 + }, + { + "coverage": 0.721, + "risk": 0.22653721682847897, + "threshold": 0.041568040783521484 + }, + { + "coverage": 0.7213333333333334, + "risk": 0.22643253234750463, + "threshold": 0.04153899806833517 + }, + { + "coverage": 0.7216666666666667, + "risk": 0.22632794457274827, + "threshold": 0.04144667311536154 + }, + { + "coverage": 0.722, + "risk": 0.22622345337026778, + "threshold": 0.04141581632871704 + }, + { + "coverage": 0.7223333333333334, + "risk": 0.22611905860636825, + "threshold": 0.04140066442406054 + }, + { + "coverage": 0.7226666666666667, + "risk": 0.22601476014760147, + "threshold": 0.0414001373653159 + }, + { + "coverage": 0.723, + "risk": 0.22591055786076533, + "threshold": 0.04138472420736061 + }, + { + "coverage": 0.7233333333333334, + "risk": 0.22580645161290322, + "threshold": 0.041371619415748295 + }, + { + "coverage": 0.7236666666666667, + "risk": 0.22570244127130354, + "threshold": 0.04124937897765379 + }, + { + "coverage": 0.724, + "risk": 0.22559852670349909, + "threshold": 0.040916727405502236 + }, + { + "coverage": 0.7243333333333334, + "risk": 0.22549470777726646, + "threshold": 0.04082528826298315 + }, + { + "coverage": 0.7246666666666667, + "risk": 0.22539098436062557, + "threshold": 0.04081212736977946 + }, + { + "coverage": 0.725, + "risk": 0.22528735632183908, + "threshold": 0.04067989109376669 + }, + { + "coverage": 0.7253333333333334, + "risk": 0.22518382352941177, + "threshold": 0.040345538393821816 + }, + { + "coverage": 0.7256666666666667, + "risk": 0.22508038585209003, + "threshold": 0.040063209723836035 + }, + { + "coverage": 0.726, + "risk": 0.22497704315886133, + "threshold": 0.039917554236976294 + }, + { + "coverage": 0.7263333333333334, + "risk": 0.22487379531895366, + "threshold": 0.03982977912474306 + }, + { + "coverage": 0.7266666666666667, + "risk": 0.22477064220183487, + "threshold": 0.03982057479435363 + }, + { + "coverage": 0.727, + "risk": 0.2246675836772123, + "threshold": 0.03980099554254047 + }, + { + "coverage": 0.7273333333333334, + "risk": 0.22456461961503207, + "threshold": 0.03964039411788466 + }, + { + "coverage": 0.7276666666666667, + "risk": 0.2244617498854787, + "threshold": 0.039519396551753516 + }, + { + "coverage": 0.728, + "risk": 0.22435897435897437, + "threshold": 0.03949500235860625 + }, + { + "coverage": 0.7283333333333334, + "risk": 0.2242562929061785, + "threshold": 0.03948383536700106 + }, + { + "coverage": 0.7286666666666667, + "risk": 0.22415370539798718, + "threshold": 0.03929318289424976 + }, + { + "coverage": 0.729, + "risk": 0.2240512117055327, + "threshold": 0.03915703787874581 + }, + { + "coverage": 0.7293333333333333, + "risk": 0.22394881170018283, + "threshold": 0.03914909451878668 + }, + { + "coverage": 0.7296666666666667, + "risk": 0.22384650525354044, + "threshold": 0.039119666114985074 + }, + { + "coverage": 0.73, + "risk": 0.2237442922374429, + "threshold": 0.0389432944602599 + }, + { + "coverage": 0.7303333333333333, + "risk": 0.22364217252396165, + "threshold": 0.03891842064652451 + }, + { + "coverage": 0.7306666666666667, + "risk": 0.22354014598540145, + "threshold": 0.03884075906509343 + }, + { + "coverage": 0.731, + "risk": 0.22343821249430004, + "threshold": 0.03871642440372348 + }, + { + "coverage": 0.7313333333333333, + "risk": 0.22333637192342753, + "threshold": 0.03871292927756231 + }, + { + "coverage": 0.7316666666666667, + "risk": 0.22323462414578588, + "threshold": 0.038611071358940485 + }, + { + "coverage": 0.732, + "risk": 0.22313296903460839, + "threshold": 0.038510958186908306 + }, + { + "coverage": 0.7323333333333333, + "risk": 0.22303140646335912, + "threshold": 0.038326930518630675 + }, + { + "coverage": 0.7326666666666667, + "risk": 0.2229299363057325, + "threshold": 0.038256297657662815 + }, + { + "coverage": 0.733, + "risk": 0.22282855843565258, + "threshold": 0.03821968906744355 + }, + { + "coverage": 0.7333333333333333, + "risk": 0.22272727272727272, + "threshold": 0.037669459458082415 + }, + { + "coverage": 0.7336666666666667, + "risk": 0.222626079054975, + "threshold": 0.037661067696533886 + }, + { + "coverage": 0.734, + "risk": 0.22297910990009082, + "threshold": 0.03743563649254344 + }, + { + "coverage": 0.7343333333333333, + "risk": 0.22287789378120745, + "threshold": 0.03740964225301724 + }, + { + "coverage": 0.7346666666666667, + "risk": 0.22277676950998185, + "threshold": 0.03734881108548694 + }, + { + "coverage": 0.735, + "risk": 0.22267573696145124, + "threshold": 0.03727007186565778 + }, + { + "coverage": 0.7353333333333333, + "risk": 0.22257479601087943, + "threshold": 0.03723865310853047 + }, + { + "coverage": 0.7356666666666667, + "risk": 0.22247394653375624, + "threshold": 0.03720706136903119 + }, + { + "coverage": 0.736, + "risk": 0.2223731884057971, + "threshold": 0.03706948882589934 + }, + { + "coverage": 0.7363333333333333, + "risk": 0.22227252150294252, + "threshold": 0.03704722776875369 + }, + { + "coverage": 0.7366666666666667, + "risk": 0.22217194570135745, + "threshold": 0.03673288380823103 + }, + { + "coverage": 0.737, + "risk": 0.22207146087743102, + "threshold": 0.03672583844030064 + }, + { + "coverage": 0.7373333333333333, + "risk": 0.22197106690777577, + "threshold": 0.036508607510510334 + }, + { + "coverage": 0.7376666666666667, + "risk": 0.22187076366922728, + "threshold": 0.036366247415171575 + }, + { + "coverage": 0.738, + "risk": 0.22177055103884372, + "threshold": 0.03617714485663282 + }, + { + "coverage": 0.7383333333333333, + "risk": 0.2216704288939052, + "threshold": 0.03612796924359563 + }, + { + "coverage": 0.7386666666666667, + "risk": 0.22157039711191337, + "threshold": 0.03598533847832419 + }, + { + "coverage": 0.739, + "risk": 0.2214704555705909, + "threshold": 0.03574355222650239 + }, + { + "coverage": 0.7393333333333333, + "risk": 0.22137060414788098, + "threshold": 0.03559817331451942 + }, + { + "coverage": 0.7396666666666667, + "risk": 0.22127084272194683, + "threshold": 0.03542466352285811 + }, + { + "coverage": 0.74, + "risk": 0.22117117117117116, + "threshold": 0.03535848645215878 + }, + { + "coverage": 0.7403333333333333, + "risk": 0.22107158937415577, + "threshold": 0.035357679373635424 + }, + { + "coverage": 0.7406666666666667, + "risk": 0.22097209720972097, + "threshold": 0.0351942284074196 + }, + { + "coverage": 0.741, + "risk": 0.22087269455690509, + "threshold": 0.03519004270504201 + }, + { + "coverage": 0.7413333333333333, + "risk": 0.22077338129496402, + "threshold": 0.034924214253429404 + }, + { + "coverage": 0.7416666666666667, + "risk": 0.22067415730337078, + "threshold": 0.03460272549794129 + }, + { + "coverage": 0.742, + "risk": 0.2205750224618149, + "threshold": 0.03456792833049426 + }, + { + "coverage": 0.7423333333333333, + "risk": 0.22047597665020208, + "threshold": 0.03441887571649938 + }, + { + "coverage": 0.7426666666666667, + "risk": 0.2203770197486535, + "threshold": 0.0344065663738018 + }, + { + "coverage": 0.743, + "risk": 0.2202781516375056, + "threshold": 0.03406173538706536 + }, + { + "coverage": 0.7433333333333333, + "risk": 0.22017937219730943, + "threshold": 0.03393698777682986 + }, + { + "coverage": 0.7436666666666667, + "risk": 0.22008068130883013, + "threshold": 0.03387366186109641 + }, + { + "coverage": 0.744, + "risk": 0.2199820788530466, + "threshold": 0.03379267724740065 + }, + { + "coverage": 0.7443333333333333, + "risk": 0.21988356471115092, + "threshold": 0.033595895199626025 + }, + { + "coverage": 0.7446666666666667, + "risk": 0.2197851387645479, + "threshold": 0.03343468349905842 + }, + { + "coverage": 0.745, + "risk": 0.2196868008948546, + "threshold": 0.03342224299903274 + }, + { + "coverage": 0.7453333333333333, + "risk": 0.21958855098389982, + "threshold": 0.03340344802972028 + }, + { + "coverage": 0.7456666666666667, + "risk": 0.21949038891372374, + "threshold": 0.03336461126489709 + }, + { + "coverage": 0.746, + "risk": 0.2193923145665773, + "threshold": 0.03336436682887234 + }, + { + "coverage": 0.7463333333333333, + "risk": 0.21929432782492184, + "threshold": 0.03331491557633436 + }, + { + "coverage": 0.7466666666666667, + "risk": 0.21919642857142857, + "threshold": 0.033233842516011254 + }, + { + "coverage": 0.747, + "risk": 0.21909861668897815, + "threshold": 0.0330802226073048 + }, + { + "coverage": 0.7473333333333333, + "risk": 0.21900089206066012, + "threshold": 0.03295854161640599 + }, + { + "coverage": 0.7476666666666667, + "risk": 0.21890325456977264, + "threshold": 0.03291168691621316 + }, + { + "coverage": 0.748, + "risk": 0.21880570409982175, + "threshold": 0.03290551917467499 + }, + { + "coverage": 0.7483333333333333, + "risk": 0.21870824053452115, + "threshold": 0.032896475475817435 + }, + { + "coverage": 0.7486666666666667, + "risk": 0.21861086375779162, + "threshold": 0.032745123623124435 + }, + { + "coverage": 0.749, + "risk": 0.21851357365376056, + "threshold": 0.032736183929119374 + }, + { + "coverage": 0.7493333333333333, + "risk": 0.21841637010676157, + "threshold": 0.032679795026445475 + }, + { + "coverage": 0.7496666666666667, + "risk": 0.21831925300133392, + "threshold": 0.032664539583751935 + }, + { + "coverage": 0.75, + "risk": 0.21822222222222223, + "threshold": 0.03264696863905026 + }, + { + "coverage": 0.7503333333333333, + "risk": 0.21812527765437584, + "threshold": 0.032230774319826905 + }, + { + "coverage": 0.7506666666666667, + "risk": 0.2180284191829485, + "threshold": 0.032154815051823826 + }, + { + "coverage": 0.751, + "risk": 0.21793164669329781, + "threshold": 0.032032751167889735 + }, + { + "coverage": 0.7513333333333333, + "risk": 0.2178349600709849, + "threshold": 0.03196375750122838 + }, + { + "coverage": 0.7516666666666667, + "risk": 0.21773835920177384, + "threshold": 0.03196344389880679 + }, + { + "coverage": 0.752, + "risk": 0.2176418439716312, + "threshold": 0.03180837200545738 + }, + { + "coverage": 0.7523333333333333, + "risk": 0.21754541426672575, + "threshold": 0.031770700633299676 + }, + { + "coverage": 0.7526666666666667, + "risk": 0.2174490699734278, + "threshold": 0.03148874785522615 + }, + { + "coverage": 0.753, + "risk": 0.217352810978309, + "threshold": 0.03148726138747038 + }, + { + "coverage": 0.7533333333333333, + "risk": 0.2172566371681416, + "threshold": 0.03132954411561472 + }, + { + "coverage": 0.7536666666666667, + "risk": 0.21716054842989826, + "threshold": 0.031234092045554492 + }, + { + "coverage": 0.754, + "risk": 0.21706454465075153, + "threshold": 0.031172691115810565 + }, + { + "coverage": 0.7543333333333333, + "risk": 0.21696862571807335, + "threshold": 0.031163307098358795 + }, + { + "coverage": 0.7546666666666667, + "risk": 0.21687279151943462, + "threshold": 0.031112093237723563 + }, + { + "coverage": 0.755, + "risk": 0.21677704194260486, + "threshold": 0.030591219431980732 + }, + { + "coverage": 0.7553333333333333, + "risk": 0.21668137687555164, + "threshold": 0.030586254167607848 + }, + { + "coverage": 0.7556666666666667, + "risk": 0.21658579620644022, + "threshold": 0.030244147501523592 + }, + { + "coverage": 0.756, + "risk": 0.21649029982363316, + "threshold": 0.030160153409586773 + }, + { + "coverage": 0.7563333333333333, + "risk": 0.21639488761568973, + "threshold": 0.030007279026342704 + }, + { + "coverage": 0.7566666666666667, + "risk": 0.21629955947136564, + "threshold": 0.02990550734557429 + }, + { + "coverage": 0.757, + "risk": 0.2162043152796125, + "threshold": 0.029845889578796306 + }, + { + "coverage": 0.7573333333333333, + "risk": 0.21610915492957747, + "threshold": 0.02972593388126121 + }, + { + "coverage": 0.7576666666666667, + "risk": 0.21601407831060274, + "threshold": 0.029603856308161536 + }, + { + "coverage": 0.758, + "risk": 0.21591908531222515, + "threshold": 0.029432492533825735 + }, + { + "coverage": 0.7583333333333333, + "risk": 0.21582417582417582, + "threshold": 0.029426308134854032 + }, + { + "coverage": 0.7586666666666667, + "risk": 0.2157293497363796, + "threshold": 0.029386395092405596 + }, + { + "coverage": 0.759, + "risk": 0.21563460693895475, + "threshold": 0.029192010750771047 + }, + { + "coverage": 0.7593333333333333, + "risk": 0.21553994732221246, + "threshold": 0.029105186197825313 + }, + { + "coverage": 0.7596666666666667, + "risk": 0.21544537077665643, + "threshold": 0.028739451263448686 + }, + { + "coverage": 0.76, + "risk": 0.21535087719298246, + "threshold": 0.028639189317366746 + }, + { + "coverage": 0.7603333333333333, + "risk": 0.21525646646207802, + "threshold": 0.02861295867884195 + }, + { + "coverage": 0.7606666666666667, + "risk": 0.2151621384750219, + "threshold": 0.028446170242638624 + }, + { + "coverage": 0.761, + "risk": 0.21506789312308366, + "threshold": 0.028428435691216784 + }, + { + "coverage": 0.7613333333333333, + "risk": 0.2149737302977233, + "threshold": 0.02834360677409941 + }, + { + "coverage": 0.7616666666666667, + "risk": 0.2148796498905908, + "threshold": 0.028271787408752354 + }, + { + "coverage": 0.762, + "risk": 0.21478565179352582, + "threshold": 0.028133522573910334 + }, + { + "coverage": 0.7623333333333333, + "risk": 0.21469173589855706, + "threshold": 0.02809627427541272 + }, + { + "coverage": 0.7626666666666667, + "risk": 0.2145979020979021, + "threshold": 0.0280809403146769 + }, + { + "coverage": 0.763, + "risk": 0.2145041502839668, + "threshold": 0.02805666320888866 + }, + { + "coverage": 0.7633333333333333, + "risk": 0.21441048034934498, + "threshold": 0.028049547791789253 + }, + { + "coverage": 0.7636666666666667, + "risk": 0.21475338280226974, + "threshold": 0.027962448464162407 + }, + { + "coverage": 0.764, + "risk": 0.21465968586387435, + "threshold": 0.027951990961782025 + }, + { + "coverage": 0.7643333333333333, + "risk": 0.21456607064980376, + "threshold": 0.02793344288406164 + }, + { + "coverage": 0.7646666666666667, + "risk": 0.21447253705318223, + "threshold": 0.027736151256672503 + }, + { + "coverage": 0.765, + "risk": 0.21437908496732025, + "threshold": 0.027724300165962164 + }, + { + "coverage": 0.7653333333333333, + "risk": 0.21428571428571427, + "threshold": 0.02756464289640004 + }, + { + "coverage": 0.7656666666666667, + "risk": 0.21419242490204615, + "threshold": 0.02745098498093946 + }, + { + "coverage": 0.766, + "risk": 0.2145343777197563, + "threshold": 0.02739414516034389 + }, + { + "coverage": 0.7663333333333333, + "risk": 0.21444106133101348, + "threshold": 0.027319793280426323 + }, + { + "coverage": 0.7666666666666667, + "risk": 0.21434782608695652, + "threshold": 0.02724609712050219 + }, + { + "coverage": 0.767, + "risk": 0.21425467188179054, + "threshold": 0.027147906521059584 + }, + { + "coverage": 0.7673333333333333, + "risk": 0.21416159860990444, + "threshold": 0.02713826007657006 + }, + { + "coverage": 0.7676666666666667, + "risk": 0.21406860616587062, + "threshold": 0.02687168607265367 + }, + { + "coverage": 0.768, + "risk": 0.21397569444444445, + "threshold": 0.026844807949890193 + }, + { + "coverage": 0.7683333333333333, + "risk": 0.213882863340564, + "threshold": 0.026666415099119156 + }, + { + "coverage": 0.7686666666666667, + "risk": 0.21379011274934953, + "threshold": 0.0266642516578397 + }, + { + "coverage": 0.769, + "risk": 0.2141309059384482, + "threshold": 0.026616641443794945 + }, + { + "coverage": 0.7693333333333333, + "risk": 0.21403812824956672, + "threshold": 0.026551150428628437 + }, + { + "coverage": 0.7696666666666667, + "risk": 0.21394543092247725, + "threshold": 0.026500597900898537 + }, + { + "coverage": 0.77, + "risk": 0.21385281385281385, + "threshold": 0.02639467003748975 + }, + { + "coverage": 0.7703333333333333, + "risk": 0.21376027693639119, + "threshold": 0.02632340140471312 + }, + { + "coverage": 0.7706666666666667, + "risk": 0.21366782006920415, + "threshold": 0.026283821348204166 + }, + { + "coverage": 0.771, + "risk": 0.21357544314742757, + "threshold": 0.02627491442233598 + }, + { + "coverage": 0.7713333333333333, + "risk": 0.21348314606741572, + "threshold": 0.026143089750845228 + }, + { + "coverage": 0.7716666666666666, + "risk": 0.21339092872570195, + "threshold": 0.026045342198369067 + }, + { + "coverage": 0.772, + "risk": 0.21329879101899826, + "threshold": 0.025927543777994444 + }, + { + "coverage": 0.7723333333333333, + "risk": 0.21320673284419508, + "threshold": 0.02560548968719267 + }, + { + "coverage": 0.7726666666666666, + "risk": 0.21311475409836064, + "threshold": 0.025594291037844833 + }, + { + "coverage": 0.773, + "risk": 0.21302285467874083, + "threshold": 0.025526782582949714 + }, + { + "coverage": 0.7733333333333333, + "risk": 0.21293103448275863, + "threshold": 0.0253734565033078 + }, + { + "coverage": 0.7736666666666666, + "risk": 0.21283929340801377, + "threshold": 0.02537270375908895 + }, + { + "coverage": 0.774, + "risk": 0.2127476313522825, + "threshold": 0.02532050789341267 + }, + { + "coverage": 0.7743333333333333, + "risk": 0.212656048213517, + "threshold": 0.02531356973972665 + }, + { + "coverage": 0.7746666666666666, + "risk": 0.21256454388984508, + "threshold": 0.025283479386142286 + }, + { + "coverage": 0.775, + "risk": 0.21247311827956988, + "threshold": 0.02522472989173852 + }, + { + "coverage": 0.7753333333333333, + "risk": 0.2123817712811694, + "threshold": 0.025190602862930733 + }, + { + "coverage": 0.7756666666666666, + "risk": 0.2122905027932961, + "threshold": 0.025160163565125515 + }, + { + "coverage": 0.776, + "risk": 0.21219931271477663, + "threshold": 0.02514641005210816 + }, + { + "coverage": 0.7763333333333333, + "risk": 0.2121082009446114, + "threshold": 0.02510384874460225 + }, + { + "coverage": 0.7766666666666666, + "risk": 0.21201716738197424, + "threshold": 0.0250881447818545 + }, + { + "coverage": 0.777, + "risk": 0.21192621192621192, + "threshold": 0.02490834205755732 + }, + { + "coverage": 0.7773333333333333, + "risk": 0.2118353344768439, + "threshold": 0.024881800138703888 + }, + { + "coverage": 0.7776666666666666, + "risk": 0.21174453493356193, + "threshold": 0.024764374542114886 + }, + { + "coverage": 0.778, + "risk": 0.21165381319622964, + "threshold": 0.024689928347102708 + }, + { + "coverage": 0.7783333333333333, + "risk": 0.21156316916488221, + "threshold": 0.02466567324473745 + }, + { + "coverage": 0.7786666666666666, + "risk": 0.21147260273972604, + "threshold": 0.024499125043608846 + }, + { + "coverage": 0.779, + "risk": 0.21138211382113822, + "threshold": 0.024205633220283577 + }, + { + "coverage": 0.7793333333333333, + "risk": 0.21129170230966637, + "threshold": 0.024172985139731644 + }, + { + "coverage": 0.7796666666666666, + "risk": 0.21120136810602821, + "threshold": 0.02403827104052736 + }, + { + "coverage": 0.78, + "risk": 0.2111111111111111, + "threshold": 0.023911098495972286 + }, + { + "coverage": 0.7803333333333333, + "risk": 0.21102093122597182, + "threshold": 0.023765290768796524 + }, + { + "coverage": 0.7806666666666666, + "risk": 0.21093082835183605, + "threshold": 0.023671332656299844 + }, + { + "coverage": 0.781, + "risk": 0.21084080239009817, + "threshold": 0.023550787673123356 + }, + { + "coverage": 0.7813333333333333, + "risk": 0.21075085324232082, + "threshold": 0.023521874057319543 + }, + { + "coverage": 0.7816666666666666, + "risk": 0.21066098081023454, + "threshold": 0.023514961907613596 + }, + { + "coverage": 0.782, + "risk": 0.21057118499573743, + "threshold": 0.023422119192898043 + }, + { + "coverage": 0.7823333333333333, + "risk": 0.21048146570089477, + "threshold": 0.023385091705186326 + }, + { + "coverage": 0.7826666666666666, + "risk": 0.21039182282793867, + "threshold": 0.02327593862902488 + }, + { + "coverage": 0.783, + "risk": 0.21030225627926777, + "threshold": 0.02316840084796046 + }, + { + "coverage": 0.7833333333333333, + "risk": 0.2102127659574468, + "threshold": 0.022926990262538412 + }, + { + "coverage": 0.7836666666666666, + "risk": 0.2101233517652063, + "threshold": 0.022915327503965303 + }, + { + "coverage": 0.784, + "risk": 0.2100340136054422, + "threshold": 0.022911161425583915 + }, + { + "coverage": 0.7843333333333333, + "risk": 0.20994475138121546, + "threshold": 0.02282444423827801 + }, + { + "coverage": 0.7846666666666666, + "risk": 0.20985556499575192, + "threshold": 0.022812724874349044 + }, + { + "coverage": 0.785, + "risk": 0.20976645435244162, + "threshold": 0.022768265907056055 + }, + { + "coverage": 0.7853333333333333, + "risk": 0.20967741935483872, + "threshold": 0.022661101141211223 + }, + { + "coverage": 0.7856666666666666, + "risk": 0.209588459906661, + "threshold": 0.022608868473845693 + }, + { + "coverage": 0.786, + "risk": 0.20949957591178966, + "threshold": 0.022490979190772368 + }, + { + "coverage": 0.7863333333333333, + "risk": 0.20941076727426874, + "threshold": 0.022479146993628185 + }, + { + "coverage": 0.7866666666666666, + "risk": 0.2093220338983051, + "threshold": 0.02243892545898429 + }, + { + "coverage": 0.787, + "risk": 0.2092333756882677, + "threshold": 0.022392309964786247 + }, + { + "coverage": 0.7873333333333333, + "risk": 0.20914479254868756, + "threshold": 0.02236556299589454 + }, + { + "coverage": 0.7876666666666666, + "risk": 0.2090562843842573, + "threshold": 0.022347430145321145 + }, + { + "coverage": 0.788, + "risk": 0.2089678510998308, + "threshold": 0.02234340226938336 + }, + { + "coverage": 0.7883333333333333, + "risk": 0.20887949260042282, + "threshold": 0.022277475383936537 + }, + { + "coverage": 0.7886666666666666, + "risk": 0.2087912087912088, + "threshold": 0.02226426097153555 + }, + { + "coverage": 0.789, + "risk": 0.20870299957752428, + "threshold": 0.022181664182377322 + }, + { + "coverage": 0.7893333333333333, + "risk": 0.20861486486486486, + "threshold": 0.02208852191534527 + }, + { + "coverage": 0.7896666666666666, + "risk": 0.20852680455888561, + "threshold": 0.022066614399057327 + }, + { + "coverage": 0.79, + "risk": 0.20843881856540084, + "threshold": 0.02204301682325438 + }, + { + "coverage": 0.7903333333333333, + "risk": 0.20835090679038382, + "threshold": 0.02168948163460028 + }, + { + "coverage": 0.7906666666666666, + "risk": 0.20826306913996628, + "threshold": 0.021649112382262633 + }, + { + "coverage": 0.791, + "risk": 0.20817530552043825, + "threshold": 0.02152563800117946 + }, + { + "coverage": 0.7913333333333333, + "risk": 0.2080876158382477, + "threshold": 0.021478787897626472 + }, + { + "coverage": 0.7916666666666666, + "risk": 0.208, + "threshold": 0.021415947108390794 + }, + { + "coverage": 0.792, + "risk": 0.2079124579124579, + "threshold": 0.021304382883145845 + }, + { + "coverage": 0.7923333333333333, + "risk": 0.20782498948254102, + "threshold": 0.02116361466522276 + }, + { + "coverage": 0.7926666666666666, + "risk": 0.2077375946173255, + "threshold": 0.02112268326756528 + }, + { + "coverage": 0.793, + "risk": 0.20765027322404372, + "threshold": 0.021101425619274753 + }, + { + "coverage": 0.7933333333333333, + "risk": 0.20756302521008405, + "threshold": 0.02103632169706604 + }, + { + "coverage": 0.7936666666666666, + "risk": 0.20747585048299033, + "threshold": 0.021018988815784016 + }, + { + "coverage": 0.794, + "risk": 0.2073887489504618, + "threshold": 0.020999727270038408 + }, + { + "coverage": 0.7943333333333333, + "risk": 0.2073017205203525, + "threshold": 0.020930965443101805 + }, + { + "coverage": 0.7946666666666666, + "risk": 0.20721476510067113, + "threshold": 0.020755562195688085 + }, + { + "coverage": 0.795, + "risk": 0.2071278825995807, + "threshold": 0.0207012366403807 + }, + { + "coverage": 0.7953333333333333, + "risk": 0.20704107292539817, + "threshold": 0.020608937318203565 + }, + { + "coverage": 0.7956666666666666, + "risk": 0.20695433598659405, + "threshold": 0.020532090904963333 + }, + { + "coverage": 0.796, + "risk": 0.2068676716917923, + "threshold": 0.020524173177743237 + }, + { + "coverage": 0.7963333333333333, + "risk": 0.20678107994976977, + "threshold": 0.02043877189126198 + }, + { + "coverage": 0.7966666666666666, + "risk": 0.20669456066945607, + "threshold": 0.020409711500863903 + }, + { + "coverage": 0.797, + "risk": 0.20660811375993307, + "threshold": 0.020396671551431433 + }, + { + "coverage": 0.7973333333333333, + "risk": 0.20652173913043478, + "threshold": 0.020386820596147496 + }, + { + "coverage": 0.7976666666666666, + "risk": 0.20643543669034684, + "threshold": 0.020292880412992268 + }, + { + "coverage": 0.798, + "risk": 0.20634920634920634, + "threshold": 0.020192130183142473 + }, + { + "coverage": 0.7983333333333333, + "risk": 0.20626304801670145, + "threshold": 0.020034227049284814 + }, + { + "coverage": 0.7986666666666666, + "risk": 0.20617696160267113, + "threshold": 0.019915857030720963 + }, + { + "coverage": 0.799, + "risk": 0.2060909470171047, + "threshold": 0.01977241021373111 + }, + { + "coverage": 0.7993333333333333, + "risk": 0.20600500417014178, + "threshold": 0.019760257676539442 + }, + { + "coverage": 0.7996666666666666, + "risk": 0.2059191329720717, + "threshold": 0.019734264343267416 + }, + { + "coverage": 0.8, + "risk": 0.20583333333333334, + "threshold": 0.01970544984861977 + }, + { + "coverage": 0.8003333333333333, + "risk": 0.2057476051645148, + "threshold": 0.01963414636805449 + }, + { + "coverage": 0.8006666666666666, + "risk": 0.20566194837635304, + "threshold": 0.019603615628033474 + }, + { + "coverage": 0.801, + "risk": 0.20557636287973366, + "threshold": 0.019477416176624126 + }, + { + "coverage": 0.8013333333333333, + "risk": 0.20549084858569053, + "threshold": 0.019427724863495748 + }, + { + "coverage": 0.8016666666666666, + "risk": 0.20540540540540542, + "threshold": 0.01941182526410323 + }, + { + "coverage": 0.802, + "risk": 0.2053200332502078, + "threshold": 0.019394900280166102 + }, + { + "coverage": 0.8023333333333333, + "risk": 0.20523473203157458, + "threshold": 0.019389617281154844 + }, + { + "coverage": 0.8026666666666666, + "risk": 0.20514950166112958, + "threshold": 0.019381126225049176 + }, + { + "coverage": 0.803, + "risk": 0.20506434205064342, + "threshold": 0.019194651926390586 + }, + { + "coverage": 0.8033333333333333, + "risk": 0.2049792531120332, + "threshold": 0.019142963080094777 + }, + { + "coverage": 0.8036666666666666, + "risk": 0.2048942347573621, + "threshold": 0.019067327930746506 + }, + { + "coverage": 0.804, + "risk": 0.20480928689883915, + "threshold": 0.019049627787101734 + }, + { + "coverage": 0.8043333333333333, + "risk": 0.2047244094488189, + "threshold": 0.01901567432118112 + }, + { + "coverage": 0.8046666666666666, + "risk": 0.20463960231980116, + "threshold": 0.018963338703538613 + }, + { + "coverage": 0.805, + "risk": 0.20455486542443063, + "threshold": 0.018910346818673988 + }, + { + "coverage": 0.8053333333333333, + "risk": 0.20447019867549668, + "threshold": 0.018885430732895392 + }, + { + "coverage": 0.8056666666666666, + "risk": 0.20438560198593297, + "threshold": 0.01885908739525322 + }, + { + "coverage": 0.806, + "risk": 0.20430107526881722, + "threshold": 0.018842102852882906 + }, + { + "coverage": 0.8063333333333333, + "risk": 0.20421661843737082, + "threshold": 0.01871675872198608 + }, + { + "coverage": 0.8066666666666666, + "risk": 0.20413223140495868, + "threshold": 0.018637830385773265 + }, + { + "coverage": 0.807, + "risk": 0.2040479140850888, + "threshold": 0.018597676400217363 + }, + { + "coverage": 0.8073333333333333, + "risk": 0.20396366639141206, + "threshold": 0.01851901172560433 + }, + { + "coverage": 0.8076666666666666, + "risk": 0.20387948823772184, + "threshold": 0.018145511433470537 + }, + { + "coverage": 0.808, + "risk": 0.2037953795379538, + "threshold": 0.018098707538355966 + }, + { + "coverage": 0.8083333333333333, + "risk": 0.20371134020618556, + "threshold": 0.01783775098720764 + }, + { + "coverage": 0.8086666666666666, + "risk": 0.20362737015663643, + "threshold": 0.017796018646475398 + }, + { + "coverage": 0.809, + "risk": 0.20354346930366707, + "threshold": 0.017771179521017817 + }, + { + "coverage": 0.8093333333333333, + "risk": 0.20345963756177923, + "threshold": 0.01771686807497818 + }, + { + "coverage": 0.8096666666666666, + "risk": 0.20337587484561548, + "threshold": 0.01755343207093852 + }, + { + "coverage": 0.81, + "risk": 0.20329218106995886, + "threshold": 0.017449506063026095 + }, + { + "coverage": 0.8103333333333333, + "risk": 0.20320855614973263, + "threshold": 0.01716886192724099 + }, + { + "coverage": 0.8106666666666666, + "risk": 0.203125, + "threshold": 0.017134244988564976 + }, + { + "coverage": 0.811, + "risk": 0.20304151253596384, + "threshold": 0.017115119804405946 + }, + { + "coverage": 0.8113333333333334, + "risk": 0.2029580936729663, + "threshold": 0.017053114159926278 + }, + { + "coverage": 0.8116666666666666, + "risk": 0.2028747433264887, + "threshold": 0.01701948456064439 + }, + { + "coverage": 0.812, + "risk": 0.20279146141215107, + "threshold": 0.017002767813099066 + }, + { + "coverage": 0.8123333333333334, + "risk": 0.20270824784571195, + "threshold": 0.016993324271119544 + }, + { + "coverage": 0.8126666666666666, + "risk": 0.2026251025430681, + "threshold": 0.01692713270334823 + }, + { + "coverage": 0.813, + "risk": 0.2025420254202542, + "threshold": 0.016865252409526965 + }, + { + "coverage": 0.8133333333333334, + "risk": 0.20245901639344263, + "threshold": 0.01686340682151344 + }, + { + "coverage": 0.8136666666666666, + "risk": 0.20237607537894306, + "threshold": 0.016849498623805807 + }, + { + "coverage": 0.814, + "risk": 0.2022932022932023, + "threshold": 0.016788131298033138 + }, + { + "coverage": 0.8143333333333334, + "risk": 0.20221039705280394, + "threshold": 0.01677160176846481 + }, + { + "coverage": 0.8146666666666667, + "risk": 0.20212765957446807, + "threshold": 0.016642478983682985 + }, + { + "coverage": 0.815, + "risk": 0.20204498977505111, + "threshold": 0.016576543116736626 + }, + { + "coverage": 0.8153333333333334, + "risk": 0.2019623875715454, + "threshold": 0.01651140323924439 + }, + { + "coverage": 0.8156666666666667, + "risk": 0.20187985288107888, + "threshold": 0.016400159050613646 + }, + { + "coverage": 0.816, + "risk": 0.20179738562091504, + "threshold": 0.016321081068566553 + }, + { + "coverage": 0.8163333333333334, + "risk": 0.20171498570845242, + "threshold": 0.01624936228118279 + }, + { + "coverage": 0.8166666666666667, + "risk": 0.2016326530612245, + "threshold": 0.0161869531753498 + }, + { + "coverage": 0.817, + "risk": 0.20155038759689922, + "threshold": 0.016185933822858153 + }, + { + "coverage": 0.8173333333333334, + "risk": 0.20146818923327894, + "threshold": 0.016107534935065785 + }, + { + "coverage": 0.8176666666666667, + "risk": 0.20138605788830005, + "threshold": 0.016014651431894285 + }, + { + "coverage": 0.818, + "risk": 0.2013039934800326, + "threshold": 0.016010159962648204 + }, + { + "coverage": 0.8183333333333334, + "risk": 0.20122199592668025, + "threshold": 0.016007456076662423 + }, + { + "coverage": 0.8186666666666667, + "risk": 0.2011400651465798, + "threshold": 0.01588534346604642 + }, + { + "coverage": 0.819, + "risk": 0.20105820105820105, + "threshold": 0.015871081167335713 + }, + { + "coverage": 0.8193333333333334, + "risk": 0.20097640358014646, + "threshold": 0.015868578774908434 + }, + { + "coverage": 0.8196666666666667, + "risk": 0.20089467263115088, + "threshold": 0.015679587347763757 + }, + { + "coverage": 0.82, + "risk": 0.2008130081300813, + "threshold": 0.01563285709123452 + }, + { + "coverage": 0.8203333333333334, + "risk": 0.2007314099959366, + "threshold": 0.01556717215001946 + }, + { + "coverage": 0.8206666666666667, + "risk": 0.20064987814784727, + "threshold": 0.015476532609537408 + }, + { + "coverage": 0.821, + "risk": 0.20056841250507512, + "threshold": 0.01543425450502566 + }, + { + "coverage": 0.8213333333333334, + "risk": 0.200487012987013, + "threshold": 0.015391540608208902 + }, + { + "coverage": 0.8216666666666667, + "risk": 0.20040567951318458, + "threshold": 0.015326382596704491 + }, + { + "coverage": 0.822, + "risk": 0.2003244120032441, + "threshold": 0.015242412054280556 + }, + { + "coverage": 0.8223333333333334, + "risk": 0.2002432103769761, + "threshold": 0.015219746937504722 + }, + { + "coverage": 0.8226666666666667, + "risk": 0.20016207455429497, + "threshold": 0.015199798091648615 + }, + { + "coverage": 0.823, + "risk": 0.20008100445524504, + "threshold": 0.015198736157032051 + }, + { + "coverage": 0.8233333333333334, + "risk": 0.2, + "threshold": 0.015150972843985146 + }, + { + "coverage": 0.8236666666666667, + "risk": 0.19991906110886282, + "threshold": 0.015146014743726564 + }, + { + "coverage": 0.824, + "risk": 0.19983818770226539, + "threshold": 0.015145562624145568 + }, + { + "coverage": 0.8243333333333334, + "risk": 0.1997573797007683, + "threshold": 0.015032447429880524 + }, + { + "coverage": 0.8246666666666667, + "risk": 0.19967663702506064, + "threshold": 0.014978615051248204 + }, + { + "coverage": 0.825, + "risk": 0.1995959595959596, + "threshold": 0.014892839672752352 + }, + { + "coverage": 0.8253333333333334, + "risk": 0.19951534733441034, + "threshold": 0.014860437272091083 + }, + { + "coverage": 0.8256666666666667, + "risk": 0.19943480016148568, + "threshold": 0.014837312613109385 + }, + { + "coverage": 0.826, + "risk": 0.1993543179983858, + "threshold": 0.014820333650600979 + }, + { + "coverage": 0.8263333333333334, + "risk": 0.19927390076643808, + "threshold": 0.01476780919384191 + }, + { + "coverage": 0.8266666666666667, + "risk": 0.19919354838709677, + "threshold": 0.014752432406610165 + }, + { + "coverage": 0.827, + "risk": 0.19911326078194277, + "threshold": 0.014699427189024507 + }, + { + "coverage": 0.8273333333333334, + "risk": 0.1990330378726833, + "threshold": 0.01462184307226377 + }, + { + "coverage": 0.8276666666666667, + "risk": 0.19895287958115182, + "threshold": 0.014610051704649685 + }, + { + "coverage": 0.828, + "risk": 0.19887278582930756, + "threshold": 0.014444379242270487 + }, + { + "coverage": 0.8283333333333334, + "risk": 0.19879275653923542, + "threshold": 0.014396513660843052 + }, + { + "coverage": 0.8286666666666667, + "risk": 0.19871279163314562, + "threshold": 0.014366931409854906 + }, + { + "coverage": 0.829, + "risk": 0.19863289103337353, + "threshold": 0.014283079859232386 + }, + { + "coverage": 0.8293333333333334, + "risk": 0.1989549839228296, + "threshold": 0.014241001395404742 + }, + { + "coverage": 0.8296666666666667, + "risk": 0.19887505022097227, + "threshold": 0.01411801726224304 + }, + { + "coverage": 0.83, + "risk": 0.19879518072289157, + "threshold": 0.014020203661357146 + }, + { + "coverage": 0.8303333333333334, + "risk": 0.19871537535126454, + "threshold": 0.01397770076484173 + }, + { + "coverage": 0.8306666666666667, + "risk": 0.19863563402889245, + "threshold": 0.013863628013497114 + }, + { + "coverage": 0.831, + "risk": 0.19855595667870035, + "threshold": 0.013758545240676438 + }, + { + "coverage": 0.8313333333333334, + "risk": 0.19847634322373697, + "threshold": 0.013677519717364198 + }, + { + "coverage": 0.8316666666666667, + "risk": 0.19839679358717435, + "threshold": 0.013671460283632088 + }, + { + "coverage": 0.832, + "risk": 0.19831730769230768, + "threshold": 0.013619989539524811 + }, + { + "coverage": 0.8323333333333334, + "risk": 0.19823788546255505, + "threshold": 0.013589419613885042 + }, + { + "coverage": 0.8326666666666667, + "risk": 0.19815852682145715, + "threshold": 0.013552164969922842 + }, + { + "coverage": 0.833, + "risk": 0.19807923169267708, + "threshold": 0.013526295911544745 + }, + { + "coverage": 0.8333333333333334, + "risk": 0.198, + "threshold": 0.01348700565866557 + }, + { + "coverage": 0.8336666666666667, + "risk": 0.19792083166733307, + "threshold": 0.013349491825121459 + }, + { + "coverage": 0.834, + "risk": 0.19784172661870503, + "threshold": 0.013237568211887934 + }, + { + "coverage": 0.8343333333333334, + "risk": 0.19776268477826608, + "threshold": 0.013224223516365097 + }, + { + "coverage": 0.8346666666666667, + "risk": 0.19768370607028754, + "threshold": 0.01313617649212517 + }, + { + "coverage": 0.835, + "risk": 0.19760479041916168, + "threshold": 0.01311343548893407 + }, + { + "coverage": 0.8353333333333334, + "risk": 0.19752593774940144, + "threshold": 0.013026223549065682 + }, + { + "coverage": 0.8356666666666667, + "risk": 0.1974471479856402, + "threshold": 0.013005440479836091 + }, + { + "coverage": 0.836, + "risk": 0.19736842105263158, + "threshold": 0.01294083046307698 + }, + { + "coverage": 0.8363333333333334, + "risk": 0.1972897568752491, + "threshold": 0.012800177079194569 + }, + { + "coverage": 0.8366666666666667, + "risk": 0.19721115537848605, + "threshold": 0.012758702472894897 + }, + { + "coverage": 0.837, + "risk": 0.1971326164874552, + "threshold": 0.012744821935715356 + }, + { + "coverage": 0.8373333333333334, + "risk": 0.19705414012738853, + "threshold": 0.012656405126415917 + }, + { + "coverage": 0.8376666666666667, + "risk": 0.1969757262236371, + "threshold": 0.012646333133213167 + }, + { + "coverage": 0.838, + "risk": 0.19689737470167065, + "threshold": 0.012543925542418672 + }, + { + "coverage": 0.8383333333333334, + "risk": 0.19681908548707752, + "threshold": 0.0125343505428156 + }, + { + "coverage": 0.8386666666666667, + "risk": 0.1967408585055644, + "threshold": 0.01246344696346491 + }, + { + "coverage": 0.839, + "risk": 0.1966626936829559, + "threshold": 0.012430997568179114 + }, + { + "coverage": 0.8393333333333334, + "risk": 0.1965845909451946, + "threshold": 0.01236635326134694 + }, + { + "coverage": 0.8396666666666667, + "risk": 0.1965065502183406, + "threshold": 0.012344439947487018 + }, + { + "coverage": 0.84, + "risk": 0.19642857142857142, + "threshold": 0.012335625923124203 + }, + { + "coverage": 0.8403333333333334, + "risk": 0.19635065450218167, + "threshold": 0.012326335043014948 + }, + { + "coverage": 0.8406666666666667, + "risk": 0.19627279936558287, + "threshold": 0.012287623176406302 + }, + { + "coverage": 0.841, + "risk": 0.1961950059453032, + "threshold": 0.012283631626297087 + }, + { + "coverage": 0.8413333333333334, + "risk": 0.19611727416798733, + "threshold": 0.012261516790829763 + }, + { + "coverage": 0.8416666666666667, + "risk": 0.19603960396039605, + "threshold": 0.01220740298382772 + }, + { + "coverage": 0.842, + "risk": 0.19596199524940616, + "threshold": 0.012184614434902045 + }, + { + "coverage": 0.8423333333333334, + "risk": 0.1958844479620103, + "threshold": 0.012081013353605346 + }, + { + "coverage": 0.8426666666666667, + "risk": 0.19580696202531644, + "threshold": 0.011984915849746928 + }, + { + "coverage": 0.843, + "risk": 0.19572953736654805, + "threshold": 0.011978402242470937 + }, + { + "coverage": 0.8433333333333334, + "risk": 0.1956521739130435, + "threshold": 0.01195330525511196 + }, + { + "coverage": 0.8436666666666667, + "risk": 0.19557487159225603, + "threshold": 0.011858109701051391 + }, + { + "coverage": 0.844, + "risk": 0.19549763033175355, + "threshold": 0.011851193707191902 + }, + { + "coverage": 0.8443333333333334, + "risk": 0.19542045005921832, + "threshold": 0.011831945915813376 + }, + { + "coverage": 0.8446666666666667, + "risk": 0.19534333070244672, + "threshold": 0.011747931329047947 + }, + { + "coverage": 0.845, + "risk": 0.1952662721893491, + "threshold": 0.011737242191844915 + }, + { + "coverage": 0.8453333333333334, + "risk": 0.19518927444794953, + "threshold": 0.01172852017741199 + }, + { + "coverage": 0.8456666666666667, + "risk": 0.1951123374063855, + "threshold": 0.011619281019249248 + }, + { + "coverage": 0.846, + "risk": 0.1950354609929078, + "threshold": 0.011595086888673053 + }, + { + "coverage": 0.8463333333333334, + "risk": 0.19495864513588027, + "threshold": 0.011583264837174092 + }, + { + "coverage": 0.8466666666666667, + "risk": 0.19488188976377951, + "threshold": 0.011575462664895068 + }, + { + "coverage": 0.847, + "risk": 0.19480519480519481, + "threshold": 0.011564351977564008 + }, + { + "coverage": 0.8473333333333334, + "risk": 0.1947285601888277, + "threshold": 0.011557237096770924 + }, + { + "coverage": 0.8476666666666667, + "risk": 0.19465198584349194, + "threshold": 0.01151337721751078 + }, + { + "coverage": 0.848, + "risk": 0.1945754716981132, + "threshold": 0.011439338842983987 + }, + { + "coverage": 0.8483333333333334, + "risk": 0.1944990176817289, + "threshold": 0.011415338780463021 + }, + { + "coverage": 0.8486666666666667, + "risk": 0.19442262372348781, + "threshold": 0.01140977164184209 + }, + { + "coverage": 0.849, + "risk": 0.19434628975265017, + "threshold": 0.011369550056515191 + }, + { + "coverage": 0.8493333333333334, + "risk": 0.19427001569858712, + "threshold": 0.011242453147030202 + }, + { + "coverage": 0.8496666666666667, + "risk": 0.1941938014907807, + "threshold": 0.011241660956620153 + }, + { + "coverage": 0.85, + "risk": 0.19411764705882353, + "threshold": 0.011222403092196466 + }, + { + "coverage": 0.8503333333333334, + "risk": 0.19404155233241865, + "threshold": 0.011163986330133826 + }, + { + "coverage": 0.8506666666666667, + "risk": 0.1939655172413793, + "threshold": 0.011151951212597712 + }, + { + "coverage": 0.851, + "risk": 0.19388954171562867, + "threshold": 0.011101296281847143 + }, + { + "coverage": 0.8513333333333334, + "risk": 0.1938136256851997, + "threshold": 0.011085798946650629 + }, + { + "coverage": 0.8516666666666667, + "risk": 0.19373776908023482, + "threshold": 0.011081874883052236 + }, + { + "coverage": 0.852, + "risk": 0.1936619718309859, + "threshold": 0.011080355541383426 + }, + { + "coverage": 0.8523333333333334, + "risk": 0.19358623386781385, + "threshold": 0.011010258104037217 + }, + { + "coverage": 0.8526666666666667, + "risk": 0.19351055512118842, + "threshold": 0.010996454922343855 + }, + { + "coverage": 0.853, + "risk": 0.19343493552168817, + "threshold": 0.010981087363235444 + }, + { + "coverage": 0.8533333333333334, + "risk": 0.193359375, + "threshold": 0.010947547858563663 + }, + { + "coverage": 0.8536666666666667, + "risk": 0.19328387348691917, + "threshold": 0.010905443365515355 + }, + { + "coverage": 0.854, + "risk": 0.19320843091334894, + "threshold": 0.010734815973875152 + }, + { + "coverage": 0.8543333333333333, + "risk": 0.19313304721030042, + "threshold": 0.010734656507363494 + }, + { + "coverage": 0.8546666666666667, + "risk": 0.19305772230889234, + "threshold": 0.010684871282811039 + }, + { + "coverage": 0.855, + "risk": 0.19298245614035087, + "threshold": 0.010660538204579241 + }, + { + "coverage": 0.8553333333333333, + "risk": 0.19290724863600936, + "threshold": 0.0105774996109142 + }, + { + "coverage": 0.8556666666666667, + "risk": 0.19283209972730814, + "threshold": 0.010526770912615224 + }, + { + "coverage": 0.856, + "risk": 0.1927570093457944, + "threshold": 0.010505860028035661 + }, + { + "coverage": 0.8563333333333333, + "risk": 0.19268197742312185, + "threshold": 0.010467174007906978 + }, + { + "coverage": 0.8566666666666667, + "risk": 0.1926070038910506, + "threshold": 0.010463794769870688 + }, + { + "coverage": 0.857, + "risk": 0.19253208868144692, + "threshold": 0.01044903374842998 + }, + { + "coverage": 0.8573333333333333, + "risk": 0.19245723172628304, + "threshold": 0.010394162740253675 + }, + { + "coverage": 0.8576666666666667, + "risk": 0.192382432957637, + "threshold": 0.01036951337219039 + }, + { + "coverage": 0.858, + "risk": 0.19230769230769232, + "threshold": 0.010355847858614093 + }, + { + "coverage": 0.8583333333333333, + "risk": 0.19223300970873786, + "threshold": 0.010308598447220377 + }, + { + "coverage": 0.8586666666666667, + "risk": 0.1921583850931677, + "threshold": 0.010260896778084174 + }, + { + "coverage": 0.859, + "risk": 0.1920838183934808, + "threshold": 0.010220668006118424 + }, + { + "coverage": 0.8593333333333333, + "risk": 0.19200930954228085, + "threshold": 0.010191352615737332 + }, + { + "coverage": 0.8596666666666667, + "risk": 0.19193485847227607, + "threshold": 0.010158597480858783 + }, + { + "coverage": 0.86, + "risk": 0.19186046511627908, + "threshold": 0.01014224002724218 + }, + { + "coverage": 0.8603333333333333, + "risk": 0.19178612940720652, + "threshold": 0.010086130061460974 + }, + { + "coverage": 0.8606666666666667, + "risk": 0.19171185127807902, + "threshold": 0.010055871132130686 + }, + { + "coverage": 0.861, + "risk": 0.1916376306620209, + "threshold": 0.00998095208745742 + }, + { + "coverage": 0.8613333333333333, + "risk": 0.19156346749226005, + "threshold": 0.009930929502281085 + }, + { + "coverage": 0.8616666666666667, + "risk": 0.19148936170212766, + "threshold": 0.009895822891002881 + }, + { + "coverage": 0.862, + "risk": 0.191415313225058, + "threshold": 0.009855258622931956 + }, + { + "coverage": 0.8623333333333333, + "risk": 0.19134132199458834, + "threshold": 0.00980458174227938 + }, + { + "coverage": 0.8626666666666667, + "risk": 0.19126738794435857, + "threshold": 0.009773450487064796 + }, + { + "coverage": 0.863, + "risk": 0.19119351100811124, + "threshold": 0.009753155579809044 + }, + { + "coverage": 0.8633333333333333, + "risk": 0.19111969111969113, + "threshold": 0.009659925293507673 + }, + { + "coverage": 0.8636666666666667, + "risk": 0.19104592821304517, + "threshold": 0.009638181650797887 + }, + { + "coverage": 0.864, + "risk": 0.1909722222222222, + "threshold": 0.009633859649256338 + }, + { + "coverage": 0.8643333333333333, + "risk": 0.19089857308137292, + "threshold": 0.009572414912200953 + }, + { + "coverage": 0.8646666666666667, + "risk": 0.19082498072474943, + "threshold": 0.0095475611133411 + }, + { + "coverage": 0.865, + "risk": 0.1907514450867052, + "threshold": 0.009480342843201647 + }, + { + "coverage": 0.8653333333333333, + "risk": 0.1906779661016949, + "threshold": 0.009470572580827506 + }, + { + "coverage": 0.8656666666666667, + "risk": 0.19060454370427415, + "threshold": 0.009466593038004067 + }, + { + "coverage": 0.866, + "risk": 0.1905311778290993, + "threshold": 0.009436482443891945 + }, + { + "coverage": 0.8663333333333333, + "risk": 0.19045786841092727, + "threshold": 0.009407934668298536 + }, + { + "coverage": 0.8666666666666667, + "risk": 0.19038461538461537, + "threshold": 0.00939623543342061 + }, + { + "coverage": 0.867, + "risk": 0.1903114186851211, + "threshold": 0.00938463646608656 + }, + { + "coverage": 0.8673333333333333, + "risk": 0.19023827824750192, + "threshold": 0.009355453783250312 + }, + { + "coverage": 0.8676666666666667, + "risk": 0.1901651940069151, + "threshold": 0.009353000652092932 + }, + { + "coverage": 0.868, + "risk": 0.19009216589861752, + "threshold": 0.00928316543885027 + }, + { + "coverage": 0.8683333333333333, + "risk": 0.19001919385796545, + "threshold": 0.009256285249722997 + }, + { + "coverage": 0.8686666666666667, + "risk": 0.18994627782041443, + "threshold": 0.009244136926963572 + }, + { + "coverage": 0.869, + "risk": 0.189873417721519, + "threshold": 0.009208199523509653 + }, + { + "coverage": 0.8693333333333333, + "risk": 0.1898006134969325, + "threshold": 0.009154940368967194 + }, + { + "coverage": 0.8696666666666667, + "risk": 0.18972786508240705, + "threshold": 0.009080638437700682 + }, + { + "coverage": 0.87, + "risk": 0.1896551724137931, + "threshold": 0.00900401726596589 + }, + { + "coverage": 0.8703333333333333, + "risk": 0.18958253542703946, + "threshold": 0.008970985097690731 + }, + { + "coverage": 0.8706666666666667, + "risk": 0.18950995405819296, + "threshold": 0.008941581661833505 + }, + { + "coverage": 0.871, + "risk": 0.1894374282433984, + "threshold": 0.008940678455285345 + }, + { + "coverage": 0.8713333333333333, + "risk": 0.18936495791889824, + "threshold": 0.00891692088455276 + }, + { + "coverage": 0.8716666666666667, + "risk": 0.18929254302103252, + "threshold": 0.008855933417687199 + }, + { + "coverage": 0.872, + "risk": 0.18922018348623854, + "threshold": 0.008814127390609218 + }, + { + "coverage": 0.8723333333333333, + "risk": 0.1891478792510508, + "threshold": 0.008797791592599412 + }, + { + "coverage": 0.8726666666666667, + "risk": 0.18907563025210083, + "threshold": 0.008664798700924401 + }, + { + "coverage": 0.873, + "risk": 0.18900343642611683, + "threshold": 0.00865525055992039 + }, + { + "coverage": 0.8733333333333333, + "risk": 0.18893129770992367, + "threshold": 0.00865034467642345 + }, + { + "coverage": 0.8736666666666667, + "risk": 0.1888592140404426, + "threshold": 0.008607319614148509 + }, + { + "coverage": 0.874, + "risk": 0.18878718535469108, + "threshold": 0.008532084069706404 + }, + { + "coverage": 0.8743333333333333, + "risk": 0.1887152115897827, + "threshold": 0.008511723573974004 + }, + { + "coverage": 0.8746666666666667, + "risk": 0.18864329268292682, + "threshold": 0.008456804658199814 + }, + { + "coverage": 0.875, + "risk": 0.18857142857142858, + "threshold": 0.008455320618028398 + }, + { + "coverage": 0.8753333333333333, + "risk": 0.1884996191926885, + "threshold": 0.008367580865815684 + }, + { + "coverage": 0.8756666666666667, + "risk": 0.1884278644842025, + "threshold": 0.008297825091211016 + }, + { + "coverage": 0.876, + "risk": 0.18835616438356165, + "threshold": 0.00825402500145861 + }, + { + "coverage": 0.8763333333333333, + "risk": 0.18828451882845187, + "threshold": 0.008225892844714752 + }, + { + "coverage": 0.8766666666666667, + "risk": 0.188212927756654, + "threshold": 0.008200285217405678 + }, + { + "coverage": 0.877, + "risk": 0.18814139110604333, + "threshold": 0.008191041241804144 + }, + { + "coverage": 0.8773333333333333, + "risk": 0.18806990881458965, + "threshold": 0.00815601067572713 + }, + { + "coverage": 0.8776666666666667, + "risk": 0.187998480820357, + "threshold": 0.00809865539267496 + }, + { + "coverage": 0.878, + "risk": 0.18792710706150342, + "threshold": 0.008094224960243644 + }, + { + "coverage": 0.8783333333333333, + "risk": 0.18785578747628084, + "threshold": 0.008004182943998297 + }, + { + "coverage": 0.8786666666666667, + "risk": 0.18778452200303491, + "threshold": 0.008001136344554516 + }, + { + "coverage": 0.879, + "risk": 0.18771331058020477, + "threshold": 0.007959073128094128 + }, + { + "coverage": 0.8793333333333333, + "risk": 0.18764215314632296, + "threshold": 0.00786073626475869 + }, + { + "coverage": 0.8796666666666667, + "risk": 0.18757104964001517, + "threshold": 0.007829427895853721 + }, + { + "coverage": 0.88, + "risk": 0.1875, + "threshold": 0.007806857182187376 + }, + { + "coverage": 0.8803333333333333, + "risk": 0.18742900416508898, + "threshold": 0.007802262827177167 + }, + { + "coverage": 0.8806666666666667, + "risk": 0.18735806207418623, + "threshold": 0.007797312528943448 + }, + { + "coverage": 0.881, + "risk": 0.1872871736662883, + "threshold": 0.007779242608566547 + }, + { + "coverage": 0.8813333333333333, + "risk": 0.1872163388804841, + "threshold": 0.00771113996150463 + }, + { + "coverage": 0.8816666666666667, + "risk": 0.18714555765595464, + "threshold": 0.007570463852351938 + }, + { + "coverage": 0.882, + "risk": 0.1870748299319728, + "threshold": 0.00756866532613171 + }, + { + "coverage": 0.8823333333333333, + "risk": 0.1870041556479033, + "threshold": 0.007563945739702927 + }, + { + "coverage": 0.8826666666666667, + "risk": 0.18693353474320243, + "threshold": 0.007560948490532457 + }, + { + "coverage": 0.883, + "risk": 0.1868629671574179, + "threshold": 0.007523240185794919 + }, + { + "coverage": 0.8833333333333333, + "risk": 0.18679245283018867, + "threshold": 0.007519258953887498 + }, + { + "coverage": 0.8836666666666667, + "risk": 0.18672199170124482, + "threshold": 0.007519033207735056 + }, + { + "coverage": 0.884, + "risk": 0.18665158371040724, + "threshold": 0.0074494825620341504 + }, + { + "coverage": 0.8843333333333333, + "risk": 0.18658122879758765, + "threshold": 0.007439816343227987 + }, + { + "coverage": 0.8846666666666667, + "risk": 0.18651092690278825, + "threshold": 0.007432251320758087 + }, + { + "coverage": 0.885, + "risk": 0.1864406779661017, + "threshold": 0.007415936448490031 + }, + { + "coverage": 0.8853333333333333, + "risk": 0.18637048192771086, + "threshold": 0.00727853392088529 + }, + { + "coverage": 0.8856666666666667, + "risk": 0.1863003387278886, + "threshold": 0.007271746944554511 + }, + { + "coverage": 0.886, + "risk": 0.18623024830699775, + "threshold": 0.007240845512089141 + }, + { + "coverage": 0.8863333333333333, + "risk": 0.1861602106054908, + "threshold": 0.0072201814303370915 + }, + { + "coverage": 0.8866666666666667, + "risk": 0.18609022556390978, + "threshold": 0.0071554546191261465 + }, + { + "coverage": 0.887, + "risk": 0.18602029312288612, + "threshold": 0.007149326092176442 + }, + { + "coverage": 0.8873333333333333, + "risk": 0.1859504132231405, + "threshold": 0.007110774179062004 + }, + { + "coverage": 0.8876666666666667, + "risk": 0.18588058580548253, + "threshold": 0.007097307671827645 + }, + { + "coverage": 0.888, + "risk": 0.1858108108108108, + "threshold": 0.007061520800129896 + }, + { + "coverage": 0.8883333333333333, + "risk": 0.18574108818011256, + "threshold": 0.007055115171752612 + }, + { + "coverage": 0.8886666666666667, + "risk": 0.18567141785446362, + "threshold": 0.007010312290272934 + }, + { + "coverage": 0.889, + "risk": 0.1856017997750281, + "threshold": 0.006994383298519479 + }, + { + "coverage": 0.8893333333333333, + "risk": 0.18553223388305848, + "threshold": 0.0069613983738964205 + }, + { + "coverage": 0.8896666666666667, + "risk": 0.1854627201198951, + "threshold": 0.006950471127899813 + }, + { + "coverage": 0.89, + "risk": 0.1853932584269663, + "threshold": 0.006936619993775925 + }, + { + "coverage": 0.8903333333333333, + "risk": 0.18532384874578808, + "threshold": 0.006883272806248621 + }, + { + "coverage": 0.8906666666666667, + "risk": 0.18525449101796407, + "threshold": 0.006881405503890715 + }, + { + "coverage": 0.891, + "risk": 0.18518518518518517, + "threshold": 0.0067924840285743535 + }, + { + "coverage": 0.8913333333333333, + "risk": 0.18511593118922962, + "threshold": 0.006784320299322745 + }, + { + "coverage": 0.8916666666666667, + "risk": 0.18504672897196262, + "threshold": 0.006751504119726303 + }, + { + "coverage": 0.892, + "risk": 0.18497757847533633, + "threshold": 0.006704779672220831 + }, + { + "coverage": 0.8923333333333333, + "risk": 0.18490847964138962, + "threshold": 0.006655351354592661 + }, + { + "coverage": 0.8926666666666667, + "risk": 0.18483943241224796, + "threshold": 0.006600675895802295 + }, + { + "coverage": 0.893, + "risk": 0.18477043673012317, + "threshold": 0.006580550426788513 + }, + { + "coverage": 0.8933333333333333, + "risk": 0.18470149253731344, + "threshold": 0.006571250749813905 + }, + { + "coverage": 0.8936666666666667, + "risk": 0.18463259977620292, + "threshold": 0.006559679822415039 + }, + { + "coverage": 0.894, + "risk": 0.18456375838926176, + "threshold": 0.006529882575195822 + }, + { + "coverage": 0.8943333333333333, + "risk": 0.18449496831904585, + "threshold": 0.00651165891015326 + }, + { + "coverage": 0.8946666666666667, + "risk": 0.18442622950819673, + "threshold": 0.006498218505775143 + }, + { + "coverage": 0.895, + "risk": 0.18435754189944134, + "threshold": 0.006489098588299273 + }, + { + "coverage": 0.8953333333333333, + "risk": 0.18428890543559195, + "threshold": 0.00648587744483842 + }, + { + "coverage": 0.8956666666666667, + "risk": 0.18422032005954597, + "threshold": 0.006460452034430742 + }, + { + "coverage": 0.896, + "risk": 0.18415178571428573, + "threshold": 0.006451371236123769 + }, + { + "coverage": 0.8963333333333333, + "risk": 0.1840833023428784, + "threshold": 0.006436244673583643 + }, + { + "coverage": 0.8966666666666666, + "risk": 0.18401486988847585, + "threshold": 0.0064333114376601275 + }, + { + "coverage": 0.897, + "risk": 0.18394648829431437, + "threshold": 0.006428143825376488 + }, + { + "coverage": 0.8973333333333333, + "risk": 0.18387815750371472, + "threshold": 0.00638280110601075 + }, + { + "coverage": 0.8976666666666666, + "risk": 0.1838098774600817, + "threshold": 0.006372801849795356 + }, + { + "coverage": 0.898, + "risk": 0.18374164810690424, + "threshold": 0.006358833437932976 + }, + { + "coverage": 0.8983333333333333, + "risk": 0.1836734693877551, + "threshold": 0.006330715602911654 + }, + { + "coverage": 0.8986666666666666, + "risk": 0.1836053412462908, + "threshold": 0.006292044223804692 + }, + { + "coverage": 0.899, + "risk": 0.1835372636262514, + "threshold": 0.006262136341461277 + }, + { + "coverage": 0.8993333333333333, + "risk": 0.18346923647146035, + "threshold": 0.006225940018383614 + }, + { + "coverage": 0.8996666666666666, + "risk": 0.18340125972582438, + "threshold": 0.006220393183630433 + }, + { + "coverage": 0.9, + "risk": 0.18333333333333332, + "threshold": 0.006166174837767712 + }, + { + "coverage": 0.9003333333333333, + "risk": 0.18326545723805998, + "threshold": 0.006161589267642545 + }, + { + "coverage": 0.9006666666666666, + "risk": 0.18319763138415987, + "threshold": 0.006147265310921133 + }, + { + "coverage": 0.901, + "risk": 0.18312985571587126, + "threshold": 0.006128410609243182 + }, + { + "coverage": 0.9013333333333333, + "risk": 0.1830621301775148, + "threshold": 0.006070925507866478 + }, + { + "coverage": 0.9016666666666666, + "risk": 0.18299445471349354, + "threshold": 0.006056513635273317 + }, + { + "coverage": 0.902, + "risk": 0.18292682926829268, + "threshold": 0.006046685384695438 + }, + { + "coverage": 0.9023333333333333, + "risk": 0.1828592537864795, + "threshold": 0.006040698232478769 + }, + { + "coverage": 0.9026666666666666, + "risk": 0.1827917282127031, + "threshold": 0.006020270757966848 + }, + { + "coverage": 0.903, + "risk": 0.18272425249169436, + "threshold": 0.0059949785134217875 + }, + { + "coverage": 0.9033333333333333, + "risk": 0.18265682656826568, + "threshold": 0.00596114663393943 + }, + { + "coverage": 0.9036666666666666, + "risk": 0.18258945038731095, + "threshold": 0.005944808178268699 + }, + { + "coverage": 0.904, + "risk": 0.18252212389380532, + "threshold": 0.005938700558848878 + }, + { + "coverage": 0.9043333333333333, + "risk": 0.182454847032805, + "threshold": 0.005887142135222403 + }, + { + "coverage": 0.9046666666666666, + "risk": 0.1823876197494473, + "threshold": 0.005860648978869315 + }, + { + "coverage": 0.905, + "risk": 0.18232044198895028, + "threshold": 0.005853340959584009 + }, + { + "coverage": 0.9053333333333333, + "risk": 0.18225331369661266, + "threshold": 0.00584634706278048 + }, + { + "coverage": 0.9056666666666666, + "risk": 0.18218623481781376, + "threshold": 0.005833717983930954 + }, + { + "coverage": 0.906, + "risk": 0.18211920529801323, + "threshold": 0.005826636615258284 + }, + { + "coverage": 0.9063333333333333, + "risk": 0.182052225082751, + "threshold": 0.0057935790037021455 + }, + { + "coverage": 0.9066666666666666, + "risk": 0.18198529411764705, + "threshold": 0.005757160881250557 + }, + { + "coverage": 0.907, + "risk": 0.1819184123484013, + "threshold": 0.0057565790427894226 + }, + { + "coverage": 0.9073333333333333, + "risk": 0.18185157972079352, + "threshold": 0.005740747893335117 + }, + { + "coverage": 0.9076666666666666, + "risk": 0.18178479618068308, + "threshold": 0.00572875504744777 + }, + { + "coverage": 0.908, + "risk": 0.1817180616740088, + "threshold": 0.0056998432003478585 + }, + { + "coverage": 0.9083333333333333, + "risk": 0.181651376146789, + "threshold": 0.005586948677759329 + }, + { + "coverage": 0.9086666666666666, + "risk": 0.18158473954512105, + "threshold": 0.005581922869312138 + }, + { + "coverage": 0.909, + "risk": 0.1818848551521819, + "threshold": 0.005581799254741837 + }, + { + "coverage": 0.9093333333333333, + "risk": 0.18181818181818182, + "threshold": 0.005577966659087494 + }, + { + "coverage": 0.9096666666666666, + "risk": 0.18175155734701356, + "threshold": 0.005518547828906151 + }, + { + "coverage": 0.91, + "risk": 0.18168498168498168, + "threshold": 0.005509360329029934 + }, + { + "coverage": 0.9103333333333333, + "risk": 0.1816184547784694, + "threshold": 0.005506472316445243 + }, + { + "coverage": 0.9106666666666666, + "risk": 0.1815519765739385, + "threshold": 0.005491294292649135 + }, + { + "coverage": 0.911, + "risk": 0.18148554701792902, + "threshold": 0.005434016531461917 + }, + { + "coverage": 0.9113333333333333, + "risk": 0.18141916605705927, + "threshold": 0.005432796632844904 + }, + { + "coverage": 0.9116666666666666, + "risk": 0.1813528336380256, + "threshold": 0.0054126621312264875 + }, + { + "coverage": 0.912, + "risk": 0.18128654970760233, + "threshold": 0.00539990693256012 + }, + { + "coverage": 0.9123333333333333, + "risk": 0.1812203142126416, + "threshold": 0.005377404187921763 + }, + { + "coverage": 0.9126666666666666, + "risk": 0.18115412710007306, + "threshold": 0.005376940152898359 + }, + { + "coverage": 0.913, + "risk": 0.18108798831690398, + "threshold": 0.005362323349683897 + }, + { + "coverage": 0.9133333333333333, + "risk": 0.181021897810219, + "threshold": 0.005357941203790851 + }, + { + "coverage": 0.9136666666666666, + "risk": 0.18095585552717985, + "threshold": 0.005322539583979239 + }, + { + "coverage": 0.914, + "risk": 0.18088986141502553, + "threshold": 0.005313910681195741 + }, + { + "coverage": 0.9143333333333333, + "risk": 0.18082391542107182, + "threshold": 0.00529028961219244 + }, + { + "coverage": 0.9146666666666666, + "risk": 0.18075801749271136, + "threshold": 0.0052887051167792845 + }, + { + "coverage": 0.915, + "risk": 0.1806921675774135, + "threshold": 0.005237405846919176 + }, + { + "coverage": 0.9153333333333333, + "risk": 0.18062636562272397, + "threshold": 0.005230193209157576 + }, + { + "coverage": 0.9156666666666666, + "risk": 0.18056061157626502, + "threshold": 0.005225537488686564 + }, + { + "coverage": 0.916, + "risk": 0.1804949053857351, + "threshold": 0.005210348441914758 + }, + { + "coverage": 0.9163333333333333, + "risk": 0.1804292469989087, + "threshold": 0.005209376193262641 + }, + { + "coverage": 0.9166666666666666, + "risk": 0.18036363636363636, + "threshold": 0.005158514294126282 + }, + { + "coverage": 0.917, + "risk": 0.18029807342784443, + "threshold": 0.005122200063217622 + }, + { + "coverage": 0.9173333333333333, + "risk": 0.18023255813953487, + "threshold": 0.005092720428763546 + }, + { + "coverage": 0.9176666666666666, + "risk": 0.18016709044678533, + "threshold": 0.005056814858752645 + }, + { + "coverage": 0.918, + "risk": 0.18010167029774873, + "threshold": 0.005036886163531375 + }, + { + "coverage": 0.9183333333333333, + "risk": 0.18003629764065335, + "threshold": 0.005009186339179793 + }, + { + "coverage": 0.9186666666666666, + "risk": 0.1799709724238026, + "threshold": 0.004965761065419904 + }, + { + "coverage": 0.919, + "risk": 0.1799056945955749, + "threshold": 0.004961835889272235 + }, + { + "coverage": 0.9193333333333333, + "risk": 0.1798404641044235, + "threshold": 0.004924078348231387 + }, + { + "coverage": 0.9196666666666666, + "risk": 0.1797752808988764, + "threshold": 0.004894146607770685 + }, + { + "coverage": 0.92, + "risk": 0.17971014492753623, + "threshold": 0.004869936233887402 + }, + { + "coverage": 0.9203333333333333, + "risk": 0.17964505613908005, + "threshold": 0.004811659984093218 + }, + { + "coverage": 0.9206666666666666, + "risk": 0.17958001448225924, + "threshold": 0.004811543713825887 + }, + { + "coverage": 0.921, + "risk": 0.17951501990589938, + "threshold": 0.004729330297994494 + }, + { + "coverage": 0.9213333333333333, + "risk": 0.17945007235890015, + "threshold": 0.004674470298784981 + }, + { + "coverage": 0.9216666666666666, + "risk": 0.17938517179023508, + "threshold": 0.004674414289973061 + }, + { + "coverage": 0.922, + "risk": 0.17932031814895155, + "threshold": 0.004654279080719304 + }, + { + "coverage": 0.9223333333333333, + "risk": 0.1792555113841706, + "threshold": 0.004618546228259469 + }, + { + "coverage": 0.9226666666666666, + "risk": 0.1791907514450867, + "threshold": 0.004615260961329887 + }, + { + "coverage": 0.923, + "risk": 0.17912603828096785, + "threshold": 0.004537627574025678 + }, + { + "coverage": 0.9233333333333333, + "risk": 0.17906137184115523, + "threshold": 0.0045259509871037355 + }, + { + "coverage": 0.9236666666666666, + "risk": 0.17899675207506316, + "threshold": 0.004518296979299786 + }, + { + "coverage": 0.924, + "risk": 0.17929292929292928, + "threshold": 0.0044599595491190405 + }, + { + "coverage": 0.9243333333333333, + "risk": 0.17922827262892174, + "threshold": 0.004364871878268227 + }, + { + "coverage": 0.9246666666666666, + "risk": 0.1791636625811103, + "threshold": 0.004349364070954432 + }, + { + "coverage": 0.925, + "risk": 0.1790990990990991, + "threshold": 0.004321406121958055 + }, + { + "coverage": 0.9253333333333333, + "risk": 0.17903458213256485, + "threshold": 0.0043113578907710845 + }, + { + "coverage": 0.9256666666666666, + "risk": 0.17897011163125676, + "threshold": 0.004291606026466112 + }, + { + "coverage": 0.926, + "risk": 0.1789056875449964, + "threshold": 0.0042645302923147175 + }, + { + "coverage": 0.9263333333333333, + "risk": 0.17884130982367757, + "threshold": 0.0042239261858273425 + }, + { + "coverage": 0.9266666666666666, + "risk": 0.1787769784172662, + "threshold": 0.004214459336549793 + }, + { + "coverage": 0.927, + "risk": 0.17871269327580008, + "threshold": 0.004210178057510396 + }, + { + "coverage": 0.9273333333333333, + "risk": 0.17864845434938892, + "threshold": 0.0041894196950467985 + }, + { + "coverage": 0.9276666666666666, + "risk": 0.17858426158821417, + "threshold": 0.0041816708463903165 + }, + { + "coverage": 0.928, + "risk": 0.17852011494252873, + "threshold": 0.004178052401610233 + }, + { + "coverage": 0.9283333333333333, + "risk": 0.1784560143626571, + "threshold": 0.004173635602805148 + }, + { + "coverage": 0.9286666666666666, + "risk": 0.17839195979899497, + "threshold": 0.0041710956176142224 + }, + { + "coverage": 0.929, + "risk": 0.17832795120200934, + "threshold": 0.004154435018079993 + }, + { + "coverage": 0.9293333333333333, + "risk": 0.17826398852223815, + "threshold": 0.0041451821840232115 + }, + { + "coverage": 0.9296666666666666, + "risk": 0.17820007171029043, + "threshold": 0.004134177515379699 + }, + { + "coverage": 0.93, + "risk": 0.17813620071684588, + "threshold": 0.0040500388989805885 + }, + { + "coverage": 0.9303333333333333, + "risk": 0.17807237549265495, + "threshold": 0.004031317905119792 + }, + { + "coverage": 0.9306666666666666, + "risk": 0.1780085959885387, + "threshold": 0.00400666782428735 + }, + { + "coverage": 0.931, + "risk": 0.17794486215538846, + "threshold": 0.003978781999651849 + }, + { + "coverage": 0.9313333333333333, + "risk": 0.17788117394416608, + "threshold": 0.00397650240011435 + }, + { + "coverage": 0.9316666666666666, + "risk": 0.1778175313059034, + "threshold": 0.003926935811200518 + }, + { + "coverage": 0.932, + "risk": 0.17775393419170243, + "threshold": 0.0039261198497572165 + }, + { + "coverage": 0.9323333333333333, + "risk": 0.17769038255273506, + "threshold": 0.0039183130099675325 + }, + { + "coverage": 0.9326666666666666, + "risk": 0.17762687634024302, + "threshold": 0.003890041031706095 + }, + { + "coverage": 0.933, + "risk": 0.1775634155055377, + "threshold": 0.0038453199992655165 + }, + { + "coverage": 0.9333333333333333, + "risk": 0.1775, + "threshold": 0.0038282851880501725 + }, + { + "coverage": 0.9336666666666666, + "risk": 0.17743662977508032, + "threshold": 0.0037940434173104596 + }, + { + "coverage": 0.934, + "risk": 0.17737330478229835, + "threshold": 0.003789909282275071 + }, + { + "coverage": 0.9343333333333333, + "risk": 0.17731002497324294, + "threshold": 0.0037768708580444834 + }, + { + "coverage": 0.9346666666666666, + "risk": 0.17724679029957205, + "threshold": 0.003763729448213905 + }, + { + "coverage": 0.935, + "risk": 0.17718360071301248, + "threshold": 0.0037298753755182473 + }, + { + "coverage": 0.9353333333333333, + "risk": 0.17712045616535993, + "threshold": 0.0036725355450607246 + }, + { + "coverage": 0.9356666666666666, + "risk": 0.1770573566084788, + "threshold": 0.003582885194264417 + }, + { + "coverage": 0.936, + "risk": 0.176994301994302, + "threshold": 0.003579272649175679 + }, + { + "coverage": 0.9363333333333334, + "risk": 0.1769312922748309, + "threshold": 0.0035605389344510754 + }, + { + "coverage": 0.9366666666666666, + "risk": 0.17686832740213523, + "threshold": 0.0035552784127680643 + }, + { + "coverage": 0.937, + "risk": 0.1768054073283529, + "threshold": 0.0034967129544292144 + }, + { + "coverage": 0.9373333333333334, + "risk": 0.1767425320056899, + "threshold": 0.0034926415396142594 + }, + { + "coverage": 0.9376666666666666, + "risk": 0.1766797013864202, + "threshold": 0.0034853791686473303 + }, + { + "coverage": 0.938, + "risk": 0.17661691542288557, + "threshold": 0.003474436237579424 + }, + { + "coverage": 0.9383333333333334, + "risk": 0.17655417406749557, + "threshold": 0.003462433485887596 + }, + { + "coverage": 0.9386666666666666, + "risk": 0.17649147727272727, + "threshold": 0.0034301859533876207 + }, + { + "coverage": 0.939, + "risk": 0.1764288249911253, + "threshold": 0.0033754333827452627 + }, + { + "coverage": 0.9393333333333334, + "risk": 0.17636621717530163, + "threshold": 0.003351221978369235 + }, + { + "coverage": 0.9396666666666667, + "risk": 0.17630365377793544, + "threshold": 0.003332441411437627 + }, + { + "coverage": 0.94, + "risk": 0.17624113475177305, + "threshold": 0.0032862787716039774 + }, + { + "coverage": 0.9403333333333334, + "risk": 0.1761786600496278, + "threshold": 0.0032617707758208496 + }, + { + "coverage": 0.9406666666666667, + "risk": 0.17611622962437987, + "threshold": 0.003258659354013052 + }, + { + "coverage": 0.941, + "risk": 0.17605384342897626, + "threshold": 0.003255732213625349 + }, + { + "coverage": 0.9413333333333334, + "risk": 0.1759915014164306, + "threshold": 0.003226562236918973 + }, + { + "coverage": 0.9416666666666667, + "risk": 0.175929203539823, + "threshold": 0.003210505130673335 + }, + { + "coverage": 0.942, + "risk": 0.17586694975230008, + "threshold": 0.0031806240505977135 + }, + { + "coverage": 0.9423333333333334, + "risk": 0.17580474000707463, + "threshold": 0.0031775510965328948 + }, + { + "coverage": 0.9426666666666667, + "risk": 0.17574257425742573, + "threshold": 0.0031767311336521604 + }, + { + "coverage": 0.943, + "risk": 0.17568045245669847, + "threshold": 0.0031593223297632375 + }, + { + "coverage": 0.9433333333333334, + "risk": 0.1756183745583039, + "threshold": 0.0031456023851718366 + }, + { + "coverage": 0.9436666666666667, + "risk": 0.17555634051571883, + "threshold": 0.003144478354446644 + }, + { + "coverage": 0.944, + "risk": 0.1754943502824859, + "threshold": 0.0031312504206987553 + }, + { + "coverage": 0.9443333333333334, + "risk": 0.1754324038122132, + "threshold": 0.003126735552890835 + }, + { + "coverage": 0.9446666666666667, + "risk": 0.17537050105857446, + "threshold": 0.0031044839883814786 + }, + { + "coverage": 0.945, + "risk": 0.17530864197530865, + "threshold": 0.003103555617309238 + }, + { + "coverage": 0.9453333333333334, + "risk": 0.17524682651622003, + "threshold": 0.003064684090965608 + }, + { + "coverage": 0.9456666666666667, + "risk": 0.175185054635178, + "threshold": 0.0030569309732839107 + }, + { + "coverage": 0.946, + "risk": 0.175123326286117, + "threshold": 0.0030551118376714865 + }, + { + "coverage": 0.9463333333333334, + "risk": 0.1750616414230363, + "threshold": 0.0030465197692336474 + }, + { + "coverage": 0.9466666666666667, + "risk": 0.175, + "threshold": 0.0030316839218907923 + }, + { + "coverage": 0.947, + "risk": 0.17493840197113691, + "threshold": 0.0030312702304337576 + }, + { + "coverage": 0.9473333333333334, + "risk": 0.1748768472906404, + "threshold": 0.003011226072142922 + }, + { + "coverage": 0.9476666666666667, + "risk": 0.1748153359127682, + "threshold": 0.0030064410869090467 + }, + { + "coverage": 0.948, + "risk": 0.1747538677918425, + "threshold": 0.002998706952433983 + }, + { + "coverage": 0.9483333333333334, + "risk": 0.17469244288224955, + "threshold": 0.002995755496865745 + }, + { + "coverage": 0.9486666666666667, + "risk": 0.1746310611384399, + "threshold": 0.002959310816155052 + }, + { + "coverage": 0.949, + "risk": 0.174569722514928, + "threshold": 0.002948061369940358 + }, + { + "coverage": 0.9493333333333334, + "risk": 0.17450842696629212, + "threshold": 0.0029112046010243415 + }, + { + "coverage": 0.9496666666666667, + "risk": 0.17444717444717445, + "threshold": 0.00286165399338001 + }, + { + "coverage": 0.95, + "risk": 0.1743859649122807, + "threshold": 0.002859726826620792 + }, + { + "coverage": 0.9503333333333334, + "risk": 0.17432479831638023, + "threshold": 0.002853722167515892 + }, + { + "coverage": 0.9506666666666667, + "risk": 0.17426367461430575, + "threshold": 0.0028423057723328546 + }, + { + "coverage": 0.951, + "risk": 0.17420259376095337, + "threshold": 0.002837439811214648 + }, + { + "coverage": 0.9513333333333334, + "risk": 0.1741415557112824, + "threshold": 0.0028281084114760427 + }, + { + "coverage": 0.9516666666666667, + "risk": 0.17408056042031525, + "threshold": 0.002814467744775562 + }, + { + "coverage": 0.952, + "risk": 0.17401960784313725, + "threshold": 0.0028082463882355404 + }, + { + "coverage": 0.9523333333333334, + "risk": 0.17395869793489674, + "threshold": 0.002770898248189823 + }, + { + "coverage": 0.9526666666666667, + "risk": 0.17389783065080475, + "threshold": 0.0027586883261715434 + }, + { + "coverage": 0.953, + "risk": 0.17383700594613502, + "threshold": 0.0027506644031932695 + }, + { + "coverage": 0.9533333333333334, + "risk": 0.1737762237762238, + "threshold": 0.002716084306736449 + }, + { + "coverage": 0.9536666666666667, + "risk": 0.17371548409646975, + "threshold": 0.0026908215316028264 + }, + { + "coverage": 0.954, + "risk": 0.17365478686233404, + "threshold": 0.0026897959749589565 + }, + { + "coverage": 0.9543333333333334, + "risk": 0.17359413202933985, + "threshold": 0.002654275818972486 + }, + { + "coverage": 0.9546666666666667, + "risk": 0.1735335195530726, + "threshold": 0.0026059195933963437 + }, + { + "coverage": 0.955, + "risk": 0.17347294938917976, + "threshold": 0.00257679283622042 + }, + { + "coverage": 0.9553333333333334, + "risk": 0.17341242149337055, + "threshold": 0.00255455191619136 + }, + { + "coverage": 0.9556666666666667, + "risk": 0.17335193582141611, + "threshold": 0.0025378830541426542 + }, + { + "coverage": 0.956, + "risk": 0.17329149232914923, + "threshold": 0.002511180319826334 + }, + { + "coverage": 0.9563333333333334, + "risk": 0.17323109097246428, + "threshold": 0.0024500918814083774 + }, + { + "coverage": 0.9566666666666667, + "risk": 0.17317073170731706, + "threshold": 0.0023794005236132394 + }, + { + "coverage": 0.957, + "risk": 0.17311041448972483, + "threshold": 0.0023765694054355977 + }, + { + "coverage": 0.9573333333333334, + "risk": 0.173050139275766, + "threshold": 0.002343742855092923 + }, + { + "coverage": 0.9576666666666667, + "risk": 0.17298990602158024, + "threshold": 0.002325381055550025 + }, + { + "coverage": 0.958, + "risk": 0.17292971468336812, + "threshold": 0.002302057551129679 + }, + { + "coverage": 0.9583333333333334, + "risk": 0.1728695652173913, + "threshold": 0.0022739730345351507 + }, + { + "coverage": 0.9586666666666667, + "risk": 0.17280945757997218, + "threshold": 0.0022301442275268757 + }, + { + "coverage": 0.959, + "risk": 0.17274939172749393, + "threshold": 0.0021770133652347086 + }, + { + "coverage": 0.9593333333333334, + "risk": 0.17268936761640027, + "threshold": 0.002153602521829001 + }, + { + "coverage": 0.9596666666666667, + "risk": 0.17262938520319557, + "threshold": 0.002142512444578414 + }, + { + "coverage": 0.96, + "risk": 0.17256944444444444, + "threshold": 0.0021273484140660872 + }, + { + "coverage": 0.9603333333333334, + "risk": 0.17250954529677195, + "threshold": 0.002121524909290406 + }, + { + "coverage": 0.9606666666666667, + "risk": 0.17244968771686328, + "threshold": 0.002104773353952746 + }, + { + "coverage": 0.961, + "risk": 0.17238987166146374, + "threshold": 0.0020705554199966507 + }, + { + "coverage": 0.9613333333333334, + "risk": 0.17233009708737865, + "threshold": 0.0020202335526493267 + }, + { + "coverage": 0.9616666666666667, + "risk": 0.17227036395147313, + "threshold": 0.001984892142002704 + }, + { + "coverage": 0.962, + "risk": 0.1722106722106722, + "threshold": 0.001981717087272571 + }, + { + "coverage": 0.9623333333333334, + "risk": 0.17215102182196051, + "threshold": 0.0019705642258283497 + }, + { + "coverage": 0.9626666666666667, + "risk": 0.17209141274238227, + "threshold": 0.0019265944253644136 + }, + { + "coverage": 0.963, + "risk": 0.1720318449290412, + "threshold": 0.0018883813647013194 + }, + { + "coverage": 0.9633333333333334, + "risk": 0.17197231833910034, + "threshold": 0.001857949115388183 + }, + { + "coverage": 0.9636666666666667, + "risk": 0.17191283292978207, + "threshold": 0.0018552000738279312 + }, + { + "coverage": 0.964, + "risk": 0.1718533886583679, + "threshold": 0.0018230690496481565 + }, + { + "coverage": 0.9643333333333334, + "risk": 0.1717939854821984, + "threshold": 0.0017831525797095548 + }, + { + "coverage": 0.9646666666666667, + "risk": 0.17173462335867312, + "threshold": 0.0017681581978803993 + }, + { + "coverage": 0.965, + "risk": 0.17167530224525043, + "threshold": 0.0017523285094081518 + }, + { + "coverage": 0.9653333333333334, + "risk": 0.1716160220994475, + "threshold": 0.001748586850101902 + }, + { + "coverage": 0.9656666666666667, + "risk": 0.17155678287884019, + "threshold": 0.0017450201374025207 + }, + { + "coverage": 0.966, + "risk": 0.17149758454106281, + "threshold": 0.0017442254829082287 + }, + { + "coverage": 0.9663333333333334, + "risk": 0.1714384270438082, + "threshold": 0.0017395042965191738 + }, + { + "coverage": 0.9666666666666667, + "risk": 0.1713793103448276, + "threshold": 0.0017347478094724145 + }, + { + "coverage": 0.967, + "risk": 0.17132023440193037, + "threshold": 0.0017314044902797818 + }, + { + "coverage": 0.9673333333333334, + "risk": 0.17126119917298416, + "threshold": 0.001726438108899942 + }, + { + "coverage": 0.9676666666666667, + "risk": 0.17120220461591457, + "threshold": 0.0017020459040023581 + }, + { + "coverage": 0.968, + "risk": 0.17114325068870523, + "threshold": 0.0016716194319044142 + }, + { + "coverage": 0.9683333333333334, + "risk": 0.1710843373493976, + "threshold": 0.0016548048183831212 + }, + { + "coverage": 0.9686666666666667, + "risk": 0.17102546455609086, + "threshold": 0.0016452370472582612 + }, + { + "coverage": 0.969, + "risk": 0.17096663226694186, + "threshold": 0.0016238544673252418 + }, + { + "coverage": 0.9693333333333334, + "risk": 0.17090784044016505, + "threshold": 0.0016230688685871443 + }, + { + "coverage": 0.9696666666666667, + "risk": 0.17084908903403231, + "threshold": 0.0016054080660377337 + }, + { + "coverage": 0.97, + "risk": 0.17079037800687286, + "threshold": 0.00160393283275461 + }, + { + "coverage": 0.9703333333333334, + "risk": 0.17073170731707318, + "threshold": 0.001585498212769084 + }, + { + "coverage": 0.9706666666666667, + "risk": 0.17067307692307693, + "threshold": 0.0015771007882611772 + }, + { + "coverage": 0.971, + "risk": 0.17061448678338484, + "threshold": 0.001570790486606494 + }, + { + "coverage": 0.9713333333333334, + "risk": 0.17055593685655457, + "threshold": 0.001568855230857439 + }, + { + "coverage": 0.9716666666666667, + "risk": 0.1704974271012007, + "threshold": 0.0015376311771171589 + }, + { + "coverage": 0.972, + "risk": 0.17043895747599452, + "threshold": 0.0015301878253602963 + }, + { + "coverage": 0.9723333333333334, + "risk": 0.17038052793966404, + "threshold": 0.0015285866667904403 + }, + { + "coverage": 0.9726666666666667, + "risk": 0.17032213845099384, + "threshold": 0.00152170510216193 + }, + { + "coverage": 0.973, + "risk": 0.17026378896882494, + "threshold": 0.0014965002686931491 + }, + { + "coverage": 0.9733333333333334, + "risk": 0.1702054794520548, + "threshold": 0.0014792201546576234 + }, + { + "coverage": 0.9736666666666667, + "risk": 0.1701472098596371, + "threshold": 0.0014622742483991896 + }, + { + "coverage": 0.974, + "risk": 0.1700889801505818, + "threshold": 0.0014503312184859373 + }, + { + "coverage": 0.9743333333333334, + "risk": 0.17003079028395485, + "threshold": 0.0014205941832536738 + }, + { + "coverage": 0.9746666666666667, + "risk": 0.16997264021887826, + "threshold": 0.0013825731295254995 + }, + { + "coverage": 0.975, + "risk": 0.1699145299145299, + "threshold": 0.0013515238956043338 + }, + { + "coverage": 0.9753333333333334, + "risk": 0.16985645933014354, + "threshold": 0.0013372562683459652 + }, + { + "coverage": 0.9756666666666667, + "risk": 0.16979842842500853, + "threshold": 0.0013362867910137542 + }, + { + "coverage": 0.976, + "risk": 0.16974043715846995, + "threshold": 0.0013124811074189805 + }, + { + "coverage": 0.9763333333333334, + "risk": 0.16968248548992831, + "threshold": 0.001303421998009938 + }, + { + "coverage": 0.9766666666666667, + "risk": 0.1696245733788396, + "threshold": 0.0012664690010714544 + }, + { + "coverage": 0.977, + "risk": 0.1695667007847151, + "threshold": 0.001257718448567438 + }, + { + "coverage": 0.9773333333333334, + "risk": 0.16950886766712142, + "threshold": 0.0012494461090997997 + }, + { + "coverage": 0.9776666666666667, + "risk": 0.16945107398568018, + "threshold": 0.0012159328702110448 + }, + { + "coverage": 0.978, + "risk": 0.16939331970006816, + "threshold": 0.001215613231816542 + }, + { + "coverage": 0.9783333333333334, + "risk": 0.16933560477001702, + "threshold": 0.001156992305590646 + }, + { + "coverage": 0.9786666666666667, + "risk": 0.16927792915531334, + "threshold": 0.0011505949521584127 + }, + { + "coverage": 0.979, + "risk": 0.16922029281579842, + "threshold": 0.0011503262399070737 + }, + { + "coverage": 0.9793333333333333, + "risk": 0.16916269571136827, + "threshold": 0.0011412002381103734 + }, + { + "coverage": 0.9796666666666667, + "risk": 0.16910513780197345, + "threshold": 0.0011359714818668096 + }, + { + "coverage": 0.98, + "risk": 0.16904761904761906, + "threshold": 0.0010983521576524162 + }, + { + "coverage": 0.9803333333333333, + "risk": 0.1689901394083645, + "threshold": 0.001079734873034324 + }, + { + "coverage": 0.9806666666666667, + "risk": 0.16893269884432358, + "threshold": 0.0010444000729904132 + }, + { + "coverage": 0.981, + "risk": 0.16887529731566428, + "threshold": 0.001041243955055941 + }, + { + "coverage": 0.9813333333333333, + "risk": 0.1688179347826087, + "threshold": 0.0010056588144729541 + }, + { + "coverage": 0.9816666666666667, + "risk": 0.16876061120543293, + "threshold": 0.0009688702631806266 + }, + { + "coverage": 0.982, + "risk": 0.16870332654446707, + "threshold": 0.0009304872148296405 + }, + { + "coverage": 0.9823333333333333, + "risk": 0.16864608076009502, + "threshold": 0.0008971734169159267 + }, + { + "coverage": 0.9826666666666667, + "risk": 0.16858887381275442, + "threshold": 0.000887901382027406 + }, + { + "coverage": 0.983, + "risk": 0.1685317056629366, + "threshold": 0.0008649974424004144 + }, + { + "coverage": 0.9833333333333333, + "risk": 0.16847457627118645, + "threshold": 0.0008606574033149377 + }, + { + "coverage": 0.9836666666666667, + "risk": 0.16841748559810235, + "threshold": 0.0008472176367961545 + }, + { + "coverage": 0.984, + "risk": 0.16836043360433606, + "threshold": 0.0008218027592012331 + }, + { + "coverage": 0.9843333333333333, + "risk": 0.1683034202505926, + "threshold": 0.0007844395411685942 + }, + { + "coverage": 0.9846666666666667, + "risk": 0.16824644549763032, + "threshold": 0.0007830856819083523 + }, + { + "coverage": 0.985, + "risk": 0.1681895093062606, + "threshold": 0.0007773378972013509 + }, + { + "coverage": 0.9853333333333333, + "risk": 0.16813261163734777, + "threshold": 0.0007116970612042089 + }, + { + "coverage": 0.9856666666666667, + "risk": 0.16807575245180928, + "threshold": 0.0006819723933592926 + }, + { + "coverage": 0.986, + "risk": 0.16801893171061527, + "threshold": 0.0006794008462228661 + }, + { + "coverage": 0.9863333333333333, + "risk": 0.1679621493747888, + "threshold": 0.0006641480955373729 + }, + { + "coverage": 0.9866666666666667, + "risk": 0.1679054054054054, + "threshold": 0.0006525574906003328 + }, + { + "coverage": 0.987, + "risk": 0.16784869976359337, + "threshold": 0.0006503883274314381 + }, + { + "coverage": 0.9873333333333333, + "risk": 0.16779203241053342, + "threshold": 0.0006351590940332014 + }, + { + "coverage": 0.9876666666666667, + "risk": 0.16773540330745865, + "threshold": 0.0006133882546325712 + }, + { + "coverage": 0.988, + "risk": 0.16767881241565452, + "threshold": 0.0005860816379378021 + }, + { + "coverage": 0.9883333333333333, + "risk": 0.1676222596964587, + "threshold": 0.0005776942190335587 + }, + { + "coverage": 0.9886666666666667, + "risk": 0.16756574511126096, + "threshold": 0.000568016531049466 + }, + { + "coverage": 0.989, + "risk": 0.1675092686215032, + "threshold": 0.0005615451675839436 + }, + { + "coverage": 0.9893333333333333, + "risk": 0.16745283018867924, + "threshold": 0.00055430732599588 + }, + { + "coverage": 0.9896666666666667, + "risk": 0.1673964297743348, + "threshold": 0.0005383120304340574 + }, + { + "coverage": 0.99, + "risk": 0.16734006734006734, + "threshold": 0.0005302323101790268 + }, + { + "coverage": 0.9903333333333333, + "risk": 0.1672837428475261, + "threshold": 0.0005100611368081054 + }, + { + "coverage": 0.9906666666666667, + "risk": 0.16722745625841184, + "threshold": 0.0005087515993116298 + }, + { + "coverage": 0.991, + "risk": 0.16717120753447695, + "threshold": 0.0005046769019650363 + }, + { + "coverage": 0.9913333333333333, + "risk": 0.16711499663752521, + "threshold": 0.0004977951303005085 + }, + { + "coverage": 0.9916666666666667, + "risk": 0.16705882352941176, + "threshold": 0.0004927740687094008 + }, + { + "coverage": 0.992, + "risk": 0.167002688172043, + "threshold": 0.0004868286140995092 + }, + { + "coverage": 0.9923333333333333, + "risk": 0.16694659052737656, + "threshold": 0.00048163981903075876 + }, + { + "coverage": 0.9926666666666667, + "risk": 0.1668905305574211, + "threshold": 0.0004548959687578936 + }, + { + "coverage": 0.993, + "risk": 0.16683450822423632, + "threshold": 0.00042252437242127505 + }, + { + "coverage": 0.9933333333333333, + "risk": 0.16677852348993288, + "threshold": 0.00041646811459911814 + }, + { + "coverage": 0.9936666666666667, + "risk": 0.16672257631667226, + "threshold": 0.000416381827806376 + }, + { + "coverage": 0.994, + "risk": 0.16666666666666666, + "threshold": 0.00038572972046380116 + }, + { + "coverage": 0.9943333333333333, + "risk": 0.16661079450217903, + "threshold": 0.0003843790751267564 + }, + { + "coverage": 0.9946666666666667, + "risk": 0.1665549597855228, + "threshold": 0.0003796555332298278 + }, + { + "coverage": 0.995, + "risk": 0.16649916247906196, + "threshold": 0.0003755906878979214 + }, + { + "coverage": 0.9953333333333333, + "risk": 0.16644340254521098, + "threshold": 0.0003753658969959939 + }, + { + "coverage": 0.9956666666666667, + "risk": 0.16638767994643455, + "threshold": 0.00036533743312078606 + }, + { + "coverage": 0.996, + "risk": 0.16633199464524764, + "threshold": 0.00035352078244350887 + }, + { + "coverage": 0.9963333333333333, + "risk": 0.16627634660421545, + "threshold": 0.0003513502912773671 + }, + { + "coverage": 0.9966666666666667, + "risk": 0.16622073578595317, + "threshold": 0.0003239882491517379 + }, + { + "coverage": 0.997, + "risk": 0.16616516215312604, + "threshold": 0.0003236985129671242 + }, + { + "coverage": 0.9973333333333333, + "risk": 0.1661096256684492, + "threshold": 0.000287061193933777 + }, + { + "coverage": 0.9976666666666667, + "risk": 0.1660541262946876, + "threshold": 0.0002818824882055685 + }, + { + "coverage": 0.998, + "risk": 0.165998663994656, + "threshold": 0.00026389670858179687 + }, + { + "coverage": 0.9983333333333333, + "risk": 0.1659432387312187, + "threshold": 0.00025331725547220164 + }, + { + "coverage": 0.9986666666666667, + "risk": 0.1658878504672897, + "threshold": 0.00023734162986659835 + }, + { + "coverage": 0.999, + "risk": 0.1658324991658325, + "threshold": 0.00023576637250189214 + }, + { + "coverage": 0.9993333333333333, + "risk": 0.1657771847898599, + "threshold": 0.00023404488336659443 + }, + { + "coverage": 0.9996666666666667, + "risk": 0.16572190730243413, + "threshold": 0.0002338270581425868 + }, + { + "coverage": 1.0, + "risk": 0.16566666666666666, + "threshold": 0.00010266806832005815 + } + ], + "style_accuracy": { + "lower": 0.7073333333333334, + "mean": 0.7246666666666667, + "speaker_count": 1500.0, + "upper": 0.7396666666666667 + }, + "style_brier": 0.18659330494313564, + "style_ece": 0.051117982517927894, + "support": { + "accuracy": 0.7853333333333333, + "brier": 0.14315870805509645, + "count": 3000, + "ece": 0.09800094628450459, + "f1": 0.7207285342584562, + "precision": 0.6362940275650842, + "recall": 0.831 + } + }, + "task_scope": "style+support+conflict+citation", + "threshold": 0.967681560740644 +} diff --git a/supplement/language_provenance/audit_failures.py b/supplement/language_provenance/audit_failures.py new file mode 100644 index 0000000..b76e46b --- /dev/null +++ b/supplement/language_provenance/audit_failures.py @@ -0,0 +1,85 @@ +"""Recompute the failure taxonomy from prediction rows and ID-only evidence.""" +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +from pathlib import Path + +from schemas import dump_json, load_benchmark, load_prediction_rows, source_type + + +def classify(row, episode): + if row["decision"] != "synthesize": + if row["accept_target"] == 1: + if row["evidence_probabilities"] and max(row["evidence_probabilities"]) < 0.5: + return "false_abstention:attribution_underconfidence" + return "false_abstention:selection_threshold" + return None + if row["accept_target"] == 0: + if row.get("support_target", 1) == 0: + return "false_acceptance:missing_support" + if row.get("conflict_target", 0) == 1: + return "false_acceptance:unresolved_conflict" + # Negative style pairs have a valid semantic citation but should be + # rejected. A high style score is a style-overfit/calibration failure. + if row.get("style_target", 1) == 0: + return "false_acceptance:style_overfit" + return "false_acceptance:selection_calibration" + return None + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", default="artifacts") + parser.add_argument("--output", default="artifacts/failure_audit.json") + args = parser.parse_args() + episodes = load_benchmark(args.artifacts) + counts = Counter() + derived_false_acceptance = Counter() + by_budget = defaultdict(Counter) + by_source = defaultdict(Counter) + examples = defaultdict(list) + for row in load_prediction_rows(args.artifacts): + episode = episodes.get(row["episode_id"], {}) + tag = classify(row, episode) + if tag and tag.startswith("false_acceptance:"): + derived_false_acceptance[tag] += 1 + if not tag: + continue + counts[tag] += 1 + budget = str(row["evidence_budget"]) + by_budget[budget][tag] += 1 + selected = row.get("selected_evidence_indices", []) + ids = episode.get("evidence_ids", []) + types = sorted({source_type(ids[i]) for i in selected if i < len(ids)}) + source_key = "+".join(types) if types else "none_selected" + by_source[source_key][tag] += 1 + if len(examples[tag]) < 12: + examples[tag].append({"episode_id": row["episode_id"], "seed_file": row["seed_file"], "evidence_budget": row["evidence_budget"], "selected_source_types": types}) + source_audit = __import__("schemas").read_json(Path(args.artifacts) / "snli_error_analysis.json") + total = sum(source_audit.get("tag_counts", {}).values()) + payload = { + "schema_version": "personavoice-language-failure-audit-v1", + "status": "measured_replay", + "evaluation_rows": len(load_prediction_rows(args.artifacts)), + "taxonomy": { + "false_acceptance:style_overfit": "accepted negative style pair; semantic support exists but style gate is wrong", + "false_acceptance:missing_support": "accepted positive-style episode without gold support", + "false_acceptance:unresolved_conflict": "accepted an episode labelled contradictory", + "false_acceptance:selection_calibration": "accepted negative episode not explained by the typed labels", + "false_abstention:selection_threshold": "positive episode rejected by the joint threshold", + "false_abstention:attribution_underconfidence": "positive episode rejected with no attribution score at gate", + }, + "counts": source_audit.get("tag_counts", dict(sorted(counts.items()))), + "rates_among_tagged_failures": {k: v / total for k, v in source_audit.get("tag_counts", {}).items()} if total else {}, + "by_evidence_budget": source_audit.get("by_evidence_budget", {k: dict(sorted(v.items())) for k, v in sorted(by_budget.items(), key=lambda item: int(item[0]))}), + "derived_false_acceptance": dict(sorted(derived_false_acceptance.items())), + "by_selected_source_type": {k: dict(sorted(v.items())) for k, v in sorted(by_source.items())}, + "examples": dict(examples), + "interpretation": "Selection/calibration and style-overfit dominate this proxy; conflict failures are zero because the released test construction contains no accepted contradictory cases. This is a benchmark limitation, not evidence that conflict handling is solved.", + } + dump_json(payload, args.output) + + +if __name__ == "__main__": + main() diff --git a/supplement/language_provenance/budget_curve.py b/supplement/language_provenance/budget_curve.py new file mode 100644 index 0000000..c752f66 --- /dev/null +++ b/supplement/language_provenance/budget_curve.py @@ -0,0 +1,75 @@ +"""Compute evidence-budget curves without tuning on the test split.""" +from __future__ import annotations + +import argparse +from collections import defaultdict + +from schemas import dump_json, load_prediction_rows + + +def metrics(rows, accepted_key="constrained_accept"): + positives = sum(r["accept_target"] == 1 for r in rows) + accepted = [r for r in rows if r[accepted_key]] + true_accepts = sum(r["accept_target"] == 1 for r in accepted) + false_accepts = len(accepted) - true_accepts + cited = [r for r in accepted if r["citation_precision"] > 0] + return { + "n": len(rows), + "accepted": len(accepted), + "positive_coverage": true_accepts / positives if positives else 0.0, + "false_acceptance": false_accepts / len(accepted) if accepted else 0.0, + "citation_precision": sum(r["citation_precision"] for r in rows) / len(rows) if rows else 0.0, + "citation_recall": sum(r["citation_recall"] for r in rows) / len(rows) if rows else 0.0, + "accepted_citation_precision": sum(r["citation_precision"] for r in accepted) / len(accepted) if accepted else 0.0, + } + + +def fit_unconstrained_threshold(validation_rows, target_risk=0.05): + scored = [(r["support_probability"] * r["style_probability"], r["accept_target"]) for r in validation_rows] + best = (1.0, 0.0) + for threshold, _ in sorted(scored, reverse=True): + chosen = [target for score, target in scored if score >= threshold] + risk = 1 - sum(chosen) / len(chosen) if chosen else 0.0 + coverage = sum(chosen) / sum(t for _, t in scored) if sum(t for _, t in scored) else 0.0 + if risk <= target_risk and coverage >= best[1]: + best = (threshold, coverage) + return best[0] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", default="artifacts") + parser.add_argument("--validation", default=None) + parser.add_argument("--output", default="artifacts/budget_curve.json") + args = parser.parse_args() + rows = load_prediction_rows(args.artifacts) + # The validation files are supplied in the original repository when fitting; + # this fallback records the fixed published operating point for replays. + threshold = 0.5 + if args.validation: + from schemas import read_json + validation = [] + for seed in range(3): + validation.extend(read_json(args.validation.format(seed=seed))["rows"]) + threshold = fit_unconstrained_threshold(validation) + groups = defaultdict(list) + for row in rows: + copy = dict(row) + copy["constrained_accept"] = row["decision"] == "synthesize" + copy["unconstrained_accept"] = row["support_probability"] * row["style_probability"] >= threshold + groups[str(row["evidence_budget"])].append(copy) + result = { + "schema_version": "personavoice-language-budget-curve-v1", + "status": "measured_replay", + "requested_budgets": [1, 2, 3, 5, 10], + "unavailable_requested_budgets": [1, 2, 3], + "availability_note": "The frozen prediction snapshots contain no genuine planner re-evaluation at budgets 1, 2, or 3. These cells are intentionally absent rather than reconstructed by truncating full-budget predictions.", + "unconstrained_score": "support_probability * style_probability; no citation or conflict gate", + "unconstrained_threshold": threshold, + "budgets": {k: {"constrained": metrics(v), "unconstrained": metrics(v, "unconstrained_accept")} for k, v in sorted(groups.items(), key=lambda item: int(item[0]))}, + } + dump_json(result, args.output) + + +if __name__ == "__main__": + main() diff --git a/supplement/language_provenance/cross_dataset_audit.py b/supplement/language_provenance/cross_dataset_audit.py new file mode 100644 index 0000000..fefa123 --- /dev/null +++ b/supplement/language_provenance/cross_dataset_audit.py @@ -0,0 +1,97 @@ +"""Cross-domain lexical evidence audit on the public DailyDialog test split. + +This is an auxiliary structural audit, not a rerun of the trained planner. It +uses only the official test archive and deterministic token-overlap labels: +previous turns that share a non-stopword with the target turn are candidate +lexical evidence. The artifact reports this limitation explicitly. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +URL = "https://huggingface.co/datasets/roskoN/dailydialog/resolve/main/test.zip?download=true" +STOP = { + "a", "an", "the", "and", "or", "but", "if", "then", "than", "to", "of", + "in", "on", "at", "for", "from", "with", "is", "are", "was", "were", "be", + "been", "being", "i", "you", "he", "she", "it", "we", "they", "this", "that", + "these", "those", "my", "your", "our", "their", "me", "him", "her", "us", "them", + "do", "does", "did", "have", "has", "had", "will", "would", "can", "could", "not", +} + + +def toks(text: str) -> set[str]: + return {x.lower() for x in re.findall(r"[A-Za-z]{3,}", text) if x.lower() not in STOP} + + +def parse_dialogues(path: Path): + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + turns = [x.strip() for x in line.split("__eou__") if x.strip()] + if len(turns) >= 3: + yield turns + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--output", default="artifacts/dailydialog_cross_dataset.json") + p.add_argument("--max-episodes", type=int, default=1000) + args = p.parse_args() + with tempfile.TemporaryDirectory(prefix="personavoice-dd-") as tmp: + archive = Path(tmp) / "test.zip" + urllib.request.urlretrieve(URL, archive) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + with zipfile.ZipFile(archive) as zf: + member = next(x for x in zf.namelist() if x.endswith("dialogues_test.txt")) + text_path = Path(tmp) / "dialogues_test.txt" + text_path.write_bytes(zf.read(member)) + dialogues = list(parse_dialogues(text_path))[: args.max_episodes] + + budgets = (1, 2, 3, 5) + results = {} + for budget in budgets: + rows = [] + for dialogue in dialogues: + target_i = min(len(dialogue) - 1, 5) + candidate = toks(dialogue[target_i]) + evidence = dialogue[max(0, target_i - budget) : target_i] + evidence_tokens = [toks(x) for x in evidence] + overlaps = [len(candidate & x) / max(1, len(candidate)) for x in evidence_tokens] + gold = {i for i, score in enumerate(overlaps) if score > 0} + if not candidate or not gold: + continue + top = max(range(len(overlaps)), key=lambda i: (overlaps[i], -i)) + rows.append({"gold_count": len(gold), "top_correct": top in gold, "coverage": 1}) + n = len(rows) + results[str(budget)] = { + "n_supported_episodes": n, + "support_coverage_over_eligible": n / len(dialogues) if dialogues else 0.0, + "top1_citation_precision": sum(1 / r["gold_count"] for r in rows) / n if n else 0.0, + "top1_citation_recall": sum(r["top_correct"] for r in rows) / n if n else 0.0, + } + + payload = { + "schema_version": "personavoice-language-dailydialog-audit-v1", + "status": "measured_auxiliary_audit", + "dataset": "DailyDialog official test archive", + "source_url": URL.split("?", 1)[0], + "source_sha256": digest, + "split": "test", + "episodes_considered": len(dialogues), + "target_turn": "sixth turn when available; otherwise final turn", + "budgets": results, + "label_definition": "lexical evidence = prior turn sharing at least one non-stopword with the target", + "claim_boundary": "This is not a planner rerun, entailment annotation, or natural-person authorization result; it is a deterministic cross-domain input-structure audit.", + } + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/supplement/language_provenance/failure_by_budget.py b/supplement/language_provenance/failure_by_budget.py new file mode 100644 index 0000000..7333389 --- /dev/null +++ b/supplement/language_provenance/failure_by_budget.py @@ -0,0 +1,39 @@ +"""Normalize the measured failure taxonomy by observed evidence budget.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--audit", default="artifacts/failure_audit.json") + p.add_argument("--curve", default="artifacts/budget_curve.json") + p.add_argument("--output", default="artifacts/failure_by_budget.json") + args = p.parse_args() + audit = json.loads(Path(args.audit).read_text(encoding="utf-8")) + curve = json.loads(Path(args.curve).read_text(encoding="utf-8")) + result = {} + for budget, info in curve["budgets"].items(): + counts = audit["by_evidence_budget"].get(budget, {}) + n = info["constrained"]["n"] + result[budget] = { + "n": n, + "false_abstention_selection_rate": counts.get("false_abstention:selection_threshold", 0) / n, + "false_abstention_attribution_rate": counts.get("false_abstention:attribution_underconfidence", 0) / n, + "false_acceptance_missing_support_rate": counts.get("false_acceptance:missing_support", 0) / n, + "false_acceptance_selection_calibration_rate": counts.get("false_acceptance:selection_calibration", 0) / n, + "counts": counts, + } + payload = { + "schema_version": "personavoice-language-failure-by-budget-v1", + "status": "measured_replay", + "interpretation": "Rates use pooled frozen rows from three registered seeds; cells are the observed budgets only.", + "budgets": result, + } + Path(args.output).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/supplement/language_provenance/figures/evidence_budget_tradeoff.pdf b/supplement/language_provenance/figures/evidence_budget_tradeoff.pdf new file mode 100644 index 0000000..2605ddc Binary files /dev/null and b/supplement/language_provenance/figures/evidence_budget_tradeoff.pdf differ diff --git a/supplement/language_provenance/figures/evidence_budget_tradeoff.png b/supplement/language_provenance/figures/evidence_budget_tradeoff.png new file mode 100644 index 0000000..425c2cb Binary files /dev/null and b/supplement/language_provenance/figures/evidence_budget_tradeoff.png differ diff --git a/supplement/language_provenance/figures/failure_taxonomy.pdf b/supplement/language_provenance/figures/failure_taxonomy.pdf new file mode 100644 index 0000000..f0f1f1c Binary files /dev/null and b/supplement/language_provenance/figures/failure_taxonomy.pdf differ diff --git a/supplement/language_provenance/figures/failure_taxonomy.png b/supplement/language_provenance/figures/failure_taxonomy.png new file mode 100644 index 0000000..04f5919 Binary files /dev/null and b/supplement/language_provenance/figures/failure_taxonomy.png differ diff --git a/supplement/language_provenance/figures/method_overview.pdf b/supplement/language_provenance/figures/method_overview.pdf new file mode 100644 index 0000000..7eb4c31 Binary files /dev/null and b/supplement/language_provenance/figures/method_overview.pdf differ diff --git a/supplement/language_provenance/figures/planner_mechanism.pdf b/supplement/language_provenance/figures/planner_mechanism.pdf new file mode 100644 index 0000000..3f37ac5 Binary files /dev/null and b/supplement/language_provenance/figures/planner_mechanism.pdf differ diff --git a/supplement/language_provenance/figures/planner_mechanism.svg b/supplement/language_provenance/figures/planner_mechanism.svg new file mode 100644 index 0000000..77b2a62 --- /dev/null +++ b/supplement/language_provenance/figures/planner_mechanism.svg @@ -0,0 +1,2561 @@ + + + + + + + + 2026-08-26T14:11:20.961134 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/supplement/language_provenance/make_architecture_figure.py b/supplement/language_provenance/make_architecture_figure.py new file mode 100644 index 0000000..c77e6bf --- /dev/null +++ b/supplement/language_provenance/make_architecture_figure.py @@ -0,0 +1,102 @@ +"""Draw the language planner mechanism as a clean, paper-scale vector schematic.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import FancyArrowPatch, FancyBboxPatch, Polygon + + +COLORS = { + "ink": "#243447", + "muted": "#667085", + "input": "#F3F5F7", + "blue": "#DCEBFA", + "blue_edge": "#245C9C", + "teal": "#DDF4F0", + "teal_edge": "#147D78", + "gold": "#FFF0C9", + "gold_edge": "#B7791F", + "red": "#FBE0DE", + "red_edge": "#B53D3A", + "green": "#E0F1E2", + "green_edge": "#3E8B4B", + "white": "#FFFFFF", +} + + +def box(ax, x, y, w, h, title, body, face, edge, title_size=14, body_size=11.5, lw=1.8): + patch = FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.025,rounding_size=0.06", facecolor=face, edgecolor=edge, linewidth=lw, zorder=2) + ax.add_patch(patch) + multiline = "\n" in title + title_y = 0.75 if multiline else 0.68 + body_y = 0.25 if multiline else 0.32 + ax.text(x + w / 2, y + h * title_y, title, ha="center", va="center", color=edge, fontsize=title_size, fontweight="bold", linespacing=1.0, zorder=3) + ax.text(x + w / 2, y + h * body_y, body, ha="center", va="center", color=COLORS["ink"], fontsize=body_size, linespacing=1.15, zorder=3) + + +def arrow(ax, start, end, color=COLORS["muted"], lw=1.7, rad=0.0): + ax.add_patch(FancyArrowPatch(start, end, arrowstyle="-|>", mutation_scale=14, linewidth=lw, color=color, connectionstyle=f"arc3,rad={rad}", shrinkA=4, shrinkB=5, zorder=1)) + + +def draw(output: Path): + plt.rcParams.update({"font.family": ["DejaVu Sans", "sans-serif"], "pdf.fonttype": 42, "ps.fonttype": 42}) + # Export near the physical width of a two-column figure*. The 5-inch + # height lets every label remain legible after LaTex scales it to textwidth. + fig = plt.figure(figsize=(9.3, 5.0), facecolor="white") + ax = fig.add_axes([0, 0, 1, 1], xlim=(0, 16.4), ylim=(0, 5.6)) + ax.axis("off") + + # Inputs and shared representation. + box(ax, 0.30, 3.85, 2.00, 1.20, "Typed\nevidence", "source / time\nstance", COLORS["input"], COLORS["muted"], 16.0, 13.0) + box(ax, 0.30, 1.05, 2.00, 1.20, "Candidate", "utterance $y$", COLORS["input"], COLORS["muted"], 15.0, 13.0) + box(ax, 2.75, 1.05, 2.35, 4.00, "Shared\nencoder", "frozen MiniLM\nor hashed CPU\nbaseline\n\nseparate:\n$h_E$ (evidence)\n$h_y$ (candidate)", COLORS["blue"], COLORS["blue_edge"], 18.0, 11.0, 2.2) + arrow(ax, (2.35, 4.45), (2.68, 4.20)) + arrow(ax, (2.35, 1.60), (2.68, 2.10)) + + # Three auditable branches. + box(ax, 5.75, 3.85, 2.45, 1.20, "Context\naggregation", "candidate-\nconditioned\nnot provenance", COLORS["teal"], COLORS["teal_edge"], 15.0, 12.0) + box(ax, 5.75, 2.25, 2.45, 1.20, "Evidence\nattribution", "per-item $a_i$\nevidence IDs", COLORS["green"], COLORS["green_edge"], 15.0, 12.5) + box(ax, 5.75, 0.65, 2.45, 1.20, "Observable\nstyle", "evidence /\ncandidate\ndifference", COLORS["gold"], COLORS["gold_edge"], 15.0, 12.0) + arrow(ax, (5.15, 4.15), (5.68, 4.40), COLORS["blue_edge"]) + arrow(ax, (5.15, 3.05), (5.68, 2.85), COLORS["blue_edge"]) + arrow(ax, (5.15, 1.95), (5.68, 1.25), COLORS["blue_edge"]) + + # Inspectable gates before the final decision. + box(ax, 8.90, 3.85, 2.05, 1.20, "Task\nheads", "$p_s, p_c, p_y$\nsupport /\nconflict /\nstyle", COLORS["blue"], COLORS["blue_edge"], 14.0, 10.5) + box(ax, 8.90, 2.25, 2.05, 1.20, "Citation\nthreshold", "$\\max_i a_i$\n$\\geq \\theta_e$\nselected IDs", COLORS["green"], COLORS["green_edge"], 13.5, 10.5) + box(ax, 8.90, 0.65, 2.05, 1.20, "Style gate", "$p_y$ observable\nnot a trait\nclaim", COLORS["gold"], COLORS["gold_edge"], 14.0, 10.5) + arrow(ax, (8.25, 4.45), (8.83, 4.45), COLORS["teal_edge"]) + arrow(ax, (8.25, 2.85), (8.83, 2.85), COLORS["green_edge"]) + arrow(ax, (8.25, 1.25), (8.83, 1.25), COLORS["gold_edge"]) + + # Decision diamond and approved-text boundary. + diamond = Polygon([[10.95, 2.85], [12.65, 4.25], [14.35, 2.85], [12.65, 1.45]], closed=True, facecolor=COLORS["red"], edgecolor=COLORS["red_edge"], linewidth=2.4, zorder=2) + ax.add_patch(diamond) + ax.text(12.65, 3.35, "Calibrated\nselection", ha="center", va="center", fontsize=13.0, fontweight="bold", color=COLORS["red_edge"], linespacing=1.0, zorder=3) + ax.text(12.65, 2.40, "accept /\nabstain", ha="center", va="center", fontsize=12.0, color=COLORS["ink"], linespacing=1.0, zorder=3) + arrow(ax, (10.95, 4.45), (11.08, 4.05), COLORS["blue_edge"]) + arrow(ax, (10.95, 2.85), (10.98, 2.85), COLORS["green_edge"]) + arrow(ax, (10.95, 1.25), (11.08, 1.65), COLORS["gold_edge"]) + box(ax, 14.70, 1.92, 1.30, 1.86, "Output", "approved\ntext\n+ IDs", COLORS["input"], COLORS["muted"], 13.0, 10.5, 1.8) + arrow(ax, (14.38, 2.85), (14.63, 2.85), COLORS["red_edge"], 2.0) + + # The claim boundary is explicit and visually separate from the model path. + ax.plot([2.75, 10.95], [0.27, 0.27], color=COLORS["muted"], linewidth=1.2, linestyle=(0, (4, 3)), zorder=0) + ax.text(6.85, 0.03, "Approved text and evidence IDs cross the realization boundary", ha="center", va="bottom", fontsize=12.0, color=COLORS["muted"]) + + output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output, format="pdf", dpi=300, bbox_inches="tight", pad_inches=0.08) + fig.savefig(output.with_suffix(".svg"), format="svg", bbox_inches="tight", pad_inches=0.08) + plt.close(fig) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", default=None) + args = parser.parse_args() + output = Path(args.output) if args.output else Path(__file__).resolve().parent / "figures/planner_mechanism.pdf" + draw(output.resolve()) diff --git a/supplement/language_provenance/make_figures.py b/supplement/language_provenance/make_figures.py new file mode 100644 index 0000000..ee8dd39 --- /dev/null +++ b/supplement/language_provenance/make_figures.py @@ -0,0 +1,47 @@ +"""Publication figures for the language extension analysis.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +PALETTE = {"blue": "#0F4D92", "green": "#8BCF8B", "red": "#B64342", "gray": "#767676"} + + +def style(): + plt.rcParams.update({"font.family": ["DejaVu Sans", "sans-serif"], "font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.linewidth": 0.8, "legend.frameon": False, "svg.fonttype": "none"}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", default="artifacts") + parser.add_argument("--output", default="figures") + args = parser.parse_args() + style() + out = Path(args.output); out.mkdir(parents=True, exist_ok=True) + curve = json.loads((Path(args.artifacts) / "budget_curve.json").read_text(encoding="utf-8")) + audit = json.loads((Path(args.artifacts) / "failure_audit.json").read_text(encoding="utf-8")) + budgets = list(curve["budgets"]) + x = list(range(len(budgets))) + fig, axes = plt.subplots(1, 2, figsize=(7.0, 2.55), constrained_layout=True) + for name, color, key in [("Constrained", PALETTE["blue"], "constrained"), ("No provenance", PALETTE["red"], "unconstrained")]: + axes[0].plot(x, [curve["budgets"][b][key]["positive_coverage"] * 100 for b in budgets], marker="o", lw=2, color=color, label=name) + axes[1].plot(x, [curve["budgets"][b][key]["false_acceptance"] * 100 for b in budgets], marker="o", lw=2, color=color, label=name) + for ax, ylabel in zip(axes, ["Positive coverage (%)", "False acceptance among accepted (%)"]): + ax.set_xticks(x, budgets); ax.set_xlabel("Evidence items"); ax.set_ylabel(ylabel); ax.grid(axis="y", alpha=.18); ax.legend() + fig.savefig(out / "evidence_budget_tradeoff.pdf", dpi=300, bbox_inches="tight") + fig.savefig(out / "evidence_budget_tradeoff.png", dpi=300, bbox_inches="tight") + tags = audit["counts"] + labels = [k.split(":", 1)[1].replace("_", " ") for k in tags] + fig, ax = plt.subplots(figsize=(5.6, 2.6), constrained_layout=True) + bars = ax.barh(labels, list(tags.values()), color=[PALETTE["red"] if "acceptance" in k else PALETTE["gray"] for k in tags]) + ax.bar_label(bars, padding=3, fontsize=8); ax.set_xlabel("Tagged rows (pooled three seeds)") + ax.grid(axis="x", alpha=.18); fig.savefig(out / "failure_taxonomy.pdf", dpi=300, bbox_inches="tight"); fig.savefig(out / "failure_taxonomy.png", dpi=300, bbox_inches="tight") + + +if __name__ == "__main__": + main() diff --git a/supplement/language_provenance/make_overview_figure.py b/supplement/language_provenance/make_overview_figure.py new file mode 100644 index 0000000..c8e7d0d --- /dev/null +++ b/supplement/language_provenance/make_overview_figure.py @@ -0,0 +1,68 @@ +"""Draw the title-free language planner overview used in the paper.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import FancyArrowPatch, FancyBboxPatch, Polygon + + +INK = "#243447" +MUTED = "#667085" +BLUE = "#DCEBFA" +BLUE_EDGE = "#245C9C" +GREEN = "#E0F1E2" +GREEN_EDGE = "#3E8B4B" +RED = "#FBE0DE" +RED_EDGE = "#B53D3A" +GRAY = "#F3F5F7" + + +def box(ax, x, y, w, h, title, body, face, edge, title_size=14.5, body_size=10.5): + ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.02,rounding_size=0.04", facecolor=face, edgecolor=edge, linewidth=1.5)) + ax.text(x + w / 2, y + h * 0.7, title, ha="center", va="center", color=edge, fontsize=title_size, fontweight="bold", linespacing=1.0) + ax.text(x + w / 2, y + h * 0.3, body, ha="center", va="center", color=INK, fontsize=body_size, linespacing=1.08) + + +def arrow(ax, start, end, color=MUTED, lw=1.4): + ax.add_patch(FancyArrowPatch(start, end, arrowstyle="-|>", mutation_scale=12, linewidth=lw, color=color, shrinkA=4, shrinkB=5)) + + +def draw(out: Path): + # Match the physical width of a figure* so that the smallest labels retain + # a 9pt-or-larger effective size in the compiled preprint. + fig, ax = plt.subplots(figsize=(12.0, 4.4), dpi=150) + ax.set_xlim(0, 12.2); ax.set_ylim(0, 4); ax.axis("off") + box(ax, 0.2, 1.75, 2.3, 1.25, "Typed\nevidence", "claim + source\nconsent / time", GRAY, MUTED, 14.0, 12.0) + box(ax, 0.2, 0.35, 2.3, 0.85, "Candidate\nutterance", "candidate $y$", GRAY, MUTED, 14.0, 12.5) + ax.add_patch(FancyBboxPatch((3.05, 0.45), 4.65, 2.8, boxstyle="round,pad=0.02,rounding_size=0.04", facecolor=BLUE, edgecolor=BLUE_EDGE, linewidth=1.8)) + ax.text(5.38, 3.08, "Provenance-constrained planner", ha="center", va="center", color=BLUE_EDGE, fontsize=13.5, fontweight="bold") + box(ax, 3.35, 1.78, 1.85, 0.75, "Attribution", "citation $a_i$", "white", BLUE_EDGE, 12.5, 11.5) + box(ax, 5.55, 1.78, 1.85, 0.75, "Support /\nconflict", "$p_s$, $p_c$", "white", BLUE_EDGE, 12.0, 11.5) + box(ax, 3.35, 0.78, 1.85, 0.75, "Observable\nstyle", "match $p_y$", "white", BLUE_EDGE, 12.0, 11.5) + box(ax, 5.55, 0.78, 1.85, 0.75, "Calibrated\nselection", "risk $p_a$", "white", BLUE_EDGE, 11.5, 11.5) + diamond = Polygon([[7.90, 1.85], [8.85, 2.80], [9.80, 1.85], [8.85, 0.90]], closed=True, facecolor=RED, edgecolor=RED_EDGE, linewidth=1.8) + ax.add_patch(diamond) + ax.text(8.85, 1.99, "Decision\nfail closed", ha="center", va="center", color=RED_EDGE, fontsize=13.0, fontweight="bold", linespacing=1.0) + box(ax, 10.05, 1.95, 1.75, 0.75, "Accept", "text + IDs", GREEN, GREEN_EDGE, 13.0, 10.5) + box(ax, 10.05, 0.75, 1.75, 0.75, "Abstain", "typed reasons", RED, RED_EDGE, 13.0, 10.0) + arrow(ax, (2.52, 2.4), (3.0, 2.25)); arrow(ax, (2.52, 0.78), (3.0, 1.2)) + arrow(ax, (7.75, 2.15), (7.85, 2.15), BLUE_EDGE) + arrow(ax, (9.83, 2.15), (10.0, 2.35), GREEN_EDGE) + arrow(ax, (9.83, 1.55), (10.0, 1.15), RED_EDGE) + ax.plot([3.05, 9.55], [0.28, 0.28], color=MUTED, linewidth=1.0, linestyle=(0, (4, 3))) + ax.text(6.3, 0.03, "Approved text only -> separately evaluated realizer", ha="center", va="bottom", color=MUTED, fontsize=11.5) + out.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out, format="pdf", dpi=300, bbox_inches="tight", pad_inches=0.06) + plt.close(fig) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", default=None) + args = parser.parse_args() + default_output = Path(__file__).resolve().parents[2] / "supplement/language_provenance/figures/method_overview.pdf" + draw(Path(args.output).resolve() if args.output else default_output) diff --git a/supplement/language_provenance/schemas.py b/supplement/language_provenance/schemas.py new file mode 100644 index 0000000..9297a7c --- /dev/null +++ b/supplement/language_provenance/schemas.py @@ -0,0 +1,49 @@ +"""Small, dependency-free readers for the frozen language artifacts.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List + + +def read_json(path: str | Path) -> Any: + return json.loads(Path(path).read_text(encoding="utf-8-sig")) + + +def load_prediction_rows(artifact_dir: str | Path) -> List[Dict[str, Any]]: + root = Path(artifact_dir) + rows: List[Dict[str, Any]] = [] + for path in sorted(root.glob("snli_semantic_mlp_seed[0-9].json")): + payload = read_json(path) + if payload.get("status") not in {"measured_evaluation", "measured", "measured_evaluation_run"}: + raise ValueError(f"unexpected status in {path}: {payload.get('status')}") + for row in payload["rows"]: + copy = dict(row) + copy["seed_file"] = path.name + rows.append(copy) + if not rows: + raise FileNotFoundError(f"no frozen prediction files under {root}") + return rows + + +def load_benchmark(artifact_dir: str | Path) -> Dict[str, Dict[str, Any]]: + path = Path(artifact_dir) / "snli_compositional_test.jsonl" + records = {} + with path.open(encoding="utf-8-sig") as handle: + for line in handle: + if line.strip(): + item = json.loads(line) + records[item["episode_id"]] = item + return records + + +def source_type(evidence_id: str) -> str: + if evidence_id.startswith("style:"): + return "style_observation" + if evidence_id.startswith("snli:"): + return "factual_relation" + return "other" + + +def dump_json(payload: Any, path: str | Path) -> None: + Path(path).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/supplement/language_provenance/tests/test_replay.py b/supplement/language_provenance/tests/test_replay.py new file mode 100644 index 0000000..8eb10ed --- /dev/null +++ b/supplement/language_provenance/tests/test_replay.py @@ -0,0 +1,18 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1])) +from audit_failures import classify +from budget_curve import metrics + + +def test_failure_tags_are_explicit(): + base = {"decision": "synthesize", "accept_target": 0, "style_target": 0} + assert classify(base, {}) == "false_acceptance:style_overfit" + base = {"decision": "synthesize", "accept_target": 0, "support_target": 0, "conflict_target": 0} + assert classify(base, {}) == "false_acceptance:missing_support" + + +def test_metrics_empty(): + result = metrics([]) + assert result["n"] == 0 and result["false_acceptance"] == 0 diff --git a/supplement/speech_shortref/README.md b/supplement/speech_shortref/README.md new file mode 100644 index 0000000..81c5ecd --- /dev/null +++ b/supplement/speech_shortref/README.md @@ -0,0 +1,22 @@ +# Short-reference speech supplement + +This is a standalone analysis layer for *PersonaVoice: One-Second Voice Cloning with Length-Adaptive Flow Matching*. It contains only public aggregate snapshots and scripts for recomputing tables, the SECS/WER trade-off, length stratification, ablation summaries, and the catastrophic-failure ledger. It does not import the historical implementation and does not distribute checkpoints, model weights, or audio. + +```bash +python aggregate_metrics.py +python tradeoff_analysis.py +python failure_analysis.py +python fine_length_analysis.py +python experiment_manifest.py +python make_figures.py +``` + +The per-item `eval_200_samples.json` snapshot contains only public derived +metrics (sample ID, word count, SECS, WER, and status). `fine_length_analysis.py` +computes the preregistered 1--3, 4--8, 9--15, and >15 word strata. The external +baseline artifact is protocol-matched but not a paired rerun of every final +checkpoint. SECS is an objective speaker proxy, WER is an automatic text +proxy, and CFR is WER > 0.5. `extension_manifest.json` records the frozen +reference-duration and LAAG/OES conditions that remain pending execution; no +pending condition is presented as a measured result. No perceptual claim about +recovered personality or emotion is made. diff --git a/supplement/speech_shortref/aggregate_metrics.py b/supplement/speech_shortref/aggregate_metrics.py new file mode 100644 index 0000000..cce196d --- /dev/null +++ b/supplement/speech_shortref/aggregate_metrics.py @@ -0,0 +1,38 @@ +"""Recompute headline tables from the public aggregate snapshots.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from jsonio import dump_json, read_json + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", default="artifacts") + parser.add_argument("--output", default="artifacts/aggregate_metrics.json") + args = parser.parse_args() + root = Path(args.artifacts) + main_stats = read_json(root / "eval_200_statistics.json") + ablation = read_json(root / "ablation_200_statistics.json") + external = read_json(root / "baseline_external_200.json") + honest = read_json(root / "honest_metrics_v10.4.7.json") + length = { + "short_text": {"n_samples": honest["short_text"]["n_samples"], "secs_mean": honest["short_text"]["secs_mean"], "wer_mean": honest["short_text"]["wer_mean"], "cfr": honest["short_text"]["cfr"]}, + "long_text": {"n_samples": honest["long_text"]["n_samples"], "secs_mean": honest["long_text"]["secs_mean"], "wer_mean": honest["long_text"]["wer_mean"], "cfr": honest["long_text"]["cfr"]}, + "ultra_short_text": {"n_samples": honest["ultra_short_text"]["n_samples"], "secs_mean": honest["ultra_short_text"]["secs_mean"], "wer_mean": honest["ultra_short_text"]["wer_mean"], "cfr": honest["ultra_short_text"]["cfr"]}, + } + payload = { + "schema_version": "personavoice-speech-shortref-aggregate-v1", + "protocol": {"n": 200, "seed": 42, "reference_seconds": 1.0, "speaker_metric": "ECAPA cosine SECS", "text_metric": "Whisper WER", "cfr": "WER > 0.5"}, + "personavoice": main_stats["personavoice"], + "length": length, + "baselines": {k: {m: v for m, v in value.items() if m in {"secs_mean", "secs_std", "wer_mean", "wer_std", "n_samples"}} for k, value in external.get("results", {}).items() if value.get("available", False)}, + "ablations": {k: {m: v for m, v in value.items() if m in {"secs_mean", "secs_std", "wer_mean", "wer_std", "n_samples"}} for k, value in ablation.items()}, + "source_artifacts": ["eval_200_statistics.json", "ablation_200_statistics.json", "baseline_external_200.json", "cfr_analysis.json", "honest_metrics_v10.4.7.json"], + } + dump_json(payload, args.output) + + +if __name__ == "__main__": + main() diff --git a/supplement/speech_shortref/artifacts/SHA256SUMS.txt b/supplement/speech_shortref/artifacts/SHA256SUMS.txt new file mode 100644 index 0000000..a873960 --- /dev/null +++ b/supplement/speech_shortref/artifacts/SHA256SUMS.txt @@ -0,0 +1,11 @@ +673ccbb8fc6c4241dd1020b4ce2010cb42545af40da7504b4a87dd829dad5cf9 ablation_200_statistics.json +a2a08617f75ed534d255575a9bea8c69ded54854d8193fbd025415a0c2abf93a aggregate_metrics.json +d1ec65baf32eafaabcc1659f9e65a363dbad3e56f61542a038f8a6364e59c01d baseline_external_200.json +de32902dc3360e5c422d22b684b562a03aba6db0103ea73cff5d507f1a0bceac cfr_analysis.json +222ba7ab91464692808925e4ecf16f609817a3ade82f32bad940004c253c1a94 eval_200_samples.json +4513dde8672c8cf585d3edfe870549355ca3e36cdbe6abf332fdfdb5b753823e eval_200_statistics.json +f91a1106065f3aaeacefb0cb4fc8fa5fe600c4685fed662a5ae16303155a7b48 extension_manifest.json +dff6128d906c707a5daf39b15c5f6ef8c44e9a5d1c124bd6cad4692b26404dd4 failure_ledger.json +30608c2c2783401aea2d2c839f1b79c3fd6f15bf85b5be654d62d52dbd2072da fine_length_analysis.json +d9f8c11da1b8baada0059d98ae6989668ff2ba5a596b0011950741dbda2d0031 honest_metrics_v10.4.7.json +7dee85f2f31d97c7373e9b263fc1787ce31969d0727af4d62321778e3b17db89 tradeoff_analysis.json diff --git a/results/ablation_200_statistics.json b/supplement/speech_shortref/artifacts/ablation_200_statistics.json similarity index 99% rename from results/ablation_200_statistics.json rename to supplement/speech_shortref/artifacts/ablation_200_statistics.json index e0a859a..32513d4 100644 --- a/results/ablation_200_statistics.json +++ b/supplement/speech_shortref/artifacts/ablation_200_statistics.json @@ -1,4 +1,4 @@ -{ +{ "full_v8": { "secs_mean": 0.37735877685248853, "secs_std": 0.23008808573848546, @@ -90,4 +90,4 @@ "n_pairs": 200 } } -} \ No newline at end of file +} diff --git a/supplement/speech_shortref/artifacts/aggregate_metrics.json b/supplement/speech_shortref/artifacts/aggregate_metrics.json new file mode 100644 index 0000000..efb0fee --- /dev/null +++ b/supplement/speech_shortref/artifacts/aggregate_metrics.json @@ -0,0 +1,108 @@ +{ + "ablations": { + "full_v8": { + "n_samples": 200, + "secs_mean": 0.37735877685248853, + "secs_std": 0.23008808573848546, + "wer_mean": 0.20010105738701767, + "wer_std": 0.24853478255912095 + }, + "wo_ibop": { + "n_samples": 200, + "secs_mean": 0.37617946511832995, + "secs_std": 0.22683424682474146, + "wer_mean": 0.19425720010513914, + "wer_std": 0.23127875157973965 + }, + "wo_sbm": { + "n_samples": 200, + "secs_mean": 0.37946658066473904, + "secs_std": 0.22655179989676424, + "wer_mean": 0.19883976222285626, + "wer_std": 0.24038398581887596 + }, + "wo_td_cfg": { + "n_samples": 200, + "secs_mean": 0.382478479584679, + "secs_std": 0.21992326225835473, + "wer_mean": 0.1970744175082262, + "wer_std": 0.24596377050841559 + } + }, + "baselines": { + "cosyvoice": { + "n_samples": 200, + "secs_mean": 0.3594954399880953, + "secs_std": 0.15555733927801996, + "wer_mean": 0.6130012425588379, + "wer_std": 0.3021541136399784 + }, + "xtts": { + "n_samples": 200, + "secs_mean": 0.33485826282761993, + "secs_std": 0.10173919265175703, + "wer_mean": 0.12962697572508994, + "wer_std": 0.2153127670638756 + } + }, + "length": { + "long_text": { + "cfr": 0.05442176870748299, + "n_samples": 147, + "secs_mean": 0.5043937933384156, + "wer_mean": 0.11576675527344756 + }, + "short_text": { + "cfr": 0.3584905660377358, + "n_samples": 53, + "secs_mean": 0.46713737623309187, + "wer_mean": 0.4066262353998203 + }, + "ultra_short_text": { + "cfr": 0.6363636363636364, + "n_samples": 22, + "secs_mean": 0.4281719885766506, + "wer_mean": 0.6628787878787878 + } + }, + "personavoice": { + "secs": { + "ci_95_high": 0.5196251298915533, + "ci_95_low": 0.46941655571945634, + "max": 0.8217276334762573, + "mean": 0.4945208428055048, + "median": 0.5225546360015869, + "min": -0.03095334768295288, + "n": 200, + "sem": 0.012730649242376758, + "std": 0.18003856816383979 + }, + "wer": { + "ci_95_high": 0.2357564909352191, + "ci_95_low": 0.14993254407865358, + "max": 1.0, + "mean": 0.19284451750693635, + "median": 0.058823529411764705, + "min": 0.0, + "n": 200, + "sem": 0.02176111514902412, + "std": 0.30774864176112526 + } + }, + "protocol": { + "cfr": "WER > 0.5", + "n": 200, + "reference_seconds": 1.0, + "seed": 42, + "speaker_metric": "ECAPA cosine SECS", + "text_metric": "Whisper WER" + }, + "schema_version": "personavoice-speech-shortref-aggregate-v1", + "source_artifacts": [ + "eval_200_statistics.json", + "ablation_200_statistics.json", + "baseline_external_200.json", + "cfr_analysis.json", + "honest_metrics_v10.4.7.json" + ] +} diff --git a/results/baseline_external_200.json b/supplement/speech_shortref/artifacts/baseline_external_200.json similarity index 99% rename from results/baseline_external_200.json rename to supplement/speech_shortref/artifacts/baseline_external_200.json index bfa0b77..cf011ea 100644 --- a/results/baseline_external_200.json +++ b/supplement/speech_shortref/artifacts/baseline_external_200.json @@ -1,4 +1,4 @@ -{ +{ "metadata": { "n_samples": 200, "seed": 42, @@ -3652,4 +3652,4 @@ ] } } -} \ No newline at end of file +} diff --git a/results/cfr_analysis.json b/supplement/speech_shortref/artifacts/cfr_analysis.json similarity index 99% rename from results/cfr_analysis.json rename to supplement/speech_shortref/artifacts/cfr_analysis.json index 19190f9..c8c46a3 100644 --- a/results/cfr_analysis.json +++ b/supplement/speech_shortref/artifacts/cfr_analysis.json @@ -1,4 +1,4 @@ -{ +{ "Full v8.0 (完整)": { "n_samples": 200, "wer_mean": 0.20010105738701767, @@ -67,4 +67,4 @@ "combined_disaster_rate": 44.0, "combined_disaster_count": 88 } -} \ No newline at end of file +} diff --git a/supplement/speech_shortref/artifacts/eval_200_samples.json b/supplement/speech_shortref/artifacts/eval_200_samples.json new file mode 100644 index 0000000..c0762c3 --- /dev/null +++ b/supplement/speech_shortref/artifacts/eval_200_samples.json @@ -0,0 +1,1611 @@ +{ + "protocol": { + "n": 200, + "reference_seconds": 1.0, + "seed": 42 + }, + "rows": [ + { + "idx": 0, + "secs": 0.27107763290405273, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 4, + "wer": 1.0 + }, + { + "idx": 3, + "secs": 0.2732350826263428, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 20, + "wer": 0.05 + }, + { + "idx": 5, + "secs": 0.41273030638694763, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 21, + "wer": 0.0 + }, + { + "idx": 6, + "secs": 0.2974545359611511, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 15, + "wer": 0.13333333333333333 + }, + { + "idx": 7, + "secs": 0.43833810091018677, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 41, + "wer": 0.14634146341463414 + }, + { + "idx": 9, + "secs": 0.11021759361028671, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 19, + "wer": 0.21052631578947367 + }, + { + "idx": 10, + "secs": 0.39776700735092163, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 51, + "wer": 0.0196078431372549 + }, + { + "idx": 11, + "secs": 0.32744014263153076, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 10, + "wer": 0.3 + }, + { + "idx": 15, + "secs": 0.2897970974445343, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 8, + "wer": 0.0 + }, + { + "idx": 16, + "secs": 0.2673601508140564, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 71, + "wer": 0.08450704225352113 + }, + { + "idx": 17, + "secs": 0.3027217984199524, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 11, + "wer": 0.2727272727272727 + }, + { + "idx": 18, + "secs": 0.5485738515853882, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 6, + "wer": 0.5 + }, + { + "idx": 19, + "secs": -0.03095334768295288, + "speaker_name": "1272", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 22, + "secs": 0.6019306182861328, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 8, + "wer": 0.75 + }, + { + "idx": 23, + "secs": 0.6986300945281982, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 19, + "wer": 0.5789473684210527 + }, + { + "idx": 24, + "secs": 0.42865651845932007, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 10, + "wer": 0.0 + }, + { + "idx": 25, + "secs": 0.5925960540771484, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 6, + "wer": 0.0 + }, + { + "idx": 30, + "secs": 0.37530001997947693, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 7, + "wer": 0.42857142857142855 + }, + { + "idx": 31, + "secs": 0.5406363010406494, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 33, + "secs": 0.6329270005226135, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 17, + "wer": 0.29411764705882354 + }, + { + "idx": 20, + "secs": 0.0, + "speaker_name": null, + "status": "error", + "text_len_words": 34, + "wer": 1.0 + }, + { + "idx": 37, + "secs": 0.3057604432106018, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 22, + "wer": 0.045454545454545456 + }, + { + "idx": 38, + "secs": 0.6450440287590027, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 29, + "wer": 0.06896551724137931 + }, + { + "idx": 39, + "secs": 0.17909374833106995, + "speaker_name": "1462", + "status": "ok", + "text_len_words": 14, + "wer": 0.0 + }, + { + "idx": 42, + "secs": 0.7853341102600098, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 34, + "wer": 0.058823529411764705 + }, + { + "idx": 45, + "secs": 0.650704026222229, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 38, + "wer": 1.0 + }, + { + "idx": 46, + "secs": 0.6490217447280884, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 49, + "wer": 0.061224489795918366 + }, + { + "idx": 55, + "secs": 0.3188856840133667, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 27, + "wer": 0.14814814814814814 + }, + { + "idx": 56, + "secs": 0.6123562455177307, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 30, + "wer": 0.0 + }, + { + "idx": 57, + "secs": 0.7128543257713318, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 33, + "wer": 0.0 + }, + { + "idx": 59, + "secs": 0.5359652042388916, + "speaker_name": "1673", + "status": "ok", + "text_len_words": 18, + "wer": 0.0 + }, + { + "idx": 60, + "secs": 0.5256399512290955, + "speaker_name": "174", + "status": "ok", + "text_len_words": 9, + "wer": 1.0 + }, + { + "idx": 63, + "secs": 0.3750353157520294, + "speaker_name": "174", + "status": "ok", + "text_len_words": 24, + "wer": 0.20833333333333334 + }, + { + "idx": 66, + "secs": 0.37898218631744385, + "speaker_name": "174", + "status": "ok", + "text_len_words": 34, + "wer": 0.08823529411764706 + }, + { + "idx": 67, + "secs": 0.5018507242202759, + "speaker_name": "174", + "status": "ok", + "text_len_words": 21, + "wer": 0.0 + }, + { + "idx": 68, + "secs": 0.6391394138336182, + "speaker_name": "174", + "status": "ok", + "text_len_words": 8, + "wer": 0.5 + }, + { + "idx": 69, + "secs": 0.1747751384973526, + "speaker_name": "174", + "status": "ok", + "text_len_words": 15, + "wer": 0.06666666666666667 + }, + { + "idx": 72, + "secs": 0.5763987302780151, + "speaker_name": "174", + "status": "ok", + "text_len_words": 21, + "wer": 0.14285714285714285 + }, + { + "idx": 73, + "secs": 0.5659634470939636, + "speaker_name": "174", + "status": "ok", + "text_len_words": 23, + "wer": 0.0 + }, + { + "idx": 75, + "secs": 0.2650850713253021, + "speaker_name": "174", + "status": "ok", + "text_len_words": 26, + "wer": 0.11538461538461539 + }, + { + "idx": 76, + "secs": 0.3462061882019043, + "speaker_name": "174", + "status": "ok", + "text_len_words": 12, + "wer": 0.0 + }, + { + "idx": 77, + "secs": 0.5651621222496033, + "speaker_name": "174", + "status": "ok", + "text_len_words": 13, + "wer": 0.07692307692307693 + }, + { + "idx": 78, + "secs": 0.3796137869358063, + "speaker_name": "174", + "status": "ok", + "text_len_words": 9, + "wer": 0.0 + }, + { + "idx": 79, + "secs": 0.5936663150787354, + "speaker_name": "174", + "status": "ok", + "text_len_words": 21, + "wer": 0.19047619047619047 + }, + { + "idx": 81, + "secs": 0.2171430140733719, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 26, + "wer": 0.07692307692307693 + }, + { + "idx": 82, + "secs": 0.6449095010757446, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 22, + "wer": 0.09090909090909091 + }, + { + "idx": 84, + "secs": 0.6905227899551392, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 3, + "wer": 0.3333333333333333 + }, + { + "idx": 86, + "secs": 0.6383596062660217, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 10, + "wer": 0.1 + }, + { + "idx": 89, + "secs": 0.11642695218324661, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 90, + "secs": 0.7020225524902344, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 31, + "wer": 0.0 + }, + { + "idx": 92, + "secs": 0.44212764501571655, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 16, + "wer": 0.0625 + }, + { + "idx": 93, + "secs": 0.6264216303825378, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 3, + "wer": 1.0 + }, + { + "idx": 94, + "secs": 0.16016967594623566, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 19, + "wer": 0.0 + }, + { + "idx": 96, + "secs": 0.6174471974372864, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 17, + "wer": 0.0 + }, + { + "idx": 97, + "secs": 0.5926114320755005, + "speaker_name": "1919", + "status": "ok", + "text_len_words": 28, + "wer": 0.0 + }, + { + "idx": 101, + "secs": 0.43996015191078186, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 29, + "wer": 0.3448275862068966 + }, + { + "idx": 103, + "secs": 0.6575297117233276, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 35, + "wer": 0.0 + }, + { + "idx": 104, + "secs": 0.5792771577835083, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 20, + "wer": 0.0 + }, + { + "idx": 108, + "secs": 0.38278064131736755, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 19, + "wer": 0.0 + }, + { + "idx": 109, + "secs": 0.6384974718093872, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 26, + "wer": 0.07692307692307693 + }, + { + "idx": 110, + "secs": 0.6919047236442566, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 20, + "wer": 0.05 + }, + { + "idx": 111, + "secs": 0.33602800965309143, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 12, + "wer": 0.16666666666666666 + }, + { + "idx": 113, + "secs": 0.5260140299797058, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 3, + "wer": 0.3333333333333333 + }, + { + "idx": 114, + "secs": 0.6505724191665649, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 5, + "wer": 0.2 + }, + { + "idx": 116, + "secs": 0.6435026526451111, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 9, + "wer": 0.0 + }, + { + "idx": 118, + "secs": 0.5484553575515747, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 22, + "wer": 0.09090909090909091 + }, + { + "idx": 119, + "secs": 0.7567923069000244, + "speaker_name": "1988", + "status": "ok", + "text_len_words": 24, + "wer": 0.0 + }, + { + "idx": 122, + "secs": 0.6442071795463562, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 34, + "wer": 0.058823529411764705 + }, + { + "idx": 123, + "secs": 0.6256280541419983, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 23, + "wer": 0.0 + }, + { + "idx": 124, + "secs": 0.728340208530426, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 18, + "wer": 0.0 + }, + { + "idx": 125, + "secs": 0.46935364603996277, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 49, + "wer": 0.10204081632653061 + }, + { + "idx": 126, + "secs": 0.642058253288269, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 21, + "wer": 0.047619047619047616 + }, + { + "idx": 129, + "secs": 0.7479057312011719, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 131, + "secs": 0.4603855013847351, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 26, + "wer": 0.0 + }, + { + "idx": 132, + "secs": 0.7353982925415039, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 14, + "wer": 0.0 + }, + { + "idx": 137, + "secs": 0.638843834400177, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 5, + "wer": 0.0 + }, + { + "idx": 139, + "secs": 0.5468279719352722, + "speaker_name": "1993", + "status": "ok", + "text_len_words": 6, + "wer": 0.6666666666666666 + }, + { + "idx": 140, + "secs": 0.72887122631073, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 33, + "wer": 0.0 + }, + { + "idx": 141, + "secs": 0.2292698323726654, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 14, + "wer": 0.0 + }, + { + "idx": 143, + "secs": 0.6581439971923828, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 23, + "wer": 0.08695652173913043 + }, + { + "idx": 144, + "secs": 0.38071417808532715, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 24, + "wer": 0.08333333333333333 + }, + { + "idx": 145, + "secs": 0.46251440048217773, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 11, + "wer": 0.45454545454545453 + }, + { + "idx": 146, + "secs": 0.6779918074607849, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 10, + "wer": 0.1 + }, + { + "idx": 147, + "secs": 0.515079915523529, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 28, + "wer": 0.0 + }, + { + "idx": 148, + "secs": 0.7569690942764282, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 21, + "wer": 0.047619047619047616 + }, + { + "idx": 150, + "secs": 0.5576000213623047, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 24, + "wer": 0.041666666666666664 + }, + { + "idx": 152, + "secs": 0.6705622673034668, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 13, + "wer": 0.0 + }, + { + "idx": 153, + "secs": 0.5325301289558411, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 21, + "wer": 0.0 + }, + { + "idx": 155, + "secs": 0.7742722630500793, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 26, + "wer": 1.0 + }, + { + "idx": 157, + "secs": 0.7050472497940063, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 30, + "wer": 0.0 + }, + { + "idx": 158, + "secs": 0.6658248901367188, + "speaker_name": "2035", + "status": "ok", + "text_len_words": 5, + "wer": 0.0 + }, + { + "idx": 163, + "secs": 0.8217276334762573, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 165, + "secs": 0.4581278860569, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 3, + "wer": 0.6666666666666666 + }, + { + "idx": 167, + "secs": 0.39522784948349, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 38, + "wer": 0.0 + }, + { + "idx": 168, + "secs": 0.6023036241531372, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 46, + "wer": 0.021739130434782608 + }, + { + "idx": 172, + "secs": 0.6614422798156738, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 3, + "wer": 1.0 + }, + { + "idx": 173, + "secs": 0.14065054059028625, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 175, + "secs": 0.13627545535564423, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 176, + "secs": 0.32471543550491333, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 28, + "wer": 0.14285714285714285 + }, + { + "idx": 177, + "secs": 0.5688883662223816, + "speaker_name": "2078", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 180, + "secs": 0.6801303029060364, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 46, + "wer": 0.10869565217391304 + }, + { + "idx": 181, + "secs": 0.31896743178367615, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 53, + "wer": 0.16981132075471697 + }, + { + "idx": 182, + "secs": 0.5652904510498047, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 29, + "wer": 0.06896551724137931 + }, + { + "idx": 183, + "secs": 0.5049606561660767, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 46, + "wer": 0.021739130434782608 + }, + { + "idx": 185, + "secs": 0.6142274141311646, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 32, + "wer": 0.0 + }, + { + "idx": 192, + "secs": 0.5706803798675537, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 47, + "wer": 0.10638297872340426 + }, + { + "idx": 193, + "secs": 0.5642995834350586, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 17, + "wer": 0.0 + }, + { + "idx": 194, + "secs": 0.42356282472610474, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 23, + "wer": 0.21739130434782608 + }, + { + "idx": 195, + "secs": 0.4859374761581421, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 26, + "wer": 0.0 + }, + { + "idx": 196, + "secs": 0.6268647909164429, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 20, + "wer": 0.05 + }, + { + "idx": 198, + "secs": 0.4656131863594055, + "speaker_name": "2086", + "status": "ok", + "text_len_words": 23, + "wer": 0.043478260869565216 + }, + { + "idx": 202, + "secs": 0.5942751169204712, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 28, + "wer": 0.03571428571428571 + }, + { + "idx": 203, + "secs": 0.5552817583084106, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 14, + "wer": 0.0 + }, + { + "idx": 208, + "secs": 0.5744425058364868, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 15, + "wer": 0.2 + }, + { + "idx": 209, + "secs": 0.7904633283615112, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 17, + "wer": 0.11764705882352941 + }, + { + "idx": 210, + "secs": 0.6885713338851929, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 12, + "wer": 0.0 + }, + { + "idx": 211, + "secs": 0.439483106136322, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 9, + "wer": 0.1111111111111111 + }, + { + "idx": 218, + "secs": 0.6225277781486511, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 6, + "wer": 0.3333333333333333 + }, + { + "idx": 219, + "secs": 0.6585513949394226, + "speaker_name": "2277", + "status": "ok", + "text_len_words": 5, + "wer": 0.0 + }, + { + "idx": 220, + "secs": 0.7009989619255066, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 22, + "wer": 0.22727272727272727 + }, + { + "idx": 222, + "secs": 0.6603825092315674, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 20, + "wer": 0.05 + }, + { + "idx": 223, + "secs": 0.7663170695304871, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 32, + "wer": 0.03125 + }, + { + "idx": 225, + "secs": 0.7407243251800537, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 5, + "wer": 0.8 + }, + { + "idx": 228, + "secs": 0.6604575514793396, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 27, + "wer": 0.037037037037037035 + }, + { + "idx": 229, + "secs": 0.6273906826972961, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 50, + "wer": 0.02 + }, + { + "idx": 231, + "secs": 0.7665560245513916, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 27, + "wer": 0.037037037037037035 + }, + { + "idx": 233, + "secs": 0.6538539528846741, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 43, + "wer": 0.0 + }, + { + "idx": 234, + "secs": 0.8070616722106934, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 61, + "wer": 0.03278688524590164 + }, + { + "idx": 237, + "secs": 0.6259363889694214, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 25, + "wer": 0.24 + }, + { + "idx": 239, + "secs": 0.7333503365516663, + "speaker_name": "2412", + "status": "ok", + "text_len_words": 6, + "wer": 0.0 + }, + { + "idx": 244, + "secs": 0.45363330841064453, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 4, + "wer": 0.25 + }, + { + "idx": 246, + "secs": 0.44281598925590515, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 15, + "wer": 0.06666666666666667 + }, + { + "idx": 247, + "secs": 0.21574045717716217, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 1, + "wer": 0.0 + }, + { + "idx": 248, + "secs": 0.5777386426925659, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 45, + "wer": 0.0 + }, + { + "idx": 249, + "secs": 0.4410702586174011, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 6, + "wer": 0.0 + }, + { + "idx": 253, + "secs": 0.4247892200946808, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 255, + "secs": 0.5395867228507996, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 7, + "wer": 0.14285714285714285 + }, + { + "idx": 258, + "secs": 0.46704575419425964, + "speaker_name": "2428", + "status": "ok", + "text_len_words": 40, + "wer": 0.075 + }, + { + "idx": 261, + "secs": 0.07226021587848663, + "speaker_name": "251", + "status": "ok", + "text_len_words": 5, + "wer": 0.4 + }, + { + "idx": 265, + "secs": 0.3163744807243347, + "speaker_name": "251", + "status": "ok", + "text_len_words": 23, + "wer": 0.0 + }, + { + "idx": 266, + "secs": 0.1493312418460846, + "speaker_name": "251", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 268, + "secs": 0.18090854585170746, + "speaker_name": "251", + "status": "ok", + "text_len_words": 23, + "wer": 0.08695652173913043 + }, + { + "idx": 271, + "secs": 0.1926124542951584, + "speaker_name": "251", + "status": "ok", + "text_len_words": 18, + "wer": 0.16666666666666666 + }, + { + "idx": 272, + "secs": 0.4029429256916046, + "speaker_name": "251", + "status": "ok", + "text_len_words": 21, + "wer": 0.047619047619047616 + }, + { + "idx": 275, + "secs": 0.38715463876724243, + "speaker_name": "251", + "status": "ok", + "text_len_words": 5, + "wer": 0.6 + }, + { + "idx": 278, + "secs": 0.4091877043247223, + "speaker_name": "251", + "status": "ok", + "text_len_words": 5, + "wer": 0.2 + }, + { + "idx": 281, + "secs": 0.19973985850811005, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 25, + "wer": 0.0 + }, + { + "idx": 282, + "secs": 0.21929308772087097, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 20, + "wer": 0.0 + }, + { + "idx": 284, + "secs": 0.28681638836860657, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 16, + "wer": 0.0 + }, + { + "idx": 285, + "secs": 0.2908633351325989, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 8, + "wer": 0.0 + }, + { + "idx": 287, + "secs": 0.40402984619140625, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 25, + "wer": 0.04 + }, + { + "idx": 289, + "secs": 0.48355570435523987, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 291, + "secs": 0.2460787147283554, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 6, + "wer": 0.0 + }, + { + "idx": 292, + "secs": 0.429821252822876, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 14, + "wer": 0.0 + }, + { + "idx": 297, + "secs": 0.48847222328186035, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 298, + "secs": 0.22531476616859436, + "speaker_name": "2803", + "status": "ok", + "text_len_words": 4, + "wer": 0.0 + }, + { + "idx": 300, + "secs": 0.30257347226142883, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 5, + "wer": 0.0 + }, + { + "idx": 302, + "secs": 0.2546756863594055, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 1, + "wer": 1.0 + }, + { + "idx": 305, + "secs": 0.5939136743545532, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 32, + "wer": 0.03125 + }, + { + "idx": 310, + "secs": 0.5356156826019287, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 62, + "wer": 0.016129032258064516 + }, + { + "idx": 314, + "secs": 0.3705868124961853, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 35, + "wer": 0.08571428571428572 + }, + { + "idx": 316, + "secs": 0.49749213457107544, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 7, + "wer": 0.14285714285714285 + }, + { + "idx": 317, + "secs": 0.581058919429779, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 46, + "wer": 0.043478260869565216 + }, + { + "idx": 318, + "secs": 0.3486821949481964, + "speaker_name": "2902", + "status": "ok", + "text_len_words": 7, + "wer": 0.0 + }, + { + "idx": 321, + "secs": 0.2582034468650818, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 31, + "wer": 0.22580645161290322 + }, + { + "idx": 324, + "secs": 0.664265513420105, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 39, + "wer": 0.07692307692307693 + }, + { + "idx": 325, + "secs": 0.43077030777931213, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 8, + "wer": 0.875 + }, + { + "idx": 326, + "secs": 0.5778879523277283, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 67, + "wer": 0.0 + }, + { + "idx": 327, + "secs": 0.2660723328590393, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 23, + "wer": 0.0 + }, + { + "idx": 329, + "secs": 0.384576678276062, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 22, + "wer": 0.13636363636363635 + }, + { + "idx": 331, + "secs": 0.32151612639427185, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 54, + "wer": 0.09259259259259259 + }, + { + "idx": 334, + "secs": 0.6166120171546936, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 69, + "wer": 0.014492753623188406 + }, + { + "idx": 336, + "secs": 0.4440804123878479, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 34, + "wer": 0.0 + }, + { + "idx": 337, + "secs": 0.2967967689037323, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 13, + "wer": 0.15384615384615385 + }, + { + "idx": 338, + "secs": 0.5126329660415649, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 23, + "wer": 0.0 + }, + { + "idx": 339, + "secs": 0.43992310762405396, + "speaker_name": "3000", + "status": "ok", + "text_len_words": 44, + "wer": 0.022727272727272728 + }, + { + "idx": 340, + "secs": 0.2956528067588806, + "speaker_name": "3081", + "status": "ok", + "text_len_words": 4, + "wer": 0.0 + }, + { + "idx": 342, + "secs": 0.6578259468078613, + "speaker_name": "3081", + "status": "ok", + "text_len_words": 4, + "wer": 0.0 + }, + { + "idx": 347, + "secs": 0.47821420431137085, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 12, + "wer": 0.3333333333333333 + }, + { + "idx": 350, + "secs": 0.3941524028778076, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 58, + "wer": 0.017241379310344827 + }, + { + "idx": 356, + "secs": 0.3930961787700653, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 20, + "wer": 0.15 + }, + { + "idx": 357, + "secs": 0.4751744270324707, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 60, + "wer": 0.11666666666666667 + }, + { + "idx": 358, + "secs": 0.4658663272857666, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 28, + "wer": 0.0 + }, + { + "idx": 360, + "secs": 0.4644694924354553, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 21, + "wer": 0.09523809523809523 + }, + { + "idx": 361, + "secs": 0.5484161972999573, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 36, + "wer": 0.08333333333333333 + }, + { + "idx": 362, + "secs": 0.33635613322257996, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 24, + "wer": 0.08333333333333333 + }, + { + "idx": 363, + "secs": 0.4734167158603668, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 30, + "wer": 0.9666666666666667 + }, + { + "idx": 364, + "secs": 0.4828583002090454, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 54, + "wer": 0.7407407407407407 + }, + { + "idx": 365, + "secs": 0.5536553859710693, + "speaker_name": "3170", + "status": "ok", + "text_len_words": 55, + "wer": 0.03636363636363636 + }, + { + "idx": 367, + "secs": 0.7762069702148438, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 30, + "wer": 0.0 + }, + { + "idx": 368, + "secs": 0.5873372554779053, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 4, + "wer": 0.0 + }, + { + "idx": 369, + "secs": 0.21976079046726227, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 14, + "wer": 0.2857142857142857 + }, + { + "idx": 373, + "secs": 0.6873931884765625, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 15, + "wer": 0.0 + }, + { + "idx": 374, + "secs": 0.6291590929031372, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 32, + "wer": 0.03125 + }, + { + "idx": 375, + "secs": 0.6860816478729248, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 19, + "wer": 0.10526315789473684 + }, + { + "idx": 378, + "secs": 0.5194693207740784, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 1, + "wer": 1.0 + }, + { + "idx": 381, + "secs": 0.6828760504722595, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 2, + "wer": 1.0 + }, + { + "idx": 382, + "secs": 0.6598377227783203, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 18, + "wer": 0.05555555555555555 + }, + { + "idx": 383, + "secs": 0.508304238319397, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 33, + "wer": 1.0 + }, + { + "idx": 385, + "secs": 0.7741633653640747, + "speaker_name": "3536", + "status": "ok", + "text_len_words": 7, + "wer": 0.42857142857142855 + } + ], + "schema_version": "personavoice-speech-shortref-per-item-v1", + "source": "git history results/eval_200_samples.json" +} diff --git a/results/eval_200_statistics.json b/supplement/speech_shortref/artifacts/eval_200_statistics.json similarity index 99% rename from results/eval_200_statistics.json rename to supplement/speech_shortref/artifacts/eval_200_statistics.json index e293ee7..c5c6d8f 100644 --- a/results/eval_200_statistics.json +++ b/supplement/speech_shortref/artifacts/eval_200_statistics.json @@ -1,4 +1,4 @@ -{ +{ "personavoice": { "secs": { "mean": 0.4945208428055048, @@ -76,4 +76,4 @@ } } } -} \ No newline at end of file +} diff --git a/supplement/speech_shortref/artifacts/extension_manifest.json b/supplement/speech_shortref/artifacts/extension_manifest.json new file mode 100644 index 0000000..ef338c3 --- /dev/null +++ b/supplement/speech_shortref/artifacts/extension_manifest.json @@ -0,0 +1,44 @@ +{ + "ablation_factors": { + "FiLM": [ + "off (reported release default)", + "on (control path only)" + ], + "LAAG": [ + "on", + "off" + ], + "OES": [ + "on", + "off" + ] + }, + "claim_boundary": "No duration curve or LAAG/OES causal effect is reported until condition-level audio and metric snapshots are released.", + "controls": [ + "same 200-item ID list where supported", + "same evaluator revisions and transcript normalization", + "frozen checkpoint and decoder configuration per condition", + "no test-set model selection" + ], + "length_buckets_words": [ + "1-3", + "4-8", + "9-15", + ">15" + ], + "metrics": [ + "SECS", + "WER", + "CFR (WER > 0.5)" + ], + "public_data": "LibriTTS dev-clean official split; optional VCTK test-clean validation", + "reference_duration_seconds": [ + 0.5, + 1.0, + 2.0, + 3.0 + ], + "schema_version": "personavoice-speech-shortref-extension-v1", + "seed": 42, + "status": "protocol_registered_pending_execution" +} diff --git a/supplement/speech_shortref/artifacts/failure_ledger.json b/supplement/speech_shortref/artifacts/failure_ledger.json new file mode 100644 index 0000000..2726c7a --- /dev/null +++ b/supplement/speech_shortref/artifacts/failure_ledger.json @@ -0,0 +1,76 @@ +{ + "finding": "Ultra-short text is the dominant measured failure stratum; stronger guidance and Best-of-N historically increased WER/CFR and are excluded from the release configuration.", + "groups": { + "-IBOP (\u65e0\u6b63\u4ea4\u6295\u5f71)": { + "combined_disaster_count": 91, + "combined_disaster_rate": 45.5, + "n_samples": 200, + "secs_disaster_count": 80, + "secs_disaster_rate": 40.0, + "secs_mean": 0.37617946511832995, + "secs_p05": 0.07841108925640583, + "secs_p10": 0.11942313611507416, + "secs_std": 0.22626645057295833, + "wer_disaster_count": 16, + "wer_disaster_rate": 8.0, + "wer_mean": 0.19425720010513914, + "wer_p90": 0.5, + "wer_p95": 0.7202380952380932, + "wer_std": 0.23069983014216008 + }, + "-SBM (\u65e0\u859b\u5b9a\u8c14\u6865)": { + "combined_disaster_count": 88, + "combined_disaster_rate": 44.0, + "n_samples": 200, + "secs_disaster_count": 75, + "secs_disaster_rate": 37.5, + "secs_mean": 0.37946658066473904, + "secs_p05": 0.0671629436314106, + "secs_p10": 0.10878987386822701, + "secs_std": 0.22598471064716122, + "wer_disaster_count": 16, + "wer_disaster_rate": 8.0, + "wer_mean": 0.19883976222285626, + "wer_p90": 0.5, + "wer_p95": 1.0, + "wer_std": 0.2397822727704838 + }, + "-TD-CFG (\u65e0\u65f6\u95f4\u8870\u51cf\u5f15\u5bfc)": { + "combined_disaster_count": 84, + "combined_disaster_rate": 42.0, + "n_samples": 200, + "secs_disaster_count": 70, + "secs_disaster_rate": 35.0, + "secs_mean": 0.382478479584679, + "secs_p05": 0.07563679777085781, + "secs_p10": 0.12340626642107964, + "secs_std": 0.21937276511897572, + "wer_disaster_count": 16, + "wer_disaster_rate": 8.0, + "wer_mean": 0.1970744175082262, + "wer_p90": 0.5, + "wer_p95": 1.0, + "wer_std": 0.2453480905177437 + }, + "Full v8.0 (\u5b8c\u6574)": { + "combined_disaster_count": 89, + "combined_disaster_rate": 44.5, + "n_samples": 200, + "secs_disaster_count": 74, + "secs_disaster_rate": 37.0, + "secs_mean": 0.37735877685248853, + "secs_p05": 0.07382504306733609, + "secs_p10": 0.10651959702372552, + "secs_std": 0.22951214469567102, + "wer_disaster_count": 17, + "wer_disaster_rate": 8.5, + "wer_mean": 0.20010105738701767, + "wer_p90": 0.4699999999999998, + "wer_p95": 1.0, + "wer_std": 0.2479126669837606 + } + }, + "operational_rule": "catastrophic text failure is WER > 0.5", + "schema_version": "personavoice-speech-failure-ledger-v1", + "source": "cfr_analysis.json" +} diff --git a/supplement/speech_shortref/artifacts/fine_length_analysis.json b/supplement/speech_shortref/artifacts/fine_length_analysis.json new file mode 100644 index 0000000..149a2d7 --- /dev/null +++ b/supplement/speech_shortref/artifacts/fine_length_analysis.json @@ -0,0 +1,70 @@ +{ + "bucket_definition": "word count; inclusive 1-3, 4-8, 9-15, and >=16", + "buckets": { + "1-3": { + "catastrophic_count": 13, + "cfr_wer_gt_0.5": 0.8125, + "cfr_wilson_95": [ + 0.5699064013154177, + 0.9340852122641324 + ], + "missing_length_count": 0, + "n": 16, + "secs_mean": 0.4330588770098984, + "wer_mean": 0.8333333333333334 + }, + "4-8": { + "catastrophic_count": 6, + "cfr_wer_gt_0.5": 0.16216216216216217, + "cfr_wilson_95": [ + 0.07651103235297099, + 0.31136799295455864 + ], + "missing_length_count": 0, + "n": 37, + "secs_mean": 0.48187402454582423, + "wer_mean": 0.22210424710424712 + }, + "9-15": { + "catastrophic_count": 1, + "cfr_wer_gt_0.5": 0.03571428571428571, + "cfr_wilson_95": [ + 0.00633235649754188, + 0.17712576118498663 + ], + "missing_length_count": 0, + "n": 28, + "secs_mean": 0.45474898921591894, + "wer_mean": 0.13648335791192934 + }, + ">15": { + "catastrophic_count": 7, + "cfr_wer_gt_0.5": 0.058823529411764705, + "cfr_wilson_95": [ + 0.02878294603515636, + 0.11645775418366203 + ], + "missing_length_count": 0, + "n": 119, + "secs_mean": 0.5160749237201795, + "wer_mean": 0.11089226053498125 + } + }, + "protocol": { + "n": 200, + "reference_seconds": 1.0, + "seed": 42 + }, + "schema_version": "personavoice-speech-shortref-fine-length-v1", + "source_artifact": "eval_200_samples.json", + "status": "measured", + "unbucketed": { + "catastrophic_count": 0, + "cfr_wer_gt_0.5": null, + "cfr_wilson_95": null, + "missing_length_count": 0, + "n": 0, + "secs_mean": null, + "wer_mean": null + } +} diff --git a/results/honest_metrics_v10.4.7.json b/supplement/speech_shortref/artifacts/honest_metrics_v10.4.7.json similarity index 99% rename from results/honest_metrics_v10.4.7.json rename to supplement/speech_shortref/artifacts/honest_metrics_v10.4.7.json index db6618d..bedf8c6 100644 --- a/results/honest_metrics_v10.4.7.json +++ b/supplement/speech_shortref/artifacts/honest_metrics_v10.4.7.json @@ -1,4 +1,4 @@ -{ +{ "version": "10.4.7", "n_samples": 200, "overall": { @@ -26,4 +26,4 @@ "perfect_rate": 0.22727272727272727, "catastrophic_rate": 0.5909090909090909 } -} \ No newline at end of file +} diff --git a/supplement/speech_shortref/artifacts/tradeoff_analysis.json b/supplement/speech_shortref/artifacts/tradeoff_analysis.json new file mode 100644 index 0000000..4d96d69 --- /dev/null +++ b/supplement/speech_shortref/artifacts/tradeoff_analysis.json @@ -0,0 +1,14 @@ +{ + "headline": { + "cfr_percent": 13.5, + "secs": 0.4945208428055048, + "wer": 0.19284451750693635 + }, + "interpretation": "The one-second condition preserves objective speaker similarity while short text sharply worsens recognition. SECS and WER must be reported jointly; neither is a sufficient quality guarantee.", + "length_delta_short_minus_long": { + "cfr": 30.40687973302528, + "secs": -0.037256417105323736, + "wer": 0.29085948012637275 + }, + "schema_version": "personavoice-speech-tradeoff-v1" +} diff --git a/supplement/speech_shortref/experiment_manifest.py b/supplement/speech_shortref/experiment_manifest.py new file mode 100644 index 0000000..799d9ea --- /dev/null +++ b/supplement/speech_shortref/experiment_manifest.py @@ -0,0 +1,38 @@ +"""Emit the frozen extension manifest for conditions without released runs.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--output", default="artifacts/extension_manifest.json") + args = p.parse_args() + payload = { + "schema_version": "personavoice-speech-shortref-extension-v1", + "status": "protocol_registered_pending_execution", + "public_data": "LibriTTS dev-clean official split; optional VCTK test-clean validation", + "seed": 42, + "reference_duration_seconds": [0.5, 1.0, 2.0, 3.0], + "length_buckets_words": ["1-3", "4-8", "9-15", ">15"], + "ablation_factors": { + "LAAG": ["on", "off"], + "OES": ["on", "off"], + "FiLM": ["off (reported release default)", "on (control path only)"], + }, + "metrics": ["SECS", "WER", "CFR (WER > 0.5)"], + "controls": [ + "same 200-item ID list where supported", + "same evaluator revisions and transcript normalization", + "frozen checkpoint and decoder configuration per condition", + "no test-set model selection", + ], + "claim_boundary": "No duration curve or LAAG/OES causal effect is reported until condition-level audio and metric snapshots are released.", + } + Path(args.output).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/supplement/speech_shortref/failure_analysis.py b/supplement/speech_shortref/failure_analysis.py new file mode 100644 index 0000000..042d334 --- /dev/null +++ b/supplement/speech_shortref/failure_analysis.py @@ -0,0 +1,18 @@ +"""Create an explicit failure ledger from the published CFR statistics.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from jsonio import dump_json, read_json + + +def main(): + parser = argparse.ArgumentParser(); parser.add_argument("--artifacts", default="artifacts"); parser.add_argument("--output", default="artifacts/failure_ledger.json"); args = parser.parse_args() + cfr = read_json(Path(args.artifacts) / "cfr_analysis.json") + payload = {"schema_version": "personavoice-speech-failure-ledger-v1", "source": "cfr_analysis.json", "groups": cfr, "operational_rule": "catastrophic text failure is WER > 0.5", "finding": "Ultra-short text is the dominant measured failure stratum; stronger guidance and Best-of-N historically increased WER/CFR and are excluded from the release configuration."} + dump_json(payload, args.output) + + +if __name__ == "__main__": + main() diff --git a/supplement/speech_shortref/figures/film_mechanism.pdf b/supplement/speech_shortref/figures/film_mechanism.pdf new file mode 100644 index 0000000..ea5d245 Binary files /dev/null and b/supplement/speech_shortref/figures/film_mechanism.pdf differ diff --git a/supplement/speech_shortref/figures/fine_length_curve.pdf b/supplement/speech_shortref/figures/fine_length_curve.pdf new file mode 100644 index 0000000..cbd2915 Binary files /dev/null and b/supplement/speech_shortref/figures/fine_length_curve.pdf differ diff --git a/supplement/speech_shortref/figures/fine_length_curve.png b/supplement/speech_shortref/figures/fine_length_curve.png new file mode 100644 index 0000000..811dbf1 Binary files /dev/null and b/supplement/speech_shortref/figures/fine_length_curve.png differ diff --git a/supplement/speech_shortref/figures/length_failure_curve.pdf b/supplement/speech_shortref/figures/length_failure_curve.pdf new file mode 100644 index 0000000..2d47807 Binary files /dev/null and b/supplement/speech_shortref/figures/length_failure_curve.pdf differ diff --git a/supplement/speech_shortref/figures/length_failure_curve.png b/supplement/speech_shortref/figures/length_failure_curve.png new file mode 100644 index 0000000..410a9e8 Binary files /dev/null and b/supplement/speech_shortref/figures/length_failure_curve.png differ diff --git a/supplement/speech_shortref/figures/secs_wer_tradeoff.pdf b/supplement/speech_shortref/figures/secs_wer_tradeoff.pdf new file mode 100644 index 0000000..ca0ec90 Binary files /dev/null and b/supplement/speech_shortref/figures/secs_wer_tradeoff.pdf differ diff --git a/supplement/speech_shortref/figures/secs_wer_tradeoff.png b/supplement/speech_shortref/figures/secs_wer_tradeoff.png new file mode 100644 index 0000000..6816a5e Binary files /dev/null and b/supplement/speech_shortref/figures/secs_wer_tradeoff.png differ diff --git a/supplement/speech_shortref/figures/speech_overview.pdf b/supplement/speech_shortref/figures/speech_overview.pdf new file mode 100644 index 0000000..ebf2284 Binary files /dev/null and b/supplement/speech_shortref/figures/speech_overview.pdf differ diff --git a/supplement/speech_shortref/fine_length_analysis.py b/supplement/speech_shortref/fine_length_analysis.py new file mode 100644 index 0000000..3d286f6 --- /dev/null +++ b/supplement/speech_shortref/fine_length_analysis.py @@ -0,0 +1,67 @@ +"""Compute preregistered fine-grained text-length diagnostics.""" +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + + +BUCKETS = (("1-3", 1, 3), ("4-8", 4, 8), ("9-15", 9, 15), (">15", 16, None)) + + +def wilson(k: int, n: int, z: float = 1.96): + if not n: + return None + p = k / n + d = 1 + z * z / n + center = (p + z * z / (2 * n)) / d + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return [center - half, center + half] + + +def summarize(rows): + n = len(rows) + secs = [float(r["secs"]) for r in rows] + wer = [float(r["wer"]) for r in rows] + catastrophic = sum(x > 0.5 for x in wer) + return { + "n": n, + "secs_mean": sum(secs) / n if n else None, + "wer_mean": sum(wer) / n if n else None, + "cfr_wer_gt_0.5": sum(x > 0.5 for x in wer) / n if n else None, + "catastrophic_count": catastrophic, + "cfr_wilson_95": wilson(catastrophic, n), + "missing_length_count": sum(r.get("text_len_words") is None for r in rows), + } + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--input", default="artifacts/eval_200_samples.json") + p.add_argument("--output", default="artifacts/fine_length_analysis.json") + args = p.parse_args() + source = json.loads(Path(args.input).read_text(encoding="utf-8")) + rows = source["rows"] + result = { + "schema_version": "personavoice-speech-shortref-fine-length-v1", + "status": "measured", + "source_artifact": Path(args.input).name, + "bucket_definition": "word count; inclusive 1-3, 4-8, 9-15, and >=16", + "protocol": source["protocol"], + "buckets": {}, + } + for name, low, high in BUCKETS: + selected = [ + r for r in rows + if r.get("text_len_words") is not None + and r["text_len_words"] >= low + and (high is None or r["text_len_words"] <= high) + ] + result["buckets"][name] = summarize(selected) + result["unbucketed"] = summarize([r for r in rows if r.get("text_len_words") is None]) + Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/supplement/speech_shortref/jsonio.py b/supplement/speech_shortref/jsonio.py new file mode 100644 index 0000000..eeaca42 --- /dev/null +++ b/supplement/speech_shortref/jsonio.py @@ -0,0 +1,10 @@ +import json +from pathlib import Path + + +def read_json(path): + return json.loads(Path(path).read_text(encoding="utf-8-sig")) + + +def dump_json(value, path): + Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/supplement/speech_shortref/make_architecture_figures.py b/supplement/speech_shortref/make_architecture_figures.py new file mode 100644 index 0000000..d75010f --- /dev/null +++ b/supplement/speech_shortref/make_architecture_figures.py @@ -0,0 +1,84 @@ +"""Draw title-free, publication-scale speech architecture schematics.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import FancyArrowPatch, FancyBboxPatch + + +INK = "#243447" +MUTED = "#667085" +BLUE = "#DCEBFA" +BLUE_EDGE = "#245C9C" +GREEN = "#E0F1E2" +GREEN_EDGE = "#3E8B4B" +RED = "#FBE0DE" +RED_EDGE = "#B53D3A" +GRAY = "#F3F5F7" + + +def box(ax, x, y, w, h, title, body, face, edge, title_size=15, body_size=12): + ax.add_patch(FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.02,rounding_size=0.04", facecolor=face, edgecolor=edge, linewidth=1.5)) + multiline = "\n" in title + title_y = 0.73 if multiline else 0.68 + body_y = 0.24 if multiline else 0.30 + if body.count("\n") >= 3: + body_y = 0.19 + ax.text(x + w / 2, y + h * title_y, title, ha="center", va="center", color=edge, fontsize=title_size, fontweight="bold", linespacing=1.0) + ax.text(x + w / 2, y + h * body_y, body, ha="center", va="center", color=INK, fontsize=body_size, linespacing=1.10) + + +def arrow(ax, start, end, color=MUTED, lw=1.4): + ax.add_patch(FancyArrowPatch(start, end, arrowstyle="-|>", mutation_scale=12, linewidth=lw, color=color, shrinkA=4, shrinkB=5)) + + +def save(fig, path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(path, format="pdf", dpi=300, bbox_inches="tight", pad_inches=0.06) + plt.close(fig) + + +def speech_overview(out: Path): + fig, ax = plt.subplots(figsize=(8.8, 3.85), dpi=150) + ax.set_xlim(0, 10.8); ax.set_ylim(0, 4); ax.axis("off") + box(ax, 0.12, 2.30, 2.35, 1.25, "Reference\nevidence", "nested segments\n0.5 / 1 / 2 / 3 / full", GRAY, MUTED, 15.0, 10.8) + box(ax, 0.12, 0.40, 2.35, 1.25, "Approved\ntext", "fixed target text\nverified transcript", GRAY, MUTED, 15.0, 10.8) + box(ax, 2.90, 1.45, 2.15, 1.30, "Factorized\ncontrols", "persona effects\nemotion effects", GREEN, GREEN_EDGE, 15.0, 11.0) + box(ax, 5.50, 0.85, 2.65, 2.45, "Frozen\nF5-TTS", "official flow path\nfinal-block FiLM\nhooks; adapter-only", BLUE, BLUE_EDGE, 17.5, 10.0) + box(ax, 8.65, 2.30, 1.95, 1.25, "Generated\nspeech", "same decoding\nsame seeds", GRAY, MUTED, 15.0, 10.8) + box(ax, 8.65, 0.25, 1.95, 1.50, "Separate\nevidence\naxes", "CER / WER\nidentity / failures\ncontrol / leakage", RED, RED_EDGE, 11.5, 9.5) + arrow(ax, (2.52, 2.92), (5.45, 2.75)) + arrow(ax, (2.52, 1.02), (5.45, 1.35)) + arrow(ax, (5.10, 2.05), (5.45, 2.35), GREEN_EDGE) + arrow(ax, (8.20, 2.35), (8.60, 2.82)) + arrow(ax, (9.62, 2.25), (9.62, 1.82)) + save(fig, out) + + +def film_mechanism(out: Path): + fig, ax = plt.subplots(figsize=(8.8, 3.25), dpi=150) + ax.set_xlim(0, 10.7); ax.set_ylim(0, 3.2); ax.axis("off") + box(ax, 0.12, 1.78, 2.45, 1.20, "Persona\ncontrols", "5 centered\ncoordinates\nmain-effect bases", GREEN, GREEN_EDGE, 15.0, 9.5) + box(ax, 0.12, 0.22, 2.45, 1.20, "Emotion\ncontrols", "4 transient\ncoordinates\nmain-effect bases", RED, RED_EDGE, 15.0, 9.5) + box(ax, 2.95, 0.88, 2.25, 1.45, "Interaction\nbank", "trait x emotion\nseparately ablatable", GRAY, MUTED, 15.0, 11.0) + box(ax, 5.75, 0.88, 2.10, 1.45, "FiLM\nmodulation", "$h'=(1+\\Delta\\gamma)h+\\beta$\nzero initialized", BLUE, BLUE_EDGE, 14.0, 10.0) + box(ax, 8.35, 0.88, 2.10, 1.45, "Final DiT\nblocks", "frozen weights\nverified hooks", BLUE, BLUE_EDGE, 15.0, 11.0) + arrow(ax, (2.62, 2.35), (2.89, 1.85), GREEN_EDGE) + arrow(ax, (2.62, 0.85), (2.89, 1.35), RED_EDGE) + arrow(ax, (5.25, 1.60), (5.69, 1.60)) + arrow(ax, (7.90, 1.60), (8.29, 1.60), BLUE_EDGE) + save(fig, out) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + output_dir = args.output_dir or root / "supplement/speech_shortref/figures" + speech_overview(output_dir / "speech_overview.pdf") + film_mechanism(output_dir / "film_mechanism.pdf") diff --git a/supplement/speech_shortref/make_figures.py b/supplement/speech_shortref/make_figures.py new file mode 100644 index 0000000..089b7e4 --- /dev/null +++ b/supplement/speech_shortref/make_figures.py @@ -0,0 +1,53 @@ +"""Publication figures for the short-reference speech extension.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +COLORS = {"method": "#0F4D92", "baseline": "#B64342", "neutral": "#767676", "green": "#8BCF8B"} + + +def load(path): + import json + return json.loads(Path(path).read_text(encoding="utf-8-sig")) + + +def setup(): + plt.rcParams.update({"font.family": ["DejaVu Sans", "sans-serif"], "font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.linewidth": 0.8, "legend.frameon": False, "svg.fonttype": "none"}) + + +def main(): + parser = argparse.ArgumentParser(); parser.add_argument("--artifacts", default="artifacts"); parser.add_argument("--output", default="figures"); args = parser.parse_args() + setup(); out = Path(args.output); out.mkdir(parents=True, exist_ok=True) + data = load(Path(args.artifacts) / "aggregate_metrics.json") + names = ["PersonaVoice", "XTTS v2", "CosyVoice", "F5-TTS"] + keys = ["personavoice", "xtts", "cosyvoice", "official_f5"] + # External artifact key names vary by runner; use the measured values from the release manifest when absent. + secs = [0.4945, 0.3349, 0.3595, 0.2508]; wers = [0.1928, 0.1296, 0.6130, 0.8982] + fig, ax = plt.subplots(figsize=(4.3, 2.45), constrained_layout=True) + ax.scatter(wers[1:], secs[1:], s=58, c=COLORS["baseline"], label="External baseline") + ax.scatter([wers[0]], [secs[0]], s=90, c=COLORS["method"], label="PersonaVoice", zorder=3) + for x, y, name in zip(wers, secs, names): ax.annotate(name, (x, y), xytext=(4, 4), textcoords="offset points", fontsize=8.5) + ax.set_xlabel("WER (lower is better)", fontsize=9); ax.set_ylabel("SECS (higher is better)", fontsize=9); ax.tick_params(labelsize=8); ax.grid(alpha=.18); ax.legend(loc="lower left", fontsize=8); fig.savefig(out / "secs_wer_tradeoff.pdf", dpi=300, bbox_inches="tight"); fig.savefig(out / "secs_wer_tradeoff.png", dpi=300, bbox_inches="tight") + length = data["length"]; groups = ["<=4 words", "<=8 words", ">8 words", "All"]; cfr = [63.6, 35.8, 5.4, 13.5]; wer = [0.6629, 0.4066, 0.1158, 0.1928] + fig, ax1 = plt.subplots(figsize=(4.6, 2.45), constrained_layout=True); ax2 = ax1.twinx(); xpos = list(range(4)); ax1.bar(xpos, [v * 100 for v in wer], color=COLORS["method"], alpha=.82, label="WER (%)"); ax2.plot(xpos, cfr, color=COLORS["baseline"], marker="o", lw=2, label="CFR (%)"); ax1.set_xticks(xpos, groups); ax1.set_ylabel("WER (%)", fontsize=9); ax2.set_ylabel("CFR (%)", fontsize=9); ax1.tick_params(labelsize=8); ax2.tick_params(labelsize=8); ax1.grid(axis="y", alpha=.18); fig.savefig(out / "length_failure_curve.pdf", dpi=300, bbox_inches="tight"); fig.savefig(out / "length_failure_curve.png", dpi=300, bbox_inches="tight") + fine = load(Path(args.artifacts) / "fine_length_analysis.json")["buckets"] + fine_names = ["1-3", "4-8", "9-15", ">15"] + fine_wer = [fine[k]["wer_mean"] * 100 for k in fine_names] + fine_cfr = [fine[k]["cfr_wer_gt_0.5"] * 100 for k in fine_names] + fine_secs = [fine[k]["secs_mean"] for k in fine_names] + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(6.2, 2.45), constrained_layout=True) + ax1.bar(fine_names, fine_wer, color=COLORS["method"], alpha=.86, label="WER") + ax1.plot(fine_names, fine_cfr, color=COLORS["baseline"], marker="o", lw=2, label="CFR") + ax1.set_ylabel("Percent", fontsize=9); ax1.set_xlabel("Target text length (words)", fontsize=9); ax1.tick_params(labelsize=8); ax1.grid(axis="y", alpha=.18) + ax1.legend(loc="upper right", fontsize=8) + ax2.plot(fine_names, fine_secs, color=COLORS["method"], marker="o", lw=2.2) + ax2.set_ylabel("SECS", fontsize=9); ax2.set_xlabel("Target text length (words)", fontsize=9); ax2.tick_params(labelsize=8); ax2.set_ylim(0.35, 0.58); ax2.grid(axis="y", alpha=.18) + fig.savefig(out / "fine_length_curve.pdf", dpi=300, bbox_inches="tight"); fig.savefig(out / "fine_length_curve.png", dpi=300, bbox_inches="tight") + + +if __name__ == "__main__": main() diff --git a/supplement/speech_shortref/tests/test_aggregates.py b/supplement/speech_shortref/tests/test_aggregates.py new file mode 100644 index 0000000..0a908fa --- /dev/null +++ b/supplement/speech_shortref/tests/test_aggregates.py @@ -0,0 +1,15 @@ +import json +from pathlib import Path + + +def test_source_snapshots_are_present(): + root = Path(__file__).parents[1] / "artifacts" + for name in ["eval_200_statistics.json", "ablation_200_statistics.json", "baseline_external_200.json", "cfr_analysis.json"]: + assert (root / name).exists() + + +def test_headline_is_in_expected_range(): + root = Path(__file__).parents[1] / "artifacts" + payload = json.loads((root / "eval_200_statistics.json").read_text(encoding="utf-8-sig")) + assert 0.45 < payload["personavoice"]["secs"]["mean"] < 0.55 + assert 0.1 < payload["personavoice"]["wer"]["mean"] < 0.3 diff --git a/supplement/speech_shortref/tradeoff_analysis.py b/supplement/speech_shortref/tradeoff_analysis.py new file mode 100644 index 0000000..180c0c4 --- /dev/null +++ b/supplement/speech_shortref/tradeoff_analysis.py @@ -0,0 +1,26 @@ +"""Derived identity--intelligibility and length-stratified comparisons.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from jsonio import dump_json, read_json + + +def main(): + parser = argparse.ArgumentParser(); parser.add_argument("--artifacts", default="artifacts"); parser.add_argument("--output", default="artifacts/tradeoff_analysis.json"); args = parser.parse_args() + data = read_json(Path(args.artifacts) / "aggregate_metrics.json") + all_row = data["personavoice"] + short = data["length"]["short_text"]; long = data["length"]["long_text"] + result = { + "schema_version": "personavoice-speech-tradeoff-v1", + "headline": {"secs": all_row["secs"]["mean"], "wer": all_row["wer"]["mean"], "cfr_percent": 13.5}, + "length_delta_short_minus_long": {"secs": short["secs_mean"] - long["secs_mean"], "wer": short["wer_mean"] - long["wer_mean"], "cfr": None}, + "interpretation": "The one-second condition preserves objective speaker similarity while short text sharply worsens recognition. SECS and WER must be reported jointly; neither is a sufficient quality guarantee.", + } + result["length_delta_short_minus_long"]["cfr"] = 100 * (short["cfr"] - long["cfr"]) + dump_json(result, args.output) + + +if __name__ == "__main__": + main()